ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Generate-data button + generation-script viewer + vector DB explorer

Browse Source 
Data Flow tab:
- Prominent "Generate data" button (500 / 2K / 10K) that inserts a fresh
  burst of business rows into all source DBs on demand via a new
  POST /api/federated/generate (fresh connections, safe alongside the
  background streamer); result toast shows what was inserted, CDC streams it.
- "Scripts" button + a "View generation scripts" action on the Data Generator
  node open a modal listing every generator script with full source, served by
  GET /api/dataflow/scripts. Sources are the real files: the live streaming
  generator (sliced live out of trino_federated.py) and the Airflow per-source
  DAGs + Faker scripts (mounted read-only from infra/airflow into the API).

Knowledge Chat:
- New "Vector DB" explorer modal: shows the ChromaDB chunking config
  (RecursiveCharacterTextSplitter 800/120, all-MiniLM-L6-v2, 384-dim, HNSW),
  collections & documents, and the actual stored chunks with text, metadata and
  an embedding preview (bars + values) so you can see exactly how files are
  split and written as vectors.

Refactor: generator row-builders shared by the streamer and the on-demand burst.
This commit is contained in:
mo
2026-06-28 22:06:27 +00:00
parent 9059006cc2
commit 213350ec75
5 changed files with 528 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/trino_federated.py
+135 -24
View File
@@ -392,7 +392,17 @@ def _gen_reset(key: str):
_gen_conns[key] = None
def _gen_orders(n: int):
# Row builders — shared by the background streamer and the on-demand "Generate
# data" button, so both produce identical, realistic business rows.
_PG_INSERT = ("INSERT INTO public.sales_orders "
"(customer_id,product_id,region,sales_channel,order_ts,amount,currency,order_status) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)")
_MYSQL_INSERT = ("INSERT INTO employee_events "
"(employee_id,department,role_name,region,event_type,salary_change,event_ts) "
"VALUES (%s,%s,%s,%s,%s,%s,%s)")
def _order_rows(n: int):
import datetime as dt
now = dt.datetime.utcnow()
by_r: dict[str, int] = {}
@@ -408,49 +418,133 @@ def _gen_orders(n: int):
by_r[r] = by_r.get(r, 0) + 1
by_s[st] = by_s.get(st, 0) + 1
val += amt
cur = _gen_pg().cursor()
cur.executemany(
"INSERT INTO public.sales_orders "
"(customer_id,product_id,region,sales_channel,order_ts,amount,currency,order_status) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", rows)
return by_r, by_s, round(val, 2)
return rows, by_r, by_s, round(val, 2)
def _gen_hr(n: int):
def _hr_rows(n: int):
import datetime as dt
now = dt.datetime.utcnow()
rows = [(_rnd.randint(1, 100000), _rnd.choice(_DEPTS), _rnd.choice(_ROLES),
return [(_rnd.randint(1, 100000), _rnd.choice(_DEPTS), _rnd.choice(_ROLES),
_rnd.choice(_REGIONS), _rnd.choice(_EVT), round(_rnd.uniform(-2000, 6000), 2), now)
for _ in range(n)]
cur = _gen_mysql().cursor()
cur.executemany(
"INSERT INTO employee_events "
"(employee_id,department,role_name,region,event_type,salary_change,event_ts) "
"VALUES (%s,%s,%s,%s,%s,%s,%s)", rows)
def _gen_supply(n: int):
def _supply_docs(n: int):
import datetime as dt
import uuid
now = dt.datetime.utcnow()
docs = [{"event_id": str(uuid.uuid4()), "type": _rnd.choice(_SUPPLY),
return [{"event_id": str(uuid.uuid4()), "type": _rnd.choice(_SUPPLY),
"region": _rnd.choice(_REGIONS), "source": _rnd.choice(_SRC),
"amount": round(_rnd.uniform(10, 40000), 2), "ts": now.isoformat()}
for _ in range(n)]
def _tel_rows(n: int):
import datetime as dt
now = dt.datetime.utcnow()
return [(f"device-{_rnd.randint(1, 99999)}", now, _rnd.choice(_METRICS),
round(_rnd.uniform(0, 100), 3), "") for _ in range(n)]
_TEL_INSERT_TPL = ("INSERT INTO {ks}.device_metrics "
"(device_id, metric_ts, metric_type, metric_value, payload) VALUES (%s,%s,%s,%s,%s)")
def _gen_orders(n: int):
rows, by_r, by_s, val = _order_rows(n)
_gen_pg().cursor().executemany(_PG_INSERT, rows)
return by_r, by_s, val
def _gen_hr(n: int):
_gen_mysql().cursor().executemany(_MYSQL_INSERT, _hr_rows(n))
def _gen_supply(n: int):
docs = _supply_docs(n)
if docs:
_gen_mongo()["events"].insert_many(docs)
def _gen_tel(n: int):
import datetime as dt
now = dt.datetime.utcnow()
sess = _gen_cass()
import sql_console as s
cql = (f"INSERT INTO {s.CASS_KS}.device_metrics "
"(device_id, metric_ts, metric_type, metric_value, payload) VALUES (%s,%s,%s,%s,%s)")
for _ in range(n):
sess.execute(cql, (f"device-{_rnd.randint(1, 99999)}", now,
_rnd.choice(_METRICS), round(_rnd.uniform(0, 100), 3), ""))
sess = _gen_cass()
cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
for row in _tel_rows(n):
sess.execute(cql, row)
def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any]:
"""On-demand burst using FRESH short-lived connections (safe to run from a
request thread alongside the background streamer). Returns inserted counts."""
import sql_console as s
out = {"orders": 0, "hr_events": 0, "supply_events": 0, "telemetry": 0}
by_r: dict[str, int] = {}
by_s: dict[str, int] = {}
val = 0.0
if orders > 0:
try:
import psycopg2
rows, by_r, by_s, val = _order_rows(orders)
c = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER, password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=8)
try:
c.autocommit = True
c.cursor().executemany(_PG_INSERT, rows)
out["orders"] = orders
finally:
c.close()
except Exception:
pass
if hr > 0:
try:
import pymysql
c = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER, password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=8, autocommit=True)
try:
c.cursor().executemany(_MYSQL_INSERT, _hr_rows(hr))
out["hr_events"] = hr
finally:
c.close()
except Exception:
pass
if supply > 0:
try:
cli = s._mongo_client()
try:
cli[s.MONGO_DB]["events"].insert_many(_supply_docs(supply))
out["supply_events"] = supply
finally:
cli.close()
except Exception:
pass
if tel > 0:
try:
cluster = s._cass_cluster()
sess = cluster.connect()
try:
cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
for row in _tel_rows(tel):
sess.execute(cql, row)
out["telemetry"] = tel
finally:
cluster.shutdown()
except Exception:
pass
# fold into the live counters + feed so the dashboard reflects it instantly
with _gen_lock:
c = _GEN["counts"]
for k in out:
c[k] += out[k]
if out["orders"]:
_GEN["by_region"] = by_r
_GEN["by_status"] = by_s
_GEN["tick_value"] = val
top = max(by_r, key=by_r.get) if by_r else "—"
_GEN["feed"].appendleft({
"ts": datetime.now(timezone.utc).isoformat(),
"text": f"⚡ manual burst: +{out['orders']} orders · €{int(val):,} · top {top} · +{out['telemetry']} telemetry · +{out['hr_events']} HR · +{out['supply_events']} supply",
})
out["revenue"] = val
return out
def _gen_tick():
@@ -528,6 +622,23 @@ async def toggle_generator(body: dict = Body(default={})):
return {"ok": True, "enabled": _GEN["enabled"], "interval": _GEN["interval"], "running": _GEN["running"]}
@router.post("/generate")
async def generate_now(body: dict = Body(default={})):
"""Manual one-shot burst into the source systems (the Data Flow "Generate
data" button). `rows` controls the order volume; the other sources scale
with it. CDC streams everything downstream automatically."""
from starlette.concurrency import run_in_threadpool
rows = int(body.get("rows", 500) or 500)
rows = max(1, min(20000, rows))
orders = rows
hr = max(1, rows // 4)
supply = max(1, rows // 4)
tel = max(1, rows // 2)
out = await run_in_threadpool(_generate_once, orders, hr, supply, tel)
total = out["orders"] + out["hr_events"] + out["supply_events"] + out["telemetry"]
return {"ok": True, "requested": rows, "inserted": out, "total": total}
@router.get("/live")
async def get_live():
import sql_console as s