feat: continuous live generator + vLLM/RAG lane in Data Flow
Live dashboard now feels truly real-time: - Background generator streams randomly-sized bursts of real rows into PostgreSQL, MySQL, MongoDB & Cassandra every ~4s (CDC picks them up). Throughput rises and falls; counters move in lock-step (base snapshot + generated). Runs only while the Live tab is polling (heartbeat-gated) so source tables do not grow unbounded; on/off toggle exposed in the UI. - New /api/federated/live/generator toggle; /live returns per-tick activity (last burst sizes, orders by region/status, event feed). - LiveDashboard: live-activity panel, orders-per-tick sparkline, event stream feed, burst-by-region/status charts, generator status + control. Data Flow graph now explains how data reaches the assistant: - Added ChromaDB -> RAG (LangChain) -> vLLM Gateway -> Knowledge Chat lane, with Trino / OpenMetadata / curated-masked feeding LLM context. Live model & embed metrics pulled from the RAG /config. New node/edge kinds + legend.
This commit is contained in:
mo
+275
-21
@@ -20,7 +20,7 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi import APIRouter, Body
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
router = APIRouter(prefix="/api/federated", tags=["federated"])
|
||||
@@ -297,45 +297,299 @@ def _live_business_aggs() -> dict[str, Any]:
|
||||
return data
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Continuous live generator — keeps the platform "alive": while the Live
|
||||
# dashboard is open it streams small, randomly-sized batches of business rows
|
||||
# into the real source databases (PostgreSQL / MySQL / MongoDB / Cassandra),
|
||||
# which CDC then propagates downstream. Batch sizes fluctuate every tick so the
|
||||
# throughput visibly goes up and down. It only runs while someone is watching
|
||||
# (the /live poll refreshes a heartbeat) so the tables don't grow unbounded.
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
import random as _rnd
|
||||
from collections import deque as _deque
|
||||
|
||||
_GEN: dict[str, Any] = {
|
||||
"enabled": True,
|
||||
"running": False,
|
||||
"interval": 4.0,
|
||||
"last_seen": 0.0,
|
||||
"last_tick": 0.0,
|
||||
"counts": {"orders": 0, "hr_events": 0, "supply_events": 0, "telemetry": 0},
|
||||
"last_batch": {"orders": 0, "hr_events": 0, "supply_events": 0, "telemetry": 0},
|
||||
"by_region": {},
|
||||
"by_status": {},
|
||||
"tick_value": 0.0,
|
||||
"feed": _deque(maxlen=14),
|
||||
"base": None,
|
||||
}
|
||||
_gen_lock = threading.Lock()
|
||||
_gen_conns: dict[str, Any] = {"pg": None, "mysql": None, "mongo": None, "cass": None}
|
||||
|
||||
_REGIONS = ["NA", "EU", "APAC", "LATAM", "EMEA", "MEA"]
|
||||
_CHANNELS = ["B2B", "B2C", "ONLINE", "PARTNER", "RETAIL"]
|
||||
_STATUSES = ["NEW", "PAID", "SHIPPED", "DELIVERED", "RETURNED", "CANCELLED"]
|
||||
_CURR = ["EUR", "USD", "GBP", "JPY"]
|
||||
_DEPTS = ["Engineering", "Sales", "Support", "Operations", "Finance", "HR", "Marketing"]
|
||||
_ROLES = ["Analyst", "Engineer", "Manager", "Lead", "Specialist", "Director"]
|
||||
_EVT = ["HIRE", "PROMOTION", "SALARY_CHANGE", "TRANSFER", "REVIEW", "EXIT"]
|
||||
_SUPPLY = ["INSERT", "UPDATE", "REPLENISH", "SHIPMENT", "RETURN"]
|
||||
_SRC = ["CRM", "ERP", "WMS", "API"]
|
||||
_METRICS = ["temperature", "humidity", "pressure", "voltage", "current"]
|
||||
|
||||
|
||||
def _gen_pg():
|
||||
import psycopg2
|
||||
import sql_console as s
|
||||
c = _gen_conns["pg"]
|
||||
if c is None or getattr(c, "closed", 1):
|
||||
c = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER,
|
||||
password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=6)
|
||||
c.autocommit = True
|
||||
_gen_conns["pg"] = c
|
||||
return c
|
||||
|
||||
|
||||
def _gen_mysql():
|
||||
import pymysql
|
||||
import sql_console as s
|
||||
c = _gen_conns["mysql"]
|
||||
if c is None:
|
||||
c = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER,
|
||||
password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=6,
|
||||
autocommit=True)
|
||||
_gen_conns["mysql"] = c
|
||||
else:
|
||||
c.ping(reconnect=True)
|
||||
return c
|
||||
|
||||
|
||||
def _gen_mongo():
|
||||
import sql_console as s
|
||||
c = _gen_conns["mongo"]
|
||||
if c is None:
|
||||
c = s._mongo_client()
|
||||
_gen_conns["mongo"] = c
|
||||
return c[s.MONGO_DB]
|
||||
|
||||
|
||||
def _gen_cass():
|
||||
import sql_console as s
|
||||
sess = _gen_conns["cass"]
|
||||
if sess is None:
|
||||
cluster = s._cass_cluster()
|
||||
sess = cluster.connect()
|
||||
_gen_conns["cass"] = sess
|
||||
return sess
|
||||
|
||||
|
||||
def _gen_reset(key: str):
|
||||
try:
|
||||
c = _gen_conns.get(key)
|
||||
if c is not None:
|
||||
c.close() if key != "cass" else c.cluster.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
_gen_conns[key] = None
|
||||
|
||||
|
||||
def _gen_orders(n: int):
|
||||
import datetime as dt
|
||||
now = dt.datetime.utcnow()
|
||||
by_r: dict[str, int] = {}
|
||||
by_s: dict[str, int] = {}
|
||||
val = 0.0
|
||||
rows = []
|
||||
for _ in range(n):
|
||||
r = _rnd.choice(_REGIONS)
|
||||
st = _rnd.choices(_STATUSES, weights=[5, 6, 5, 8, 2, 2])[0]
|
||||
ch = _rnd.choice(_CHANNELS)
|
||||
amt = round(_rnd.uniform(15, 9500), 2)
|
||||
rows.append((_rnd.randint(1, 20000), _rnd.randint(1, 5000), r, ch, now, amt, _rnd.choice(_CURR), st))
|
||||
by_r[r] = by_r.get(r, 0) + 1
|
||||
by_s[st] = by_s.get(st, 0) + 1
|
||||
val += amt
|
||||
cur = _gen_pg().cursor()
|
||||
cur.executemany(
|
||||
"INSERT INTO public.sales_orders "
|
||||
"(customer_id,product_id,region,sales_channel,order_ts,amount,currency,order_status) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", rows)
|
||||
return by_r, by_s, round(val, 2)
|
||||
|
||||
|
||||
def _gen_hr(n: int):
|
||||
import datetime as dt
|
||||
now = dt.datetime.utcnow()
|
||||
rows = [(_rnd.randint(1, 100000), _rnd.choice(_DEPTS), _rnd.choice(_ROLES),
|
||||
_rnd.choice(_REGIONS), _rnd.choice(_EVT), round(_rnd.uniform(-2000, 6000), 2), now)
|
||||
for _ in range(n)]
|
||||
cur = _gen_mysql().cursor()
|
||||
cur.executemany(
|
||||
"INSERT INTO employee_events "
|
||||
"(employee_id,department,role_name,region,event_type,salary_change,event_ts) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s)", rows)
|
||||
|
||||
|
||||
def _gen_supply(n: int):
|
||||
import datetime as dt
|
||||
import uuid
|
||||
now = dt.datetime.utcnow()
|
||||
docs = [{"event_id": str(uuid.uuid4()), "type": _rnd.choice(_SUPPLY),
|
||||
"region": _rnd.choice(_REGIONS), "source": _rnd.choice(_SRC),
|
||||
"amount": round(_rnd.uniform(10, 40000), 2), "ts": now.isoformat()}
|
||||
for _ in range(n)]
|
||||
if docs:
|
||||
_gen_mongo()["events"].insert_many(docs)
|
||||
|
||||
|
||||
def _gen_tel(n: int):
|
||||
import datetime as dt
|
||||
now = dt.datetime.utcnow()
|
||||
sess = _gen_cass()
|
||||
import sql_console as s
|
||||
cql = (f"INSERT INTO {s.CASS_KS}.device_metrics "
|
||||
"(device_id, metric_ts, metric_type, metric_value, payload) VALUES (%s,%s,%s,%s,%s)")
|
||||
for _ in range(n):
|
||||
sess.execute(cql, (f"device-{_rnd.randint(1, 99999)}", now,
|
||||
_rnd.choice(_METRICS), round(_rnd.uniform(0, 100), 3), ""))
|
||||
|
||||
|
||||
def _gen_tick():
|
||||
# fluctuating batch sizes, with the occasional spike, so throughput moves up & down
|
||||
no = _rnd.randint(2, 40)
|
||||
if _rnd.random() < 0.18:
|
||||
no += _rnd.randint(25, 70)
|
||||
nh = _rnd.randint(0, 18)
|
||||
ns = _rnd.randint(0, 16)
|
||||
nt = _rnd.randint(8, 55)
|
||||
by_r: dict[str, int] = {}
|
||||
by_s: dict[str, int] = {}
|
||||
val = 0.0
|
||||
try:
|
||||
by_r, by_s, val = _gen_orders(no)
|
||||
except Exception:
|
||||
_gen_reset("pg"); no = 0
|
||||
try:
|
||||
_gen_hr(nh)
|
||||
except Exception:
|
||||
_gen_reset("mysql"); nh = 0
|
||||
try:
|
||||
_gen_supply(ns)
|
||||
except Exception:
|
||||
_gen_reset("mongo"); ns = 0
|
||||
try:
|
||||
_gen_tel(nt)
|
||||
except Exception:
|
||||
_gen_reset("cass"); nt = 0
|
||||
with _gen_lock:
|
||||
c = _GEN["counts"]
|
||||
c["orders"] += no
|
||||
c["hr_events"] += nh
|
||||
c["supply_events"] += ns
|
||||
c["telemetry"] += nt
|
||||
_GEN["last_batch"] = {"orders": no, "hr_events": nh, "supply_events": ns, "telemetry": nt}
|
||||
_GEN["by_region"] = by_r
|
||||
_GEN["by_status"] = by_s
|
||||
_GEN["tick_value"] = val
|
||||
_GEN["last_tick"] = time.time()
|
||||
if no:
|
||||
top = max(by_r, key=by_r.get) if by_r else "—"
|
||||
_GEN["feed"].appendleft({
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"text": f"+{no} orders · €{int(val):,} · top {top} ({by_r.get(top, 0)}) · +{nt} telemetry · +{nh} HR",
|
||||
})
|
||||
|
||||
|
||||
def _gen_loop():
|
||||
while True:
|
||||
try:
|
||||
if _GEN["enabled"] and (time.time() - _GEN["last_seen"] < 25):
|
||||
_GEN["running"] = True
|
||||
_gen_tick()
|
||||
else:
|
||||
_GEN["running"] = False
|
||||
except Exception:
|
||||
_GEN["running"] = False
|
||||
time.sleep(max(2.0, float(_GEN["interval"])))
|
||||
|
||||
|
||||
threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
|
||||
|
||||
|
||||
@router.post("/live/generator")
|
||||
async def toggle_generator(body: dict = Body(default={})):
|
||||
if "enabled" in body:
|
||||
_GEN["enabled"] = bool(body["enabled"])
|
||||
if "interval" in body:
|
||||
try:
|
||||
_GEN["interval"] = max(2.0, min(30.0, float(body["interval"])))
|
||||
except Exception:
|
||||
pass
|
||||
_GEN["last_seen"] = time.time()
|
||||
return {"ok": True, "enabled": _GEN["enabled"], "interval": _GEN["interval"], "running": _GEN["running"]}
|
||||
|
||||
|
||||
@router.get("/live")
|
||||
async def get_live():
|
||||
import sql_console as s
|
||||
orders = s._table_row_count("postgres", "public.sales_orders") or 0
|
||||
# MySQL event_id is monotonic, so max(event_id) tracks inserts in real time
|
||||
# (the planner estimate only refreshes after ANALYZE).
|
||||
hr_res = _trino("SELECT max(event_id) FROM mysql_hr.hr.employee_events", 1)
|
||||
hr = 0
|
||||
if hr_res.get("ok"):
|
||||
try:
|
||||
hr = int((hr_res.get("rows") or [[0]])[0][0] or 0)
|
||||
except Exception:
|
||||
hr = 0
|
||||
if not hr:
|
||||
hr = s._table_row_count("mysql", "hr.employee_events") or 0
|
||||
supply = s._table_row_count("mongodb", "supplychain.events") or 0
|
||||
_GEN["last_seen"] = time.time() # heartbeat: keeps the generator running while watched
|
||||
|
||||
# Cassandra has no cheap estimate — reuse the exact count from the cached
|
||||
# federated matrix query when available.
|
||||
telemetry = 0
|
||||
if _marquee.get("data") is None:
|
||||
_load_marquee()
|
||||
mq = _marquee.get("data") or {}
|
||||
cass_base = 0
|
||||
for r in ((mq.get("matrix") or {}).get("rows") or []):
|
||||
if r.get("catalog") == "cassandra_telemetry":
|
||||
try:
|
||||
telemetry = int(r.get("records") or 0)
|
||||
cass_base = int(r.get("records") or 0)
|
||||
except Exception:
|
||||
telemetry = 0
|
||||
cass_base = 0
|
||||
|
||||
# One-time base snapshot of source sizes; every subsequent reading is
|
||||
# base + rows the generator has streamed in, so the counters move smoothly
|
||||
# and in lock-step with the live activity feed.
|
||||
with _gen_lock:
|
||||
if _GEN["base"] is None:
|
||||
_GEN["base"] = {
|
||||
"orders": s._table_row_count("postgres", "public.sales_orders") or 0,
|
||||
"hr_events": s._table_row_count("mysql", "hr.employee_events") or 0,
|
||||
"supply_events": s._table_row_count("mongodb", "supplychain.events") or 0,
|
||||
"telemetry": cass_base,
|
||||
}
|
||||
elif cass_base and not _GEN["base"].get("telemetry"):
|
||||
_GEN["base"]["telemetry"] = cass_base
|
||||
base = dict(_GEN["base"])
|
||||
gc = dict(_GEN["counts"])
|
||||
gen_view = {
|
||||
"enabled": _GEN["enabled"],
|
||||
"running": _GEN["running"],
|
||||
"interval": _GEN["interval"],
|
||||
"counts": dict(_GEN["counts"]),
|
||||
"last_batch": dict(_GEN["last_batch"]),
|
||||
"tick_value": _GEN["tick_value"],
|
||||
"by_region": [{"key": k, "count": v} for k, v in sorted(_GEN["by_region"].items(), key=lambda kv: -kv[1])],
|
||||
"by_status": [{"key": k, "count": v} for k, v in sorted(_GEN["by_status"].items(), key=lambda kv: -kv[1])],
|
||||
"feed": list(_GEN["feed"]),
|
||||
}
|
||||
|
||||
orders = base["orders"] + gc["orders"]
|
||||
hr = base["hr_events"] + gc["hr_events"]
|
||||
supply = base["supply_events"] + gc["supply_events"]
|
||||
telemetry = base["telemetry"] + gc["telemetry"]
|
||||
|
||||
avg_order = _avg_order_value()
|
||||
sources = [
|
||||
{"key": "orders", "label": "Orders", "engine": "PostgreSQL", "catalog": "postgres_sales", "rows": orders, "color": "#fbbf24"},
|
||||
{"key": "hr_events", "label": "HR events", "engine": "MySQL", "catalog": "mysql_hr", "rows": hr, "color": "#60a5fa"},
|
||||
{"key": "supply_events", "label": "Supply events", "engine": "MongoDB", "catalog": "mongodb_supplychain", "rows": supply, "color": "#a78bfa"},
|
||||
{"key": "telemetry", "label": "Telemetry", "engine": "Cassandra", "catalog": "cassandra_telemetry", "rows": telemetry, "color": "#22d3ee"},
|
||||
{"key": "orders", "label": "Orders", "engine": "PostgreSQL", "catalog": "postgres_sales", "rows": orders, "added": gc["orders"], "color": "#fbbf24"},
|
||||
{"key": "hr_events", "label": "HR events", "engine": "MySQL", "catalog": "mysql_hr", "rows": hr, "added": gc["hr_events"], "color": "#60a5fa"},
|
||||
{"key": "supply_events", "label": "Supply events", "engine": "MongoDB", "catalog": "mongodb_supplychain", "rows": supply, "added": gc["supply_events"], "color": "#a78bfa"},
|
||||
{"key": "telemetry", "label": "Telemetry", "engine": "Cassandra", "catalog": "cassandra_telemetry", "rows": telemetry, "added": gc["telemetry"], "color": "#22d3ee"},
|
||||
]
|
||||
return {
|
||||
"ok": True,
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"sources": sources,
|
||||
"generator": gen_view,
|
||||
"totals": {
|
||||
"records": orders + hr + supply + telemetry,
|
||||
"revenue_est": round(orders * avg_order, 2),
|
||||
|
||||
Reference in New Issue
Block a user