feat: continuous live generator + vLLM/RAG lane in Data Flow

Live dashboard now feels truly real-time:
- Background generator streams randomly-sized bursts of real rows into
  PostgreSQL, MySQL, MongoDB & Cassandra every ~4s (CDC picks them up).
  Throughput rises and falls; counters move in lock-step (base snapshot +
  generated). Runs only while the Live tab is polling (heartbeat-gated) so
  source tables do not grow unbounded; on/off toggle exposed in the UI.
- New /api/federated/live/generator toggle; /live returns per-tick activity
  (last burst sizes, orders by region/status, event feed).
- LiveDashboard: live-activity panel, orders-per-tick sparkline, event
  stream feed, burst-by-region/status charts, generator status + control.

Data Flow graph now explains how data reaches the assistant:
- Added ChromaDB -> RAG (LangChain) -> vLLM Gateway -> Knowledge Chat lane,
  with Trino / OpenMetadata / curated-masked feeding LLM context. Live model
  & embed metrics pulled from the RAG /config. New node/edge kinds + legend.
This commit is contained in:
mo
2026-06-28 21:37:41 +00:00
parent 8c72d1dc63
commit 9059006cc2
4 changed files with 429 additions and 26 deletions
+48
View File
@@ -52,6 +52,11 @@ NODES: list[dict[str, Any]] = [
# governance — bottom centre # governance — bottom centre
{"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance", {"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance",
"x": 47, "y": 93, "url": "http://10.0.21.47:8585"}, "x": 47, "y": 93, "url": "http://10.0.21.47:8585"},
# AI serving lane — how governed data reaches the LLM & the Command Center chat
{"id": "chromadb", "label": "ChromaDB", "sub": "vectors · embeddings", "kind": "vector", "x": 60, "y": 65},
{"id": "rag", "label": "RAG · LangChain", "sub": "retrieve · augment · agent", "kind": "rag", "x": 74, "y": 65},
{"id": "vllm", "label": "vLLM Gateway", "sub": "Llama3-70B · GPT-4o", "kind": "llm", "x": 88, "y": 68},
{"id": "chat", "label": "Knowledge Chat", "sub": "Command Center", "kind": "chat", "x": 92, "y": 90},
] ]
# Edges. movement_id (optional) links to movements.py so the edge is triggerable. # Edges. movement_id (optional) links to movements.py so the edge is triggerable.
@@ -86,6 +91,14 @@ EDGES: list[dict[str, Any]] = [
{"from": "cassandra", "to": "openmetadata", "kind": "catalog"}, {"from": "cassandra", "to": "openmetadata", "kind": "catalog"},
{"from": "neo4j", "to": "openmetadata", "kind": "catalog"}, {"from": "neo4j", "to": "openmetadata", "kind": "catalog"},
{"from": "trino", "to": "openmetadata", "kind": "catalog"}, {"from": "trino", "to": "openmetadata", "kind": "catalog"},
# AI serving lane: governed business data + catalog + vectors → RAG → vLLM → chat
{"from": "trino", "to": "rag", "kind": "context"},
{"from": "openmetadata", "to": "rag", "kind": "context"},
{"from": "iceberg_curated", "to": "rag", "kind": "context"},
{"from": "chromadb", "to": "rag", "kind": "retrieve"},
{"from": "rag", "to": "vllm", "kind": "prompt"},
{"from": "vllm", "to": "chat", "kind": "answer"},
{"from": "rag", "to": "chat", "kind": "answer"},
] ]
_cache: dict[str, Any] = {"ts": 0.0, "data": None} _cache: dict[str, Any] = {"ts": 0.0, "data": None}
@@ -117,6 +130,25 @@ def _trino_scalar(sql: str, deadline_s: float = 8.0) -> int | None:
return None return None
RAG_URL = os.getenv("RAG_URL", "http://rag-api:5020").rstrip("/")
_rag_cache: dict[str, Any] = {"ts": 0.0, "val": None}
def _rag_info() -> dict[str, Any]:
now = time.time()
if _rag_cache["val"] is not None and now - _rag_cache["ts"] < 60:
return _rag_cache["val"]
info: dict[str, Any] = {}
try:
with httpx.Client(timeout=2.0) as client:
info = client.get(f"{RAG_URL}/config").json() or {}
except Exception:
info = {}
_rag_cache["val"] = info
_rag_cache["ts"] = now
return info
def _iceberg_hadoop_count() -> int | None: def _iceberg_hadoop_count() -> int | None:
now = time.time() now = time.time()
if _count_cache["val"] is not None and now - _count_cache["ts"] < 60: if _count_cache["val"] is not None and now - _count_cache["ts"] < 60:
@@ -180,6 +212,19 @@ async def _build() -> dict[str, Any]:
metric = f"{c:,} rows" if c is not None else "iceberg table" metric = f"{c:,} rows" if c is not None else "iceberg table"
elif n["id"] == "generator": elif n["id"] == "generator":
metric = "Airflow gen DAGs" metric = "Airflow gen DAGs"
elif n["id"] in ("vllm", "rag", "chromadb"):
info = _rag_info()
model = info.get("llm_model") or "gpt-4o"
embed = (info.get("embed_model") or "all-MiniLM-L6-v2").split("/")[-1]
if n["id"] == "vllm":
metric = f"{model} · OpenAI-compat"
elif n["id"] == "rag":
metric = f"LangChain · {embed}"
else:
metric = f"embeddings · {embed}"
node["level"] = "ok" if info else "warn"
elif n["id"] == "chat":
metric = "RAG chat · agent mode"
# PII overlay # PII overlay
p = pii_by_node.get(n["id"]) p = pii_by_node.get(n["id"])
if p: if p:
@@ -219,6 +264,9 @@ async def _build() -> dict[str, Any]:
edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active") edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
elif e.get("from") == "spark" and e.get("to") == "s3_cdc": elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active") edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
# AI serving lane pulses while governed data is being served to the LLM
edge["active"] = bool(_rag_info())
if e.get("offload"): if e.get("offload"):
try: try:
from agent_ops import custodian_recent from agent_ops import custodian_recent
+275 -21
View File
@@ -20,7 +20,7 @@ from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from fastapi import APIRouter from fastapi import APIRouter, Body
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
router = APIRouter(prefix="/api/federated", tags=["federated"]) router = APIRouter(prefix="/api/federated", tags=["federated"])
@@ -297,45 +297,299 @@ def _live_business_aggs() -> dict[str, Any]:
return data return data
# ──────────────────────────────────────────────────────────────────────────────
# Continuous live generator — keeps the platform "alive": while the Live
# dashboard is open it streams small, randomly-sized batches of business rows
# into the real source databases (PostgreSQL / MySQL / MongoDB / Cassandra),
# which CDC then propagates downstream. Batch sizes fluctuate every tick so the
# throughput visibly goes up and down. It only runs while someone is watching
# (the /live poll refreshes a heartbeat) so the tables don't grow unbounded.
# ──────────────────────────────────────────────────────────────────────────────
import random as _rnd
from collections import deque as _deque
_GEN: dict[str, Any] = {
"enabled": True,
"running": False,
"interval": 4.0,
"last_seen": 0.0,
"last_tick": 0.0,
"counts": {"orders": 0, "hr_events": 0, "supply_events": 0, "telemetry": 0},
"last_batch": {"orders": 0, "hr_events": 0, "supply_events": 0, "telemetry": 0},
"by_region": {},
"by_status": {},
"tick_value": 0.0,
"feed": _deque(maxlen=14),
"base": None,
}
_gen_lock = threading.Lock()
_gen_conns: dict[str, Any] = {"pg": None, "mysql": None, "mongo": None, "cass": None}
_REGIONS = ["NA", "EU", "APAC", "LATAM", "EMEA", "MEA"]
_CHANNELS = ["B2B", "B2C", "ONLINE", "PARTNER", "RETAIL"]
_STATUSES = ["NEW", "PAID", "SHIPPED", "DELIVERED", "RETURNED", "CANCELLED"]
_CURR = ["EUR", "USD", "GBP", "JPY"]
_DEPTS = ["Engineering", "Sales", "Support", "Operations", "Finance", "HR", "Marketing"]
_ROLES = ["Analyst", "Engineer", "Manager", "Lead", "Specialist", "Director"]
_EVT = ["HIRE", "PROMOTION", "SALARY_CHANGE", "TRANSFER", "REVIEW", "EXIT"]
_SUPPLY = ["INSERT", "UPDATE", "REPLENISH", "SHIPMENT", "RETURN"]
_SRC = ["CRM", "ERP", "WMS", "API"]
_METRICS = ["temperature", "humidity", "pressure", "voltage", "current"]
def _gen_pg():
import psycopg2
import sql_console as s
c = _gen_conns["pg"]
if c is None or getattr(c, "closed", 1):
c = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER,
password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=6)
c.autocommit = True
_gen_conns["pg"] = c
return c
def _gen_mysql():
import pymysql
import sql_console as s
c = _gen_conns["mysql"]
if c is None:
c = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER,
password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=6,
autocommit=True)
_gen_conns["mysql"] = c
else:
c.ping(reconnect=True)
return c
def _gen_mongo():
import sql_console as s
c = _gen_conns["mongo"]
if c is None:
c = s._mongo_client()
_gen_conns["mongo"] = c
return c[s.MONGO_DB]
def _gen_cass():
import sql_console as s
sess = _gen_conns["cass"]
if sess is None:
cluster = s._cass_cluster()
sess = cluster.connect()
_gen_conns["cass"] = sess
return sess
def _gen_reset(key: str):
try:
c = _gen_conns.get(key)
if c is not None:
c.close() if key != "cass" else c.cluster.shutdown()
except Exception:
pass
_gen_conns[key] = None
def _gen_orders(n: int):
import datetime as dt
now = dt.datetime.utcnow()
by_r: dict[str, int] = {}
by_s: dict[str, int] = {}
val = 0.0
rows = []
for _ in range(n):
r = _rnd.choice(_REGIONS)
st = _rnd.choices(_STATUSES, weights=[5, 6, 5, 8, 2, 2])[0]
ch = _rnd.choice(_CHANNELS)
amt = round(_rnd.uniform(15, 9500), 2)
rows.append((_rnd.randint(1, 20000), _rnd.randint(1, 5000), r, ch, now, amt, _rnd.choice(_CURR), st))
by_r[r] = by_r.get(r, 0) + 1
by_s[st] = by_s.get(st, 0) + 1
val += amt
cur = _gen_pg().cursor()
cur.executemany(
"INSERT INTO public.sales_orders "
"(customer_id,product_id,region,sales_channel,order_ts,amount,currency,order_status) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", rows)
return by_r, by_s, round(val, 2)
def _gen_hr(n: int):
import datetime as dt
now = dt.datetime.utcnow()
rows = [(_rnd.randint(1, 100000), _rnd.choice(_DEPTS), _rnd.choice(_ROLES),
_rnd.choice(_REGIONS), _rnd.choice(_EVT), round(_rnd.uniform(-2000, 6000), 2), now)
for _ in range(n)]
cur = _gen_mysql().cursor()
cur.executemany(
"INSERT INTO employee_events "
"(employee_id,department,role_name,region,event_type,salary_change,event_ts) "
"VALUES (%s,%s,%s,%s,%s,%s,%s)", rows)
def _gen_supply(n: int):
import datetime as dt
import uuid
now = dt.datetime.utcnow()
docs = [{"event_id": str(uuid.uuid4()), "type": _rnd.choice(_SUPPLY),
"region": _rnd.choice(_REGIONS), "source": _rnd.choice(_SRC),
"amount": round(_rnd.uniform(10, 40000), 2), "ts": now.isoformat()}
for _ in range(n)]
if docs:
_gen_mongo()["events"].insert_many(docs)
def _gen_tel(n: int):
import datetime as dt
now = dt.datetime.utcnow()
sess = _gen_cass()
import sql_console as s
cql = (f"INSERT INTO {s.CASS_KS}.device_metrics "
"(device_id, metric_ts, metric_type, metric_value, payload) VALUES (%s,%s,%s,%s,%s)")
for _ in range(n):
sess.execute(cql, (f"device-{_rnd.randint(1, 99999)}", now,
_rnd.choice(_METRICS), round(_rnd.uniform(0, 100), 3), ""))
def _gen_tick():
# fluctuating batch sizes, with the occasional spike, so throughput moves up & down
no = _rnd.randint(2, 40)
if _rnd.random() < 0.18:
no += _rnd.randint(25, 70)
nh = _rnd.randint(0, 18)
ns = _rnd.randint(0, 16)
nt = _rnd.randint(8, 55)
by_r: dict[str, int] = {}
by_s: dict[str, int] = {}
val = 0.0
try:
by_r, by_s, val = _gen_orders(no)
except Exception:
_gen_reset("pg"); no = 0
try:
_gen_hr(nh)
except Exception:
_gen_reset("mysql"); nh = 0
try:
_gen_supply(ns)
except Exception:
_gen_reset("mongo"); ns = 0
try:
_gen_tel(nt)
except Exception:
_gen_reset("cass"); nt = 0
with _gen_lock:
c = _GEN["counts"]
c["orders"] += no
c["hr_events"] += nh
c["supply_events"] += ns
c["telemetry"] += nt
_GEN["last_batch"] = {"orders": no, "hr_events": nh, "supply_events": ns, "telemetry": nt}
_GEN["by_region"] = by_r
_GEN["by_status"] = by_s
_GEN["tick_value"] = val
_GEN["last_tick"] = time.time()
if no:
top = max(by_r, key=by_r.get) if by_r else ""
_GEN["feed"].appendleft({
"ts": datetime.now(timezone.utc).isoformat(),
"text": f"+{no} orders · €{int(val):,} · top {top} ({by_r.get(top, 0)}) · +{nt} telemetry · +{nh} HR",
})
def _gen_loop():
while True:
try:
if _GEN["enabled"] and (time.time() - _GEN["last_seen"] < 25):
_GEN["running"] = True
_gen_tick()
else:
_GEN["running"] = False
except Exception:
_GEN["running"] = False
time.sleep(max(2.0, float(_GEN["interval"])))
threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
@router.post("/live/generator")
async def toggle_generator(body: dict = Body(default={})):
if "enabled" in body:
_GEN["enabled"] = bool(body["enabled"])
if "interval" in body:
try:
_GEN["interval"] = max(2.0, min(30.0, float(body["interval"])))
except Exception:
pass
_GEN["last_seen"] = time.time()
return {"ok": True, "enabled": _GEN["enabled"], "interval": _GEN["interval"], "running": _GEN["running"]}
@router.get("/live") @router.get("/live")
async def get_live(): async def get_live():
import sql_console as s import sql_console as s
orders = s._table_row_count("postgres", "public.sales_orders") or 0 _GEN["last_seen"] = time.time() # heartbeat: keeps the generator running while watched
# MySQL event_id is monotonic, so max(event_id) tracks inserts in real time
# (the planner estimate only refreshes after ANALYZE).
hr_res = _trino("SELECT max(event_id) FROM mysql_hr.hr.employee_events", 1)
hr = 0
if hr_res.get("ok"):
try:
hr = int((hr_res.get("rows") or [[0]])[0][0] or 0)
except Exception:
hr = 0
if not hr:
hr = s._table_row_count("mysql", "hr.employee_events") or 0
supply = s._table_row_count("mongodb", "supplychain.events") or 0
# Cassandra has no cheap estimate — reuse the exact count from the cached # Cassandra has no cheap estimate — reuse the exact count from the cached
# federated matrix query when available. # federated matrix query when available.
telemetry = 0
if _marquee.get("data") is None: if _marquee.get("data") is None:
_load_marquee() _load_marquee()
mq = _marquee.get("data") or {} mq = _marquee.get("data") or {}
cass_base = 0
for r in ((mq.get("matrix") or {}).get("rows") or []): for r in ((mq.get("matrix") or {}).get("rows") or []):
if r.get("catalog") == "cassandra_telemetry": if r.get("catalog") == "cassandra_telemetry":
try: try:
telemetry = int(r.get("records") or 0) cass_base = int(r.get("records") or 0)
except Exception: except Exception:
telemetry = 0 cass_base = 0
# One-time base snapshot of source sizes; every subsequent reading is
# base + rows the generator has streamed in, so the counters move smoothly
# and in lock-step with the live activity feed.
with _gen_lock:
if _GEN["base"] is None:
_GEN["base"] = {
"orders": s._table_row_count("postgres", "public.sales_orders") or 0,
"hr_events": s._table_row_count("mysql", "hr.employee_events") or 0,
"supply_events": s._table_row_count("mongodb", "supplychain.events") or 0,
"telemetry": cass_base,
}
elif cass_base and not _GEN["base"].get("telemetry"):
_GEN["base"]["telemetry"] = cass_base
base = dict(_GEN["base"])
gc = dict(_GEN["counts"])
gen_view = {
"enabled": _GEN["enabled"],
"running": _GEN["running"],
"interval": _GEN["interval"],
"counts": dict(_GEN["counts"]),
"last_batch": dict(_GEN["last_batch"]),
"tick_value": _GEN["tick_value"],
"by_region": [{"key": k, "count": v} for k, v in sorted(_GEN["by_region"].items(), key=lambda kv: -kv[1])],
"by_status": [{"key": k, "count": v} for k, v in sorted(_GEN["by_status"].items(), key=lambda kv: -kv[1])],
"feed": list(_GEN["feed"]),
}
orders = base["orders"] + gc["orders"]
hr = base["hr_events"] + gc["hr_events"]
supply = base["supply_events"] + gc["supply_events"]
telemetry = base["telemetry"] + gc["telemetry"]
avg_order = _avg_order_value() avg_order = _avg_order_value()
sources = [ sources = [
{"key": "orders", "label": "Orders", "engine": "PostgreSQL", "catalog": "postgres_sales", "rows": orders, "color": "#fbbf24"}, {"key": "orders", "label": "Orders", "engine": "PostgreSQL", "catalog": "postgres_sales", "rows": orders, "added": gc["orders"], "color": "#fbbf24"},
{"key": "hr_events", "label": "HR events", "engine": "MySQL", "catalog": "mysql_hr", "rows": hr, "color": "#60a5fa"}, {"key": "hr_events", "label": "HR events", "engine": "MySQL", "catalog": "mysql_hr", "rows": hr, "added": gc["hr_events"], "color": "#60a5fa"},
{"key": "supply_events", "label": "Supply events", "engine": "MongoDB", "catalog": "mongodb_supplychain", "rows": supply, "color": "#a78bfa"}, {"key": "supply_events", "label": "Supply events", "engine": "MongoDB", "catalog": "mongodb_supplychain", "rows": supply, "added": gc["supply_events"], "color": "#a78bfa"},
{"key": "telemetry", "label": "Telemetry", "engine": "Cassandra", "catalog": "cassandra_telemetry", "rows": telemetry, "color": "#22d3ee"}, {"key": "telemetry", "label": "Telemetry", "engine": "Cassandra", "catalog": "cassandra_telemetry", "rows": telemetry, "added": gc["telemetry"], "color": "#22d3ee"},
] ]
return { return {
"ok": True, "ok": True,
"ts": datetime.now(timezone.utc).isoformat(), "ts": datetime.now(timezone.utc).isoformat(),
"sources": sources, "sources": sources,
"generator": gen_view,
"totals": { "totals": {
"records": orders + hr + supply + telemetry, "records": orders + hr + supply + telemetry,
"revenue_est": round(orders * avg_order, 2), "revenue_est": round(orders * avg_order, 2),
@@ -18,6 +18,10 @@ const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' }, compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' }, engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' }, governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
vector: { ring: 'border-teal-400/60', chip: 'bg-teal-500/15 text-teal-300 border-teal-400/40', dot: '#2dd4bf' },
rag: { ring: 'border-pink-400/60', chip: 'bg-pink-500/15 text-pink-300 border-pink-400/40', dot: '#f472b6' },
llm: { ring: 'border-rose-400/70', chip: 'bg-rose-500/15 text-rose-200 border-rose-400/50', dot: '#fb7185' },
chat: { ring: 'border-indigo-400/60', chip: 'bg-indigo-500/15 text-indigo-300 border-indigo-400/40', dot: '#818cf8' },
} }
const EDGE_COLOR: Record<string, string> = { const EDGE_COLOR: Record<string, string> = {
@@ -28,6 +32,10 @@ const EDGE_COLOR: Record<string, string> = {
mask: '#fb7185', mask: '#fb7185',
query: '#818cf8', query: '#818cf8',
catalog: '#d946ef', catalog: '#d946ef',
context: '#2dd4bf',
retrieve: '#f472b6',
prompt: '#fb7185',
answer: '#818cf8',
} }
const EDGE_LEGEND: { kind: string; label: string }[] = [ const EDGE_LEGEND: { kind: string; label: string }[] = [
@@ -38,6 +46,10 @@ const EDGE_LEGEND: { kind: string; label: string }[] = [
{ kind: 'mask', label: 'PII masking' }, { kind: 'mask', label: 'PII masking' },
{ kind: 'query', label: 'Query' }, { kind: 'query', label: 'Query' },
{ kind: 'catalog', label: 'Catalog (OpenMetadata)' }, { kind: 'catalog', label: 'Catalog (OpenMetadata)' },
{ kind: 'context', label: 'LLM context' },
{ kind: 'retrieve', label: 'Vector retrieve' },
{ kind: 'prompt', label: 'Prompt' },
{ kind: 'answer', label: 'Answer → chat' },
] ]
type Anchor = { x: number; y: number; w: number; h: number } type Anchor = { x: number; y: number; w: number; h: number }
+94 -5
View File
@@ -1,14 +1,26 @@
import { useCallback, useEffect, useRef, useState } from 'react' import { useCallback, useEffect, useRef, useState } from 'react'
import { Activity, Pause, Play, ShoppingCart, Users, Boxes, Cpu, DollarSign, Database, Gauge } from 'lucide-react' import { Activity, Pause, Play, ShoppingCart, Users, Boxes, Cpu, DollarSign, Database, Gauge, Zap, Sparkles, Radio } from 'lucide-react'
import { cn } from '../../lib/utils' import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number } type Bucket = { key: string; count: number; value?: number }
type Source = { key: string; label: string; engine: string; catalog: string; rows: number; color: string } type Source = { key: string; label: string; engine: string; catalog: string; rows: number; added?: number; color: string }
type RegionRow = { region: string; orders: number; revenue: number; hr_events: number; supply_events: number } type RegionRow = { region: string; orders: number; revenue: number; hr_events: number; supply_events: number }
type Gen = {
enabled: boolean
running: boolean
interval: number
counts: { orders: number; hr_events: number; supply_events: number; telemetry: number }
last_batch: { orders: number; hr_events: number; supply_events: number; telemetry: number }
tick_value: number
by_region: Bucket[]
by_status: Bucket[]
feed: { ts: string; text: string }[]
}
type Live = { type Live = {
ok: boolean ok: boolean
ts: string ts: string
sources: Source[] sources: Source[]
generator?: Gen
totals: { records: number; revenue_est: number; avg_order: number } totals: { records: number; revenue_est: number; avg_order: number }
business: { business: {
orders_by_region: Bucket[] orders_by_region: Bucket[]
@@ -174,9 +186,11 @@ export function LiveDashboard() {
const [paused, setPaused] = useState(false) const [paused, setPaused] = useState(false)
const [err, setErr] = useState(false) const [err, setErr] = useState(false)
const [totalHist, setTotalHist] = useState<number[]>([]) const [totalHist, setTotalHist] = useState<number[]>([])
const [ordHist, setOrdHist] = useState<number[]>([])
const [rateBySrc, setRateBySrc] = useState<Record<string, number>>({}) const [rateBySrc, setRateBySrc] = useState<Record<string, number>>({})
const [added, setAdded] = useState(0) const [added, setAdded] = useState(0)
const prev = useRef<{ ts: number; rows: Record<string, number>; total: number } | null>(null) const [genBusy, setGenBusy] = useState(false)
const prev = useRef<{ ts: number; rows: Record<string, number>; total: number; genOrders: number } | null>(null)
const startTotal = useRef<number | null>(null) const startTotal = useRef<number | null>(null)
const poll = useCallback(async () => { const poll = useCallback(async () => {
@@ -187,10 +201,13 @@ export function LiveDashboard() {
setErr(false) setErr(false)
const now = Date.parse(d.ts) || Date.now() const now = Date.parse(d.ts) || Date.now()
const total = d.totals.records const total = d.totals.records
const genOrders = d.generator?.counts.orders ?? 0
if (prev.current) { if (prev.current) {
const dt = Math.max(0.5, (now - prev.current.ts) / 1000) const dt = Math.max(0.5, (now - prev.current.ts) / 1000)
const totRate = Math.max(0, (total - prev.current.total) / dt) const totRate = Math.max(0, (total - prev.current.total) / dt)
setTotalHist((h) => [...h, totRate].slice(-90)) setTotalHist((h) => [...h, totRate].slice(-90))
// orders added between polls — fluctuates up and down with each batch
setOrdHist((h) => [...h, Math.max(0, genOrders - prev.current!.genOrders)].slice(-60))
const rmap: Record<string, number> = {} const rmap: Record<string, number> = {}
d.sources.forEach((s) => { d.sources.forEach((s) => {
const p = prev.current!.rows[s.key] ?? s.rows const p = prev.current!.rows[s.key] ?? s.rows
@@ -200,13 +217,23 @@ export function LiveDashboard() {
} }
if (startTotal.current == null) startTotal.current = total if (startTotal.current == null) startTotal.current = total
setAdded(Math.max(0, total - (startTotal.current || total))) setAdded(Math.max(0, total - (startTotal.current || total)))
prev.current = { ts: now, rows: Object.fromEntries(d.sources.map((s) => [s.key, s.rows])), total } prev.current = { ts: now, rows: Object.fromEntries(d.sources.map((s) => [s.key, s.rows])), total, genOrders }
setData(d) setData(d)
} catch { } catch {
setErr(true) setErr(true)
} }
}, []) }, [])
const toggleGen = useCallback(async (enabled: boolean) => {
setGenBusy(true)
try {
await fetch('/api/federated/live/generator', {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ enabled }),
})
await poll()
} catch { /* ignore */ } finally { setGenBusy(false) }
}, [poll])
useEffect(() => { useEffect(() => {
poll() poll()
if (paused) return if (paused) return
@@ -215,7 +242,10 @@ export function LiveDashboard() {
}, [poll, paused]) }, [poll, paused])
const b = data?.business const b = data?.business
const gen = data?.generator
const lb = gen?.last_batch
const totalRate = totalHist.length ? totalHist[totalHist.length - 1] : 0 const totalRate = totalHist.length ? totalHist[totalHist.length - 1] : 0
const ordPeak = Math.max(1, ...ordHist)
const matrix = b?.region_matrix || [] const matrix = b?.region_matrix || []
const maxRev = Math.max(1, ...matrix.map((m) => Number(m.revenue) || 0)) const maxRev = Math.max(1, ...matrix.map((m) => Number(m.revenue) || 0))
@@ -236,7 +266,20 @@ export function LiveDashboard() {
<span className="text-[10px] text-foreground-muted">·</span> <span className="text-[10px] text-foreground-muted">·</span>
<span className="text-[10px] text-foreground-muted">+{fmtNum(added)} since opened</span> <span className="text-[10px] text-foreground-muted">+{fmtNum(added)} since opened</span>
{err && <span className="text-[10px] text-amber-400">reconnecting</span>} {err && <span className="text-[10px] text-amber-400">reconnecting</span>}
{gen && (
<span className={cn('inline-flex items-center gap-1 rounded-full border px-2 py-0.5 text-[10px] font-medium',
gen.running ? 'border-amber-500/40 bg-amber-500/10 text-amber-300' : 'border-border text-foreground-faint')}>
<Zap className={cn('h-3 w-3', gen.running && 'animate-pulse')} /> stream {gen.running ? `every ${gen.interval}s` : 'idle'}
</span>
)}
<span className="ml-auto text-[9px] text-foreground-faint">{data ? `updated ${new Date(data.ts).toLocaleTimeString()}` : 'connecting…'}</span> <span className="ml-auto text-[9px] text-foreground-faint">{data ? `updated ${new Date(data.ts).toLocaleTimeString()}` : 'connecting…'}</span>
{gen && (
<button type="button" disabled={genBusy} onClick={() => toggleGen(!gen.enabled)}
className={cn('inline-flex items-center gap-1 rounded-md border px-2 py-1 text-[10px] transition-colors disabled:opacity-60',
gen.enabled ? 'border-amber-500/40 bg-amber-500/10 text-amber-300 hover:bg-amber-500/20' : 'border-border text-foreground-muted hover:bg-surface-overlay')}>
<Zap className="h-3 w-3" /> {gen.enabled ? 'Generator on' : 'Generator off'}
</button>
)}
<button type="button" onClick={() => setPaused((p) => !p)} className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay"> <button type="button" onClick={() => setPaused((p) => !p)} className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
{paused ? <Play className="h-3 w-3" /> : <Pause className="h-3 w-3" />} {paused ? 'Resume' : 'Pause'} {paused ? <Play className="h-3 w-3" /> : <Pause className="h-3 w-3" />} {paused ? 'Resume' : 'Pause'}
</button> </button>
@@ -251,6 +294,52 @@ export function LiveDashboard() {
))} ))}
</div> </div>
{/* live activity — the per-tick pulse: orders & events streaming into the sources right now */}
{gen && (
<div className="grid shrink-0 gap-2 lg:grid-cols-3">
<Panel title="Live activity — last burst" subtitle={gen.running ? `every ${gen.interval}s` : 'paused'} icon={Sparkles} className="lg:col-span-2">
<div className="mb-2 grid grid-cols-2 gap-2 sm:grid-cols-4">
{([
{ k: 'orders', label: 'orders', color: '#fbbf24', icon: ShoppingCart },
{ k: 'telemetry', label: 'telemetry', color: '#22d3ee', icon: Cpu },
{ k: 'hr_events', label: 'HR', color: '#60a5fa', icon: Users },
{ k: 'supply_events', label: 'supply', color: '#a78bfa', icon: Boxes },
] as const).map(({ k, label, color, icon: Icon }) => {
const v = lb ? (lb as Record<string, number>)[k] : 0
return (
<div key={k} className="rounded-lg border border-border bg-surface-overlay/40 px-2 py-1.5">
<span className="flex items-center gap-1 text-[9px] uppercase tracking-wider text-foreground-muted"><Icon className="h-3 w-3" style={{ color }} />{label}</span>
<span className="font-mono text-base font-bold" style={{ color }}>+{fmtNum(v)}</span>
</div>
)
})}
</div>
<Spark data={ordHist} color="#fbbf24" height={56} />
<div className="mt-1 flex justify-between text-[9px] text-foreground-faint">
<span>orders per {(POLL_MS / 1000).toFixed(1)}s tick watch it rise &amp; fall</span>
<span>peak {fmtNum(ordPeak)} · {fmtNum(gen.tick_value)} last burst</span>
</div>
</Panel>
<Panel title="Event stream" subtitle="newest first" icon={Radio}>
<div className="max-h-40 space-y-1 overflow-y-auto">
{(gen.feed || []).length ? gen.feed.map((f, i) => (
<div key={`${f.ts}-${i}`} className={cn('rounded border-l-2 px-2 py-1 text-[9px] font-mono', i === 0 ? 'border-amber-400 bg-amber-500/10 text-amber-200' : 'border-border bg-surface-overlay/30 text-foreground-muted')}>
<span className="text-foreground-faint">{new Date(f.ts).toLocaleTimeString()}</span> · {f.text}
</div>
)) : <p className="py-4 text-center text-[10px] text-foreground-faint">waiting for the next burst</p>}
</div>
</Panel>
</div>
)}
{/* orders by region this burst (changes every tick) */}
{gen && (gen.by_region?.length || gen.by_status?.length) ? (
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Orders this burst — by region" subtitle="live distribution" icon={Database}><Bars data={gen.by_region} colorByIndex /></Panel>
<Panel title="Orders this burst — by status" subtitle="live distribution" icon={ShoppingCart}><Donut data={gen.by_status} /></Panel>
</div>
) : null}
{/* throughput + per-source rates */} {/* throughput + per-source rates */}
<div className="grid shrink-0 gap-2 lg:grid-cols-3"> <div className="grid shrink-0 gap-2 lg:grid-cols-3">
<Panel title="Ingestion throughput" subtitle="rows/sec · live" icon={Activity} className="lg:col-span-2"> <Panel title="Ingestion throughput" subtitle="rows/sec · live" icon={Activity} className="lg:col-span-2">
@@ -326,7 +415,7 @@ export function LiveDashboard() {
<Panel title="Supply events by type" icon={Boxes}><Bars data={b?.supply_by_type} colorByIndex /></Panel> <Panel title="Supply events by type" icon={Boxes}><Bars data={b?.supply_by_type} colorByIndex /></Panel>
</div> </div>
<p className="shrink-0 px-1 pb-2 text-[9px] text-foreground-faint"> <p className="shrink-0 px-1 pb-2 text-[9px] text-foreground-faint">
Counters &amp; throughput are live source estimates (Trino over PostgreSQL, MySQL, MongoDB &amp; Cassandra); breakdown charts aggregate the materialized Hadoop lake. Polling every {POLL_MS / 1000}s. While this tab is open a live generator streams randomly-sized bursts of real rows into PostgreSQL, MySQL, MongoDB &amp; Cassandra (picked up by CDC) counters &amp; the activity feed move every {gen?.interval ?? 4}s; the region scorecard &amp; breakdown charts aggregate the materialized Hadoop lake. Polling every {POLL_MS / 1000}s.
</p> </p>
</div> </div>
) )