feat: pulse generator edges on data gen + New & Changed Data dashboard

Generated data is now visible flowing into the source systems on the Data
Flow graph: a manual "Generate data" burst (and the continuous live loop)
stamps a last_tick, and a new generator_active() helper lights up the
generator -> postgres/mysql/mongodb/cassandra edges for ~12s. Neo4j stays
dark since the live generator does not write to it.

Redesigned the Changes tab into a "New & Changed Data" dashboard driven by
the live CDC stream: animated KPI cards (new/updated/deleted + throughput),
a change-volume area chart, an operation-mix donut, per-system and
top-table breakdown bars, plus the existing filterable change feed.
This commit is contained in:
mo
2026-06-28 22:41:48 +00:00
parent aa9ee66966
commit dfd5d4da8a
3 changed files with 254 additions and 51 deletions
+11 -1
View File
@@ -173,6 +173,11 @@ async def _build() -> dict[str, Any]:
cdc = cdc_snapshot(15)
except Exception:
cdc = {"connected": False, "consumed": 0, "by_source": {}, "window_total": 0}
try:
from trino_federated import generator_active
gen_active = generator_active()
except Exception:
gen_active = False
try:
from pii_catalog import get_pii
pii = get_pii()
@@ -255,7 +260,12 @@ async def _build() -> dict[str, Any]:
edge["last_rows"] = lr.get("rows")
edge["last_duration_s"] = lr.get("duration_s")
edge["active"] = lr.get("state") == "running"
if e["kind"] == "cdc":
if e["kind"] == "generate":
# The live generator (continuous loop or the 'Generate data' burst)
# writes into these four sources; pulse the edge while it is active.
if e["to"] in ("postgres", "mysql", "mongodb", "cassandra"):
edge["active"] = gen_active or bool(edge.get("active"))
elif e["kind"] == "cdc":
edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
elif e.get("from") == "hdfs" and e.get("to") == "kafka":
edge["active"] = bool(edge_live.get("hdfs→kafka"))
+10
View File
@@ -534,6 +534,9 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
c = _GEN["counts"]
for k in out:
c[k] += out[k]
_GEN["last_batch"] = {"orders": out["orders"], "hr_events": out["hr_events"],
"supply_events": out["supply_events"], "telemetry": out["telemetry"]}
_GEN["last_tick"] = time.time() # marks recent activity → Data Flow CDC edges pulse
if out["orders"]:
_GEN["by_region"] = by_r
_GEN["by_status"] = by_s
@@ -609,6 +612,13 @@ def _gen_loop():
threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
def generator_active(window_s: float = 12.0) -> bool:
"""True when the live generator (continuous loop OR a manual 'Generate data'
burst) wrote rows very recently. The Data Flow graph uses this to pulse the
generator→source edges so generated data is visible flowing into the sources."""
return (time.time() - float(_GEN.get("last_tick") or 0.0)) < window_s
@router.post("/live/generator")
async def toggle_generator(body: dict = Body(default={})):
if "enabled" in body: