feat: pulse generator edges on data gen + New & Changed Data dashboard
Generated data is now visible flowing into the source systems on the Data Flow graph: a manual "Generate data" burst (and the continuous live loop) stamps a last_tick, and a new generator_active() helper lights up the generator -> postgres/mysql/mongodb/cassandra edges for ~12s. Neo4j stays dark since the live generator does not write to it. Redesigned the Changes tab into a "New & Changed Data" dashboard driven by the live CDC stream: animated KPI cards (new/updated/deleted + throughput), a change-volume area chart, an operation-mix donut, per-system and top-table breakdown bars, plus the existing filterable change feed.
This commit is contained in:
+11
-1
@@ -173,6 +173,11 @@ async def _build() -> dict[str, Any]:
|
||||
cdc = cdc_snapshot(15)
|
||||
except Exception:
|
||||
cdc = {"connected": False, "consumed": 0, "by_source": {}, "window_total": 0}
|
||||
try:
|
||||
from trino_federated import generator_active
|
||||
gen_active = generator_active()
|
||||
except Exception:
|
||||
gen_active = False
|
||||
try:
|
||||
from pii_catalog import get_pii
|
||||
pii = get_pii()
|
||||
@@ -255,7 +260,12 @@ async def _build() -> dict[str, Any]:
|
||||
edge["last_rows"] = lr.get("rows")
|
||||
edge["last_duration_s"] = lr.get("duration_s")
|
||||
edge["active"] = lr.get("state") == "running"
|
||||
if e["kind"] == "cdc":
|
||||
if e["kind"] == "generate":
|
||||
# The live generator (continuous loop or the 'Generate data' burst)
|
||||
# writes into these four sources; pulse the edge while it is active.
|
||||
if e["to"] in ("postgres", "mysql", "mongodb", "cassandra"):
|
||||
edge["active"] = gen_active or bool(edge.get("active"))
|
||||
elif e["kind"] == "cdc":
|
||||
edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
|
||||
elif e.get("from") == "hdfs" and e.get("to") == "kafka":
|
||||
edge["active"] = bool(edge_live.get("hdfs→kafka"))
|
||||
|
||||
@@ -534,6 +534,9 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
|
||||
c = _GEN["counts"]
|
||||
for k in out:
|
||||
c[k] += out[k]
|
||||
_GEN["last_batch"] = {"orders": out["orders"], "hr_events": out["hr_events"],
|
||||
"supply_events": out["supply_events"], "telemetry": out["telemetry"]}
|
||||
_GEN["last_tick"] = time.time() # marks recent activity → Data Flow CDC edges pulse
|
||||
if out["orders"]:
|
||||
_GEN["by_region"] = by_r
|
||||
_GEN["by_status"] = by_s
|
||||
@@ -609,6 +612,13 @@ def _gen_loop():
|
||||
threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
|
||||
|
||||
|
||||
def generator_active(window_s: float = 12.0) -> bool:
|
||||
"""True when the live generator (continuous loop OR a manual 'Generate data'
|
||||
burst) wrote rows very recently. The Data Flow graph uses this to pulse the
|
||||
generator→source edges so generated data is visible flowing into the sources."""
|
||||
return (time.time() - float(_GEN.get("last_tick") or 0.0)) < window_s
|
||||
|
||||
|
||||
@router.post("/live/generator")
|
||||
async def toggle_generator(body: dict = Body(default={})):
|
||||
if "enabled" in body:
|
||||
|
||||
Reference in New Issue
Block a user