feat(dataflow): live Data Flow graph API + PII catalog
api/dataflow.py: node-link landscape (generators->sources->CDC/Kafka->sinks,
HDFS->Iceberg, sources->curated_masked) with live overlays (movement run state,
CDC volume, Trino counts) and PII overlay. api/pii_catalog.py: column PII
classification via Trino information_schema (OM-ready). Endpoints /api/dataflow,
/api/dataflow/{id}/run, /api/pii.
This commit is contained in:
@@ -180,6 +180,23 @@ async def cdc_consumer_loop() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def snapshot(minutes: int = 15) -> dict[str, Any]:
|
||||
"""Lightweight CDC snapshot for other modules (Data Flow graph)."""
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - minutes * 60
|
||||
by_source: dict[str, int] = {}
|
||||
total = 0
|
||||
for c in _ring:
|
||||
try:
|
||||
if datetime.fromisoformat(c["ts"]).timestamp() < cutoff:
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
total += 1
|
||||
by_source[c["source"]] = by_source.get(c["source"], 0) + 1
|
||||
return {"connected": _state["connected"], "consumed": _state["consumed"],
|
||||
"buffered": len(_ring), "window_total": total, "by_source": by_source}
|
||||
|
||||
|
||||
# ── Endpoints ────────────────────────────────────────────────────────────────
|
||||
@router.get("")
|
||||
async def list_changes(
|
||||
|
||||
Reference in New Issue
Block a user