feat(dataflow): live Data Flow graph API + PII catalog

api/dataflow.py: node-link landscape (generators->sources->CDC/Kafka->sinks,
HDFS->Iceberg, sources->curated_masked) with live overlays (movement run state,
CDC volume, Trino counts) and PII overlay. api/pii_catalog.py: column PII
classification via Trino information_schema (OM-ready). Endpoints /api/dataflow,
/api/dataflow/{id}/run, /api/pii.
This commit is contained in:
mo
2026-06-27 02:04:46 +02:00
parent a724615a9a
commit ee9357aa31
5 changed files with 349 additions and 1 deletions
+17
View File
@@ -180,6 +180,23 @@ async def cdc_consumer_loop() -> None:
pass
def snapshot(minutes: int = 15) -> dict[str, Any]:
"""Lightweight CDC snapshot for other modules (Data Flow graph)."""
cutoff = datetime.now(timezone.utc).timestamp() - minutes * 60
by_source: dict[str, int] = {}
total = 0
for c in _ring:
try:
if datetime.fromisoformat(c["ts"]).timestamp() < cutoff:
continue
except Exception:
continue
total += 1
by_source[c["source"]] = by_source.get(c["source"], 0) + 1
return {"connected": _state["connected"], "consumed": _state["consumed"],
"buffered": len(_ring), "window_total": total, "by_source": by_source}
# ── Endpoints ────────────────────────────────────────────────────────────────
@router.get("")
async def list_changes(