feat: neo4j in topology + agent-driven datagen with activity log; fix all-sources generate

api/pipeline_ops.py

@@ -35,6 +35,23 @@ SOURCE_DAG = {
     "hadoop": "gen_hadoop_history",
 }
 
+# UI source key -> the agent responsible for that part of the platform
+SOURCE_AGENT = {
+    "postgres": "data-custodian",
+    "mysql": "data-custodian",
+    "mongodb": "data-custodian",
+    "cassandra": "data-custodian",
+    "neo4j": "data-custodian",
+    "all": "data-custodian",
+    "hadoop": "hadoop-ranger",
+}
+AGENT_NAME = {
+    "data-custodian": "Data Custodian",
+    "hadoop-ranger": "Hadoop Ranger",
+    "etl-guardian": "ETL Guardian",
+    "lakehouse-ops": "Lakehouse Ops",
+}
+
 # UI source key -> Trino fully-qualified table for live row counts
 SOURCE_COUNT_SQL = {
     "postgres": "SELECT count(*) FROM postgres_sales.public.sales_orders",
@@ -85,6 +102,43 @@ async def _airflow_token(client: httpx.AsyncClient) -> str:
     return tok
 
 
+def _feed(agent_id: str, message: str, level: str = "info") -> None:
+    """Write an entry to the shared agent activity feed (Comms log)."""
+    try:
+        from main import add_feed  # lazy: main is fully loaded by request time
+        add_feed(agent_id, message, level)
+    except Exception:
+        pass
+
+
+async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: int | None) -> None:
+    """Poll an Airflow run to completion and log the outcome to the feed."""
+    name = AGENT_NAME.get(agent_id, agent_id)
+    label = f"{rows} rijen" if rows else "data"
+    try:
+        async with httpx.AsyncClient() as client:
+            tok = await _airflow_token(client)
+            headers = {"Authorization": f"Bearer {tok}"}
+            for _ in range(180):  # up to ~15 min
+                await asyncio.sleep(5)
+                try:
+                    r = await client.get(
+                        f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns/{run_id}",
+                        headers=headers, timeout=10,
+                    )
+                    state = r.json().get("state")
+                except Exception:
+                    continue
+                if state == "success":
+                    _feed(agent_id, f"[datagen] {name} genereerde {label} in {source} — klaar, data stroomt via CDC naar Kafka/S3", "info")
+                    return
+                if state == "failed":
+                    _feed(agent_id, f"[datagen] {name}: generatie voor {source} is mislukt (zie Airflow logs)", "err")
+                    return
+    except Exception:
+        pass
+
+
 async def _trino_scalar(sql: str, deadline_s: float = 8.0) -> int | None:
     """Run a scalar Trino query with a hard wall-clock deadline.
 
@@ -138,6 +192,9 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON
             conf["rows"] = max(1, min(int(rows), 2_000_000))
         except (TypeError, ValueError):
             return JSONResponse({"ok": False, "error": "rows must be an integer"}, status_code=400)
+    agent_id = body.get("agent_id") or SOURCE_AGENT.get(source, "data-custodian")
+    autonomous = bool(body.get("autonomous"))
+    name = AGENT_NAME.get(agent_id, agent_id)
     try:
         async with httpx.AsyncClient() as client:
             tok = await _airflow_token(client)
@@ -148,20 +205,66 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON
                 timeout=15,
             )
         if r.status_code >= 400:
+            _feed(agent_id, f"[datagen] {name}: kon generatie voor {source} niet starten (Airflow {r.status_code})", "err")
             return JSONResponse({"ok": False, "error": f"Airflow {r.status_code}: {r.text[:300]}"}, status_code=200)
         j = r.json()
+        run_id = j.get("dag_run_id")
+        verb = "genereert zelf" if autonomous else "startte generatie:"
+        rows_txt = f"{conf['rows']} rijen" if conf.get("rows") else "data"
+        _feed(agent_id, f"[datagen] {name} {verb} {rows_txt} in {source}", "info")
+        if run_id:
+            asyncio.create_task(_watch_run(source, dag_id, run_id, agent_id, conf.get("rows")))
         return JSONResponse({
             "ok": True,
             "source": source,
             "dag_id": dag_id,
-            "run_id": j.get("dag_run_id"),
+            "run_id": run_id,
             "state": j.get("state"),
             "rows": conf.get("rows"),
+            "agent_id": agent_id,
+            "agent_name": name,
         })
     except Exception as exc:
         return JSONResponse({"ok": False, "error": str(exc)}, status_code=200)
 
 
+@router.get("/agents")
+async def agents() -> JSONResponse:
+    """Which agent is responsible for generating each source."""
+    out = {src: {"agent_id": aid, "agent_name": AGENT_NAME.get(aid, aid)}
+           for src, aid in SOURCE_AGENT.items()}
+    return JSONResponse({"ok": True, "agents": out})
+
+
+@router.get("/activity")
+async def activity(limit: int = Query(25)) -> JSONResponse:
+    """Recent data-generation activity performed by agents (from the feed)."""
+    try:
+        from main import FeedEntry
+        from db import SessionLocal
+        from sqlalchemy import select
+        with SessionLocal() as db:
+            rows = db.execute(
+                select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(400)
+            ).scalars().all()
+        items = []
+        for r in rows:
+            if r.message and "[datagen]" in r.message:
+                items.append({
+                    "id": r.id,
+                    "ts": r.ts.isoformat() if r.ts else None,
+                    "agent_id": r.agent_id,
+                    "agent_name": AGENT_NAME.get(r.agent_id, r.agent_id),
+                    "message": r.message.replace("[datagen] ", ""),
+                    "level": r.level,
+                })
+            if len(items) >= limit:
+                break
+        return JSONResponse({"ok": True, "activity": items})
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc), "activity": []}, status_code=200)
+
+
 @router.get("/runs/{source}")
 async def runs(source: str, limit: int = Query(5)) -> JSONResponse:
     dag_id = SOURCE_DAG.get(source)