ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

Browse Source 
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations
This commit is contained in:
mo
2026-06-27 19:37:50 +00:00
parent 5828113f53
commit 46b9c50e73
39 changed files with 5476 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/pipeline_ops.py
+6 -6
View File
@@ -114,7 +114,7 @@ def _feed(agent_id: str, message: str, level: str = "info") -> None:
async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: int | None) -> None:
"""Poll an Airflow run to completion and log the outcome to the feed."""
name = AGENT_NAME.get(agent_id, agent_id)
label = f"{rows} rijen" if rows else "data"
label = f"{rows} rows" if rows else "data"
try:
async with httpx.AsyncClient() as client:
tok = await _airflow_token(client)
@@ -130,10 +130,10 @@ async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows:
except Exception:
continue
if state == "success":
_feed(agent_id, f"[datagen] {name} genereerde {label} in {source} — klaar, data stroomt via CDC naar Kafka/S3", "info")
_feed(agent_id, f"[datagen] {name} generated {label} in {source} — complete, data flowing via CDC to Kafka/S3", "info")
return
if state == "failed":
_feed(agent_id, f"[datagen] {name}: generatie voor {source} is mislukt (zie Airflow logs)", "err")
_feed(agent_id, f"[datagen] {name}: generation for {source} failed (see Airflow logs)", "err")
return
except Exception:
pass
@@ -205,12 +205,12 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON
timeout=15,
)
if r.status_code >= 400:
_feed(agent_id, f"[datagen] {name}: kon generatie voor {source} niet starten (Airflow {r.status_code})", "err")
_feed(agent_id, f"[datagen] {name}: could not start generation for {source} (Airflow {r.status_code})", "err")
return JSONResponse({"ok": False, "error": f"Airflow {r.status_code}: {r.text[:300]}"}, status_code=200)
j = r.json()
run_id = j.get("dag_run_id")
verb = "genereert zelf" if autonomous else "startte generatie:"
rows_txt = f"{conf['rows']} rijen" if conf.get("rows") else "data"
verb = "generating autonomously" if autonomous else "started generation of"
rows_txt = f"{conf['rows']} rows" if conf.get("rows") else "data"
_feed(agent_id, f"[datagen] {name} {verb} {rows_txt} in {source}", "info")
if run_id:
asyncio.create_task(_watch_run(source, dag_id, run_id, agent_id, conf.get("rows")))