feat(etl): Hadoop->Trino movement DAG + movements registry + autonomous ETL-agents

- DAG hadoop_to_trino (deployed to Airflow) + worker hadoop_to_trino.py (on the
  Hadoop master) move HDFS historical_sales -> iceberg.hadoop.historical_sales_hdfs.
- api/movements.py: movement registry, Airflow trigger+watch, run tracking
  (state/duration/rows), endpoints /api/movements, /{id}/run, /runs.
- agent_ops.py: ETL-agent loop autonomously triggers movements on an interval
  and logs each run; /api/agent-ops/etl/toggle + etl status.
This commit is contained in:
mo
2026-06-27 01:59:35 +02:00
parent 921342442f
commit a724615a9a
4 changed files with 257 additions and 3 deletions
+53 -1
View File
@@ -49,6 +49,10 @@ DML_AGENT = "data-custodian"
_INTERVAL = float(os.getenv("AGENT_DML_INTERVAL_SECONDS", "45"))
_MAX_ROWS = int(os.getenv("AGENT_DML_MAX_ROWS", "5"))
# ETL-agent: autonomously trigger data movements on an interval.
_ETL_INTERVAL = float(os.getenv("ETL_AGENT_INTERVAL_SECONDS", "300"))
# Movements the ETL-agent cycles through autonomously (must exist in movements.py).
_ETL_ROTATION = [m for m in os.getenv("ETL_AGENT_MOVEMENTS", "hadoop_to_trino,gen_postgres,gen_mysql,gen_mongodb").split(",") if m]
# How many agent-created rows we keep around per source before favouring DELETE.
_POOL_CAP = int(os.getenv("AGENT_DML_POOL_CAP", "400"))
@@ -64,6 +68,16 @@ _state: dict[str, Any] = {
"by_op": {"insert": 0, "update": 0, "delete": 0},
}
_etl_state: dict[str, Any] = {
"enabled": os.getenv("ETL_AGENT_ENABLED", "1") not in ("0", "false", "False", ""),
"interval": _ETL_INTERVAL,
"rotation": _ETL_ROTATION,
"idx": 0,
"runs_total": 0,
"last": None,
"started": False,
}
# ── Realistic value domains ─────────────────────────────────────────────────
REGIONS = ["NA", "EU", "EMEA", "APAC", "LATAM"]
CHANNELS = ["online", "retail", "partner", "wholesale", "direct"]
@@ -291,11 +305,49 @@ async def agent_dml_loop() -> None:
await asyncio.sleep(max(5.0, float(_state["interval"])))
async def etl_agent_loop() -> None:
"""Background loop: ETL-agents autonomously trigger data movements and log
each run (rows, duration, status) to the feed."""
_etl_state["started"] = True
await asyncio.sleep(30) # let the platform settle
while True:
try:
if _etl_state["enabled"] and _etl_state["rotation"]:
from movements import trigger_and_watch, MOVEMENT_BY_ID
mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])]
_etl_state["idx"] += 1
if mid in MOVEMENT_BY_ID:
result = await trigger_and_watch(mid, autonomous=True)
_etl_state["runs_total"] += 1
_etl_state["last"] = {"movement_id": mid, "state": result.get("state"),
"rows": result.get("rows"), "duration_s": result.get("duration_s"),
"ts": datetime.now(timezone.utc).isoformat()}
except Exception as exc:
_etl_state["last"] = {"error": str(exc), "ts": datetime.now(timezone.utc).isoformat()}
await asyncio.sleep(max(30.0, float(_etl_state["interval"])))
# ── Endpoints ────────────────────────────────────────────────────────────────
@router.get("/status")
async def status() -> JSONResponse:
pools = {k: len(v) for k, v in _pools.items()}
return JSONResponse({"ok": True, "pools": pools, **_state})
return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, **_state})
@router.post("/etl/toggle")
async def etl_toggle(body: dict[str, Any] = Body(default={})) -> JSONResponse:
if "enabled" in body:
_etl_state["enabled"] = bool(body["enabled"])
else:
_etl_state["enabled"] = not _etl_state["enabled"]
if "interval" in body:
try:
_etl_state["interval"] = max(30.0, float(body["interval"]))
except (TypeError, ValueError):
pass
await _emit(f"[etl-agent] Autonomous ETL movements {'ENABLED' if _etl_state['enabled'] else 'PAUSED'} by operator", "warn")
return JSONResponse({"ok": True, "enabled": _etl_state["enabled"], "interval": _etl_state["interval"]})
@router.post("/toggle")