feat(etl): Hadoop->Trino movement DAG + movements registry + autonomous ETL-agents
- DAG hadoop_to_trino (deployed to Airflow) + worker hadoop_to_trino.py (on the
Hadoop master) move HDFS historical_sales -> iceberg.hadoop.historical_sales_hdfs.
- api/movements.py: movement registry, Airflow trigger+watch, run tracking
(state/duration/rows), endpoints /api/movements, /{id}/run, /runs.
- agent_ops.py: ETL-agent loop autonomously triggers movements on an interval
and logs each run; /api/agent-ops/etl/toggle + etl status.
This commit is contained in:
+53
-1
@@ -49,6 +49,10 @@ DML_AGENT = "data-custodian"
|
||||
|
||||
_INTERVAL = float(os.getenv("AGENT_DML_INTERVAL_SECONDS", "45"))
|
||||
_MAX_ROWS = int(os.getenv("AGENT_DML_MAX_ROWS", "5"))
|
||||
# ETL-agent: autonomously trigger data movements on an interval.
|
||||
_ETL_INTERVAL = float(os.getenv("ETL_AGENT_INTERVAL_SECONDS", "300"))
|
||||
# Movements the ETL-agent cycles through autonomously (must exist in movements.py).
|
||||
_ETL_ROTATION = [m for m in os.getenv("ETL_AGENT_MOVEMENTS", "hadoop_to_trino,gen_postgres,gen_mysql,gen_mongodb").split(",") if m]
|
||||
# How many agent-created rows we keep around per source before favouring DELETE.
|
||||
_POOL_CAP = int(os.getenv("AGENT_DML_POOL_CAP", "400"))
|
||||
|
||||
@@ -64,6 +68,16 @@ _state: dict[str, Any] = {
|
||||
"by_op": {"insert": 0, "update": 0, "delete": 0},
|
||||
}
|
||||
|
||||
_etl_state: dict[str, Any] = {
|
||||
"enabled": os.getenv("ETL_AGENT_ENABLED", "1") not in ("0", "false", "False", ""),
|
||||
"interval": _ETL_INTERVAL,
|
||||
"rotation": _ETL_ROTATION,
|
||||
"idx": 0,
|
||||
"runs_total": 0,
|
||||
"last": None,
|
||||
"started": False,
|
||||
}
|
||||
|
||||
# ── Realistic value domains ─────────────────────────────────────────────────
|
||||
REGIONS = ["NA", "EU", "EMEA", "APAC", "LATAM"]
|
||||
CHANNELS = ["online", "retail", "partner", "wholesale", "direct"]
|
||||
@@ -291,11 +305,49 @@ async def agent_dml_loop() -> None:
|
||||
await asyncio.sleep(max(5.0, float(_state["interval"])))
|
||||
|
||||
|
||||
async def etl_agent_loop() -> None:
|
||||
"""Background loop: ETL-agents autonomously trigger data movements and log
|
||||
each run (rows, duration, status) to the feed."""
|
||||
_etl_state["started"] = True
|
||||
await asyncio.sleep(30) # let the platform settle
|
||||
while True:
|
||||
try:
|
||||
if _etl_state["enabled"] and _etl_state["rotation"]:
|
||||
from movements import trigger_and_watch, MOVEMENT_BY_ID
|
||||
|
||||
mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])]
|
||||
_etl_state["idx"] += 1
|
||||
if mid in MOVEMENT_BY_ID:
|
||||
result = await trigger_and_watch(mid, autonomous=True)
|
||||
_etl_state["runs_total"] += 1
|
||||
_etl_state["last"] = {"movement_id": mid, "state": result.get("state"),
|
||||
"rows": result.get("rows"), "duration_s": result.get("duration_s"),
|
||||
"ts": datetime.now(timezone.utc).isoformat()}
|
||||
except Exception as exc:
|
||||
_etl_state["last"] = {"error": str(exc), "ts": datetime.now(timezone.utc).isoformat()}
|
||||
await asyncio.sleep(max(30.0, float(_etl_state["interval"])))
|
||||
|
||||
|
||||
# ── Endpoints ────────────────────────────────────────────────────────────────
|
||||
@router.get("/status")
|
||||
async def status() -> JSONResponse:
|
||||
pools = {k: len(v) for k, v in _pools.items()}
|
||||
return JSONResponse({"ok": True, "pools": pools, **_state})
|
||||
return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, **_state})
|
||||
|
||||
|
||||
@router.post("/etl/toggle")
|
||||
async def etl_toggle(body: dict[str, Any] = Body(default={})) -> JSONResponse:
|
||||
if "enabled" in body:
|
||||
_etl_state["enabled"] = bool(body["enabled"])
|
||||
else:
|
||||
_etl_state["enabled"] = not _etl_state["enabled"]
|
||||
if "interval" in body:
|
||||
try:
|
||||
_etl_state["interval"] = max(30.0, float(body["interval"]))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
await _emit(f"[etl-agent] Autonomous ETL movements {'ENABLED' if _etl_state['enabled'] else 'PAUSED'} by operator", "warn")
|
||||
return JSONResponse({"ok": True, "enabled": _etl_state["enabled"], "interval": _etl_state["interval"]})
|
||||
|
||||
|
||||
@router.post("/toggle")
|
||||
|
||||
Reference in New Issue
Block a user