feat(etl): Hadoop->Trino movement DAG + movements registry + autonomous ETL-agents

- DAG hadoop_to_trino (deployed to Airflow) + worker hadoop_to_trino.py (on the
  Hadoop master) move HDFS historical_sales -> iceberg.hadoop.historical_sales_hdfs.
- api/movements.py: movement registry, Airflow trigger+watch, run tracking
  (state/duration/rows), endpoints /api/movements, /{id}/run, /runs.
- agent_ops.py: ETL-agent loop autonomously triggers movements on an interval
  and logs each run; /api/agent-ops/etl/toggle + etl status.
This commit is contained in:
mo
2026-06-27 01:59:35 +02:00
parent 921342442f
commit a724615a9a
4 changed files with 257 additions and 3 deletions
+5 -1
View File
@@ -42,8 +42,9 @@ from pipeline_ops import router as pipeline_router
from hadoop_analytics import router as hadoop_router
from elasticsearch_api import router as elasticsearch_router
from sql_console import router as sql_router
from agent_ops import router as agent_ops_router, agent_dml_loop
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop
from cdc_consumer import router as cdc_router, cdc_consumer_loop
from movements import router as movements_router
from ssh_terminal import ssh_session
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
from node_ops import build_node_detail, probe_node, run_node_probe_task
@@ -715,11 +716,13 @@ async def lifespan(app: FastAPI):
task = asyncio.create_task(heartbeat_loop())
dml_task = asyncio.create_task(agent_dml_loop())
cdc_task = asyncio.create_task(cdc_consumer_loop())
etl_task = asyncio.create_task(etl_agent_loop())
add_feed("infra-sentinel", "ATC Command Center API online", "info")
yield
task.cancel()
dml_task.cancel()
cdc_task.cancel()
etl_task.cancel()
if redis_client:
await redis_client.close()
@@ -733,6 +736,7 @@ app.include_router(elasticsearch_router)
app.include_router(sql_router)
app.include_router(agent_ops_router)
app.include_router(cdc_router)
app.include_router(movements_router)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],