feat(etl): Hadoop->Trino movement DAG + movements registry + autonomous ETL-agents
- DAG hadoop_to_trino (deployed to Airflow) + worker hadoop_to_trino.py (on the
Hadoop master) move HDFS historical_sales -> iceberg.hadoop.historical_sales_hdfs.
- api/movements.py: movement registry, Airflow trigger+watch, run tracking
(state/duration/rows), endpoints /api/movements, /{id}/run, /runs.
- agent_ops.py: ETL-agent loop autonomously triggers movements on an interval
and logs each run; /api/agent-ops/etl/toggle + etl status.
This commit is contained in:
+5
-1
@@ -42,8 +42,9 @@ from pipeline_ops import router as pipeline_router
|
||||
from hadoop_analytics import router as hadoop_router
|
||||
from elasticsearch_api import router as elasticsearch_router
|
||||
from sql_console import router as sql_router
|
||||
from agent_ops import router as agent_ops_router, agent_dml_loop
|
||||
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop
|
||||
from cdc_consumer import router as cdc_router, cdc_consumer_loop
|
||||
from movements import router as movements_router
|
||||
from ssh_terminal import ssh_session
|
||||
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
|
||||
from node_ops import build_node_detail, probe_node, run_node_probe_task
|
||||
@@ -715,11 +716,13 @@ async def lifespan(app: FastAPI):
|
||||
task = asyncio.create_task(heartbeat_loop())
|
||||
dml_task = asyncio.create_task(agent_dml_loop())
|
||||
cdc_task = asyncio.create_task(cdc_consumer_loop())
|
||||
etl_task = asyncio.create_task(etl_agent_loop())
|
||||
add_feed("infra-sentinel", "ATC Command Center API online", "info")
|
||||
yield
|
||||
task.cancel()
|
||||
dml_task.cancel()
|
||||
cdc_task.cancel()
|
||||
etl_task.cancel()
|
||||
if redis_client:
|
||||
await redis_client.close()
|
||||
|
||||
@@ -733,6 +736,7 @@ app.include_router(elasticsearch_router)
|
||||
app.include_router(sql_router)
|
||||
app.include_router(agent_ops_router)
|
||||
app.include_router(cdc_router)
|
||||
app.include_router(movements_router)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
|
||||
Reference in New Issue
Block a user