feat: live per-agent terminal activity with real scripts/SQL
Agent terminals were idle (one-shot probe) while agents were busy in the background. Now every agent streams what it is actually doing: - agent_terminal: emit_threadsafe() so background threads can stream lines. - agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL Guardian announces each orchestrated movement. - movements: trigger_and_watch streams the Airflow DAG / API call, conf, before/after Trino counts and result to the owning agent terminal. - etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle summaries stream to the ETL Guardian terminal. - agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host load) and Network Watcher (VLAN 20/21 path checks). - fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61). - ui: terminal dock merges the selected agent ops stream with the node probe.
This commit is contained in:
+24
-2
@@ -83,6 +83,15 @@ async def _publish(event: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _term(agent_id: str, text: str, level: str = "info", phase: str = "etl") -> None:
|
||||
"""Stream a movement step to the owning agent's terminal."""
|
||||
try:
|
||||
from agent_terminal import terminal_log
|
||||
await terminal_log(agent_id, text, level=level, phase=phase)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _airflow_token(client: httpx.AsyncClient) -> str:
|
||||
now = time.time()
|
||||
if _token_cache["token"] and _token_cache["exp"] > now + 30:
|
||||
@@ -128,9 +137,12 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
|
||||
mv = MOVEMENT_BY_ID.get(mid)
|
||||
if not mv:
|
||||
return {"ok": False, "error": f"unknown movement {mid}"}
|
||||
agent = mv["agent"]
|
||||
if mv.get("api"):
|
||||
try:
|
||||
payload = {**(mv.get("default_conf") or {}), **(conf or {})}
|
||||
await _term(agent, f"$ POST {mv['api']} # {mv['label']}", level="cmd")
|
||||
await _term(agent, f" payload={payload}", level="cmd")
|
||||
t0 = time.time()
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
|
||||
@@ -145,21 +157,26 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
|
||||
_last_runs[mid] = run
|
||||
await _publish({"type": "movement", **run})
|
||||
lvl = "info" if state == "success" else "err"
|
||||
await _term(agent, f" ← {state} · {rows if rows is not None else '?'} rows in {dur}s", level="ok" if state == "success" else "err")
|
||||
_feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
|
||||
return {"ok": state == "success", **run}
|
||||
except Exception as exc:
|
||||
_last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
|
||||
await _term(agent, f" ✗ error {str(exc)[:140]}", level="err")
|
||||
_feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
conf = {**(mv.get("default_conf") or {}), **(conf or {})}
|
||||
agent = mv["agent"]
|
||||
count_sql = mv.get("count_sql")
|
||||
verb = "autonomously triggered" if autonomous else "triggered"
|
||||
await _term(agent, f"$ airflow dags trigger {mv['dag_id']} # {mv['label']} ({verb})", level="cmd")
|
||||
await _term(agent, f" conf={conf}", level="cmd")
|
||||
before = await _trino_scalar(count_sql) if count_sql else None
|
||||
if count_sql:
|
||||
await _term(agent, f" trino: {count_sql} → {before if before is not None else '?'} rows (before)", level="info")
|
||||
|
||||
_last_runs[mid] = {**_last_runs.get(mid, {}), "state": "running", "started_at": datetime.now(timezone.utc).isoformat()}
|
||||
await _publish({"type": "movement", "movement_id": mid, "state": "running"})
|
||||
verb = "autonomously triggered" if autonomous else "triggered"
|
||||
_feed(agent, f"[etl] {mv['label']}: {verb} (conf={conf})", "info")
|
||||
|
||||
try:
|
||||
@@ -182,8 +199,10 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
|
||||
if state in ("success", "failed"):
|
||||
break
|
||||
dur = round(time.time() - t0, 1)
|
||||
await _term(agent, f" ← Airflow run {run_id} finished state={state} in {dur}s", level="ok" if state == "success" else "err")
|
||||
except Exception as exc:
|
||||
_last_runs[mid] = {**_last_runs[mid], "state": "failed", "error": str(exc)}
|
||||
await _term(agent, f" ✗ error {str(exc)[:140]}", level="err")
|
||||
_feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
@@ -201,6 +220,9 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
|
||||
await _publish({"type": "movement", **run})
|
||||
rtxt = f"{rows} rows" if rows is not None else "data"
|
||||
lvl = "info" if state == "success" else "err"
|
||||
if count_sql:
|
||||
await _term(agent, f" trino: count {before if before is not None else '?'} → {after if after is not None else '?'} ({rtxt})", level="info")
|
||||
await _term(agent, f" ← {mv['label']}: {state} — {rtxt} in {dur}s", level="ok" if state == "success" else "err")
|
||||
_feed(agent, f"[etl] {mv['label']}: {state} — {rtxt} in {dur}s", lvl)
|
||||
return {"ok": state == "success", **run}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user