feat: live per-agent terminal activity with real scripts/SQL

Agent terminals were idle (one-shot probe) while agents were busy in the
background. Now every agent streams what it is actually doing:

- agent_terminal: emit_threadsafe() so background threads can stream lines.
- agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL
  (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL
  Guardian announces each orchestrated movement.
- movements: trigger_and_watch streams the Airflow DAG / API call, conf,
  before/after Trino counts and result to the owning agent terminal.
- etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle
  summaries stream to the ETL Guardian terminal.
- agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop
  Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host
  load) and Network Watcher (VLAN 20/21 path checks).
- fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61).
- ui: terminal dock merges the selected agent ops stream with the node probe.
This commit is contained in:
mo
2026-06-29 12:54:46 +00:00
parent b6d7d3dc74
commit 1e2cfe80f2
9 changed files with 388 additions and 52 deletions
+24 -2
View File
@@ -83,6 +83,15 @@ async def _publish(event: dict[str, Any]) -> None:
pass
async def _term(agent_id: str, text: str, level: str = "info", phase: str = "etl") -> None:
"""Stream a movement step to the owning agent's terminal."""
try:
from agent_terminal import terminal_log
await terminal_log(agent_id, text, level=level, phase=phase)
except Exception:
pass
async def _airflow_token(client: httpx.AsyncClient) -> str:
now = time.time()
if _token_cache["token"] and _token_cache["exp"] > now + 30:
@@ -128,9 +137,12 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
mv = MOVEMENT_BY_ID.get(mid)
if not mv:
return {"ok": False, "error": f"unknown movement {mid}"}
agent = mv["agent"]
if mv.get("api"):
try:
payload = {**(mv.get("default_conf") or {}), **(conf or {})}
await _term(agent, f"$ POST {mv['api']} # {mv['label']}", level="cmd")
await _term(agent, f" payload={payload}", level="cmd")
t0 = time.time()
async with httpx.AsyncClient(timeout=120.0) as client:
r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
@@ -145,21 +157,26 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
_last_runs[mid] = run
await _publish({"type": "movement", **run})
lvl = "info" if state == "success" else "err"
await _term(agent, f"{state} · {rows if rows is not None else '?'} rows in {dur}s", level="ok" if state == "success" else "err")
_feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
return {"ok": state == "success", **run}
except Exception as exc:
_last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
await _term(agent, f" ✗ error {str(exc)[:140]}", level="err")
_feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
return {"ok": False, "error": str(exc)}
conf = {**(mv.get("default_conf") or {}), **(conf or {})}
agent = mv["agent"]
count_sql = mv.get("count_sql")
verb = "autonomously triggered" if autonomous else "triggered"
await _term(agent, f"$ airflow dags trigger {mv['dag_id']} # {mv['label']} ({verb})", level="cmd")
await _term(agent, f" conf={conf}", level="cmd")
before = await _trino_scalar(count_sql) if count_sql else None
if count_sql:
await _term(agent, f" trino: {count_sql}{before if before is not None else '?'} rows (before)", level="info")
_last_runs[mid] = {**_last_runs.get(mid, {}), "state": "running", "started_at": datetime.now(timezone.utc).isoformat()}
await _publish({"type": "movement", "movement_id": mid, "state": "running"})
verb = "autonomously triggered" if autonomous else "triggered"
_feed(agent, f"[etl] {mv['label']}: {verb} (conf={conf})", "info")
try:
@@ -182,8 +199,10 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
if state in ("success", "failed"):
break
dur = round(time.time() - t0, 1)
await _term(agent, f" ← Airflow run {run_id} finished state={state} in {dur}s", level="ok" if state == "success" else "err")
except Exception as exc:
_last_runs[mid] = {**_last_runs[mid], "state": "failed", "error": str(exc)}
await _term(agent, f" ✗ error {str(exc)[:140]}", level="err")
_feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
return {"ok": False, "error": str(exc)}
@@ -201,6 +220,9 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
await _publish({"type": "movement", **run})
rtxt = f"{rows} rows" if rows is not None else "data"
lvl = "info" if state == "success" else "err"
if count_sql:
await _term(agent, f" trino: count {before if before is not None else '?'}{after if after is not None else '?'} ({rtxt})", level="info")
await _term(agent, f"{mv['label']}: {state}{rtxt} in {dur}s", level="ok" if state == "success" else "err")
_feed(agent, f"[etl] {mv['label']}: {state}{rtxt} in {dur}s", lvl)
return {"ok": state == "success", **run}