feat: live per-agent terminal activity with real scripts/SQL

Agent terminals were idle (one-shot probe) while agents were busy in the
background. Now every agent streams what it is actually doing:

- agent_terminal: emit_threadsafe() so background threads can stream lines.
- agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL
  (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL
  Guardian announces each orchestrated movement.
- movements: trigger_and_watch streams the Airflow DAG / API call, conf,
  before/after Trino counts and result to the owning agent terminal.
- etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle
  summaries stream to the ETL Guardian terminal.
- agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop
  Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host
  load) and Network Watcher (VLAN 20/21 path checks).
- fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61).
- ui: terminal dock merges the selected agent ops stream with the node probe.
This commit is contained in:
mo
2026-06-29 12:54:46 +00:00
parent b6d7d3dc74
commit 1e2cfe80f2
9 changed files with 388 additions and 52 deletions
+23
View File
@@ -104,6 +104,15 @@ def _feed(text: str, level: str = "info") -> None:
pass
def _term(text: str, level: str = "info", phase: str = "offload") -> None:
"""Stream a line to the ETL Guardian terminal from this background thread."""
try:
from agent_terminal import emit_threadsafe
emit_threadsafe("etl-guardian", text, level=level, phase=phase)
except Exception:
pass
def _scalar(v: Any) -> Any:
import datetime as _dt
import decimal
@@ -312,6 +321,9 @@ def _offload_dataset(key: str) -> dict[str, Any]:
st = _state["datasets"][key]
chunk = int(_state["chunk"])
res = {"rows": 0, "bytes": 0, "order_rows": 0, "revenue": 0.0}
cur_txt = st["cursor"] if st.get("cursor") not in (None, "") else "<start>"
_term(f"$ read {st.get('engine', key)} · {st.get('label', key)} [cursor={cur_txt} · LIMIT {chunk}]",
level="cmd", phase="extract")
try:
if key == "hr_events":
rows, new_cursor, done = _read_hr(st["cursor"], chunk, st)
@@ -319,6 +331,7 @@ def _offload_dataset(key: str) -> dict[str, Any]:
rows, new_cursor, done = _READERS[key](st["cursor"], chunk)
except Exception as exc:
st["error"] = str(exc)[:160]
_term(f" ✗ extract failed: {str(exc)[:140]}", level="err", phase="extract")
return res
st["error"] = None
if not rows:
@@ -331,7 +344,10 @@ def _offload_dataset(key: str) -> dict[str, Any]:
obj_key, nbytes = _write_parquet(key, rows)
except Exception as exc:
st["error"] = f"parquet/s3: {str(exc)[:140]}"
_term(f" ✗ parquet/s3 write failed: {str(exc)[:140]}", level="err", phase="load")
return res
_term(f" ← pyarrow.write_table → s3://data/{obj_key} ({len(rows)} rows · {nbytes / 1024:.1f} KB)",
level="ok", phase="load")
o_rows, o_rev = _agg(key, rows)
with _lock:
st["parts"] += 1
@@ -378,6 +394,8 @@ def run_cycle() -> dict[str, Any]:
order_rows = 0
revenue = 0.0
try:
_term(f"═══ ETL offload cycle {_state['cycles'] + 1} — source DBs → S3 Parquet lake ═══",
level="info", phase="cycle")
_source_totals()
for d in DATASETS:
if not _state["enabled"]:
@@ -404,7 +422,12 @@ def run_cycle() -> dict[str, Any]:
_state["series"].append(point)
with _lock:
_business["ts"].append(point)
_term(f"═══ cycle {_state['cycles']} done — {total:,} rows · {cbytes / 1024:.1f} KB · €{int(revenue):,} revenue ═══",
level="ok", phase="cycle")
_feed(f"offloaded {total:,} rows → S3 Parquet · €{int(revenue):,} (cycle {_state['cycles']})")
else:
_term(f" cycle {_state['cycles']} — no new rows (all datasets caught up, tailing)",
level="info", phase="cycle")
finally:
_state["running_cycle"] = False
return {"ok": True, "rows": total, "bytes": cbytes, "revenue": round(revenue, 2)}