feat: live per-agent terminal activity with real scripts/SQL
Agent terminals were idle (one-shot probe) while agents were busy in the background. Now every agent streams what it is actually doing: - agent_terminal: emit_threadsafe() so background threads can stream lines. - agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL Guardian announces each orchestrated movement. - movements: trigger_and_watch streams the Airflow DAG / API call, conf, before/after Trino counts and result to the owning agent terminal. - etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle summaries stream to the ETL Guardian terminal. - agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host load) and Network Watcher (VLAN 20/21 path checks). - fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61). - ui: terminal dock merges the selected agent ops stream with the node probe.
This commit is contained in:
mo
@@ -0,0 +1,204 @@
|
||||
"""Continuous autonomous activity for the field agents that don't drive a
|
||||
data-movement loop of their own (Lakehouse Ops, Hadoop Ranger, Infra Sentinel,
|
||||
Network Watcher).
|
||||
|
||||
Every tick the loop runs ONE real, lightweight probe for the next agent in the
|
||||
rotation and streams the exact command + result into that agent's terminal, so
|
||||
the operator can always see what each agent is doing in the background instead
|
||||
of an idle prompt. All probes hit live endpoints (Trino, WebHDFS JMX, YARN,
|
||||
Dockhand, VLAN hosts) and are individually guarded so a single failure never
|
||||
breaks the loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
|
||||
YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088").rstrip("/")
|
||||
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082").rstrip("/")
|
||||
OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")).rstrip("/")
|
||||
|
||||
TICK_SECONDS = float(os.getenv("AGENT_ACTIVITY_TICK_SECONDS", "8"))
|
||||
|
||||
# VLAN data paths the Network Watcher keeps an eye on.
|
||||
_NET_TARGETS = [
|
||||
("Kafka UI", "http://10.0.21.36:9000"),
|
||||
("Trino", f"{TRINO_URL}/v1/info"),
|
||||
("HDFS NameNode", f"{HDFS_NN_URL}/dfshealth.html"),
|
||||
("Airflow", os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080")),
|
||||
("ObjectScale S3", OBJECTSCALE_URL),
|
||||
]
|
||||
|
||||
|
||||
async def _term(agent_id: str, text: str, level: str = "info", phase: str = "ops") -> None:
|
||||
try:
|
||||
from agent_terminal import terminal_log
|
||||
await terminal_log(agent_id, text, level=level, phase=phase)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _trino(sql: str, timeout: float = 8.0) -> list[list[Any]]:
|
||||
rows: list[list[Any]] = []
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
d = (await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(),
|
||||
headers={"X-Trino-User": TRINO_USER})).json()
|
||||
for _ in range(40):
|
||||
if d.get("error"):
|
||||
raise RuntimeError(d["error"].get("message", "trino error"))
|
||||
rows += d.get("data") or []
|
||||
nxt = d.get("nextUri")
|
||||
if not nxt:
|
||||
break
|
||||
d = (await client.get(nxt)).json()
|
||||
return rows
|
||||
|
||||
|
||||
# ── per-agent probes ─────────────────────────────────────────────────────────
|
||||
async def _lakehouse_ops() -> None:
|
||||
aid = "lakehouse-ops"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||||
info = (await client.get(f"{TRINO_URL}/v1/info")).json()
|
||||
ver = info.get("nodeVersion", {}).get("version", "?")
|
||||
up = info.get("uptime", "?")
|
||||
await _term(aid, f"$ curl -s {TRINO_URL}/v1/info # Trino coordinator health", level="cmd", phase="trino")
|
||||
await _term(aid, f" ← Trino {ver} · uptime {up} · serving federated queries", level="ok", phase="trino")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ✗ Trino unreachable: {str(exc)[:100]}", level="err", phase="trino")
|
||||
try:
|
||||
cats = await _trino("SHOW CATALOGS")
|
||||
names = ", ".join(sorted(c[0] for c in cats))
|
||||
await _term(aid, "$ trino --execute 'SHOW CATALOGS'", level="cmd", phase="trino")
|
||||
await _term(aid, f" ← {len(cats)} catalogs federated: {names}", level="ok", phase="trino")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ✗ SHOW CATALOGS failed: {str(exc)[:100]}", level="err", phase="trino")
|
||||
try:
|
||||
sql = "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"
|
||||
rows = await _trino(sql, timeout=12.0)
|
||||
n = int(rows[0][0]) if rows else 0
|
||||
await _term(aid, f"$ trino --execute '{sql}' # curated Iceberg lakehouse", level="cmd", phase="iceberg")
|
||||
await _term(aid, f" ← {n:,} masked rows in iceberg.curated_masked (PII-safe layer)", level="ok", phase="iceberg")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ✗ Iceberg count failed: {str(exc)[:100]}", level="err", phase="iceberg")
|
||||
|
||||
|
||||
def _g(d: dict, *keys) -> Any:
|
||||
for k in keys:
|
||||
if k in d:
|
||||
return d[k]
|
||||
return None
|
||||
|
||||
|
||||
async def _hadoop_ranger() -> None:
|
||||
aid = "hadoop-ranger"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||||
fs = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem")).json()
|
||||
state = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")).json()
|
||||
fsb = (fs.get("beans") or [{}])[0]
|
||||
stb = (state.get("beans") or [{}])[0]
|
||||
cap_total = float(_g(fsb, "CapacityTotalGB") or 0)
|
||||
cap_used = float(_g(fsb, "CapacityUsedGB") or 0)
|
||||
blocks = int(_g(fsb, "BlocksTotal", "TotalBlocks") or _g(stb, "BlocksTotal") or 0)
|
||||
live = int(_g(stb, "NumLiveDataNodes") or 0)
|
||||
dead = int(_g(stb, "NumDeadDataNodes") or 0)
|
||||
pct = (100.0 * cap_used / cap_total) if cap_total else 0.0
|
||||
await _term(aid, f"$ curl -s {HDFS_NN_URL}/jmx?qry=...FSNamesystemState # NameNode health", level="cmd", phase="hdfs")
|
||||
await _term(aid, f" ← live datanodes={live} dead={dead} · blocks={blocks:,} · "
|
||||
f"used {cap_used:.1f}/{cap_total:.1f} GB ({pct:.0f}%)", level="ok", phase="hdfs")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ✗ NameNode JMX unreachable: {str(exc)[:100]}", level="err", phase="hdfs")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||||
m = (await client.get(f"{YARN_URL}/ws/v1/cluster/metrics")).json().get("clusterMetrics", {})
|
||||
await _term(aid, f"$ yarn application -list # ResourceManager {YARN_URL}", level="cmd", phase="yarn")
|
||||
await _term(aid, f" ← apps running={m.get('appsRunning', 0)} pending={m.get('appsPending', 0)} · "
|
||||
f"available {round(m.get('availableMB', 0) / 1024, 1)} GB / "
|
||||
f"{m.get('totalNodes', 0)} nodes", level="ok", phase="yarn")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ⚠ YARN RM not responding ({str(exc)[:60]}) — HDFS storage layer still healthy", level="warn", phase="yarn")
|
||||
|
||||
|
||||
async def _infra_sentinel() -> None:
|
||||
aid = "infra-sentinel"
|
||||
await _term(aid, f"$ dockhand ps --all-envs # container inventory via {DOCKHAND_URL}", level="cmd", phase="docker")
|
||||
try:
|
||||
from dockhand_envs import DOCKHAND_ENVS
|
||||
envs = DOCKHAND_ENVS
|
||||
except Exception:
|
||||
envs = {"docker01": 1, "docker02": 2, "lakehouse": 9, "airflow": 10, "db02": 5}
|
||||
total = running = 0
|
||||
reached = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=4.0) as client:
|
||||
async def _one(name: str, eid: int):
|
||||
try:
|
||||
r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": eid})
|
||||
if r.status_code >= 400:
|
||||
return None
|
||||
d = r.json()
|
||||
return d if isinstance(d, list) else d.get("containers", [])
|
||||
except Exception:
|
||||
return None
|
||||
results = await asyncio.gather(*[_one(n, e) for n, e in envs.items()])
|
||||
for conts in results:
|
||||
if conts is None:
|
||||
continue
|
||||
reached += 1
|
||||
total += len(conts)
|
||||
running += sum(1 for c in conts
|
||||
if str(c.get("state", c.get("status", ""))).lower().startswith(("run", "up")))
|
||||
if reached:
|
||||
await _term(aid, f" ← {running}/{total} containers up across {reached} Dockhand environments", level="ok", phase="docker")
|
||||
else:
|
||||
await _term(aid, " ✗ Dockhand returned no environments", level="warn", phase="docker")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" ✗ Dockhand unreachable: {str(exc)[:100]}", level="err", phase="docker")
|
||||
try:
|
||||
load = os.getloadavg()
|
||||
await _term(aid, "$ cat /proc/loadavg # command-center host", level="cmd", phase="host")
|
||||
await _term(aid, f" ← load avg {load[0]:.2f} {load[1]:.2f} {load[2]:.2f} (1/5/15m)", level="ok", phase="host")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _network_watcher() -> None:
|
||||
aid = "network-watcher"
|
||||
await _term(aid, "$ probe VLAN 20/21 data paths # ingress/egress reachability", level="cmd", phase="net")
|
||||
async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
|
||||
for label, url in _NET_TARGETS:
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = await client.get(url)
|
||||
ms = int((time.time() - t0) * 1000)
|
||||
lvl = "ok" if r.status_code < 500 else "warn"
|
||||
await _term(aid, f" → {label:<16} {url} {r.status_code} {ms}ms", level=lvl, phase="net")
|
||||
except Exception as exc:
|
||||
await _term(aid, f" → {label:<16} {url} DOWN ({str(exc)[:60]})", level="err", phase="net")
|
||||
|
||||
|
||||
_ROTATION = [_lakehouse_ops, _hadoop_ranger, _infra_sentinel, _network_watcher]
|
||||
|
||||
|
||||
async def agent_activity_loop() -> None:
|
||||
"""Round-robin: run one agent's live probe per tick so each agent terminal
|
||||
shows fresh real activity roughly every (len(rotation) * TICK_SECONDS)s."""
|
||||
await asyncio.sleep(15) # let the platform settle
|
||||
idx = 0
|
||||
while True:
|
||||
probe = _ROTATION[idx % len(_ROTATION)]
|
||||
idx += 1
|
||||
try:
|
||||
await probe()
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(max(3.0, TICK_SECONDS))
|
||||
Reference in New Issue
Block a user