feat: live per-agent terminal activity with real scripts/SQL
Agent terminals were idle (one-shot probe) while agents were busy in the background. Now every agent streams what it is actually doing: - agent_terminal: emit_threadsafe() so background threads can stream lines. - agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL Guardian announces each orchestrated movement. - movements: trigger_and_watch streams the Airflow DAG / API call, conf, before/after Trino counts and result to the owning agent terminal. - etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle summaries stream to the ETL Guardian terminal. - agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host load) and Network Watcher (VLAN 20/21 path checks). - fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61). - ui: terminal dock merges the selected agent ops stream with the node probe.
This commit is contained in:
mo
+82
-46
@@ -103,6 +103,15 @@ async def _emit(message: str, level: str = "info") -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _term(agent_id: str, message: str, level: str = "info", phase: str = "ops") -> None:
|
||||
"""Stream a line to a specific agent terminal (no feed entry)."""
|
||||
try:
|
||||
from agent_terminal import terminal_log
|
||||
await terminal_log(agent_id, message, level=level, phase=phase)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Synchronous DB operations (run in a thread) ──────────────────────────────
|
||||
def _pg_conn():
|
||||
import psycopg2
|
||||
@@ -152,7 +161,7 @@ def _resolve_op(op: str, pool: deque) -> str:
|
||||
return op
|
||||
|
||||
|
||||
def _pg_dml(op: str) -> str:
|
||||
def _pg_dml(op: str) -> dict[str, str]:
|
||||
pool = _pools["postgres"]
|
||||
op = _resolve_op(op, pool)
|
||||
conn = _pg_conn()
|
||||
@@ -161,35 +170,39 @@ def _pg_dml(op: str) -> str:
|
||||
with conn.cursor() as cur:
|
||||
if op == "insert":
|
||||
rid = uuid.uuid4().hex[:8]
|
||||
cid, pid = random.randint(1, 50000), random.randint(1, 2000)
|
||||
region, channel = random.choice(REGIONS), random.choice(CHANNELS)
|
||||
amt, curr, status = round(random.uniform(10, 9999), 2), random.choice(CURRENCIES), random.choice(ORDER_STATUS)
|
||||
cur.execute(
|
||||
"""INSERT INTO public.sales_orders
|
||||
(customer_id, product_id, region, sales_channel, order_ts, amount, currency, order_status, notes)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING order_id""",
|
||||
(
|
||||
random.randint(1, 50000), random.randint(1, 2000),
|
||||
random.choice(REGIONS), random.choice(CHANNELS),
|
||||
datetime.now(timezone.utc), round(random.uniform(10, 9999), 2),
|
||||
random.choice(CURRENCIES), random.choice(ORDER_STATUS), _tag(rid),
|
||||
),
|
||||
(cid, pid, region, channel, datetime.now(timezone.utc), amt, curr, status, _tag(rid)),
|
||||
)
|
||||
oid = cur.fetchone()[0]
|
||||
pool.append(oid)
|
||||
return f"INSERT sales_orders order_id={oid} ({_tag(rid)})"
|
||||
sql = (f"INSERT INTO public.sales_orders (customer_id,product_id,region,sales_channel,"
|
||||
f"amount,currency,order_status,notes) VALUES ({cid},{pid},'{region}','{channel}',"
|
||||
f"{amt},'{curr}','{status}','{_tag(rid)}');")
|
||||
return {"op": "insert", "detail": f"INSERT sales_orders order_id={oid} ({_tag(rid)})", "sql": sql}
|
||||
if op == "update":
|
||||
oid = random.choice(list(pool))
|
||||
status, mult = random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2)
|
||||
cur.execute(
|
||||
"UPDATE public.sales_orders SET order_status=%s, amount=round(amount*%s,2) WHERE order_id=%s",
|
||||
(random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2), oid),
|
||||
(status, mult, oid),
|
||||
)
|
||||
return f"UPDATE sales_orders order_id={oid}"
|
||||
sql = f"UPDATE public.sales_orders SET order_status='{status}', amount=round(amount*{mult},2) WHERE order_id={oid};"
|
||||
return {"op": "update", "detail": f"UPDATE sales_orders order_id={oid}", "sql": sql}
|
||||
oid = pool.popleft()
|
||||
cur.execute("DELETE FROM public.sales_orders WHERE order_id=%s", (oid,))
|
||||
return f"DELETE sales_orders order_id={oid}"
|
||||
return {"op": "delete", "detail": f"DELETE sales_orders order_id={oid}",
|
||||
"sql": f"DELETE FROM public.sales_orders WHERE order_id={oid};"}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _mysql_dml(op: str) -> str:
|
||||
def _mysql_dml(op: str) -> dict[str, str]:
|
||||
pool = _pools["mysql"]
|
||||
op = _resolve_op(op, pool)
|
||||
conn = _mysql_conn()
|
||||
@@ -197,64 +210,64 @@ def _mysql_dml(op: str) -> str:
|
||||
with conn.cursor() as cur:
|
||||
if op == "insert":
|
||||
rid = uuid.uuid4().hex[:8]
|
||||
eid_in, dept, role = random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES)
|
||||
region, etype, sal = random.choice(REGIONS), random.choice(EVENT_TYPES), round(random.uniform(-5000, 15000), 2)
|
||||
cur.execute(
|
||||
"""INSERT INTO hr.employee_events
|
||||
(employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(
|
||||
random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES),
|
||||
random.choice(REGIONS), random.choice(EVENT_TYPES),
|
||||
round(random.uniform(-5000, 15000), 2), datetime.now(timezone.utc), _tag(rid),
|
||||
),
|
||||
(eid_in, dept, role, region, etype, sal, datetime.now(timezone.utc), _tag(rid)),
|
||||
)
|
||||
eid = cur.lastrowid
|
||||
pool.append(eid)
|
||||
return f"INSERT employee_events event_id={eid} ({_tag(rid)})"
|
||||
sql = (f"INSERT INTO hr.employee_events (employee_id,department,role_name,region,event_type,"
|
||||
f"salary_change,notes) VALUES ({eid_in},'{dept}','{role}','{region}','{etype}',{sal},'{_tag(rid)}');")
|
||||
return {"op": "insert", "detail": f"INSERT employee_events event_id={eid} ({_tag(rid)})", "sql": sql}
|
||||
if op == "update":
|
||||
eid = random.choice(list(pool))
|
||||
sal, etype = round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES)
|
||||
cur.execute(
|
||||
"UPDATE hr.employee_events SET salary_change=%s, event_type=%s WHERE event_id=%s",
|
||||
(round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES), eid),
|
||||
(sal, etype, eid),
|
||||
)
|
||||
return f"UPDATE employee_events event_id={eid}"
|
||||
sql = f"UPDATE hr.employee_events SET salary_change={sal}, event_type='{etype}' WHERE event_id={eid};"
|
||||
return {"op": "update", "detail": f"UPDATE employee_events event_id={eid}", "sql": sql}
|
||||
eid = pool.popleft()
|
||||
cur.execute("DELETE FROM hr.employee_events WHERE event_id=%s", (eid,))
|
||||
return f"DELETE employee_events event_id={eid}"
|
||||
return {"op": "delete", "detail": f"DELETE employee_events event_id={eid}",
|
||||
"sql": f"DELETE FROM hr.employee_events WHERE event_id={eid};"}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _mongo_dml(op: str) -> str:
|
||||
def _mongo_dml(op: str) -> dict[str, str]:
|
||||
pool = _pools["mongodb"]
|
||||
op = _resolve_op(op, pool)
|
||||
client, coll = _mongo_coll()
|
||||
try:
|
||||
if op == "insert":
|
||||
rid = uuid.uuid4().hex[:8]
|
||||
mtype, region, msrc = random.choice(MONGO_TYPES), random.choice(REGIONS), random.choice(MONGO_SOURCES)
|
||||
amt = round(random.uniform(10, 50000), 4)
|
||||
doc = {
|
||||
"event_id": str(uuid.uuid4()),
|
||||
"type": random.choice(MONGO_TYPES),
|
||||
"region": random.choice(REGIONS),
|
||||
"source": random.choice(MONGO_SOURCES),
|
||||
"amount": round(random.uniform(10, 50000), 4),
|
||||
"ts": datetime.now(timezone.utc),
|
||||
"payload": "X" * 200,
|
||||
"atc_agent": True,
|
||||
"agent_run": rid,
|
||||
"event_id": str(uuid.uuid4()), "type": mtype, "region": region, "source": msrc,
|
||||
"amount": amt, "ts": datetime.now(timezone.utc), "payload": "X" * 200,
|
||||
"atc_agent": True, "agent_run": rid,
|
||||
}
|
||||
res = coll.insert_one(doc)
|
||||
pool.append(res.inserted_id)
|
||||
return f"INSERT events _id={res.inserted_id} (agent_run={rid})"
|
||||
sql = (f"db.events.insertOne({{type:'{mtype}', region:'{region}', source:'{msrc}', "
|
||||
f"amount:{amt}, atc_agent:true, agent_run:'{rid}'}})")
|
||||
return {"op": "insert", "detail": f"INSERT events _id={res.inserted_id} (agent_run={rid})", "sql": sql}
|
||||
if op == "update":
|
||||
oid = random.choice(list(pool))
|
||||
coll.update_one(
|
||||
{"_id": oid},
|
||||
{"$set": {"type": random.choice(MONGO_TYPES), "amount": round(random.uniform(10, 50000), 4)}},
|
||||
)
|
||||
return f"UPDATE events _id={oid}"
|
||||
mtype, amt = random.choice(MONGO_TYPES), round(random.uniform(10, 50000), 4)
|
||||
coll.update_one({"_id": oid}, {"$set": {"type": mtype, "amount": amt}})
|
||||
sql = f"db.events.updateOne({{_id:{oid!r}}}, {{$set:{{type:'{mtype}', amount:{amt}}}}})"
|
||||
return {"op": "update", "detail": f"UPDATE events _id={oid}", "sql": sql}
|
||||
oid = pool.popleft()
|
||||
coll.delete_one({"_id": oid})
|
||||
return f"DELETE events _id={oid}"
|
||||
return {"op": "delete", "detail": f"DELETE events _id={oid}", "sql": f"db.events.deleteOne({{_id:{oid!r}}})"}
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
@@ -268,6 +281,9 @@ def _pick_op() -> str:
|
||||
return random.choices(["insert", "update", "delete"], weights=[0.5, 0.3, 0.2], k=1)[0]
|
||||
|
||||
|
||||
_CLIENT_CMD = {"postgres": "psql sales", "mysql": "mysql hr", "mongodb": "mongosh supplychain"}
|
||||
|
||||
|
||||
async def _run_one(source: str | None = None, op: str | None = None) -> dict[str, Any]:
|
||||
source = source or random.choice(list(_DISPATCH.keys()))
|
||||
op = op or _pick_op()
|
||||
@@ -275,18 +291,25 @@ async def _run_one(source: str | None = None, op: str | None = None) -> dict[str
|
||||
if not fn:
|
||||
return {"ok": False, "error": f"unknown source {source}"}
|
||||
try:
|
||||
detail = await asyncio.to_thread(fn, op)
|
||||
# Show the operator exactly what the Data Custodian is about to run.
|
||||
await _term(DML_AGENT, f"$ {_CLIENT_CMD.get(source, source)} # autonomous DML on {_SRC_LABEL[source]}",
|
||||
level="cmd", phase="dml")
|
||||
res = await asyncio.to_thread(fn, op)
|
||||
detail, sql, actual_op = res["detail"], res["sql"], res["op"]
|
||||
await _term(DML_AGENT, f" {sql}", level="cmd", phase="dml")
|
||||
_state["ops_total"] += 1
|
||||
_state["by_source"][source] = _state["by_source"].get(source, 0) + 1
|
||||
actual_op = detail.split(" ", 1)[0].lower()
|
||||
if actual_op in _state["by_op"]:
|
||||
_state["by_op"][actual_op] += 1
|
||||
_state["last_op"] = {"source": source, "op": op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
|
||||
_state["last_op"] = {"source": source, "op": actual_op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
|
||||
_state["last_error"] = None
|
||||
await _term(DML_AGENT, f" ← {detail} · Debezium CDC will stream this to Kafka", level="ok", phase="dml")
|
||||
# Keep the supervisor feed concise (single summary entry).
|
||||
await _emit(f"[agent-dml] {_SRC_LABEL[source]}: {detail} — Debezium will capture this change", "info")
|
||||
return {"ok": True, "source": source, "op": op, "detail": detail}
|
||||
return {"ok": True, "source": source, "op": actual_op, "detail": detail}
|
||||
except Exception as exc:
|
||||
_state["last_error"] = str(exc)
|
||||
await _term(DML_AGENT, f" ✗ {source} {op} failed: {str(exc)[:140]}", level="err", phase="dml")
|
||||
await _emit(f"[agent-dml] {_SRC_LABEL.get(source, source)}: operation failed: {str(exc)[:120]}", "err")
|
||||
return {"ok": False, "source": source, "op": op, "error": str(exc)}
|
||||
|
||||
@@ -318,7 +341,14 @@ async def etl_agent_loop() -> None:
|
||||
mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])]
|
||||
_etl_state["idx"] += 1
|
||||
if mid in MOVEMENT_BY_ID:
|
||||
mv = MOVEMENT_BY_ID[mid]
|
||||
await _term("etl-guardian",
|
||||
f"$ orchestrate movement '{mid}' ({mv.get('label')}) {mv.get('from')}→{mv.get('to')}",
|
||||
level="cmd", phase="orchestrate")
|
||||
result = await trigger_and_watch(mid, autonomous=True)
|
||||
await _term("etl-guardian",
|
||||
f" ← {mv.get('label')}: {result.get('state')} · {result.get('rows')} rows in {result.get('duration_s')}s",
|
||||
level="ok" if result.get("ok") else "err", phase="orchestrate")
|
||||
_etl_state["runs_total"] += 1
|
||||
_etl_state["last"] = {"movement_id": mid, "state": result.get("state"),
|
||||
"rows": result.get("rows"), "duration_s": result.get("duration_s"),
|
||||
@@ -355,10 +385,14 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
|
||||
i = _custodian_state["idx"] if idx is None else idx
|
||||
tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
|
||||
_custodian_state["idx"] = i + 1
|
||||
await _trino_collect(
|
||||
f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1)
|
||||
ins = await _trino_collect(
|
||||
f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1)
|
||||
ddl = f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0"
|
||||
dml = f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}"
|
||||
await _term(DML_AGENT, f"$ trino --catalog iceberg # Hadoop offload: {tgt['label']} → {tgt['target']}",
|
||||
level="cmd", phase="offload")
|
||||
await _term(DML_AGENT, f" {ddl};", level="cmd", phase="offload")
|
||||
await _trino_collect(ddl, 1)
|
||||
await _term(DML_AGENT, f" {dml};", level="cmd", phase="offload")
|
||||
ins = await _trino_collect(dml, 1)
|
||||
ok = bool(ins.get("ok"))
|
||||
_custodian_state["runs_total"] += 1
|
||||
_custodian_state["last"] = {
|
||||
@@ -367,8 +401,10 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
|
||||
"ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"),
|
||||
}
|
||||
if ok:
|
||||
await _term(DML_AGENT, f" ← offloaded ~{_CUST_BATCH} rows into the Hadoop Iceberg lake", level="ok", phase="offload")
|
||||
await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info")
|
||||
else:
|
||||
await _term(DML_AGENT, f" ✗ offload failed: {str(ins.get('error'))[:140]}", level="err", phase="offload")
|
||||
await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err")
|
||||
return _custodian_state["last"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user