feat: live per-agent terminal activity with real scripts/SQL

Agent terminals were idle (one-shot probe) while agents were busy in the
background. Now every agent streams what it is actually doing:

- agent_terminal: emit_threadsafe() so background threads can stream lines.
- agent_ops: Data Custodian DML loop logs the real INSERT/UPDATE/DELETE SQL
  (+ Mongo ops) and Hadoop-offload Trino CTAS/INSERT to its terminal; ETL
  Guardian announces each orchestrated movement.
- movements: trigger_and_watch streams the Airflow DAG / API call, conf,
  before/after Trino counts and result to the owning agent terminal.
- etl_offload: per-dataset read + pyarrow->S3 parquet writes and cycle
  summaries stream to the ETL Guardian terminal.
- agent_activity (new): round-robin live probes for Lakehouse Ops, Hadoop
  Ranger (NameNode JMX + YARN), Infra Sentinel (Dockhand inventory + host
  load) and Network Watcher (VLAN 20/21 path checks).
- fix: YARN ResourceManager runs on 10.0.21.62:8088 (was .61).
- ui: terminal dock merges the selected agent ops stream with the node probe.

api/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
-COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py hive_bench_seed.json .
+COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py agent_activity.py hive_bench_seed.json .
 RUN mkdir -p /data
 ENV DATABASE_URL=sqlite:////data/atc-agents.db
 EXPOSE 3201

api/agent_activity.py

@@ -0,0 +1,204 @@
+"""Continuous autonomous activity for the field agents that don't drive a
+data-movement loop of their own (Lakehouse Ops, Hadoop Ranger, Infra Sentinel,
+Network Watcher).
+
+Every tick the loop runs ONE real, lightweight probe for the next agent in the
+rotation and streams the exact command + result into that agent's terminal, so
+the operator can always see what each agent is doing in the background instead
+of an idle prompt. All probes hit live endpoints (Trino, WebHDFS JMX, YARN,
+Dockhand, VLAN hosts) and are individually guarded so a single failure never
+breaks the loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from typing import Any
+
+import httpx
+
+TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
+TRINO_USER = os.getenv("TRINO_USER", "mo")
+HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
+YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088").rstrip("/")
+DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082").rstrip("/")
+OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")).rstrip("/")
+
+TICK_SECONDS = float(os.getenv("AGENT_ACTIVITY_TICK_SECONDS", "8"))
+
+# VLAN data paths the Network Watcher keeps an eye on.
+_NET_TARGETS = [
+    ("Kafka UI", "http://10.0.21.36:9000"),
+    ("Trino", f"{TRINO_URL}/v1/info"),
+    ("HDFS NameNode", f"{HDFS_NN_URL}/dfshealth.html"),
+    ("Airflow", os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080")),
+    ("ObjectScale S3", OBJECTSCALE_URL),
+]
+
+
+async def _term(agent_id: str, text: str, level: str = "info", phase: str = "ops") -> None:
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+async def _trino(sql: str, timeout: float = 8.0) -> list[list[Any]]:
+    rows: list[list[Any]] = []
+    async with httpx.AsyncClient(timeout=timeout) as client:
+        d = (await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(),
+                               headers={"X-Trino-User": TRINO_USER})).json()
+        for _ in range(40):
+            if d.get("error"):
+                raise RuntimeError(d["error"].get("message", "trino error"))
+            rows += d.get("data") or []
+            nxt = d.get("nextUri")
+            if not nxt:
+                break
+            d = (await client.get(nxt)).json()
+    return rows
+
+
+# ── per-agent probes ─────────────────────────────────────────────────────────
+async def _lakehouse_ops() -> None:
+    aid = "lakehouse-ops"
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            info = (await client.get(f"{TRINO_URL}/v1/info")).json()
+        ver = info.get("nodeVersion", {}).get("version", "?")
+        up = info.get("uptime", "?")
+        await _term(aid, f"$ curl -s {TRINO_URL}/v1/info  # Trino coordinator health", level="cmd", phase="trino")
+        await _term(aid, f"  ← Trino {ver} · uptime {up} · serving federated queries", level="ok", phase="trino")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Trino unreachable: {str(exc)[:100]}", level="err", phase="trino")
+    try:
+        cats = await _trino("SHOW CATALOGS")
+        names = ", ".join(sorted(c[0] for c in cats))
+        await _term(aid, "$ trino --execute 'SHOW CATALOGS'", level="cmd", phase="trino")
+        await _term(aid, f"  ← {len(cats)} catalogs federated: {names}", level="ok", phase="trino")
+    except Exception as exc:
+        await _term(aid, f"  ✗ SHOW CATALOGS failed: {str(exc)[:100]}", level="err", phase="trino")
+    try:
+        sql = "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"
+        rows = await _trino(sql, timeout=12.0)
+        n = int(rows[0][0]) if rows else 0
+        await _term(aid, f"$ trino --execute '{sql}'  # curated Iceberg lakehouse", level="cmd", phase="iceberg")
+        await _term(aid, f"  ← {n:,} masked rows in iceberg.curated_masked (PII-safe layer)", level="ok", phase="iceberg")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Iceberg count failed: {str(exc)[:100]}", level="err", phase="iceberg")
+
+
+def _g(d: dict, *keys) -> Any:
+    for k in keys:
+        if k in d:
+            return d[k]
+    return None
+
+
+async def _hadoop_ranger() -> None:
+    aid = "hadoop-ranger"
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            fs = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem")).json()
+            state = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")).json()
+        fsb = (fs.get("beans") or [{}])[0]
+        stb = (state.get("beans") or [{}])[0]
+        cap_total = float(_g(fsb, "CapacityTotalGB") or 0)
+        cap_used = float(_g(fsb, "CapacityUsedGB") or 0)
+        blocks = int(_g(fsb, "BlocksTotal", "TotalBlocks") or _g(stb, "BlocksTotal") or 0)
+        live = int(_g(stb, "NumLiveDataNodes") or 0)
+        dead = int(_g(stb, "NumDeadDataNodes") or 0)
+        pct = (100.0 * cap_used / cap_total) if cap_total else 0.0
+        await _term(aid, f"$ curl -s {HDFS_NN_URL}/jmx?qry=...FSNamesystemState  # NameNode health", level="cmd", phase="hdfs")
+        await _term(aid, f"  ← live datanodes={live} dead={dead} · blocks={blocks:,} · "
+                         f"used {cap_used:.1f}/{cap_total:.1f} GB ({pct:.0f}%)", level="ok", phase="hdfs")
+    except Exception as exc:
+        await _term(aid, f"  ✗ NameNode JMX unreachable: {str(exc)[:100]}", level="err", phase="hdfs")
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            m = (await client.get(f"{YARN_URL}/ws/v1/cluster/metrics")).json().get("clusterMetrics", {})
+        await _term(aid, f"$ yarn application -list  # ResourceManager {YARN_URL}", level="cmd", phase="yarn")
+        await _term(aid, f"  ← apps running={m.get('appsRunning', 0)} pending={m.get('appsPending', 0)} · "
+                         f"available {round(m.get('availableMB', 0) / 1024, 1)} GB / "
+                         f"{m.get('totalNodes', 0)} nodes", level="ok", phase="yarn")
+    except Exception as exc:
+        await _term(aid, f"  ⚠ YARN RM not responding ({str(exc)[:60]}) — HDFS storage layer still healthy", level="warn", phase="yarn")
+
+
+async def _infra_sentinel() -> None:
+    aid = "infra-sentinel"
+    await _term(aid, f"$ dockhand ps --all-envs  # container inventory via {DOCKHAND_URL}", level="cmd", phase="docker")
+    try:
+        from dockhand_envs import DOCKHAND_ENVS
+        envs = DOCKHAND_ENVS
+    except Exception:
+        envs = {"docker01": 1, "docker02": 2, "lakehouse": 9, "airflow": 10, "db02": 5}
+    total = running = 0
+    reached = 0
+    try:
+        async with httpx.AsyncClient(timeout=4.0) as client:
+            async def _one(name: str, eid: int):
+                try:
+                    r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": eid})
+                    if r.status_code >= 400:
+                        return None
+                    d = r.json()
+                    return d if isinstance(d, list) else d.get("containers", [])
+                except Exception:
+                    return None
+            results = await asyncio.gather(*[_one(n, e) for n, e in envs.items()])
+        for conts in results:
+            if conts is None:
+                continue
+            reached += 1
+            total += len(conts)
+            running += sum(1 for c in conts
+                           if str(c.get("state", c.get("status", ""))).lower().startswith(("run", "up")))
+        if reached:
+            await _term(aid, f"  ← {running}/{total} containers up across {reached} Dockhand environments", level="ok", phase="docker")
+        else:
+            await _term(aid, "  ✗ Dockhand returned no environments", level="warn", phase="docker")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Dockhand unreachable: {str(exc)[:100]}", level="err", phase="docker")
+    try:
+        load = os.getloadavg()
+        await _term(aid, "$ cat /proc/loadavg  # command-center host", level="cmd", phase="host")
+        await _term(aid, f"  ← load avg {load[0]:.2f} {load[1]:.2f} {load[2]:.2f} (1/5/15m)", level="ok", phase="host")
+    except Exception:
+        pass
+
+
+async def _network_watcher() -> None:
+    aid = "network-watcher"
+    await _term(aid, "$ probe VLAN 20/21 data paths  # ingress/egress reachability", level="cmd", phase="net")
+    async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
+        for label, url in _NET_TARGETS:
+            t0 = time.time()
+            try:
+                r = await client.get(url)
+                ms = int((time.time() - t0) * 1000)
+                lvl = "ok" if r.status_code < 500 else "warn"
+                await _term(aid, f"  → {label:<16} {url}  {r.status_code} {ms}ms", level=lvl, phase="net")
+            except Exception as exc:
+                await _term(aid, f"  → {label:<16} {url}  DOWN ({str(exc)[:60]})", level="err", phase="net")
+
+
+_ROTATION = [_lakehouse_ops, _hadoop_ranger, _infra_sentinel, _network_watcher]
+
+
+async def agent_activity_loop() -> None:
+    """Round-robin: run one agent's live probe per tick so each agent terminal
+    shows fresh real activity roughly every (len(rotation) * TICK_SECONDS)s."""
+    await asyncio.sleep(15)  # let the platform settle
+    idx = 0
+    while True:
+        probe = _ROTATION[idx % len(_ROTATION)]
+        idx += 1
+        try:
+            await probe()
+        except Exception:
+            pass
+        await asyncio.sleep(max(3.0, TICK_SECONDS))

api/agent_ops.py

@@ -103,6 +103,15 @@ async def _emit(message: str, level: str = "info") -> None:
         pass
 
 
+async def _term(agent_id: str, message: str, level: str = "info", phase: str = "ops") -> None:
+    """Stream a line to a specific agent terminal (no feed entry)."""
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, message, level=level, phase=phase)
+    except Exception:
+        pass
+
+
 # ── Synchronous DB operations (run in a thread) ──────────────────────────────
 def _pg_conn():
     import psycopg2
@@ -152,7 +161,7 @@ def _resolve_op(op: str, pool: deque) -> str:
     return op
 
 
-def _pg_dml(op: str) -> str:
+def _pg_dml(op: str) -> dict[str, str]:
     pool = _pools["postgres"]
     op = _resolve_op(op, pool)
     conn = _pg_conn()
@@ -161,35 +170,39 @@ def _pg_dml(op: str) -> str:
         with conn.cursor() as cur:
             if op == "insert":
                 rid = uuid.uuid4().hex[:8]
+                cid, pid = random.randint(1, 50000), random.randint(1, 2000)
+                region, channel = random.choice(REGIONS), random.choice(CHANNELS)
+                amt, curr, status = round(random.uniform(10, 9999), 2), random.choice(CURRENCIES), random.choice(ORDER_STATUS)
                 cur.execute(
                     """INSERT INTO public.sales_orders
                        (customer_id, product_id, region, sales_channel, order_ts, amount, currency, order_status, notes)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING order_id""",
-                    (
-                        random.randint(1, 50000), random.randint(1, 2000),
-                        random.choice(REGIONS), random.choice(CHANNELS),
-                        datetime.now(timezone.utc), round(random.uniform(10, 9999), 2),
-                        random.choice(CURRENCIES), random.choice(ORDER_STATUS), _tag(rid),
-                    ),
+                    (cid, pid, region, channel, datetime.now(timezone.utc), amt, curr, status, _tag(rid)),
                 )
                 oid = cur.fetchone()[0]
                 pool.append(oid)
-                return f"INSERT sales_orders order_id={oid} ({_tag(rid)})"
+                sql = (f"INSERT INTO public.sales_orders (customer_id,product_id,region,sales_channel,"
+                       f"amount,currency,order_status,notes) VALUES ({cid},{pid},'{region}','{channel}',"
+                       f"{amt},'{curr}','{status}','{_tag(rid)}');")
+                return {"op": "insert", "detail": f"INSERT sales_orders order_id={oid} ({_tag(rid)})", "sql": sql}
             if op == "update":
                 oid = random.choice(list(pool))
+                status, mult = random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2)
                 cur.execute(
                     "UPDATE public.sales_orders SET order_status=%s, amount=round(amount*%s,2) WHERE order_id=%s",
-                    (random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2), oid),
+                    (status, mult, oid),
                 )
-                return f"UPDATE sales_orders order_id={oid}"
+                sql = f"UPDATE public.sales_orders SET order_status='{status}', amount=round(amount*{mult},2) WHERE order_id={oid};"
+                return {"op": "update", "detail": f"UPDATE sales_orders order_id={oid}", "sql": sql}
             oid = pool.popleft()
             cur.execute("DELETE FROM public.sales_orders WHERE order_id=%s", (oid,))
-            return f"DELETE sales_orders order_id={oid}"
+            return {"op": "delete", "detail": f"DELETE sales_orders order_id={oid}",
+                    "sql": f"DELETE FROM public.sales_orders WHERE order_id={oid};"}
     finally:
         conn.close()
 
 
-def _mysql_dml(op: str) -> str:
+def _mysql_dml(op: str) -> dict[str, str]:
     pool = _pools["mysql"]
     op = _resolve_op(op, pool)
     conn = _mysql_conn()
@@ -197,64 +210,64 @@ def _mysql_dml(op: str) -> str:
         with conn.cursor() as cur:
             if op == "insert":
                 rid = uuid.uuid4().hex[:8]
+                eid_in, dept, role = random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES)
+                region, etype, sal = random.choice(REGIONS), random.choice(EVENT_TYPES), round(random.uniform(-5000, 15000), 2)
                 cur.execute(
                     """INSERT INTO hr.employee_events
                        (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
-                    (
-                        random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES),
-                        random.choice(REGIONS), random.choice(EVENT_TYPES),
-                        round(random.uniform(-5000, 15000), 2), datetime.now(timezone.utc), _tag(rid),
-                    ),
+                    (eid_in, dept, role, region, etype, sal, datetime.now(timezone.utc), _tag(rid)),
                 )
                 eid = cur.lastrowid
                 pool.append(eid)
-                return f"INSERT employee_events event_id={eid} ({_tag(rid)})"
+                sql = (f"INSERT INTO hr.employee_events (employee_id,department,role_name,region,event_type,"
+                       f"salary_change,notes) VALUES ({eid_in},'{dept}','{role}','{region}','{etype}',{sal},'{_tag(rid)}');")
+                return {"op": "insert", "detail": f"INSERT employee_events event_id={eid} ({_tag(rid)})", "sql": sql}
             if op == "update":
                 eid = random.choice(list(pool))
+                sal, etype = round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES)
                 cur.execute(
                     "UPDATE hr.employee_events SET salary_change=%s, event_type=%s WHERE event_id=%s",
-                    (round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES), eid),
+                    (sal, etype, eid),
                 )
-                return f"UPDATE employee_events event_id={eid}"
+                sql = f"UPDATE hr.employee_events SET salary_change={sal}, event_type='{etype}' WHERE event_id={eid};"
+                return {"op": "update", "detail": f"UPDATE employee_events event_id={eid}", "sql": sql}
             eid = pool.popleft()
             cur.execute("DELETE FROM hr.employee_events WHERE event_id=%s", (eid,))
-            return f"DELETE employee_events event_id={eid}"
+            return {"op": "delete", "detail": f"DELETE employee_events event_id={eid}",
+                    "sql": f"DELETE FROM hr.employee_events WHERE event_id={eid};"}
     finally:
         conn.close()
 
 
-def _mongo_dml(op: str) -> str:
+def _mongo_dml(op: str) -> dict[str, str]:
     pool = _pools["mongodb"]
     op = _resolve_op(op, pool)
     client, coll = _mongo_coll()
     try:
         if op == "insert":
             rid = uuid.uuid4().hex[:8]
+            mtype, region, msrc = random.choice(MONGO_TYPES), random.choice(REGIONS), random.choice(MONGO_SOURCES)
+            amt = round(random.uniform(10, 50000), 4)
             doc = {
-                "event_id": str(uuid.uuid4()),
-                "type": random.choice(MONGO_TYPES),
-                "region": random.choice(REGIONS),
-                "source": random.choice(MONGO_SOURCES),
-                "amount": round(random.uniform(10, 50000), 4),
-                "ts": datetime.now(timezone.utc),
-                "payload": "X" * 200,
-                "atc_agent": True,
-                "agent_run": rid,
+                "event_id": str(uuid.uuid4()), "type": mtype, "region": region, "source": msrc,
+                "amount": amt, "ts": datetime.now(timezone.utc), "payload": "X" * 200,
+                "atc_agent": True, "agent_run": rid,
             }
             res = coll.insert_one(doc)
             pool.append(res.inserted_id)
-            return f"INSERT events _id={res.inserted_id} (agent_run={rid})"
+            sql = (f"db.events.insertOne({{type:'{mtype}', region:'{region}', source:'{msrc}', "
+                   f"amount:{amt}, atc_agent:true, agent_run:'{rid}'}})")
+            return {"op": "insert", "detail": f"INSERT events _id={res.inserted_id} (agent_run={rid})", "sql": sql}
         if op == "update":
             oid = random.choice(list(pool))
-            coll.update_one(
-                {"_id": oid},
-                {"$set": {"type": random.choice(MONGO_TYPES), "amount": round(random.uniform(10, 50000), 4)}},
-            )
-            return f"UPDATE events _id={oid}"
+            mtype, amt = random.choice(MONGO_TYPES), round(random.uniform(10, 50000), 4)
+            coll.update_one({"_id": oid}, {"$set": {"type": mtype, "amount": amt}})
+            sql = f"db.events.updateOne({{_id:{oid!r}}}, {{$set:{{type:'{mtype}', amount:{amt}}}}})"
+            return {"op": "update", "detail": f"UPDATE events _id={oid}", "sql": sql}
         oid = pool.popleft()
         coll.delete_one({"_id": oid})
-        return f"DELETE events _id={oid}"
+        return {"op": "delete", "detail": f"DELETE events _id={oid}", "sql": f"db.events.deleteOne({{_id:{oid!r}}})"}
     finally:
         client.close()
 
@@ -268,6 +281,9 @@ def _pick_op() -> str:
     return random.choices(["insert", "update", "delete"], weights=[0.5, 0.3, 0.2], k=1)[0]
 
 
+_CLIENT_CMD = {"postgres": "psql sales", "mysql": "mysql hr", "mongodb": "mongosh supplychain"}
+
+
 async def _run_one(source: str | None = None, op: str | None = None) -> dict[str, Any]:
     source = source or random.choice(list(_DISPATCH.keys()))
     op = op or _pick_op()
@@ -275,18 +291,25 @@ async def _run_one(source: str | None = None, op: str | None = None) -> dict[str
     if not fn:
         return {"ok": False, "error": f"unknown source {source}"}
     try:
-        detail = await asyncio.to_thread(fn, op)
+        # Show the operator exactly what the Data Custodian is about to run.
+        await _term(DML_AGENT, f"$ {_CLIENT_CMD.get(source, source)}  # autonomous DML on {_SRC_LABEL[source]}",
+                    level="cmd", phase="dml")
+        res = await asyncio.to_thread(fn, op)
+        detail, sql, actual_op = res["detail"], res["sql"], res["op"]
+        await _term(DML_AGENT, f"  {sql}", level="cmd", phase="dml")
         _state["ops_total"] += 1
         _state["by_source"][source] = _state["by_source"].get(source, 0) + 1
-        actual_op = detail.split(" ", 1)[0].lower()
         if actual_op in _state["by_op"]:
             _state["by_op"][actual_op] += 1
-        _state["last_op"] = {"source": source, "op": op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
+        _state["last_op"] = {"source": source, "op": actual_op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
         _state["last_error"] = None
+        await _term(DML_AGENT, f"  ← {detail} · Debezium CDC will stream this to Kafka", level="ok", phase="dml")
+        # Keep the supervisor feed concise (single summary entry).
         await _emit(f"[agent-dml] {_SRC_LABEL[source]}: {detail} — Debezium will capture this change", "info")
-        return {"ok": True, "source": source, "op": op, "detail": detail}
+        return {"ok": True, "source": source, "op": actual_op, "detail": detail}
     except Exception as exc:
         _state["last_error"] = str(exc)
+        await _term(DML_AGENT, f"  ✗ {source} {op} failed: {str(exc)[:140]}", level="err", phase="dml")
         await _emit(f"[agent-dml] {_SRC_LABEL.get(source, source)}: operation failed: {str(exc)[:120]}", "err")
         return {"ok": False, "source": source, "op": op, "error": str(exc)}
 
@@ -318,7 +341,14 @@ async def etl_agent_loop() -> None:
                 mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])]
                 _etl_state["idx"] += 1
                 if mid in MOVEMENT_BY_ID:
+                    mv = MOVEMENT_BY_ID[mid]
+                    await _term("etl-guardian",
+                                f"$ orchestrate movement '{mid}' ({mv.get('label')}) {mv.get('from')}→{mv.get('to')}",
+                                level="cmd", phase="orchestrate")
                     result = await trigger_and_watch(mid, autonomous=True)
+                    await _term("etl-guardian",
+                                f"  ← {mv.get('label')}: {result.get('state')} · {result.get('rows')} rows in {result.get('duration_s')}s",
+                                level="ok" if result.get("ok") else "err", phase="orchestrate")
                     _etl_state["runs_total"] += 1
                     _etl_state["last"] = {"movement_id": mid, "state": result.get("state"),
                                           "rows": result.get("rows"), "duration_s": result.get("duration_s"),
@@ -355,10 +385,14 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
     i = _custodian_state["idx"] if idx is None else idx
     tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
     _custodian_state["idx"] = i + 1
-    await _trino_collect(
-        f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1)
-    ins = await _trino_collect(
-        f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1)
+    ddl = f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0"
+    dml = f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}"
+    await _term(DML_AGENT, f"$ trino --catalog iceberg  # Hadoop offload: {tgt['label']} → {tgt['target']}",
+                level="cmd", phase="offload")
+    await _term(DML_AGENT, f"  {ddl};", level="cmd", phase="offload")
+    await _trino_collect(ddl, 1)
+    await _term(DML_AGENT, f"  {dml};", level="cmd", phase="offload")
+    ins = await _trino_collect(dml, 1)
     ok = bool(ins.get("ok"))
     _custodian_state["runs_total"] += 1
     _custodian_state["last"] = {
@@ -367,8 +401,10 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
         "ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"),
     }
     if ok:
+        await _term(DML_AGENT, f"  ← offloaded ~{_CUST_BATCH} rows into the Hadoop Iceberg lake", level="ok", phase="offload")
         await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info")
     else:
+        await _term(DML_AGENT, f"  ✗ offload failed: {str(ins.get('error'))[:140]}", level="err", phase="offload")
         await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err")
     return _custodian_state["last"]
 

api/agent_terminal.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import uuid
 from collections import deque
 from datetime import datetime, timezone
@@ -13,6 +14,7 @@ PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
 
 _buffers: dict[str, deque[dict[str, Any]]] = {}
 _publish: PublishFn | None = None
+_loop: asyncio.AbstractEventLoop | None = None
 
 
 def init_terminals(agent_ids: list[str]) -> None:
@@ -22,8 +24,38 @@ def init_terminals(agent_ids: list[str]) -> None:
 
 
 def set_terminal_publisher(fn: PublishFn) -> None:
-    global _publish
+    global _publish, _loop
     _publish = fn
+    try:  # capture the main event loop so background threads can stream too
+        _loop = asyncio.get_running_loop()
+    except RuntimeError:
+        _loop = None
+
+
+def _build_line(agent_id: str, text: str, level: str, phase: str, prompt_id: str | None) -> dict[str, Any]:
+    return {
+        "id": str(uuid.uuid4())[:8],
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "agent_id": agent_id,
+        "level": level,
+        "phase": phase,
+        "text": text,
+        "prompt_id": prompt_id,
+    }
+
+
+def emit_threadsafe(agent_id: str, text: str, *, level: str = "info", phase: str = "ops") -> dict[str, Any]:
+    """Append a terminal line and broadcast it from a non-async context (e.g. a
+    background ``threading.Thread``). Safe to call from any thread."""
+    init_terminals([agent_id])
+    line = _build_line(agent_id, text, level, phase, None)
+    _buffers[agent_id].append(line)
+    if _publish and _loop and not _loop.is_closed():
+        try:
+            asyncio.run_coroutine_threadsafe(_publish({"type": "terminal", "line": line}), _loop)
+        except Exception:
+            pass
+    return line
 
 
 def get_terminal_lines(agent_id: str, limit: int = 200) -> list[dict[str, Any]]:

api/etl_offload.py

@@ -104,6 +104,15 @@ def _feed(text: str, level: str = "info") -> None:
         pass
 
 
+def _term(text: str, level: str = "info", phase: str = "offload") -> None:
+    """Stream a line to the ETL Guardian terminal from this background thread."""
+    try:
+        from agent_terminal import emit_threadsafe
+        emit_threadsafe("etl-guardian", text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
 def _scalar(v: Any) -> Any:
     import datetime as _dt
     import decimal
@@ -312,6 +321,9 @@ def _offload_dataset(key: str) -> dict[str, Any]:
     st = _state["datasets"][key]
     chunk = int(_state["chunk"])
     res = {"rows": 0, "bytes": 0, "order_rows": 0, "revenue": 0.0}
+    cur_txt = st["cursor"] if st.get("cursor") not in (None, "") else "<start>"
+    _term(f"$ read {st.get('engine', key)} · {st.get('label', key)}  [cursor={cur_txt} · LIMIT {chunk}]",
+          level="cmd", phase="extract")
     try:
         if key == "hr_events":
             rows, new_cursor, done = _read_hr(st["cursor"], chunk, st)
@@ -319,6 +331,7 @@ def _offload_dataset(key: str) -> dict[str, Any]:
             rows, new_cursor, done = _READERS[key](st["cursor"], chunk)
     except Exception as exc:
         st["error"] = str(exc)[:160]
+        _term(f"  ✗ extract failed: {str(exc)[:140]}", level="err", phase="extract")
         return res
     st["error"] = None
     if not rows:
@@ -331,7 +344,10 @@ def _offload_dataset(key: str) -> dict[str, Any]:
         obj_key, nbytes = _write_parquet(key, rows)
     except Exception as exc:
         st["error"] = f"parquet/s3: {str(exc)[:140]}"
+        _term(f"  ✗ parquet/s3 write failed: {str(exc)[:140]}", level="err", phase="load")
         return res
+    _term(f"  ← pyarrow.write_table → s3://data/{obj_key}  ({len(rows)} rows · {nbytes / 1024:.1f} KB)",
+          level="ok", phase="load")
     o_rows, o_rev = _agg(key, rows)
     with _lock:
         st["parts"] += 1
@@ -378,6 +394,8 @@ def run_cycle() -> dict[str, Any]:
     order_rows = 0
     revenue = 0.0
     try:
+        _term(f"═══ ETL offload cycle {_state['cycles'] + 1} — source DBs → S3 Parquet lake ═══",
+              level="info", phase="cycle")
         _source_totals()
         for d in DATASETS:
             if not _state["enabled"]:
@@ -404,7 +422,12 @@ def run_cycle() -> dict[str, Any]:
             _state["series"].append(point)
             with _lock:
                 _business["ts"].append(point)
+            _term(f"═══ cycle {_state['cycles']} done — {total:,} rows · {cbytes / 1024:.1f} KB · €{int(revenue):,} revenue ═══",
+                  level="ok", phase="cycle")
             _feed(f"offloaded {total:,} rows → S3 Parquet · €{int(revenue):,} (cycle {_state['cycles']})")
+        else:
+            _term(f"  cycle {_state['cycles']} — no new rows (all datasets caught up, tailing)",
+                  level="info", phase="cycle")
     finally:
         _state["running_cycle"] = False
     return {"ok": True, "rows": total, "bytes": cbytes, "revenue": round(revenue, 2)}

api/lab_context.py

@@ -26,7 +26,7 @@ TRINO_URL = os.getenv("TRINO_URL", f"http://{LAKEHOUSE_HOST}:8089")
 SPARK_UI_URL = os.getenv("SPARK_UI_URL", f"http://{LAKEHOUSE_HOST}:8080")
 GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
 OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", "http://10.0.20.111:9020")
-YARN_URL = os.getenv("YARN_URL", "http://10.0.21.61:8088")
+YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088")
 
 AGENT_PRIMARY_DOMAIN = {
     "infra-sentinel": "docker",

api/main.py

@@ -46,6 +46,7 @@ from hadoop_analytics import router as hadoop_router
 from elasticsearch_api import router as elasticsearch_router
 from sql_console import router as sql_router
 from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
+from agent_activity import agent_activity_loop
 from cdc_consumer import router as cdc_router, cdc_consumer_loop
 from movements import router as movements_router
 from movements import MOVEMENT_BY_ID, trigger_and_watch
@@ -735,12 +736,15 @@ async def lifespan(app: FastAPI):
     cdc_task = asyncio.create_task(cdc_consumer_loop())
     etl_task = asyncio.create_task(etl_agent_loop())
     cust_task = asyncio.create_task(custodian_offload_loop())
+    act_task = asyncio.create_task(agent_activity_loop())
     add_feed("infra-sentinel", "ATC Command Center API online", "info")
     yield
     task.cancel()
     dml_task.cancel()
     cdc_task.cancel()
     etl_task.cancel()
+    cust_task.cancel()
+    act_task.cancel()
     if redis_client:
         await redis_client.close()
 

api/movements.py

@@ -83,6 +83,15 @@ async def _publish(event: dict[str, Any]) -> None:
         pass
 
 
+async def _term(agent_id: str, text: str, level: str = "info", phase: str = "etl") -> None:
+    """Stream a movement step to the owning agent's terminal."""
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
 async def _airflow_token(client: httpx.AsyncClient) -> str:
     now = time.time()
     if _token_cache["token"] and _token_cache["exp"] > now + 30:
@@ -128,9 +137,12 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     mv = MOVEMENT_BY_ID.get(mid)
     if not mv:
         return {"ok": False, "error": f"unknown movement {mid}"}
+    agent = mv["agent"]
     if mv.get("api"):
         try:
             payload = {**(mv.get("default_conf") or {}), **(conf or {})}
+            await _term(agent, f"$ POST {mv['api']}  # {mv['label']}", level="cmd")
+            await _term(agent, f"  payload={payload}", level="cmd")
             t0 = time.time()
             async with httpx.AsyncClient(timeout=120.0) as client:
                 r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
@@ -145,21 +157,26 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
             _last_runs[mid] = run
             await _publish({"type": "movement", **run})
             lvl = "info" if state == "success" else "err"
+            await _term(agent, f"  ← {state} · {rows if rows is not None else '?'} rows in {dur}s", level="ok" if state == "success" else "err")
             _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
             return {"ok": state == "success", **run}
         except Exception as exc:
             _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
+            await _term(agent, f"  ✗ error {str(exc)[:140]}", level="err")
             _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
             return {"ok": False, "error": str(exc)}
 
     conf = {**(mv.get("default_conf") or {}), **(conf or {})}
-    agent = mv["agent"]
     count_sql = mv.get("count_sql")
+    verb = "autonomously triggered" if autonomous else "triggered"
+    await _term(agent, f"$ airflow dags trigger {mv['dag_id']}  # {mv['label']} ({verb})", level="cmd")
+    await _term(agent, f"  conf={conf}", level="cmd")
     before = await _trino_scalar(count_sql) if count_sql else None
+    if count_sql:
+        await _term(agent, f"  trino: {count_sql} → {before if before is not None else '?'} rows (before)", level="info")
 
     _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "running", "started_at": datetime.now(timezone.utc).isoformat()}
     await _publish({"type": "movement", "movement_id": mid, "state": "running"})
-    verb = "autonomously triggered" if autonomous else "triggered"
     _feed(agent, f"[etl] {mv['label']}: {verb} (conf={conf})", "info")
 
     try:
@@ -182,8 +199,10 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
                 if state in ("success", "failed"):
                     break
             dur = round(time.time() - t0, 1)
+            await _term(agent, f"  ← Airflow run {run_id} finished state={state} in {dur}s", level="ok" if state == "success" else "err")
     except Exception as exc:
         _last_runs[mid] = {**_last_runs[mid], "state": "failed", "error": str(exc)}
+        await _term(agent, f"  ✗ error {str(exc)[:140]}", level="err")
         _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
         return {"ok": False, "error": str(exc)}
 
@@ -201,6 +220,9 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     await _publish({"type": "movement", **run})
     rtxt = f"{rows} rows" if rows is not None else "data"
     lvl = "info" if state == "success" else "err"
+    if count_sql:
+        await _term(agent, f"  trino: count {before if before is not None else '?'} → {after if after is not None else '?'} ({rtxt})", level="info")
+    await _term(agent, f"  ← {mv['label']}: {state} — {rtxt} in {dur}s", level="ok" if state == "success" else "err")
     _feed(agent, f"[etl] {mv['label']}: {state} — {rtxt} in {dur}s", lvl)
     return {"ok": state == "success", **run}
 

ui/src/hooks/useCommandCenter.ts

@@ -313,7 +313,22 @@ export function useCommandCenter() {
   const inspectorLines = useMemo(() => {
     if (!terminalSubjectId) return []
     const probeId = resolveProbeId(terminalSubjectId)
-    return terminals[probeId] || terminals[terminalSubjectId] || terminals[selectedAgentId || ''] || []
+    const nodeLines = terminals[probeId] || terminals[terminalSubjectId] || []
+    const agentLines = selectedAgentId ? terminals[selectedAgentId] || [] : []
+    // When an agent is selected we also show its own autonomous ops stream
+    // (DML/ETL/probes) merged chronologically with the node probe output, so
+    // the terminal reflects what the agent is doing in the background — not just
+    // a one-shot probe.
+    if (!agentLines.length) return nodeLines
+    if (!nodeLines.length) return agentLines
+    const seen = new Set<string>()
+    const merged = [...nodeLines, ...agentLines].filter((l) => {
+      if (seen.has(l.id)) return false
+      seen.add(l.id)
+      return true
+    })
+    merged.sort((a, b) => ((a.ts || '') < (b.ts || '') ? -1 : (a.ts || '') > (b.ts || '') ? 1 : 0))
+    return merged.slice(-300)
   }, [terminalSubjectId, terminals, selectedAgentId])
 
   const focusApprovals = useCallback(() => {