diff --git a/api/Dockerfile b/api/Dockerfile index 6014407..ac360e8 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py hive_bench_seed.json . +COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py agent_activity.py hive_bench_seed.json . RUN mkdir -p /data ENV DATABASE_URL=sqlite:////data/atc-agents.db EXPOSE 3201 diff --git a/api/agent_activity.py b/api/agent_activity.py new file mode 100644 index 0000000..7e86e86 --- /dev/null +++ b/api/agent_activity.py @@ -0,0 +1,204 @@ +"""Continuous autonomous activity for the field agents that don't drive a +data-movement loop of their own (Lakehouse Ops, Hadoop Ranger, Infra Sentinel, +Network Watcher). + +Every tick the loop runs ONE real, lightweight probe for the next agent in the +rotation and streams the exact command + result into that agent's terminal, so +the operator can always see what each agent is doing in the background instead +of an idle prompt. All probes hit live endpoints (Trino, WebHDFS JMX, YARN, +Dockhand, VLAN hosts) and are individually guarded so a single failure never +breaks the loop. +""" + +from __future__ import annotations + +import asyncio +import os +import time +from typing import Any + +import httpx + +TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/") +TRINO_USER = os.getenv("TRINO_USER", "mo") +HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/") +YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088").rstrip("/") +DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082").rstrip("/") +OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")).rstrip("/") + +TICK_SECONDS = float(os.getenv("AGENT_ACTIVITY_TICK_SECONDS", "8")) + +# VLAN data paths the Network Watcher keeps an eye on. +_NET_TARGETS = [ + ("Kafka UI", "http://10.0.21.36:9000"), + ("Trino", f"{TRINO_URL}/v1/info"), + ("HDFS NameNode", f"{HDFS_NN_URL}/dfshealth.html"), + ("Airflow", os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080")), + ("ObjectScale S3", OBJECTSCALE_URL), +] + + +async def _term(agent_id: str, text: str, level: str = "info", phase: str = "ops") -> None: + try: + from agent_terminal import terminal_log + await terminal_log(agent_id, text, level=level, phase=phase) + except Exception: + pass + + +async def _trino(sql: str, timeout: float = 8.0) -> list[list[Any]]: + rows: list[list[Any]] = [] + async with httpx.AsyncClient(timeout=timeout) as client: + d = (await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), + headers={"X-Trino-User": TRINO_USER})).json() + for _ in range(40): + if d.get("error"): + raise RuntimeError(d["error"].get("message", "trino error")) + rows += d.get("data") or [] + nxt = d.get("nextUri") + if not nxt: + break + d = (await client.get(nxt)).json() + return rows + + +# ── per-agent probes ───────────────────────────────────────────────────────── +async def _lakehouse_ops() -> None: + aid = "lakehouse-ops" + try: + async with httpx.AsyncClient(timeout=6.0) as client: + info = (await client.get(f"{TRINO_URL}/v1/info")).json() + ver = info.get("nodeVersion", {}).get("version", "?") + up = info.get("uptime", "?") + await _term(aid, f"$ curl -s {TRINO_URL}/v1/info # Trino coordinator health", level="cmd", phase="trino") + await _term(aid, f" ← Trino {ver} · uptime {up} · serving federated queries", level="ok", phase="trino") + except Exception as exc: + await _term(aid, f" ✗ Trino unreachable: {str(exc)[:100]}", level="err", phase="trino") + try: + cats = await _trino("SHOW CATALOGS") + names = ", ".join(sorted(c[0] for c in cats)) + await _term(aid, "$ trino --execute 'SHOW CATALOGS'", level="cmd", phase="trino") + await _term(aid, f" ← {len(cats)} catalogs federated: {names}", level="ok", phase="trino") + except Exception as exc: + await _term(aid, f" ✗ SHOW CATALOGS failed: {str(exc)[:100]}", level="err", phase="trino") + try: + sql = "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked" + rows = await _trino(sql, timeout=12.0) + n = int(rows[0][0]) if rows else 0 + await _term(aid, f"$ trino --execute '{sql}' # curated Iceberg lakehouse", level="cmd", phase="iceberg") + await _term(aid, f" ← {n:,} masked rows in iceberg.curated_masked (PII-safe layer)", level="ok", phase="iceberg") + except Exception as exc: + await _term(aid, f" ✗ Iceberg count failed: {str(exc)[:100]}", level="err", phase="iceberg") + + +def _g(d: dict, *keys) -> Any: + for k in keys: + if k in d: + return d[k] + return None + + +async def _hadoop_ranger() -> None: + aid = "hadoop-ranger" + try: + async with httpx.AsyncClient(timeout=6.0) as client: + fs = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem")).json() + state = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")).json() + fsb = (fs.get("beans") or [{}])[0] + stb = (state.get("beans") or [{}])[0] + cap_total = float(_g(fsb, "CapacityTotalGB") or 0) + cap_used = float(_g(fsb, "CapacityUsedGB") or 0) + blocks = int(_g(fsb, "BlocksTotal", "TotalBlocks") or _g(stb, "BlocksTotal") or 0) + live = int(_g(stb, "NumLiveDataNodes") or 0) + dead = int(_g(stb, "NumDeadDataNodes") or 0) + pct = (100.0 * cap_used / cap_total) if cap_total else 0.0 + await _term(aid, f"$ curl -s {HDFS_NN_URL}/jmx?qry=...FSNamesystemState # NameNode health", level="cmd", phase="hdfs") + await _term(aid, f" ← live datanodes={live} dead={dead} · blocks={blocks:,} · " + f"used {cap_used:.1f}/{cap_total:.1f} GB ({pct:.0f}%)", level="ok", phase="hdfs") + except Exception as exc: + await _term(aid, f" ✗ NameNode JMX unreachable: {str(exc)[:100]}", level="err", phase="hdfs") + try: + async with httpx.AsyncClient(timeout=6.0) as client: + m = (await client.get(f"{YARN_URL}/ws/v1/cluster/metrics")).json().get("clusterMetrics", {}) + await _term(aid, f"$ yarn application -list # ResourceManager {YARN_URL}", level="cmd", phase="yarn") + await _term(aid, f" ← apps running={m.get('appsRunning', 0)} pending={m.get('appsPending', 0)} · " + f"available {round(m.get('availableMB', 0) / 1024, 1)} GB / " + f"{m.get('totalNodes', 0)} nodes", level="ok", phase="yarn") + except Exception as exc: + await _term(aid, f" ⚠ YARN RM not responding ({str(exc)[:60]}) — HDFS storage layer still healthy", level="warn", phase="yarn") + + +async def _infra_sentinel() -> None: + aid = "infra-sentinel" + await _term(aid, f"$ dockhand ps --all-envs # container inventory via {DOCKHAND_URL}", level="cmd", phase="docker") + try: + from dockhand_envs import DOCKHAND_ENVS + envs = DOCKHAND_ENVS + except Exception: + envs = {"docker01": 1, "docker02": 2, "lakehouse": 9, "airflow": 10, "db02": 5} + total = running = 0 + reached = 0 + try: + async with httpx.AsyncClient(timeout=4.0) as client: + async def _one(name: str, eid: int): + try: + r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": eid}) + if r.status_code >= 400: + return None + d = r.json() + return d if isinstance(d, list) else d.get("containers", []) + except Exception: + return None + results = await asyncio.gather(*[_one(n, e) for n, e in envs.items()]) + for conts in results: + if conts is None: + continue + reached += 1 + total += len(conts) + running += sum(1 for c in conts + if str(c.get("state", c.get("status", ""))).lower().startswith(("run", "up"))) + if reached: + await _term(aid, f" ← {running}/{total} containers up across {reached} Dockhand environments", level="ok", phase="docker") + else: + await _term(aid, " ✗ Dockhand returned no environments", level="warn", phase="docker") + except Exception as exc: + await _term(aid, f" ✗ Dockhand unreachable: {str(exc)[:100]}", level="err", phase="docker") + try: + load = os.getloadavg() + await _term(aid, "$ cat /proc/loadavg # command-center host", level="cmd", phase="host") + await _term(aid, f" ← load avg {load[0]:.2f} {load[1]:.2f} {load[2]:.2f} (1/5/15m)", level="ok", phase="host") + except Exception: + pass + + +async def _network_watcher() -> None: + aid = "network-watcher" + await _term(aid, "$ probe VLAN 20/21 data paths # ingress/egress reachability", level="cmd", phase="net") + async with httpx.AsyncClient(timeout=4.0, verify=False) as client: + for label, url in _NET_TARGETS: + t0 = time.time() + try: + r = await client.get(url) + ms = int((time.time() - t0) * 1000) + lvl = "ok" if r.status_code < 500 else "warn" + await _term(aid, f" → {label:<16} {url} {r.status_code} {ms}ms", level=lvl, phase="net") + except Exception as exc: + await _term(aid, f" → {label:<16} {url} DOWN ({str(exc)[:60]})", level="err", phase="net") + + +_ROTATION = [_lakehouse_ops, _hadoop_ranger, _infra_sentinel, _network_watcher] + + +async def agent_activity_loop() -> None: + """Round-robin: run one agent's live probe per tick so each agent terminal + shows fresh real activity roughly every (len(rotation) * TICK_SECONDS)s.""" + await asyncio.sleep(15) # let the platform settle + idx = 0 + while True: + probe = _ROTATION[idx % len(_ROTATION)] + idx += 1 + try: + await probe() + except Exception: + pass + await asyncio.sleep(max(3.0, TICK_SECONDS)) diff --git a/api/agent_ops.py b/api/agent_ops.py index c1c8033..b4d1f1f 100644 --- a/api/agent_ops.py +++ b/api/agent_ops.py @@ -103,6 +103,15 @@ async def _emit(message: str, level: str = "info") -> None: pass +async def _term(agent_id: str, message: str, level: str = "info", phase: str = "ops") -> None: + """Stream a line to a specific agent terminal (no feed entry).""" + try: + from agent_terminal import terminal_log + await terminal_log(agent_id, message, level=level, phase=phase) + except Exception: + pass + + # ── Synchronous DB operations (run in a thread) ────────────────────────────── def _pg_conn(): import psycopg2 @@ -152,7 +161,7 @@ def _resolve_op(op: str, pool: deque) -> str: return op -def _pg_dml(op: str) -> str: +def _pg_dml(op: str) -> dict[str, str]: pool = _pools["postgres"] op = _resolve_op(op, pool) conn = _pg_conn() @@ -161,35 +170,39 @@ def _pg_dml(op: str) -> str: with conn.cursor() as cur: if op == "insert": rid = uuid.uuid4().hex[:8] + cid, pid = random.randint(1, 50000), random.randint(1, 2000) + region, channel = random.choice(REGIONS), random.choice(CHANNELS) + amt, curr, status = round(random.uniform(10, 9999), 2), random.choice(CURRENCIES), random.choice(ORDER_STATUS) cur.execute( """INSERT INTO public.sales_orders (customer_id, product_id, region, sales_channel, order_ts, amount, currency, order_status, notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING order_id""", - ( - random.randint(1, 50000), random.randint(1, 2000), - random.choice(REGIONS), random.choice(CHANNELS), - datetime.now(timezone.utc), round(random.uniform(10, 9999), 2), - random.choice(CURRENCIES), random.choice(ORDER_STATUS), _tag(rid), - ), + (cid, pid, region, channel, datetime.now(timezone.utc), amt, curr, status, _tag(rid)), ) oid = cur.fetchone()[0] pool.append(oid) - return f"INSERT sales_orders order_id={oid} ({_tag(rid)})" + sql = (f"INSERT INTO public.sales_orders (customer_id,product_id,region,sales_channel," + f"amount,currency,order_status,notes) VALUES ({cid},{pid},'{region}','{channel}'," + f"{amt},'{curr}','{status}','{_tag(rid)}');") + return {"op": "insert", "detail": f"INSERT sales_orders order_id={oid} ({_tag(rid)})", "sql": sql} if op == "update": oid = random.choice(list(pool)) + status, mult = random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2) cur.execute( "UPDATE public.sales_orders SET order_status=%s, amount=round(amount*%s,2) WHERE order_id=%s", - (random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2), oid), + (status, mult, oid), ) - return f"UPDATE sales_orders order_id={oid}" + sql = f"UPDATE public.sales_orders SET order_status='{status}', amount=round(amount*{mult},2) WHERE order_id={oid};" + return {"op": "update", "detail": f"UPDATE sales_orders order_id={oid}", "sql": sql} oid = pool.popleft() cur.execute("DELETE FROM public.sales_orders WHERE order_id=%s", (oid,)) - return f"DELETE sales_orders order_id={oid}" + return {"op": "delete", "detail": f"DELETE sales_orders order_id={oid}", + "sql": f"DELETE FROM public.sales_orders WHERE order_id={oid};"} finally: conn.close() -def _mysql_dml(op: str) -> str: +def _mysql_dml(op: str) -> dict[str, str]: pool = _pools["mysql"] op = _resolve_op(op, pool) conn = _mysql_conn() @@ -197,64 +210,64 @@ def _mysql_dml(op: str) -> str: with conn.cursor() as cur: if op == "insert": rid = uuid.uuid4().hex[:8] + eid_in, dept, role = random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES) + region, etype, sal = random.choice(REGIONS), random.choice(EVENT_TYPES), round(random.uniform(-5000, 15000), 2) cur.execute( """INSERT INTO hr.employee_events (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", - ( - random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES), - random.choice(REGIONS), random.choice(EVENT_TYPES), - round(random.uniform(-5000, 15000), 2), datetime.now(timezone.utc), _tag(rid), - ), + (eid_in, dept, role, region, etype, sal, datetime.now(timezone.utc), _tag(rid)), ) eid = cur.lastrowid pool.append(eid) - return f"INSERT employee_events event_id={eid} ({_tag(rid)})" + sql = (f"INSERT INTO hr.employee_events (employee_id,department,role_name,region,event_type," + f"salary_change,notes) VALUES ({eid_in},'{dept}','{role}','{region}','{etype}',{sal},'{_tag(rid)}');") + return {"op": "insert", "detail": f"INSERT employee_events event_id={eid} ({_tag(rid)})", "sql": sql} if op == "update": eid = random.choice(list(pool)) + sal, etype = round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES) cur.execute( "UPDATE hr.employee_events SET salary_change=%s, event_type=%s WHERE event_id=%s", - (round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES), eid), + (sal, etype, eid), ) - return f"UPDATE employee_events event_id={eid}" + sql = f"UPDATE hr.employee_events SET salary_change={sal}, event_type='{etype}' WHERE event_id={eid};" + return {"op": "update", "detail": f"UPDATE employee_events event_id={eid}", "sql": sql} eid = pool.popleft() cur.execute("DELETE FROM hr.employee_events WHERE event_id=%s", (eid,)) - return f"DELETE employee_events event_id={eid}" + return {"op": "delete", "detail": f"DELETE employee_events event_id={eid}", + "sql": f"DELETE FROM hr.employee_events WHERE event_id={eid};"} finally: conn.close() -def _mongo_dml(op: str) -> str: +def _mongo_dml(op: str) -> dict[str, str]: pool = _pools["mongodb"] op = _resolve_op(op, pool) client, coll = _mongo_coll() try: if op == "insert": rid = uuid.uuid4().hex[:8] + mtype, region, msrc = random.choice(MONGO_TYPES), random.choice(REGIONS), random.choice(MONGO_SOURCES) + amt = round(random.uniform(10, 50000), 4) doc = { - "event_id": str(uuid.uuid4()), - "type": random.choice(MONGO_TYPES), - "region": random.choice(REGIONS), - "source": random.choice(MONGO_SOURCES), - "amount": round(random.uniform(10, 50000), 4), - "ts": datetime.now(timezone.utc), - "payload": "X" * 200, - "atc_agent": True, - "agent_run": rid, + "event_id": str(uuid.uuid4()), "type": mtype, "region": region, "source": msrc, + "amount": amt, "ts": datetime.now(timezone.utc), "payload": "X" * 200, + "atc_agent": True, "agent_run": rid, } res = coll.insert_one(doc) pool.append(res.inserted_id) - return f"INSERT events _id={res.inserted_id} (agent_run={rid})" + sql = (f"db.events.insertOne({{type:'{mtype}', region:'{region}', source:'{msrc}', " + f"amount:{amt}, atc_agent:true, agent_run:'{rid}'}})") + return {"op": "insert", "detail": f"INSERT events _id={res.inserted_id} (agent_run={rid})", "sql": sql} if op == "update": oid = random.choice(list(pool)) - coll.update_one( - {"_id": oid}, - {"$set": {"type": random.choice(MONGO_TYPES), "amount": round(random.uniform(10, 50000), 4)}}, - ) - return f"UPDATE events _id={oid}" + mtype, amt = random.choice(MONGO_TYPES), round(random.uniform(10, 50000), 4) + coll.update_one({"_id": oid}, {"$set": {"type": mtype, "amount": amt}}) + sql = f"db.events.updateOne({{_id:{oid!r}}}, {{$set:{{type:'{mtype}', amount:{amt}}}}})" + return {"op": "update", "detail": f"UPDATE events _id={oid}", "sql": sql} oid = pool.popleft() coll.delete_one({"_id": oid}) - return f"DELETE events _id={oid}" + return {"op": "delete", "detail": f"DELETE events _id={oid}", "sql": f"db.events.deleteOne({{_id:{oid!r}}})"} finally: client.close() @@ -268,6 +281,9 @@ def _pick_op() -> str: return random.choices(["insert", "update", "delete"], weights=[0.5, 0.3, 0.2], k=1)[0] +_CLIENT_CMD = {"postgres": "psql sales", "mysql": "mysql hr", "mongodb": "mongosh supplychain"} + + async def _run_one(source: str | None = None, op: str | None = None) -> dict[str, Any]: source = source or random.choice(list(_DISPATCH.keys())) op = op or _pick_op() @@ -275,18 +291,25 @@ async def _run_one(source: str | None = None, op: str | None = None) -> dict[str if not fn: return {"ok": False, "error": f"unknown source {source}"} try: - detail = await asyncio.to_thread(fn, op) + # Show the operator exactly what the Data Custodian is about to run. + await _term(DML_AGENT, f"$ {_CLIENT_CMD.get(source, source)} # autonomous DML on {_SRC_LABEL[source]}", + level="cmd", phase="dml") + res = await asyncio.to_thread(fn, op) + detail, sql, actual_op = res["detail"], res["sql"], res["op"] + await _term(DML_AGENT, f" {sql}", level="cmd", phase="dml") _state["ops_total"] += 1 _state["by_source"][source] = _state["by_source"].get(source, 0) + 1 - actual_op = detail.split(" ", 1)[0].lower() if actual_op in _state["by_op"]: _state["by_op"][actual_op] += 1 - _state["last_op"] = {"source": source, "op": op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()} + _state["last_op"] = {"source": source, "op": actual_op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()} _state["last_error"] = None + await _term(DML_AGENT, f" ← {detail} · Debezium CDC will stream this to Kafka", level="ok", phase="dml") + # Keep the supervisor feed concise (single summary entry). await _emit(f"[agent-dml] {_SRC_LABEL[source]}: {detail} — Debezium will capture this change", "info") - return {"ok": True, "source": source, "op": op, "detail": detail} + return {"ok": True, "source": source, "op": actual_op, "detail": detail} except Exception as exc: _state["last_error"] = str(exc) + await _term(DML_AGENT, f" ✗ {source} {op} failed: {str(exc)[:140]}", level="err", phase="dml") await _emit(f"[agent-dml] {_SRC_LABEL.get(source, source)}: operation failed: {str(exc)[:120]}", "err") return {"ok": False, "source": source, "op": op, "error": str(exc)} @@ -318,7 +341,14 @@ async def etl_agent_loop() -> None: mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])] _etl_state["idx"] += 1 if mid in MOVEMENT_BY_ID: + mv = MOVEMENT_BY_ID[mid] + await _term("etl-guardian", + f"$ orchestrate movement '{mid}' ({mv.get('label')}) {mv.get('from')}→{mv.get('to')}", + level="cmd", phase="orchestrate") result = await trigger_and_watch(mid, autonomous=True) + await _term("etl-guardian", + f" ← {mv.get('label')}: {result.get('state')} · {result.get('rows')} rows in {result.get('duration_s')}s", + level="ok" if result.get("ok") else "err", phase="orchestrate") _etl_state["runs_total"] += 1 _etl_state["last"] = {"movement_id": mid, "state": result.get("state"), "rows": result.get("rows"), "duration_s": result.get("duration_s"), @@ -355,10 +385,14 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]: i = _custodian_state["idx"] if idx is None else idx tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)] _custodian_state["idx"] = i + 1 - await _trino_collect( - f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1) - ins = await _trino_collect( - f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1) + ddl = f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0" + dml = f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}" + await _term(DML_AGENT, f"$ trino --catalog iceberg # Hadoop offload: {tgt['label']} → {tgt['target']}", + level="cmd", phase="offload") + await _term(DML_AGENT, f" {ddl};", level="cmd", phase="offload") + await _trino_collect(ddl, 1) + await _term(DML_AGENT, f" {dml};", level="cmd", phase="offload") + ins = await _trino_collect(dml, 1) ok = bool(ins.get("ok")) _custodian_state["runs_total"] += 1 _custodian_state["last"] = { @@ -367,8 +401,10 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]: "ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"), } if ok: + await _term(DML_AGENT, f" ← offloaded ~{_CUST_BATCH} rows into the Hadoop Iceberg lake", level="ok", phase="offload") await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info") else: + await _term(DML_AGENT, f" ✗ offload failed: {str(ins.get('error'))[:140]}", level="err", phase="offload") await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err") return _custodian_state["last"] diff --git a/api/agent_terminal.py b/api/agent_terminal.py index 66189ba..c30238c 100644 --- a/api/agent_terminal.py +++ b/api/agent_terminal.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import uuid from collections import deque from datetime import datetime, timezone @@ -13,6 +14,7 @@ PublishFn = Callable[[dict[str, Any]], Awaitable[None]] _buffers: dict[str, deque[dict[str, Any]]] = {} _publish: PublishFn | None = None +_loop: asyncio.AbstractEventLoop | None = None def init_terminals(agent_ids: list[str]) -> None: @@ -22,8 +24,38 @@ def init_terminals(agent_ids: list[str]) -> None: def set_terminal_publisher(fn: PublishFn) -> None: - global _publish + global _publish, _loop _publish = fn + try: # capture the main event loop so background threads can stream too + _loop = asyncio.get_running_loop() + except RuntimeError: + _loop = None + + +def _build_line(agent_id: str, text: str, level: str, phase: str, prompt_id: str | None) -> dict[str, Any]: + return { + "id": str(uuid.uuid4())[:8], + "ts": datetime.now(timezone.utc).isoformat(), + "agent_id": agent_id, + "level": level, + "phase": phase, + "text": text, + "prompt_id": prompt_id, + } + + +def emit_threadsafe(agent_id: str, text: str, *, level: str = "info", phase: str = "ops") -> dict[str, Any]: + """Append a terminal line and broadcast it from a non-async context (e.g. a + background ``threading.Thread``). Safe to call from any thread.""" + init_terminals([agent_id]) + line = _build_line(agent_id, text, level, phase, None) + _buffers[agent_id].append(line) + if _publish and _loop and not _loop.is_closed(): + try: + asyncio.run_coroutine_threadsafe(_publish({"type": "terminal", "line": line}), _loop) + except Exception: + pass + return line def get_terminal_lines(agent_id: str, limit: int = 200) -> list[dict[str, Any]]: diff --git a/api/etl_offload.py b/api/etl_offload.py index 1aeca0d..d73b7ac 100644 --- a/api/etl_offload.py +++ b/api/etl_offload.py @@ -104,6 +104,15 @@ def _feed(text: str, level: str = "info") -> None: pass +def _term(text: str, level: str = "info", phase: str = "offload") -> None: + """Stream a line to the ETL Guardian terminal from this background thread.""" + try: + from agent_terminal import emit_threadsafe + emit_threadsafe("etl-guardian", text, level=level, phase=phase) + except Exception: + pass + + def _scalar(v: Any) -> Any: import datetime as _dt import decimal @@ -312,6 +321,9 @@ def _offload_dataset(key: str) -> dict[str, Any]: st = _state["datasets"][key] chunk = int(_state["chunk"]) res = {"rows": 0, "bytes": 0, "order_rows": 0, "revenue": 0.0} + cur_txt = st["cursor"] if st.get("cursor") not in (None, "") else "" + _term(f"$ read {st.get('engine', key)} · {st.get('label', key)} [cursor={cur_txt} · LIMIT {chunk}]", + level="cmd", phase="extract") try: if key == "hr_events": rows, new_cursor, done = _read_hr(st["cursor"], chunk, st) @@ -319,6 +331,7 @@ def _offload_dataset(key: str) -> dict[str, Any]: rows, new_cursor, done = _READERS[key](st["cursor"], chunk) except Exception as exc: st["error"] = str(exc)[:160] + _term(f" ✗ extract failed: {str(exc)[:140]}", level="err", phase="extract") return res st["error"] = None if not rows: @@ -331,7 +344,10 @@ def _offload_dataset(key: str) -> dict[str, Any]: obj_key, nbytes = _write_parquet(key, rows) except Exception as exc: st["error"] = f"parquet/s3: {str(exc)[:140]}" + _term(f" ✗ parquet/s3 write failed: {str(exc)[:140]}", level="err", phase="load") return res + _term(f" ← pyarrow.write_table → s3://data/{obj_key} ({len(rows)} rows · {nbytes / 1024:.1f} KB)", + level="ok", phase="load") o_rows, o_rev = _agg(key, rows) with _lock: st["parts"] += 1 @@ -378,6 +394,8 @@ def run_cycle() -> dict[str, Any]: order_rows = 0 revenue = 0.0 try: + _term(f"═══ ETL offload cycle {_state['cycles'] + 1} — source DBs → S3 Parquet lake ═══", + level="info", phase="cycle") _source_totals() for d in DATASETS: if not _state["enabled"]: @@ -404,7 +422,12 @@ def run_cycle() -> dict[str, Any]: _state["series"].append(point) with _lock: _business["ts"].append(point) + _term(f"═══ cycle {_state['cycles']} done — {total:,} rows · {cbytes / 1024:.1f} KB · €{int(revenue):,} revenue ═══", + level="ok", phase="cycle") _feed(f"offloaded {total:,} rows → S3 Parquet · €{int(revenue):,} (cycle {_state['cycles']})") + else: + _term(f" cycle {_state['cycles']} — no new rows (all datasets caught up, tailing)", + level="info", phase="cycle") finally: _state["running_cycle"] = False return {"ok": True, "rows": total, "bytes": cbytes, "revenue": round(revenue, 2)} diff --git a/api/lab_context.py b/api/lab_context.py index 2823617..a8a3506 100644 --- a/api/lab_context.py +++ b/api/lab_context.py @@ -26,7 +26,7 @@ TRINO_URL = os.getenv("TRINO_URL", f"http://{LAKEHOUSE_HOST}:8089") SPARK_UI_URL = os.getenv("SPARK_UI_URL", f"http://{LAKEHOUSE_HOST}:8080") GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", "http://10.0.20.111:9020") -YARN_URL = os.getenv("YARN_URL", "http://10.0.21.61:8088") +YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088") AGENT_PRIMARY_DOMAIN = { "infra-sentinel": "docker", diff --git a/api/main.py b/api/main.py index 3d24539..84caa05 100644 --- a/api/main.py +++ b/api/main.py @@ -46,6 +46,7 @@ from hadoop_analytics import router as hadoop_router from elasticsearch_api import router as elasticsearch_router from sql_console import router as sql_router from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop +from agent_activity import agent_activity_loop from cdc_consumer import router as cdc_router, cdc_consumer_loop from movements import router as movements_router from movements import MOVEMENT_BY_ID, trigger_and_watch @@ -735,12 +736,15 @@ async def lifespan(app: FastAPI): cdc_task = asyncio.create_task(cdc_consumer_loop()) etl_task = asyncio.create_task(etl_agent_loop()) cust_task = asyncio.create_task(custodian_offload_loop()) + act_task = asyncio.create_task(agent_activity_loop()) add_feed("infra-sentinel", "ATC Command Center API online", "info") yield task.cancel() dml_task.cancel() cdc_task.cancel() etl_task.cancel() + cust_task.cancel() + act_task.cancel() if redis_client: await redis_client.close() diff --git a/api/movements.py b/api/movements.py index 15f7034..53e8101 100644 --- a/api/movements.py +++ b/api/movements.py @@ -83,6 +83,15 @@ async def _publish(event: dict[str, Any]) -> None: pass +async def _term(agent_id: str, text: str, level: str = "info", phase: str = "etl") -> None: + """Stream a movement step to the owning agent's terminal.""" + try: + from agent_terminal import terminal_log + await terminal_log(agent_id, text, level=level, phase=phase) + except Exception: + pass + + async def _airflow_token(client: httpx.AsyncClient) -> str: now = time.time() if _token_cache["token"] and _token_cache["exp"] > now + 30: @@ -128,9 +137,12 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut mv = MOVEMENT_BY_ID.get(mid) if not mv: return {"ok": False, "error": f"unknown movement {mid}"} + agent = mv["agent"] if mv.get("api"): try: payload = {**(mv.get("default_conf") or {}), **(conf or {})} + await _term(agent, f"$ POST {mv['api']} # {mv['label']}", level="cmd") + await _term(agent, f" payload={payload}", level="cmd") t0 = time.time() async with httpx.AsyncClient(timeout=120.0) as client: r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload) @@ -145,21 +157,26 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut _last_runs[mid] = run await _publish({"type": "movement", **run}) lvl = "info" if state == "success" else "err" + await _term(agent, f" ← {state} · {rows if rows is not None else '?'} rows in {dur}s", level="ok" if state == "success" else "err") _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl) return {"ok": state == "success", **run} except Exception as exc: _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)} + await _term(agent, f" ✗ error {str(exc)[:140]}", level="err") _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err") return {"ok": False, "error": str(exc)} conf = {**(mv.get("default_conf") or {}), **(conf or {})} - agent = mv["agent"] count_sql = mv.get("count_sql") + verb = "autonomously triggered" if autonomous else "triggered" + await _term(agent, f"$ airflow dags trigger {mv['dag_id']} # {mv['label']} ({verb})", level="cmd") + await _term(agent, f" conf={conf}", level="cmd") before = await _trino_scalar(count_sql) if count_sql else None + if count_sql: + await _term(agent, f" trino: {count_sql} → {before if before is not None else '?'} rows (before)", level="info") _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "running", "started_at": datetime.now(timezone.utc).isoformat()} await _publish({"type": "movement", "movement_id": mid, "state": "running"}) - verb = "autonomously triggered" if autonomous else "triggered" _feed(agent, f"[etl] {mv['label']}: {verb} (conf={conf})", "info") try: @@ -182,8 +199,10 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut if state in ("success", "failed"): break dur = round(time.time() - t0, 1) + await _term(agent, f" ← Airflow run {run_id} finished state={state} in {dur}s", level="ok" if state == "success" else "err") except Exception as exc: _last_runs[mid] = {**_last_runs[mid], "state": "failed", "error": str(exc)} + await _term(agent, f" ✗ error {str(exc)[:140]}", level="err") _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err") return {"ok": False, "error": str(exc)} @@ -201,6 +220,9 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut await _publish({"type": "movement", **run}) rtxt = f"{rows} rows" if rows is not None else "data" lvl = "info" if state == "success" else "err" + if count_sql: + await _term(agent, f" trino: count {before if before is not None else '?'} → {after if after is not None else '?'} ({rtxt})", level="info") + await _term(agent, f" ← {mv['label']}: {state} — {rtxt} in {dur}s", level="ok" if state == "success" else "err") _feed(agent, f"[etl] {mv['label']}: {state} — {rtxt} in {dur}s", lvl) return {"ok": state == "success", **run} diff --git a/ui/src/hooks/useCommandCenter.ts b/ui/src/hooks/useCommandCenter.ts index fd0dd37..d2bb738 100644 --- a/ui/src/hooks/useCommandCenter.ts +++ b/ui/src/hooks/useCommandCenter.ts @@ -313,7 +313,22 @@ export function useCommandCenter() { const inspectorLines = useMemo(() => { if (!terminalSubjectId) return [] const probeId = resolveProbeId(terminalSubjectId) - return terminals[probeId] || terminals[terminalSubjectId] || terminals[selectedAgentId || ''] || [] + const nodeLines = terminals[probeId] || terminals[terminalSubjectId] || [] + const agentLines = selectedAgentId ? terminals[selectedAgentId] || [] : [] + // When an agent is selected we also show its own autonomous ops stream + // (DML/ETL/probes) merged chronologically with the node probe output, so + // the terminal reflects what the agent is doing in the background — not just + // a one-shot probe. + if (!agentLines.length) return nodeLines + if (!nodeLines.length) return agentLines + const seen = new Set() + const merged = [...nodeLines, ...agentLines].filter((l) => { + if (seen.has(l.id)) return false + seen.add(l.id) + return true + }) + merged.sort((a, b) => ((a.ts || '') < (b.ts || '') ? -1 : (a.ts || '') > (b.ts || '') ? 1 : 0)) + return merged.slice(-300) }, [terminalSubjectId, terminals, selectedAgentId]) const focusApprovals = useCallback(() => {