Merge origin/main into running Command Center

Keep the deployed tree on conflict; integrate the remote Dockhand/CDC tip.

api/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
-COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py hive_bench_seed.json .
+COPY main.py auth.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py agent_activity.py lake_meta.py lineage.py dq_monitor.py observability.py catalog_governance.py gpu_config.py hive_bench_seed.json .
 RUN mkdir -p /data
 ENV DATABASE_URL=sqlite:////data/atc-agents.db
 EXPOSE 3201

api/agent_activity.py

@@ -0,0 +1,206 @@
+"""Continuous autonomous activity for the field agents that don't drive a
+data-movement loop of their own (Lakehouse Ops, Hadoop Ranger, Infra Sentinel,
+Network Watcher).
+
+Every tick the loop runs ONE real, lightweight probe for the next agent in the
+rotation and streams the exact command + result into that agent's terminal, so
+the operator can always see what each agent is doing in the background instead
+of an idle prompt. All probes hit live endpoints (Trino, WebHDFS JMX, YARN,
+Dockhand, VLAN hosts) and are individually guarded so a single failure never
+breaks the loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from typing import Any
+
+import httpx
+
+TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
+TRINO_USER = os.getenv("TRINO_USER", "mo")
+HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
+YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088").rstrip("/")
+DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082").rstrip("/")
+DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
+OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")).rstrip("/")
+
+TICK_SECONDS = float(os.getenv("AGENT_ACTIVITY_TICK_SECONDS", "8"))
+
+# VLAN data paths the Network Watcher keeps an eye on.
+_NET_TARGETS = [
+    ("Kafka UI", "http://10.0.21.36:9000"),
+    ("Trino", f"{TRINO_URL}/v1/info"),
+    ("HDFS NameNode", f"{HDFS_NN_URL}/dfshealth.html"),
+    ("Airflow", os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080")),
+    ("ObjectScale S3", OBJECTSCALE_URL),
+]
+
+
+async def _term(agent_id: str, text: str, level: str = "info", phase: str = "ops") -> None:
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+async def _trino(sql: str, timeout: float = 8.0) -> list[list[Any]]:
+    rows: list[list[Any]] = []
+    async with httpx.AsyncClient(timeout=timeout) as client:
+        d = (await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(),
+                               headers={"X-Trino-User": TRINO_USER})).json()
+        for _ in range(40):
+            if d.get("error"):
+                raise RuntimeError(d["error"].get("message", "trino error"))
+            rows += d.get("data") or []
+            nxt = d.get("nextUri")
+            if not nxt:
+                break
+            d = (await client.get(nxt)).json()
+    return rows
+
+
+# ── per-agent probes ─────────────────────────────────────────────────────────
+async def _lakehouse_ops() -> None:
+    aid = "lakehouse-ops"
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            info = (await client.get(f"{TRINO_URL}/v1/info")).json()
+        ver = info.get("nodeVersion", {}).get("version", "?")
+        up = info.get("uptime", "?")
+        await _term(aid, f"$ curl -s {TRINO_URL}/v1/info  # Trino coordinator health", level="cmd", phase="trino")
+        await _term(aid, f"  ← Trino {ver} · uptime {up} · serving federated queries", level="ok", phase="trino")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Trino unreachable: {str(exc)[:100]}", level="err", phase="trino")
+    try:
+        cats = await _trino("SHOW CATALOGS")
+        names = ", ".join(sorted(c[0] for c in cats))
+        await _term(aid, "$ trino --execute 'SHOW CATALOGS'", level="cmd", phase="trino")
+        await _term(aid, f"  ← {len(cats)} catalogs federated: {names}", level="ok", phase="trino")
+    except Exception as exc:
+        await _term(aid, f"  ✗ SHOW CATALOGS failed: {str(exc)[:100]}", level="err", phase="trino")
+    try:
+        sql = "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"
+        rows = await _trino(sql, timeout=12.0)
+        n = int(rows[0][0]) if rows else 0
+        await _term(aid, f"$ trino --execute '{sql}'  # curated Iceberg lakehouse", level="cmd", phase="iceberg")
+        await _term(aid, f"  ← {n:,} masked rows in iceberg.curated_masked (PII-safe layer)", level="ok", phase="iceberg")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Iceberg count failed: {str(exc)[:100]}", level="err", phase="iceberg")
+
+
+def _g(d: dict, *keys) -> Any:
+    for k in keys:
+        if k in d:
+            return d[k]
+    return None
+
+
+async def _hadoop_ranger() -> None:
+    aid = "hadoop-ranger"
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            fs = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem")).json()
+            state = (await client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")).json()
+        fsb = (fs.get("beans") or [{}])[0]
+        stb = (state.get("beans") or [{}])[0]
+        cap_total = float(_g(fsb, "CapacityTotalGB") or 0)
+        cap_used = float(_g(fsb, "CapacityUsedGB") or 0)
+        blocks = int(_g(fsb, "BlocksTotal", "TotalBlocks") or _g(stb, "BlocksTotal") or 0)
+        live = int(_g(stb, "NumLiveDataNodes") or 0)
+        dead = int(_g(stb, "NumDeadDataNodes") or 0)
+        pct = (100.0 * cap_used / cap_total) if cap_total else 0.0
+        await _term(aid, f"$ curl -s {HDFS_NN_URL}/jmx?qry=...FSNamesystemState  # NameNode health", level="cmd", phase="hdfs")
+        await _term(aid, f"  ← live datanodes={live} dead={dead} · blocks={blocks:,} · "
+                         f"used {cap_used:.1f}/{cap_total:.1f} GB ({pct:.0f}%)", level="ok", phase="hdfs")
+    except Exception as exc:
+        await _term(aid, f"  ✗ NameNode JMX unreachable: {str(exc)[:100]}", level="err", phase="hdfs")
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            m = (await client.get(f"{YARN_URL}/ws/v1/cluster/metrics")).json().get("clusterMetrics", {})
+        await _term(aid, f"$ yarn application -list  # ResourceManager {YARN_URL}", level="cmd", phase="yarn")
+        await _term(aid, f"  ← apps running={m.get('appsRunning', 0)} pending={m.get('appsPending', 0)} · "
+                         f"available {round(m.get('availableMB', 0) / 1024, 1)} GB / "
+                         f"{m.get('totalNodes', 0)} nodes", level="ok", phase="yarn")
+    except Exception as exc:
+        await _term(aid, f"  ⚠ YARN RM not responding ({str(exc)[:60]}) — HDFS storage layer still healthy", level="warn", phase="yarn")
+
+
+async def _infra_sentinel() -> None:
+    aid = "infra-sentinel"
+    await _term(aid, f"$ dockhand ps --all-envs  # container inventory via {DOCKHAND_URL}", level="cmd", phase="docker")
+    try:
+        from dockhand_envs import DOCKHAND_ENVS
+        envs = DOCKHAND_ENVS
+    except Exception:
+        envs = {"docker01": 1, "docker02": 2, "lakehouse": 9, "airflow": 10, "db02": 5}
+    total = running = 0
+    reached = 0
+    try:
+        async with httpx.AsyncClient(timeout=4.0) as client:
+            async def _one(name: str, eid: int):
+                try:
+                    hdr = {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"} if DOCKHAND_API_TOKEN else {}
+                    r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": eid}, headers=hdr)
+                    if r.status_code >= 400:
+                        return None
+                    d = r.json()
+                    return d if isinstance(d, list) else d.get("containers", [])
+                except Exception:
+                    return None
+            results = await asyncio.gather(*[_one(n, e) for n, e in envs.items()])
+        for conts in results:
+            if conts is None:
+                continue
+            reached += 1
+            total += len(conts)
+            running += sum(1 for c in conts
+                           if str(c.get("state", c.get("status", ""))).lower().startswith(("run", "up")))
+        if reached:
+            await _term(aid, f"  ← {running}/{total} containers up across {reached} Dockhand environments", level="ok", phase="docker")
+        else:
+            await _term(aid, "  ✗ Dockhand returned no environments", level="warn", phase="docker")
+    except Exception as exc:
+        await _term(aid, f"  ✗ Dockhand unreachable: {str(exc)[:100]}", level="err", phase="docker")
+    try:
+        load = os.getloadavg()
+        await _term(aid, "$ cat /proc/loadavg  # command-center host", level="cmd", phase="host")
+        await _term(aid, f"  ← load avg {load[0]:.2f} {load[1]:.2f} {load[2]:.2f} (1/5/15m)", level="ok", phase="host")
+    except Exception:
+        pass
+
+
+async def _network_watcher() -> None:
+    aid = "network-watcher"
+    await _term(aid, "$ probe VLAN 20/21 data paths  # ingress/egress reachability", level="cmd", phase="net")
+    async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
+        for label, url in _NET_TARGETS:
+            t0 = time.time()
+            try:
+                r = await client.get(url)
+                ms = int((time.time() - t0) * 1000)
+                lvl = "ok" if r.status_code < 500 else "warn"
+                await _term(aid, f"  → {label:<16} {url}  {r.status_code} {ms}ms", level=lvl, phase="net")
+            except Exception as exc:
+                await _term(aid, f"  → {label:<16} {url}  DOWN ({str(exc)[:60]})", level="err", phase="net")
+
+
+_ROTATION = [_lakehouse_ops, _hadoop_ranger, _infra_sentinel, _network_watcher]
+
+
+async def agent_activity_loop() -> None:
+    """Round-robin: run one agent's live probe per tick so each agent terminal
+    shows fresh real activity roughly every (len(rotation) * TICK_SECONDS)s."""
+    await asyncio.sleep(15)  # let the platform settle
+    idx = 0
+    while True:
+        probe = _ROTATION[idx % len(_ROTATION)]
+        idx += 1
+        try:
+            await probe()
+        except Exception:
+            pass
+        await asyncio.sleep(max(3.0, TICK_SECONDS))

api/agent_ops.py

@@ -103,6 +103,15 @@ async def _emit(message: str, level: str = "info") -> None:
         pass
 
 
+async def _term(agent_id: str, message: str, level: str = "info", phase: str = "ops") -> None:
+    """Stream a line to a specific agent terminal (no feed entry)."""
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, message, level=level, phase=phase)
+    except Exception:
+        pass
+
+
 # ── Synchronous DB operations (run in a thread) ──────────────────────────────
 def _pg_conn():
     import psycopg2
@@ -152,7 +161,7 @@ def _resolve_op(op: str, pool: deque) -> str:
     return op
 
 
-def _pg_dml(op: str) -> str:
+def _pg_dml(op: str) -> dict[str, str]:
     pool = _pools["postgres"]
     op = _resolve_op(op, pool)
     conn = _pg_conn()
@@ -161,35 +170,39 @@ def _pg_dml(op: str) -> str:
         with conn.cursor() as cur:
             if op == "insert":
                 rid = uuid.uuid4().hex[:8]
+                cid, pid = random.randint(1, 50000), random.randint(1, 2000)
+                region, channel = random.choice(REGIONS), random.choice(CHANNELS)
+                amt, curr, status = round(random.uniform(10, 9999), 2), random.choice(CURRENCIES), random.choice(ORDER_STATUS)
                 cur.execute(
                     """INSERT INTO public.sales_orders
                        (customer_id, product_id, region, sales_channel, order_ts, amount, currency, order_status, notes)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING order_id""",
-                    (
-                        random.randint(1, 50000), random.randint(1, 2000),
-                        random.choice(REGIONS), random.choice(CHANNELS),
-                        datetime.now(timezone.utc), round(random.uniform(10, 9999), 2),
-                        random.choice(CURRENCIES), random.choice(ORDER_STATUS), _tag(rid),
-                    ),
+                    (cid, pid, region, channel, datetime.now(timezone.utc), amt, curr, status, _tag(rid)),
                 )
                 oid = cur.fetchone()[0]
                 pool.append(oid)
-                return f"INSERT sales_orders order_id={oid} ({_tag(rid)})"
+                sql = (f"INSERT INTO public.sales_orders (customer_id,product_id,region,sales_channel,"
+                       f"amount,currency,order_status,notes) VALUES ({cid},{pid},'{region}','{channel}',"
+                       f"{amt},'{curr}','{status}','{_tag(rid)}');")
+                return {"op": "insert", "detail": f"INSERT sales_orders order_id={oid} ({_tag(rid)})", "sql": sql}
             if op == "update":
                 oid = random.choice(list(pool))
+                status, mult = random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2)
                 cur.execute(
                     "UPDATE public.sales_orders SET order_status=%s, amount=round(amount*%s,2) WHERE order_id=%s",
-                    (random.choice(ORDER_STATUS), round(random.uniform(0.9, 1.2), 2), oid),
+                    (status, mult, oid),
                 )
-                return f"UPDATE sales_orders order_id={oid}"
+                sql = f"UPDATE public.sales_orders SET order_status='{status}', amount=round(amount*{mult},2) WHERE order_id={oid};"
+                return {"op": "update", "detail": f"UPDATE sales_orders order_id={oid}", "sql": sql}
             oid = pool.popleft()
             cur.execute("DELETE FROM public.sales_orders WHERE order_id=%s", (oid,))
-            return f"DELETE sales_orders order_id={oid}"
+            return {"op": "delete", "detail": f"DELETE sales_orders order_id={oid}",
+                    "sql": f"DELETE FROM public.sales_orders WHERE order_id={oid};"}
     finally:
         conn.close()
 
 
-def _mysql_dml(op: str) -> str:
+def _mysql_dml(op: str) -> dict[str, str]:
     pool = _pools["mysql"]
     op = _resolve_op(op, pool)
     conn = _mysql_conn()
@@ -197,64 +210,64 @@ def _mysql_dml(op: str) -> str:
         with conn.cursor() as cur:
             if op == "insert":
                 rid = uuid.uuid4().hex[:8]
+                eid_in, dept, role = random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES)
+                region, etype, sal = random.choice(REGIONS), random.choice(EVENT_TYPES), round(random.uniform(-5000, 15000), 2)
                 cur.execute(
                     """INSERT INTO hr.employee_events
                        (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
-                    (
-                        random.randint(1, 20000), random.choice(DEPARTMENTS), random.choice(ROLES),
-                        random.choice(REGIONS), random.choice(EVENT_TYPES),
-                        round(random.uniform(-5000, 15000), 2), datetime.now(timezone.utc), _tag(rid),
-                    ),
+                    (eid_in, dept, role, region, etype, sal, datetime.now(timezone.utc), _tag(rid)),
                 )
                 eid = cur.lastrowid
                 pool.append(eid)
-                return f"INSERT employee_events event_id={eid} ({_tag(rid)})"
+                sql = (f"INSERT INTO hr.employee_events (employee_id,department,role_name,region,event_type,"
+                       f"salary_change,notes) VALUES ({eid_in},'{dept}','{role}','{region}','{etype}',{sal},'{_tag(rid)}');")
+                return {"op": "insert", "detail": f"INSERT employee_events event_id={eid} ({_tag(rid)})", "sql": sql}
             if op == "update":
                 eid = random.choice(list(pool))
+                sal, etype = round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES)
                 cur.execute(
                     "UPDATE hr.employee_events SET salary_change=%s, event_type=%s WHERE event_id=%s",
-                    (round(random.uniform(-5000, 15000), 2), random.choice(EVENT_TYPES), eid),
+                    (sal, etype, eid),
                 )
-                return f"UPDATE employee_events event_id={eid}"
+                sql = f"UPDATE hr.employee_events SET salary_change={sal}, event_type='{etype}' WHERE event_id={eid};"
+                return {"op": "update", "detail": f"UPDATE employee_events event_id={eid}", "sql": sql}
             eid = pool.popleft()
             cur.execute("DELETE FROM hr.employee_events WHERE event_id=%s", (eid,))
-            return f"DELETE employee_events event_id={eid}"
+            return {"op": "delete", "detail": f"DELETE employee_events event_id={eid}",
+                    "sql": f"DELETE FROM hr.employee_events WHERE event_id={eid};"}
     finally:
         conn.close()
 
 
-def _mongo_dml(op: str) -> str:
+def _mongo_dml(op: str) -> dict[str, str]:
     pool = _pools["mongodb"]
     op = _resolve_op(op, pool)
     client, coll = _mongo_coll()
     try:
         if op == "insert":
             rid = uuid.uuid4().hex[:8]
+            mtype, region, msrc = random.choice(MONGO_TYPES), random.choice(REGIONS), random.choice(MONGO_SOURCES)
+            amt = round(random.uniform(10, 50000), 4)
             doc = {
-                "event_id": str(uuid.uuid4()),
-                "type": random.choice(MONGO_TYPES),
-                "region": random.choice(REGIONS),
-                "source": random.choice(MONGO_SOURCES),
-                "amount": round(random.uniform(10, 50000), 4),
-                "ts": datetime.now(timezone.utc),
-                "payload": "X" * 200,
-                "atc_agent": True,
-                "agent_run": rid,
+                "event_id": str(uuid.uuid4()), "type": mtype, "region": region, "source": msrc,
+                "amount": amt, "ts": datetime.now(timezone.utc), "payload": "X" * 200,
+                "atc_agent": True, "agent_run": rid,
             }
             res = coll.insert_one(doc)
             pool.append(res.inserted_id)
-            return f"INSERT events _id={res.inserted_id} (agent_run={rid})"
+            sql = (f"db.events.insertOne({{type:'{mtype}', region:'{region}', source:'{msrc}', "
+                   f"amount:{amt}, atc_agent:true, agent_run:'{rid}'}})")
+            return {"op": "insert", "detail": f"INSERT events _id={res.inserted_id} (agent_run={rid})", "sql": sql}
         if op == "update":
             oid = random.choice(list(pool))
-            coll.update_one(
-                {"_id": oid},
-                {"$set": {"type": random.choice(MONGO_TYPES), "amount": round(random.uniform(10, 50000), 4)}},
-            )
-            return f"UPDATE events _id={oid}"
+            mtype, amt = random.choice(MONGO_TYPES), round(random.uniform(10, 50000), 4)
+            coll.update_one({"_id": oid}, {"$set": {"type": mtype, "amount": amt}})
+            sql = f"db.events.updateOne({{_id:{oid!r}}}, {{$set:{{type:'{mtype}', amount:{amt}}}}})"
+            return {"op": "update", "detail": f"UPDATE events _id={oid}", "sql": sql}
         oid = pool.popleft()
         coll.delete_one({"_id": oid})
-        return f"DELETE events _id={oid}"
+        return {"op": "delete", "detail": f"DELETE events _id={oid}", "sql": f"db.events.deleteOne({{_id:{oid!r}}})"}
     finally:
         client.close()
 
@@ -268,6 +281,9 @@ def _pick_op() -> str:
     return random.choices(["insert", "update", "delete"], weights=[0.5, 0.3, 0.2], k=1)[0]
 
 
+_CLIENT_CMD = {"postgres": "psql sales", "mysql": "mysql hr", "mongodb": "mongosh supplychain"}
+
+
 async def _run_one(source: str | None = None, op: str | None = None) -> dict[str, Any]:
     source = source or random.choice(list(_DISPATCH.keys()))
     op = op or _pick_op()
@@ -275,18 +291,25 @@ async def _run_one(source: str | None = None, op: str | None = None) -> dict[str
     if not fn:
         return {"ok": False, "error": f"unknown source {source}"}
     try:
-        detail = await asyncio.to_thread(fn, op)
+        # Show the operator exactly what the Data Custodian is about to run.
+        await _term(DML_AGENT, f"$ {_CLIENT_CMD.get(source, source)}  # autonomous DML on {_SRC_LABEL[source]}",
+                    level="cmd", phase="dml")
+        res = await asyncio.to_thread(fn, op)
+        detail, sql, actual_op = res["detail"], res["sql"], res["op"]
+        await _term(DML_AGENT, f"  {sql}", level="cmd", phase="dml")
         _state["ops_total"] += 1
         _state["by_source"][source] = _state["by_source"].get(source, 0) + 1
-        actual_op = detail.split(" ", 1)[0].lower()
         if actual_op in _state["by_op"]:
             _state["by_op"][actual_op] += 1
-        _state["last_op"] = {"source": source, "op": op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
+        _state["last_op"] = {"source": source, "op": actual_op, "detail": detail, "ts": datetime.now(timezone.utc).isoformat()}
         _state["last_error"] = None
+        await _term(DML_AGENT, f"  ← {detail} · Debezium CDC will stream this to Kafka", level="ok", phase="dml")
+        # Keep the supervisor feed concise (single summary entry).
         await _emit(f"[agent-dml] {_SRC_LABEL[source]}: {detail} — Debezium will capture this change", "info")
-        return {"ok": True, "source": source, "op": op, "detail": detail}
+        return {"ok": True, "source": source, "op": actual_op, "detail": detail}
     except Exception as exc:
         _state["last_error"] = str(exc)
+        await _term(DML_AGENT, f"  ✗ {source} {op} failed: {str(exc)[:140]}", level="err", phase="dml")
         await _emit(f"[agent-dml] {_SRC_LABEL.get(source, source)}: operation failed: {str(exc)[:120]}", "err")
         return {"ok": False, "source": source, "op": op, "error": str(exc)}
 
@@ -318,7 +341,14 @@ async def etl_agent_loop() -> None:
                 mid = _etl_state["rotation"][_etl_state["idx"] % len(_etl_state["rotation"])]
                 _etl_state["idx"] += 1
                 if mid in MOVEMENT_BY_ID:
+                    mv = MOVEMENT_BY_ID[mid]
+                    await _term("etl-guardian",
+                                f"$ orchestrate movement '{mid}' ({mv.get('label')}) {mv.get('from')}→{mv.get('to')}",
+                                level="cmd", phase="orchestrate")
                     result = await trigger_and_watch(mid, autonomous=True)
+                    await _term("etl-guardian",
+                                f"  ← {mv.get('label')}: {result.get('state')} · {result.get('rows')} rows in {result.get('duration_s')}s",
+                                level="ok" if result.get("ok") else "err", phase="orchestrate")
                     _etl_state["runs_total"] += 1
                     _etl_state["last"] = {"movement_id": mid, "state": result.get("state"),
                                           "rows": result.get("rows"), "duration_s": result.get("duration_s"),
@@ -334,10 +364,36 @@ _CUST_INTERVAL = float(os.getenv("CUSTODIAN_OFFLOAD_INTERVAL_SECONDS", "120"))
 _CUST_BATCH = int(os.getenv("CUSTODIAN_OFFLOAD_BATCH", "200"))
 _CUST_TARGETS = [
     {"label": "postgres sales_orders", "src": "postgres_sales.public.sales_orders",
-     "target": "iceberg.hadoop.sales_orders_offload"},
+     "target": "iceberg.hadoop.sales_orders_offload",
+     "create_sql": (
+         "CREATE TABLE iceberg.hadoop.sales_orders_offload AS "
+         "SELECT * FROM postgres_sales.public.sales_orders WHERE 1=0"
+     ),
+     "insert_sql": (
+         "INSERT INTO iceberg.hadoop.sales_orders_offload "
+         "SELECT * FROM postgres_sales.public.sales_orders LIMIT {batch}"
+     )},
     {"label": "mysql employee_events", "src": "mysql_hr.hr.employee_events",
-     "target": "iceberg.hadoop.employee_events_offload"},
+     "target": "iceberg.hadoop.employee_events_offload",
+     "create_sql": (
+         "CREATE TABLE iceberg.hadoop.employee_events_offload AS SELECT "
+         "event_id, employee_id, department, role_name, region, event_type, "
+         "CAST(salary_change AS double) AS salary_change, "
+         "CAST(event_ts AS timestamp(6)) AS event_ts, notes, "
+         "employee_name, employee_email, employee_phone, national_id, home_address, "
+         "CAST(date_of_birth AS date) AS date_of_birth "
+         "FROM mysql_hr.hr.employee_events WHERE 1=0"
+     ),
+     "insert_sql": (
+         "INSERT INTO iceberg.hadoop.employee_events_offload SELECT "
+         "event_id, employee_id, department, role_name, region, event_type, "
+         "CAST(salary_change AS double), CAST(event_ts AS timestamp(6)), notes, "
+         "employee_name, employee_email, employee_phone, national_id, home_address, "
+         "CAST(date_of_birth AS date) "
+         "FROM mysql_hr.hr.employee_events LIMIT {batch}"
+     )},
 ]
+
 _custodian_state: dict[str, Any] = {
     "enabled": os.getenv("CUSTODIAN_OFFLOAD_ENABLED", "1") not in ("0", "false", "False", ""),
     "interval": _CUST_INTERVAL,
@@ -355,10 +411,30 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
     i = _custodian_state["idx"] if idx is None else idx
     tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
     _custodian_state["idx"] = i + 1
-    await _trino_collect(
-        f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1)
-    ins = await _trino_collect(
-        f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1)
+    await _trino_collect("CREATE SCHEMA IF NOT EXISTS iceberg.hadoop", 1)
+    probe = await _trino_collect(f"SELECT 1 FROM {tgt['target']} WHERE 1=0", 1)
+    if not probe.get("ok"):
+        ddl = tgt["create_sql"]
+        await _term(DML_AGENT, f"$ trino --catalog iceberg  # Hadoop offload: {tgt['label']} → {tgt['target']}",
+                    level="cmd", phase="offload")
+        await _term(DML_AGENT, f"  {ddl};", level="cmd", phase="offload")
+        created = await _trino_collect(ddl, 1)
+        if not created.get("ok"):
+            err = created.get("error")
+            await _term(DML_AGENT, f"  ✗ offload failed: {str(err)[:140]}", level="err", phase="offload")
+            await _emit(f"[custodian-offload] {tgt['label']} failed: {str(err)[:120]}", "err")
+            _custodian_state["runs_total"] += 1
+            _custodian_state["last"] = {
+                "target": tgt["target"], "src": tgt["src"], "ok": False,
+                "rows": 0, "ts": datetime.now(timezone.utc).isoformat(), "error": err,
+            }
+            return _custodian_state["last"]
+    else:
+        await _term(DML_AGENT, f"$ trino --catalog iceberg  # Hadoop offload: {tgt['label']} → {tgt['target']}",
+                    level="cmd", phase="offload")
+    dml = tgt["insert_sql"].format(batch=_CUST_BATCH)
+    await _term(DML_AGENT, f"  {dml};", level="cmd", phase="offload")
+    ins = await _trino_collect(dml, 1)
     ok = bool(ins.get("ok"))
     _custodian_state["runs_total"] += 1
     _custodian_state["last"] = {
@@ -367,8 +443,10 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
         "ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"),
     }
     if ok:
+        await _term(DML_AGENT, f"  ← offloaded ~{_CUST_BATCH} rows into the Hadoop Iceberg lake", level="ok", phase="offload")
         await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info")
     else:
+        await _term(DML_AGENT, f"  ✗ offload failed: {str(ins.get('error'))[:140]}", level="err", phase="offload")
         await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err")
     return _custodian_state["last"]
 

api/agent_terminal.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import uuid
 from collections import deque
 from datetime import datetime, timezone
@@ -13,6 +14,7 @@ PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
 
 _buffers: dict[str, deque[dict[str, Any]]] = {}
 _publish: PublishFn | None = None
+_loop: asyncio.AbstractEventLoop | None = None
 
 
 def init_terminals(agent_ids: list[str]) -> None:
@@ -22,8 +24,38 @@ def init_terminals(agent_ids: list[str]) -> None:
 
 
 def set_terminal_publisher(fn: PublishFn) -> None:
-    global _publish
+    global _publish, _loop
     _publish = fn
+    try:  # capture the main event loop so background threads can stream too
+        _loop = asyncio.get_running_loop()
+    except RuntimeError:
+        _loop = None
+
+
+def _build_line(agent_id: str, text: str, level: str, phase: str, prompt_id: str | None) -> dict[str, Any]:
+    return {
+        "id": str(uuid.uuid4())[:8],
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "agent_id": agent_id,
+        "level": level,
+        "phase": phase,
+        "text": text,
+        "prompt_id": prompt_id,
+    }
+
+
+def emit_threadsafe(agent_id: str, text: str, *, level: str = "info", phase: str = "ops") -> dict[str, Any]:
+    """Append a terminal line and broadcast it from a non-async context (e.g. a
+    background ``threading.Thread``). Safe to call from any thread."""
+    init_terminals([agent_id])
+    line = _build_line(agent_id, text, level, phase, None)
+    _buffers[agent_id].append(line)
+    if _publish and _loop and not _loop.is_closed():
+        try:
+            asyncio.run_coroutine_threadsafe(_publish({"type": "terminal", "line": line}), _loop)
+        except Exception:
+            pass
+    return line
 
 
 def get_terminal_lines(agent_id: str, limit: int = 200) -> list[dict[str, Any]]:

api/auth.py

@@ -0,0 +1,229 @@
+"""Authentik OIDC login and session cookie for Command Center."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+from urllib.parse import quote, urlencode
+
+import httpx
+from authlib.common.security import generate_token
+from authlib.integrations.starlette_client import OAuth
+from authlib.oauth2.rfc7636 import create_s256_code_challenge
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import JSONResponse, RedirectResponse
+from starlette.middleware.sessions import SessionMiddleware
+
+log = logging.getLogger("atc-agents.auth")
+
+AUTH_ENABLED = os.getenv("AUTH_ENABLED", "true").lower() in ("1", "true", "yes")
+AUTHENTIK_ISSUER = os.getenv(
+    "AUTHENTIK_ISSUER",
+    "http://atc-mgt01.dell-atc.lan:9000/application/o/command-center/",
+).rstrip("/") + "/"
+AUTHENTIK_CLIENT_ID = os.getenv("AUTHENTIK_CLIENT_ID", "")
+AUTHENTIK_CLIENT_SECRET = os.getenv("AUTHENTIK_CLIENT_SECRET", "")
+AUTHENTIK_REDIRECT_URI = os.getenv(
+    "AUTHENTIK_REDIRECT_URI",
+    "http://10.0.21.33/auth/callback",
+)
+SESSION_SECRET = os.getenv("SESSION_SECRET", "dev-insecure-change-me")
+SESSION_COOKIE = "cc_session"
+
+oauth = OAuth()
+_oauth_ready = False
+
+PUBLIC_PREFIXES = (
+    "/auth/",
+    "/api/health",
+    "/api/auth/me",
+)
+
+
+def is_auth_enabled() -> bool:
+    return AUTH_ENABLED
+
+
+def build_session_user(claims: dict[str, Any]) -> dict[str, Any]:
+    name = claims.get("name") or claims.get("preferred_username") or claims.get("email") or "user"
+    email = claims.get("email") or ""
+    username = claims.get("preferred_username") or claims.get("nickname") or email or str(claims.get("sub") or "user")
+    return {
+        "sub": claims.get("sub"),
+        "email": email,
+        "name": name,
+        "preferred_username": username,
+    }
+
+
+def get_session_user(request: Request) -> dict[str, Any] | None:
+    if not AUTH_ENABLED:
+        return {
+            "sub": "dev:local",
+            "email": "",
+            "name": "Dev User",
+            "preferred_username": "dev",
+            "dev": True,
+        }
+    user = request.session.get("user")
+    return user if isinstance(user, dict) else None
+
+
+def auth_me_payload(request: Request) -> dict[str, Any]:
+    user = get_session_user(request)
+    if not user:
+        raise HTTPException(status_code=401, detail="Not authenticated")
+    return {
+        "user": user.get("sub"),
+        "email": user.get("email"),
+        "name": user.get("name"),
+        "preferred_username": user.get("preferred_username"),
+        "auth_enabled": AUTH_ENABLED,
+    }
+
+
+def _ensure_oauth() -> None:
+    global _oauth_ready
+    if _oauth_ready or not AUTH_ENABLED:
+        return
+    if not AUTHENTIK_CLIENT_ID or not AUTHENTIK_CLIENT_SECRET:
+        log.warning("AUTH_ENABLED but Authentik client credentials missing")
+        return
+    meta_url = AUTHENTIK_ISSUER + ".well-known/openid-configuration"
+    oauth.register(
+        name="authentik",
+        client_id=AUTHENTIK_CLIENT_ID,
+        client_secret=AUTHENTIK_CLIENT_SECRET,
+        server_metadata_url=meta_url,
+        client_kwargs={"scope": "openid profile email"},
+    )
+    _oauth_ready = True
+
+
+def _is_public_path(path: str) -> bool:
+    return any(path == p or path.startswith(p) for p in PUBLIC_PREFIXES)
+
+
+async def auth_guard_middleware(request: Request, call_next):
+    if not AUTH_ENABLED:
+        return await call_next(request)
+    path = request.url.path
+    if _is_public_path(path):
+        return await call_next(request)
+    if path.startswith("/api/"):
+        if get_session_user(request) is None:
+            return JSONResponse({"detail": "Not authenticated"}, status_code=401)
+    return await call_next(request)
+
+
+def init_auth_middleware(app: FastAPI) -> None:
+    """Session must wrap auth guard so request.session is populated first."""
+    app.middleware("http")(auth_guard_middleware)
+    app.add_middleware(
+        SessionMiddleware,
+        secret_key=SESSION_SECRET,
+        session_cookie=SESSION_COOKIE,
+        max_age=86400 * 7,
+        same_site="lax",
+        https_only=False,
+    )
+
+
+def setup_auth(app: FastAPI) -> None:
+    _ensure_oauth()
+    _register_routes(app)
+
+
+def _token_endpoint() -> str:
+    return AUTHENTIK_ISSUER.rstrip("/").rsplit("/application/o/", 1)[0] + "/application/o/token/"
+
+
+def _userinfo_endpoint() -> str:
+    return AUTHENTIK_ISSUER.rstrip("/").rsplit("/application/o/", 1)[0] + "/application/o/userinfo/"
+
+
+async def _exchange_code_for_userinfo(code: str, code_verifier: str | None) -> dict[str, Any]:
+    data = {
+        "grant_type": "authorization_code",
+        "code": code,
+        "redirect_uri": AUTHENTIK_REDIRECT_URI,
+        "client_id": AUTHENTIK_CLIENT_ID,
+        "client_secret": AUTHENTIK_CLIENT_SECRET,
+    }
+    if code_verifier:
+        data["code_verifier"] = code_verifier
+    async with httpx.AsyncClient(timeout=20.0) as client:
+        tok = await client.post(_token_endpoint(), data=data)
+        if tok.status_code >= 400:
+            log.warning("Token exchange failed: %s %s", tok.status_code, tok.text[:300])
+            tok.raise_for_status()
+        payload = tok.json()
+        access = payload.get("access_token")
+        if not access:
+            raise RuntimeError("No access_token in token response")
+        ui = await client.get(
+            _userinfo_endpoint(),
+            headers={"Authorization": f"Bearer {access}"},
+        )
+        if ui.status_code >= 400:
+            log.warning("Userinfo failed: %s %s", ui.status_code, ui.text[:300])
+            ui.raise_for_status()
+        return ui.json()
+
+
+def _register_routes(app: FastAPI) -> None:
+    @app.get("/api/auth/me")
+    async def api_auth_me(request: Request):
+        return auth_me_payload(request)
+
+    @app.get("/auth/login")
+    async def auth_login(request: Request):
+        if not AUTH_ENABLED:
+            return RedirectResponse("/", status_code=302)
+        _ensure_oauth()
+        if not _oauth_ready:
+            raise HTTPException(503, "Authentik not configured")
+        code_verifier = generate_token(48)
+        request.session["pkce_code_verifier"] = code_verifier
+        return await oauth.authentik.authorize_redirect(
+            request,
+            AUTHENTIK_REDIRECT_URI,
+            code_challenge=create_s256_code_challenge(code_verifier),
+            code_challenge_method="S256",
+            code_verifier=code_verifier,
+        )
+
+    @app.get("/auth/callback")
+    async def auth_callback(request: Request):
+        if not AUTH_ENABLED:
+            return RedirectResponse("/", status_code=302)
+        err = request.query_params.get("error")
+        if err:
+            desc = request.query_params.get("error_description") or err
+            log.warning("OIDC provider error: %s — %s", err, desc)
+            return RedirectResponse(f"/?error={quote(desc)}", status_code=302)
+        code = request.query_params.get("code")
+        if not code:
+            return RedirectResponse("/?error=missing_code", status_code=302)
+        state = request.query_params.get("state")
+        if state:
+            request.session.pop(f"_state_authentik_{state}", None)
+        code_verifier = request.session.pop("pkce_code_verifier", None)
+        try:
+            userinfo = await _exchange_code_for_userinfo(code, code_verifier)
+        except Exception as e:
+            log.warning("OIDC callback failed: %s", e)
+            return RedirectResponse("/?error=login_failed", status_code=302)
+        request.session["user"] = build_session_user(userinfo or {})
+        return RedirectResponse("/", status_code=302)
+
+    @app.get("/auth/logout")
+    async def auth_logout(request: Request):
+        request.session.clear()
+        if not AUTH_ENABLED:
+            return RedirectResponse("/", status_code=302)
+        end_session = AUTHENTIK_ISSUER + "end-session/"
+        post_logout = "http://10.0.21.33/"
+        params = urlencode({"post_logout_redirect_uri": post_logout})
+        return RedirectResponse(f"{end_session}?{params}", status_code=302)

api/catalog_governance.py

@@ -0,0 +1,314 @@
+"""Data governance: ownership, stewardship, glossary & posture
+(Diseases #3 Ownership and #5 Governance).
+
+Ownership/steward assignments are kept in a local store (so the feature always
+works for the demo regardless of OpenMetadata ingestion state) and are
+best-effort mirrored to OpenMetadata. Users/teams for the assignment dropdowns
+are read from OpenMetadata when reachable. The governance *posture* view
+combines, per dataset: ownership, PII/masking (pii_catalog), live data-quality
+score (dq_monitor), observability alerts and a simple data-contract check.
+
+Endpoints:
+  GET  /api/governance/datasets   -> ownership matrix (+ orphan flag)
+  GET  /api/governance/users      -> assignable users/teams
+  POST /api/governance/assign     -> set owner/steward/team/tier (native + OM)
+  GET  /api/governance/glossary    -> business glossary terms
+  GET  /api/governance/posture     -> combined governance posture per dataset
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+
+from lake_meta import DATASETS, DATASET_BY_KEY
+
+router = APIRouter(prefix="/api/governance", tags=["governance"])
+
+OPENMETADATA_URL = os.getenv("OPENMETADATA_URL", "").rstrip("/")
+OPENMETADATA_TOKEN = os.getenv("OPENMETADATA_TOKEN", "")
+OWNERS_PATH = Path(os.getenv("GOVERNANCE_OWNERS_PATH", "/data/governance_owners.json"))
+CONTRACTS_PATH = Path(os.getenv("GOVERNANCE_CONTRACTS_PATH", "/data/governance_contracts.json"))
+
+# Default data contracts (quality SLAs) per dataset — used when none stored.
+DEFAULT_CONTRACTS = {
+    "_default": {"min_score": 80, "min_completeness": 95, "min_freshness_min": 60,
+                 "max_critical_alerts": 0},
+}
+
+# Seed business glossary so the term list is never empty even before OM ingestion.
+SEED_GLOSSARY = [
+    {"name": "Customer", "description": "A person or organization that places sales orders.",
+     "related": ["customer_id", "customer_name", "customer_email"], "domain": "Sales"},
+    {"name": "Order", "description": "A sales transaction with an amount, channel and status.",
+     "related": ["order_id", "amount", "order_status"], "domain": "Sales"},
+    {"name": "Revenue", "description": "Sum of order amounts over a period.",
+     "related": ["amount", "currency"], "domain": "Sales"},
+    {"name": "Employee", "description": "A member of the workforce tracked via HR events.",
+     "related": ["employee_id", "department", "role_name"], "domain": "People"},
+    {"name": "PII", "description": "Personally Identifiable Information — masked per policy.",
+     "related": ["customer_email", "national_id", "billing_iban"], "domain": "Governance"},
+    {"name": "Telemetry", "description": "Device metric readings over time.",
+     "related": ["device_id", "metric_type", "metric_value"], "domain": "IoT"},
+]
+
+
+def _headers() -> dict[str, str]:
+    h = {"Accept": "application/json"}
+    if OPENMETADATA_TOKEN:
+        h["Authorization"] = f"Bearer {OPENMETADATA_TOKEN}"
+    return h
+
+
+def _load(path: Path) -> dict[str, Any]:
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return {}
+
+
+def _save(path: Path, data: dict[str, Any]) -> None:
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(data, indent=2))
+    except Exception:
+        pass
+
+
+# ── OpenMetadata best-effort ────────────────────────────────────────────────
+def _om_users() -> list[dict[str, Any]]:
+    if not OPENMETADATA_URL:
+        return []
+    out: list[dict[str, Any]] = []
+    try:
+        with httpx.Client(timeout=8.0, verify=False) as c:
+            r = c.get(f"{OPENMETADATA_URL}/api/v1/users?limit=50&isBot=false", headers=_headers())
+            if r.status_code == 200:
+                for u in r.json().get("data", []):
+                    out.append({"id": u.get("id"), "name": u.get("name"),
+                                "display": u.get("displayName") or u.get("name"), "type": "user"})
+            rt = c.get(f"{OPENMETADATA_URL}/api/v1/teams?limit=50", headers=_headers())
+            if rt.status_code == 200:
+                for t in rt.json().get("data", []):
+                    out.append({"id": t.get("id"), "name": t.get("name"),
+                                "display": t.get("displayName") or t.get("name"), "type": "team"})
+    except Exception:
+        pass
+    return out
+
+
+def _om_glossary() -> list[dict[str, Any]]:
+    if not OPENMETADATA_URL:
+        return []
+    try:
+        with httpx.Client(timeout=8.0, verify=False) as c:
+            r = c.get(f"{OPENMETADATA_URL}/api/v1/glossaryTerms?limit=100", headers=_headers())
+            if r.status_code == 200:
+                return [{"name": t.get("name"), "description": t.get("description", ""),
+                         "domain": "OpenMetadata", "related": []}
+                        for t in r.json().get("data", [])]
+    except Exception:
+        pass
+    return []
+
+
+def _om_patch_owner(om_fqn: str, user: dict[str, Any]) -> bool:
+    """Best-effort: set table owner in OpenMetadata via JSON-Patch."""
+    if not OPENMETADATA_URL or not om_fqn or not user.get("id"):
+        return False
+    try:
+        with httpx.Client(timeout=8.0, verify=False) as c:
+            g = c.get(f"{OPENMETADATA_URL}/api/v1/tables/name/{om_fqn}", headers=_headers())
+            if g.status_code != 200:
+                return False
+            patch = [{"op": "add", "path": "/owners/0",
+                      "value": {"id": user["id"], "type": user.get("type", "user")}}]
+            h = {**_headers(), "Content-Type": "application/json-patch+json"}
+            p = c.patch(f"{OPENMETADATA_URL}/api/v1/tables/name/{om_fqn}",
+                        headers=h, content=json.dumps(patch))
+            return p.status_code in (200, 201)
+    except Exception:
+        return False
+
+
+# ── helpers pulling from sibling modules (all best-effort) ──────────────────
+def _pii_summary(pii_key: str) -> dict[str, Any]:
+    try:
+        from pii_catalog import get_pii
+        for d in get_pii().get("datasets", []):
+            if d.get("key") == pii_key:
+                return {"pii_count": d.get("pii_count", 0),
+                        "all_masked": d.get("all_masked", False),
+                        "masked": sum(1 for c in d.get("pii_columns", []) if c.get("masked")),
+                        "unmasked": sum(1 for c in d.get("pii_columns", []) if not c.get("masked"))}
+    except Exception:
+        pass
+    return {"pii_count": 0, "all_masked": False, "masked": 0, "unmasked": 0}
+
+
+def _dq_card(key: str) -> dict[str, Any]:
+    try:
+        from dq_monitor import scorecards_view
+        for c in scorecards_view().get("cards", []):
+            if c.get("key") == key:
+                return {"score": c.get("score"), "issues": c.get("issues", []),
+                        "freshness_age_min": c.get("freshness_age_min")}
+    except Exception:
+        pass
+    return {"score": None, "issues": []}
+
+
+def _obs_alerts(key: str) -> list[dict[str, Any]]:
+    try:
+        from observability import _state, _lock
+        with _lock:
+            return [{"type": a["type"], "severity": a["severity"], "message": a["message"]}
+                    for a in _state["alerts_active"].values() if a["dataset"] == key]
+    except Exception:
+        return []
+
+
+# ── views ───────────────────────────────────────────────────────────────────
+def datasets_view() -> dict[str, Any]:
+    owners = _load(OWNERS_PATH)
+    rows = []
+    orphans = 0
+    stewarded = 0
+    for ds in DATASETS:
+        rec = owners.get(ds["key"], {})
+        owner = rec.get("owner")
+        steward = rec.get("steward")
+        if not owner:
+            orphans += 1
+        if steward:
+            stewarded += 1
+        rows.append({
+            "key": ds["key"], "label": ds["label"], "engine": ds["engine"], "color": ds["color"],
+            "table": ds["fqtn"], "domain": rec.get("domain") or ds.get("domain"),
+            "owner": owner, "steward": steward, "team": rec.get("team"),
+            "tier": rec.get("tier"), "classification": rec.get("classification"),
+            "updated_at": rec.get("updated_at"),
+            "orphan": not owner,
+            "pii": _pii_summary(ds.get("pii_key", ds["key"])),
+        })
+    return {"ok": True, "datasets": rows,
+            "summary": {"total": len(rows), "orphans": orphans, "stewarded": stewarded,
+                        "owned": len(rows) - orphans},
+            "om_connected": bool(OPENMETADATA_URL)}
+
+
+class AssignRequest(BaseModel):
+    key: str
+    owner: str | None = None
+    steward: str | None = None
+    team: str | None = None
+    tier: str | None = None
+    classification: str | None = None
+
+
+def posture_view() -> dict[str, Any]:
+    owners = _load(OWNERS_PATH)
+    contracts = _load(CONTRACTS_PATH)
+    default_c = DEFAULT_CONTRACTS["_default"]
+    out = []
+    compliant = 0
+    for ds in DATASETS:
+        key = ds["key"]
+        rec = owners.get(key, {})
+        contract = {**default_c, **(contracts.get(key, {}))}
+        pii = _pii_summary(ds.get("pii_key", key))
+        dq = _dq_card(key)
+        alerts = _obs_alerts(key)
+        crit = sum(1 for a in alerts if a["severity"] == "critical")
+        checks = []
+        score = dq.get("score")
+        checks.append({"name": "DQ score", "ok": score is not None and score >= contract["min_score"],
+                       "value": score, "target": contract["min_score"]})
+        checks.append({"name": "Owner assigned", "ok": bool(rec.get("owner")),
+                       "value": rec.get("owner") or "—", "target": "assigned"})
+        checks.append({"name": "PII masked", "ok": pii["unmasked"] == 0,
+                       "value": f'{pii["masked"]}/{pii["pii_count"]}', "target": "all"})
+        checks.append({"name": "Critical alerts", "ok": crit <= contract["max_critical_alerts"],
+                       "value": crit, "target": contract["max_critical_alerts"]})
+        ok = all(c["ok"] for c in checks)
+        if ok:
+            compliant += 1
+        out.append({
+            "key": key, "label": ds["label"], "engine": ds["engine"], "color": ds["color"],
+            "table": ds["fqtn"], "owner": rec.get("owner"), "steward": rec.get("steward"),
+            "tier": rec.get("tier"), "pii": pii, "dq_score": score, "issues": dq.get("issues", []),
+            "alerts": alerts, "contract": contract, "checks": checks, "compliant": ok,
+        })
+    return {"ok": True, "datasets": out,
+            "summary": {"total": len(out), "compliant": compliant,
+                        "non_compliant": len(out) - compliant}}
+
+
+def summary_for_llm() -> dict[str, Any]:
+    owners = _load(OWNERS_PATH)
+    return {
+        "owners": {k: {"owner": v.get("owner"), "steward": v.get("steward"), "tier": v.get("tier")}
+                   for k, v in owners.items()},
+        "orphan_datasets": [d["key"] for d in DATASETS if not owners.get(d["key"], {}).get("owner")],
+    }
+
+
+@router.get("/datasets")
+async def get_datasets() -> JSONResponse:
+    return JSONResponse(datasets_view())
+
+
+@router.get("/users")
+async def get_users() -> JSONResponse:
+    users = _om_users()
+    if not users:
+        users = [{"id": None, "name": n, "display": n, "type": "user"}
+                 for n in ("admin", "bart", "mo")] + \
+                [{"id": None, "name": "Organization", "display": "Organization", "type": "team"}]
+    return JSONResponse({"ok": True, "users": users, "om_connected": bool(OPENMETADATA_URL)})
+
+
+@router.post("/assign")
+async def assign(body: AssignRequest) -> JSONResponse:
+    if body.key not in DATASET_BY_KEY:
+        return JSONResponse({"ok": False, "error": f"unknown dataset {body.key}"}, status_code=400)
+    owners = _load(OWNERS_PATH)
+    rec = dict(owners.get(body.key, {}))
+    for field in ("owner", "steward", "team", "tier", "classification"):
+        val = getattr(body, field)
+        if val is not None:
+            rec[field] = val or None
+    rec["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    owners[body.key] = rec
+    _save(OWNERS_PATH, owners)
+
+    om_synced = False
+    if body.owner:
+        ds = DATASET_BY_KEY[body.key]
+        user = next((u for u in _om_users() if u.get("display") == body.owner or u.get("name") == body.owner), None)
+        if user:
+            om_synced = _om_patch_owner(ds.get("om_fqn", ""), user)
+    return JSONResponse({"ok": True, "key": body.key, "record": rec, "om_synced": om_synced})
+
+
+@router.get("/glossary")
+async def get_glossary() -> JSONResponse:
+    terms = _om_glossary()
+    source = "openmetadata"
+    if not terms:
+        terms = SEED_GLOSSARY
+        source = "seed"
+    return JSONResponse({"ok": True, "source": source, "count": len(terms), "terms": terms})
+
+
+@router.get("/posture")
+async def get_posture() -> JSONResponse:
+    return JSONResponse(posture_view())

api/cdc_consumer.py

@@ -18,7 +18,7 @@ import json
 import os
 import re
 import uuid
-from collections import deque
+from collections import defaultdict, deque
 from datetime import datetime, timezone
 from typing import Any
 
@@ -28,7 +28,8 @@ from fastapi.responses import JSONResponse
 router = APIRouter(prefix="/api/changes", tags=["changes"])
 
 KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "10.0.21.36:9092")
-RING_SIZE = int(os.getenv("CDC_RING_SIZE", "1000"))
+RING_SIZE = int(os.getenv("CDC_RING_SIZE", "5000"))
+METRICS_RETENTION_MIN = int(os.getenv("CDC_METRICS_RETENTION_MIN", str(24 * 60)))
 
 # Active CDC topic prefixes -> logical source. Matches Debezium topic.prefix.
 PREFIX_SOURCE = {
@@ -45,6 +46,8 @@ TOPIC_PATTERN = re.compile(
 OP_MAP = {"c": "insert", "u": "update", "d": "delete", "r": "snapshot"}
 
 _ring: deque[dict[str, Any]] = deque(maxlen=RING_SIZE)
+# Minute-level rollups — accurate counts beyond the display ring buffer cap.
+_minute_rollups: dict[int, dict[str, Any]] = {}
 _state: dict[str, Any] = {
     "started": False,
     "connected": False,
@@ -127,6 +130,83 @@ def _parse(topic: str, value: bytes | None) -> dict[str, Any] | None:
     }
 
 
+def _record_metrics(entry: dict[str, Any]) -> None:
+    """Track per-minute aggregates for accurate window stats (not ring-limited)."""
+    try:
+        ts = datetime.fromisoformat(entry["ts"].replace("Z", "+00:00"))
+    except Exception:
+        ts = datetime.now(timezone.utc)
+    minute_key = int(ts.timestamp()) // 60
+    bucket = _minute_rollups.setdefault(
+        minute_key,
+        {"total": 0, "by_source": defaultdict(int), "by_op": defaultdict(int), "by_table": defaultdict(int)},
+    )
+    bucket["total"] += 1
+    bucket["by_source"][entry["source"]] += 1
+    bucket["by_op"][entry["op"]] += 1
+    bucket["by_table"][entry["table"]] += 1
+
+    cutoff = minute_key - METRICS_RETENTION_MIN
+    for old_key in list(_minute_rollups.keys()):
+        if old_key < cutoff:
+            del _minute_rollups[old_key]
+
+
+def _aggregate_window(minutes: int) -> dict[str, Any]:
+    """Aggregate rollup buckets for the requested time window."""
+    now = datetime.now(timezone.utc)
+    cutoff_minute = int((now.timestamp() - minutes * 60) // 60)
+    use_hour_buckets = minutes >= 120
+
+    by_source: dict[str, int] = defaultdict(int)
+    by_op: dict[str, int] = defaultdict(int)
+    by_table: dict[str, int] = defaultdict(int)
+    buckets: dict[str, int] = defaultdict(int)
+    total = 0
+
+    for minute_key, bucket in _minute_rollups.items():
+        if minute_key < cutoff_minute:
+            continue
+        total += bucket["total"]
+        for k, v in bucket["by_source"].items():
+            by_source[k] += v
+        for k, v in bucket["by_op"].items():
+            by_op[k] += v
+        for k, v in bucket["by_table"].items():
+            by_table[k] += v
+        ts = datetime.fromtimestamp(minute_key * 60, timezone.utc)
+        label = ts.strftime("%H:00") if use_hour_buckets else ts.strftime("%H:%M")
+        buckets[label] += bucket["total"]
+
+    # Fill chart timeline with zero buckets so the window span is visible.
+    if use_hour_buckets:
+        span = max(1, (minutes + 59) // 60)
+        filled: dict[str, int] = {}
+        for i in range(span):
+            t = datetime.fromtimestamp(now.timestamp() - (span - 1 - i) * 3600, timezone.utc)
+            filled[t.strftime("%H:00")] = buckets.get(t.strftime("%H:00"), 0)
+        buckets = filled
+    else:
+        span = max(1, minutes)
+        filled = {}
+        for i in range(span):
+            t = datetime.fromtimestamp(now.timestamp() - (span - 1 - i) * 60, timezone.utc)
+            filled[t.strftime("%H:%M")] = buckets.get(t.strftime("%H:%M"), 0)
+        buckets = filled
+
+    return {
+        "total": total,
+        "by_source": dict(by_source),
+        "by_op": dict(by_op),
+        "by_table": dict(by_table),
+        "buckets": [{"t": k, "n": v} for k, v in sorted(buckets.items())],
+        "inserts": by_op.get("insert", 0),
+        "updates": by_op.get("update", 0),
+        "deletes": by_op.get("delete", 0),
+        "rate_per_min": round(total / max(1, minutes), 1),
+    }
+
+
 async def cdc_consumer_loop() -> None:
     """Background loop: consume CDC topics and publish each change live."""
     _state["started"] = True
@@ -158,6 +238,7 @@ async def cdc_consumer_loop() -> None:
                 if not entry:
                     continue
                 _ring.append(entry)
+                _record_metrics(entry)
                 _state["consumed"] += 1
                 _state["last_ts"] = entry["ts"]
                 try:
@@ -182,30 +263,36 @@ async def cdc_consumer_loop() -> None:
 
 def snapshot(minutes: int = 15) -> dict[str, Any]:
     """Lightweight CDC snapshot for other modules (Data Flow graph)."""
-    cutoff = datetime.now(timezone.utc).timestamp() - minutes * 60
-    by_source: dict[str, int] = {}
-    total = 0
-    for c in _ring:
-        try:
-            if datetime.fromisoformat(c["ts"]).timestamp() < cutoff:
-                continue
-        except Exception:
-            continue
-        total += 1
-        by_source[c["source"]] = by_source.get(c["source"], 0) + 1
-    return {"connected": _state["connected"], "consumed": _state["consumed"],
-            "buffered": len(_ring), "window_total": total, "by_source": by_source}
+    agg = _aggregate_window(minutes)
+    return {
+        "connected": _state["connected"],
+        "consumed": _state["consumed"],
+        "buffered": len(_ring),
+        "window_total": agg["total"],
+        "by_source": agg["by_source"],
+    }
 
 
 # ── Endpoints ────────────────────────────────────────────────────────────────
 @router.get("")
 async def list_changes(
-    limit: int = Query(100, le=500),
+    limit: int = Query(200, le=1000),
     source: str | None = None,
     op: str | None = None,
     table: str | None = None,
+    minutes: int | None = Query(None, le=1440),
 ) -> JSONResponse:
     items = list(_ring)
+    if minutes is not None:
+        cutoff = datetime.now(timezone.utc).timestamp() - minutes * 60
+        filtered_by_time: list[dict[str, Any]] = []
+        for c in items:
+            try:
+                if datetime.fromisoformat(c["ts"].replace("Z", "+00:00")).timestamp() >= cutoff:
+                    filtered_by_time.append(c)
+            except Exception:
+                continue
+        items = filtered_by_time
     if source:
         items = [c for c in items if c["source"] == source]
     if op:
@@ -217,47 +304,29 @@ async def list_changes(
         "ok": True,
         "changes": items,
         "buffered": len(_ring),
+        "buffer_cap": RING_SIZE,
         "connected": _state["connected"],
         "consumed": _state["consumed"],
         "last_error": _state["last_error"],
+        "last_ts": _state["last_ts"],
     })
 
 
 @router.get("/stats")
-async def change_stats(minutes: int = Query(15, le=240)) -> JSONResponse:
-    now = datetime.now(timezone.utc)
-    cutoff = now.timestamp() - minutes * 60
-    by_source: dict[str, int] = {}
-    by_op: dict[str, int] = {}
-    by_table: dict[str, int] = {}
-    buckets: dict[str, int] = {}
-    total = 0
-    for c in _ring:
-        try:
-            t = datetime.fromisoformat(c["ts"]).timestamp()
-        except Exception:
-            continue
-        if t < cutoff:
-            continue
-        total += 1
-        by_source[c["source"]] = by_source.get(c["source"], 0) + 1
-        by_op[c["op"]] = by_op.get(c["op"], 0) + 1
-        by_table[c["table"]] = by_table.get(c["table"], 0) + 1
-        bucket = datetime.fromtimestamp(t, timezone.utc).strftime("%H:%M")
-        buckets[bucket] = buckets.get(bucket, 0) + 1
+async def change_stats(minutes: int = Query(15, le=1440)) -> JSONResponse:
+    agg = _aggregate_window(minutes)
     return JSONResponse({
         "ok": True,
         "window_minutes": minutes,
-        "total": total,
-        "by_source": by_source,
-        "by_op": by_op,
-        "by_table": by_table,
-        "buckets": [{"t": k, "n": v} for k, v in sorted(buckets.items())],
+        **agg,
         "connected": _state["connected"],
         "consumed": _state["consumed"],
+        "buffered": len(_ring),
+        "buffer_cap": RING_SIZE,
+        "last_ts": _state["last_ts"],
     })
 
 
 @router.get("/status")
 async def changes_status() -> JSONResponse:
-    return JSONResponse({"ok": True, "buffered": len(_ring), **_state})
+    return JSONResponse({"ok": True, "buffered": len(_ring), "buffer_cap": RING_SIZE, **_state})

api/dataflow.py

@@ -173,6 +173,16 @@ async def _build() -> dict[str, Any]:
         cdc = cdc_snapshot(15)
     except Exception:
         cdc = {"connected": False, "consumed": 0, "by_source": {}, "window_total": 0}
+    try:
+        from trino_federated import generator_active
+        gen_active = generator_active()
+    except Exception:
+        gen_active = False
+    try:
+        from storage_s3 import archive_active
+        arch_active = archive_active()
+    except Exception:
+        arch_active = False
     try:
         from pii_catalog import get_pii
         pii = get_pii()
@@ -255,7 +265,12 @@ async def _build() -> dict[str, Any]:
             edge["last_rows"] = lr.get("rows")
             edge["last_duration_s"] = lr.get("duration_s")
             edge["active"] = lr.get("state") == "running"
-        if e["kind"] == "cdc":
+        if e["kind"] == "generate":
+            # The live generator (continuous loop or the 'Generate data' burst)
+            # writes into these four sources; pulse the edge while it is active.
+            if e["to"] in ("postgres", "mysql", "mongodb", "cassandra"):
+                edge["active"] = gen_active or bool(edge.get("active"))
+        elif e["kind"] == "cdc":
             edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
         elif e.get("from") == "hdfs" and e.get("to") == "kafka":
             edge["active"] = bool(edge_live.get("hdfs→kafka"))
@@ -264,7 +279,10 @@ async def _build() -> dict[str, Any]:
         elif e.get("from") == "spark" and e.get("to") == "iceberg_curated":
             edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
         elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
-            edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
+            edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active") or arch_active
+        elif e["kind"] == "archive" and e.get("from") == "kafka" and e.get("to") == "s3_cdc":
+            # Kafka → S3 CDC archive pulses while the pipeline lands objects in S3
+            edge["active"] = arch_active
         elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
             # AI serving lane pulses while governed data is being served to the LLM
             edge["active"] = bool(_rag_info())

api/dq_monitor.py

@@ -0,0 +1,345 @@
+"""Continuous data-quality monitoring (Disease #2 — Dirty Data).
+
+Where dq-api runs Great-Expectations/Soda on *uploaded* files, this agent runs
+quality checks continuously against the *live* business tables through Trino and
+produces a per-dataset scorecard (0-100) across five dimensions:
+
+  completeness  – non-null ratio across columns
+  uniqueness    – distinct/key ratio (duplicate detection)
+  validity      – domain rules (non-negative amounts, present timestamps)
+  freshness     – age of the newest record vs a threshold
+  volume        – row count + delta vs the previous cycle
+
+A rolling score history powers trend sparklines. The loop streams the exact
+SQL it runs into the Data Custodian terminal so operators can see the checks.
+
+Endpoints:
+  GET  /api/dq/scorecards          -> all datasets, dimensions, score, trend
+  GET  /api/dq/scorecard/{key}     -> one dataset detail
+  POST /api/dq/run                 -> run one cycle now
+  POST /api/dq/config              -> {enabled, interval_s, sample}
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections import deque
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse
+
+from lake_meta import DATASETS, DATASET_BY_KEY, trino, trino_scalar, discover_columns
+
+router = APIRouter(prefix="/api/dq", tags=["data-quality"])
+
+FRESHNESS_THRESHOLD_MIN = 60.0  # newest row older than this => freshness degraded
+MAX_COLS = 24
+
+_lock = threading.Lock()
+_state: dict[str, Any] = {
+    "enabled": True,
+    "interval_s": 180.0,
+    "sample": 20000,
+    "running": False,
+    "cycles": 0,
+    "last_cycle_ts": 0.0,
+    "next_run_ts": 0.0,
+    "started_at": None,
+    "cards": {},                      # key -> latest scorecard
+    "history": {d["key"]: deque(maxlen=60) for d in DATASETS},
+    "cols_cache": {},                 # key -> [{name,type}]
+    "feed": deque(maxlen=40),
+}
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _term(text: str, level: str = "info", phase: str = "dq") -> None:
+    try:
+        from agent_terminal import emit_threadsafe
+        emit_threadsafe("data-custodian", text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+def _feed(text: str, level: str = "info") -> None:
+    _state["feed"].appendleft({"ts": _now().isoformat(), "text": text, "level": level})
+
+
+def _score_color(score: float) -> str:
+    if score >= 90:
+        return "#34d399"
+    if score >= 75:
+        return "#fbbf24"
+    if score >= 50:
+        return "#fb923c"
+    return "#f87171"
+
+
+def _obs_volume_freshness(key: str) -> tuple[int | None, float | None]:
+    """Read row count + freshness from the observability monitor (in-memory)."""
+    try:
+        from observability import _state, _lock
+        with _lock:
+            st = _state["datasets"].get(key) or {}
+            return st.get("rows"), st.get("freshness_age_min")
+    except Exception:
+        return None, None
+
+
+def _columns(ds: dict[str, Any]) -> list[dict[str, str]]:
+    key = ds["key"]
+    cols = _state["cols_cache"].get(key)
+    if not cols:
+        cols = discover_columns(ds)
+        if cols:
+            _state["cols_cache"][key] = cols
+    return cols or []
+
+
+def _assess(ds: dict[str, Any], sample: int) -> dict[str, Any]:
+    key = ds["key"]
+    fq = ds["fqtn"]
+    cols = _columns(ds)
+    col_names = [c["name"] for c in cols][:MAX_COLS]
+    dims: dict[str, Any] = {}
+    issues: list[str] = []
+    col_profiles: list[dict[str, Any]] = []
+
+    # ── one bounded query: completeness + uniqueness + validity over a sample ──
+    has_unique_key = ds.get("unique_key", True)
+    key_col = ds.get("key_col") if ds.get("key_col") in col_names else (col_names[0] if col_names else None)
+    if not has_unique_key:
+        key_col = None  # no single-column unique key (e.g. composite PK) → skip dedup
+    ts_col = ds.get("ts_col") if ds.get("ts_col") in col_names else None
+    amt_col = ds.get("amount_col") if ds.get("amount_col") in col_names else None
+
+    selects = ["count(*) AS n"]
+    for i, c in enumerate(col_names):
+        selects.append(f'count("{c}") AS c{i}')
+    if key_col:
+        selects.append(f'count(DISTINCT "{key_col}") AS dk')
+    if amt_col:
+        selects.append(f'count_if("{amt_col}" >= 0) AS amt_ok')
+    if ts_col:
+        selects.append(f'count_if("{ts_col}" IS NOT NULL) AS ts_ok')
+
+    sql = f"SELECT {', '.join(selects)} FROM (SELECT * FROM {fq} LIMIT {sample}) s"
+    _term(f"$ trino: profile {key}  ({len(col_names)} cols · sample {sample:,})", level="cmd", phase="dq")
+    cols_out, rows = trino(sql, timeout=40.0)
+    rec = dict(zip(cols_out, rows[0])) if rows else {}
+    n = int(rec.get("n") or 0)
+
+    if n > 0:
+        # completeness
+        non_null_ratios = []
+        for i, c in enumerate(col_names):
+            cnt = int(rec.get(f"c{i}") or 0)
+            ratio = cnt / n
+            non_null_ratios.append(ratio)
+            col_profiles.append({"name": c, "completeness": round(ratio * 100, 1),
+                                 "nulls": n - cnt})
+        completeness = round(100.0 * sum(non_null_ratios) / max(1, len(non_null_ratios)), 1)
+        dims["completeness"] = completeness
+        worst = sorted(col_profiles, key=lambda x: x["completeness"])[:3]
+        for w in worst:
+            if w["completeness"] < 95:
+                issues.append(f'{w["name"]} {100 - w["completeness"]:.0f}% null')
+
+        # uniqueness / dedup
+        if key_col and rec.get("dk") is not None:
+            dk = int(rec["dk"])
+            uniq = round(100.0 * dk / n, 1)
+            dims["uniqueness"] = uniq
+            dups = n - dk
+            if dups > 0:
+                issues.append(f"{dups:,} duplicate {key_col} in sample")
+
+        # validity
+        valid_parts = []
+        if amt_col and rec.get("amt_ok") is not None:
+            valid_parts.append(int(rec["amt_ok"]) / n)
+        if ts_col and rec.get("ts_ok") is not None:
+            valid_parts.append(int(rec["ts_ok"]) / n)
+        if valid_parts:
+            validity = round(100.0 * sum(valid_parts) / len(valid_parts), 1)
+            dims["validity"] = validity
+            if validity < 99 and amt_col:
+                issues.append(f"negative/invalid {amt_col}")
+
+    # ── volume + freshness: reuse the observability monitor's in-memory probes
+    # (it already polls count(*) and max(ts)). We deliberately do NOT issue our
+    # own count/max here — those are expensive full scans on 25-54M-row tables —
+    # so a DQ cycle stays fast and the score is driven by the cheap sample. ──
+    volume, fresh_age_min = _obs_volume_freshness(key)
+    if volume is None:
+        volume = n  # sample size until observability reports the real count
+    if fresh_age_min is not None:
+        fresh_score = 100.0 if fresh_age_min <= FRESHNESS_THRESHOLD_MIN else max(
+            0.0, 100.0 - (fresh_age_min - FRESHNESS_THRESHOLD_MIN) / 5.0)
+        dims["freshness"] = round(fresh_score, 1)
+        if fresh_age_min > FRESHNESS_THRESHOLD_MIN and not ds.get("curated"):
+            issues.append(f"stale {fresh_age_min:.0f}m")
+
+    score = round(sum(dims.values()) / len(dims), 1) if dims else 0.0
+    prev = _state["cards"].get(key, {})
+    prev_vol = prev.get("volume")
+    delta = (volume - prev_vol) if (volume is not None and prev_vol is not None) else None
+
+    card = {
+        "key": key, "label": ds["label"], "engine": ds["engine"], "color": ds["color"],
+        "table": fq, "domain": ds.get("domain"),
+        "score": score, "score_color": _score_color(score),
+        "dimensions": dims, "volume": volume, "volume_delta": delta,
+        "freshness_age_min": round(fresh_age_min, 1) if fresh_age_min is not None else None,
+        "issues": issues[:5], "columns": len(col_names),
+        "worst_columns": sorted(col_profiles, key=lambda x: x["completeness"])[:5],
+        "ts": _now().isoformat(),
+    }
+    lvl = "ok" if score >= 90 else ("warn" if score >= 60 else "err")
+    _term(f"  ← {key}: score {score} · " + " · ".join(f"{k} {v}" for k, v in dims.items())
+          + (f" · {len(issues)} issue(s)" if issues else ""), level=lvl, phase="dq")
+    return card
+
+
+def run_cycle() -> dict[str, Any]:
+    if _state["running"]:
+        return {"ok": True, "skipped": "already running"}
+    _state["running"] = True
+    sample = int(_state["sample"])
+    n_ok = 0
+    try:
+        _term(f"═══ DQ monitor cycle {_state['cycles'] + 1} — live tables via Trino ═══",
+              level="info", phase="cycle")
+        for ds in DATASETS:
+            if not _state["enabled"]:
+                break
+            try:
+                card = _assess(ds, sample)
+            except Exception as exc:
+                card = {"key": ds["key"], "label": ds["label"], "engine": ds["engine"],
+                        "color": ds["color"], "table": ds["fqtn"], "score": None,
+                        "score_color": "#64748b", "dimensions": {}, "error": str(exc)[:160],
+                        "issues": [f"check failed: {str(exc)[:80]}"], "ts": _now().isoformat()}
+                _term(f"  ✗ {ds['key']}: {str(exc)[:120]}", level="err", phase="dq")
+            with _lock:
+                _state["cards"][ds["key"]] = card
+                if card.get("score") is not None:
+                    _state["history"][ds["key"]].append({"t": _now().strftime("%H:%M"), "score": card["score"]})
+                    n_ok += 1
+        _state["cycles"] += 1
+        _state["last_cycle_ts"] = time.time()
+        _state["next_run_ts"] = time.time() + float(_state["interval_s"])
+        avg = _overall_score()
+        _feed(f"cycle {_state['cycles']} — {n_ok}/{len(DATASETS)} datasets scored · platform DQ {avg}")
+        _term(f"═══ cycle {_state['cycles']} done — platform DQ score {avg} ═══", level="ok", phase="cycle")
+    finally:
+        _state["running"] = False
+    return {"ok": True, "scored": n_ok, "platform_score": _overall_score()}
+
+
+def _overall_score() -> float | None:
+    vals = [c["score"] for c in _state["cards"].values() if c.get("score") is not None]
+    return round(sum(vals) / len(vals), 1) if vals else None
+
+
+def _loop() -> None:
+    _state["started_at"] = _now().isoformat()
+    time.sleep(20)
+    while True:
+        try:
+            if _state["enabled"]:
+                run_cycle()
+        except Exception:
+            pass
+        time.sleep(max(30.0, float(_state["interval_s"])))
+
+
+threading.Thread(target=_loop, daemon=True, name="dq-monitor").start()
+
+
+def scorecards_view() -> dict[str, Any]:
+    with _lock:
+        cards = []
+        for ds in DATASETS:
+            c = _state["cards"].get(ds["key"])
+            if c:
+                cards.append({**c, "trend": list(_state["history"][ds["key"]])})
+            else:
+                cards.append({"key": ds["key"], "label": ds["label"], "engine": ds["engine"],
+                              "color": ds["color"], "table": ds["fqtn"], "score": None,
+                              "score_color": "#64748b", "dimensions": {}, "issues": [],
+                              "trend": [], "pending": True})
+        # platform dimension averages
+        dim_avg: dict[str, list[float]] = {}
+        for c in cards:
+            for k, v in (c.get("dimensions") or {}).items():
+                dim_avg.setdefault(k, []).append(v)
+        return {
+            "ok": True,
+            "enabled": _state["enabled"],
+            "interval_s": _state["interval_s"],
+            "sample": _state["sample"],
+            "running": _state["running"],
+            "cycles": _state["cycles"],
+            "last_cycle_ts": _state["last_cycle_ts"],
+            "next_run_ts": _state["next_run_ts"],
+            "platform_score": _overall_score(),
+            "dimension_averages": {k: round(sum(v) / len(v), 1) for k, v in dim_avg.items()},
+            "cards": cards,
+            "feed": list(_state["feed"])[:20],
+        }
+
+
+def summary_for_llm() -> dict[str, Any]:
+    with _lock:
+        return {
+            "platform_dq_score": _overall_score(),
+            "datasets": [{"key": c["key"], "score": c.get("score"),
+                          "issues": c.get("issues", []), "freshness_age_min": c.get("freshness_age_min")}
+                         for c in _state["cards"].values()],
+        }
+
+
+@router.get("/scorecards")
+async def get_scorecards() -> JSONResponse:
+    return JSONResponse(scorecards_view())
+
+
+@router.get("/scorecard/{key}")
+async def get_scorecard(key: str) -> JSONResponse:
+    with _lock:
+        c = _state["cards"].get(key)
+        if not c:
+            return JSONResponse({"ok": False, "error": "unknown or not yet scored"}, status_code=404)
+        return JSONResponse({"ok": True, "card": {**c, "trend": list(_state["history"].get(key, []))}})
+
+
+@router.post("/run")
+async def post_run() -> JSONResponse:
+    from starlette.concurrency import run_in_threadpool
+    res = await run_in_threadpool(run_cycle)
+    return JSONResponse({**res, "scorecards": scorecards_view()})
+
+
+@router.post("/config")
+async def post_config(body: dict = Body(default={})) -> JSONResponse:
+    if "enabled" in body:
+        _state["enabled"] = bool(body["enabled"])
+    if "interval_s" in body:
+        try:
+            _state["interval_s"] = max(30.0, min(1800.0, float(body["interval_s"])))
+        except Exception:
+            pass
+    if "sample" in body:
+        try:
+            _state["sample"] = max(1000, min(200000, int(body["sample"])))
+        except Exception:
+            pass
+    return JSONResponse({"ok": True, "enabled": _state["enabled"],
+                         "interval_s": _state["interval_s"], "sample": _state["sample"]})

api/etl_offload.py

@@ -0,0 +1,583 @@
+"""Autonomous ETL offload agent — source databases → S3 Parquet lake.
+
+A background agent ("Lakehouse Loader") that, on a fixed cadence, pulls the next
+small chunk of rows from every source database and writes it to the S3 object
+store as a partitioned Parquet part:
+
+    lake/<dataset>/dt=YYYY-MM-DD/part-<ts>.parquet
+
+It progressively *backfills* the entire history in small chunks (so a 50M-row
+table lands as thousands of small files) and then *tails* newly generated rows,
+so the object store keeps filling and the analytics stay realtime. While
+offloading it also accumulates a live federated business matrix (revenue by
+region/status/channel, HR by dept, supply by type, telemetry by metric, …)
+straight from the rows it actually moved — which therefore always reflects the
+newest generated data, and powers the realtime Trino / Object-store dashboards.
+
+Endpoints:
+  GET  /api/etl/status     -> per-dataset progress, ingest rate, recent parts, feed
+  GET  /api/etl/business   -> accumulated realtime federated business matrix
+  POST /api/etl/run        -> trigger one offload cycle now
+  POST /api/etl/config     -> {enabled, interval_s, chunk}
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import json
+import random
+import threading
+import time
+from collections import deque
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse
+
+router = APIRouter(prefix="/api/etl", tags=["etl"])
+
+DATASETS = [
+    {"key": "orders", "label": "Sales orders", "engine": "PostgreSQL", "color": "#fbbf24"},
+    {"key": "hr_events", "label": "HR events", "engine": "MySQL", "color": "#60a5fa"},
+    {"key": "supply_events", "label": "Supply events", "engine": "MongoDB", "color": "#a78bfa"},
+    {"key": "telemetry", "label": "Device telemetry", "engine": "Cassandra", "color": "#22d3ee"},
+]
+
+_lock = threading.Lock()
+_state: dict[str, Any] = {
+    "enabled": os.getenv("ETL_OFFLOAD_ENABLED", "1") not in ("0", "false", "False"),
+    "interval_s": float(os.getenv("ETL_OFFLOAD_INTERVAL_SECONDS", "30")),
+    "chunk": int(os.getenv("ETL_OFFLOAD_CHUNK", "20000")),
+    "running_cycle": False,
+    "started_at": None,
+    "last_cycle_ts": 0.0,
+    "last_cycle_rows": 0,
+    "next_run_ts": 0.0,
+    "cycles": 0,
+    "datasets": {
+        d["key"]: {"label": d["label"], "engine": d["engine"], "color": d["color"],
+                   "parts": 0, "rows": 0, "bytes": 0, "backfilled": False, "cursor": None,
+                   "keycol": None, "mode": None, "total_source": None, "last_ts": None,
+                   "last_rows": 0, "last_key": None, "error": None}
+        for d in DATASETS
+    },
+    "feed": deque(maxlen=60),
+    "minute": deque(maxlen=60),      # (minute_epoch, rows, bytes)
+    "series": deque(maxlen=48),      # per-cycle points for the realtime chart
+}
+
+# Accumulated federated business matrix, built from the rows we actually offload.
+_business: dict[str, Any] = {
+    "orders": {"count": 0, "revenue": 0.0, "by_region": {}, "by_status": {}, "by_channel": {}, "by_currency": {}},
+    "hr": {"count": 0, "by_department": {}, "by_event": {}, "by_region": {}},
+    "supply": {"count": 0, "amount": 0.0, "by_type": {}, "by_region": {}, "by_source": {}},
+    "telemetry": {"count": 0, "by_metric": {}},
+    "ts": deque(maxlen=48),          # {t, orders, revenue, rows}
+}
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _bump(d: dict, k: Any, n: int = 1, val: float | None = None) -> None:
+    if k is None or k == "":
+        return
+    k = str(k)
+    if val is None:
+        d[k] = d.get(k, 0) + n
+    else:
+        cur = d.get(k) or {"count": 0, "value": 0.0}
+        cur["count"] += n
+        cur["value"] += val
+        d[k] = cur
+
+
+def _feed(text: str, level: str = "info") -> None:
+    _state["feed"].appendleft({"ts": _now().isoformat(), "text": text, "level": level})
+    try:
+        from main import add_feed
+        add_feed("etl-guardian", f"[lakehouse] {text}", level)
+    except Exception:
+        pass
+
+
+def _term(text: str, level: str = "info", phase: str = "offload") -> None:
+    """Stream a line to the ETL Guardian terminal from this background thread."""
+    try:
+        from agent_terminal import emit_threadsafe
+        emit_threadsafe("etl-guardian", text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+def _scalar(v: Any) -> Any:
+    import datetime as _dt
+    import decimal
+    if v is None or isinstance(v, (str, int, float, bool)):
+        return v
+    if isinstance(v, decimal.Decimal):
+        return float(v)
+    if isinstance(v, (_dt.datetime, _dt.date)):
+        return v.isoformat()
+    if isinstance(v, (dict, list)):
+        return json.dumps(v, default=str)
+    return str(v)
+
+
+def _normalize(rows: list[dict]) -> list[dict]:
+    keys: list[str] = []
+    seen: set[str] = set()
+    for r in rows:
+        for k in r.keys():
+            if k not in seen:
+                seen.add(k)
+                keys.append(k)
+    return [{k: _scalar(r.get(k)) for k in keys} for r in rows]
+
+
+def _write_parquet(dataset_key: str, rows: list[dict]) -> tuple[str, int]:
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    table = pa.Table.from_pylist(_normalize(rows))
+    buf = io.BytesIO()
+    pq.write_table(table, buf, compression="snappy")
+    body = buf.getvalue()
+    now = _now()
+    key = (f"lake/{dataset_key}/dt={now.strftime('%Y-%m-%d')}/"
+           f"part-{int(now.timestamp() * 1000)}-{random.randint(1000, 9999)}.parquet")
+    from storage_s3 import put_object_bytes
+    put_object_bytes(key, body, "application/vnd.apache.parquet")
+    return key, len(body)
+
+
+# ── source readers: return (rows, new_cursor, done) ─────────────────────────────
+def _read_orders(cursor, n):
+    import psycopg2
+    import psycopg2.extras
+    import sql_console as s
+    conn = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER, password=s.PG_PASS,
+                            dbname=s.PG_DB, connect_timeout=8)
+    try:
+        conn.autocommit = True
+        cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+        wm = int(cursor or 0)
+        cur.execute("SELECT * FROM public.sales_orders WHERE order_id > %s ORDER BY order_id LIMIT %s", (wm, n))
+        rows = [dict(r) for r in cur.fetchall()]
+        new_cursor = rows[-1].get("order_id", wm) if rows else wm
+        return rows, new_cursor, len(rows) < n
+    finally:
+        conn.close()
+
+
+def _mysql_keycol(cur) -> tuple[str, str]:
+    import sql_console as s
+    try:
+        cur.execute(
+            "SELECT column_name FROM information_schema.columns WHERE table_schema=%s AND table_name='employee_events' "
+            "AND extra LIKE '%%auto_increment%%' LIMIT 1", (s.MYSQL_DB,))
+        r = cur.fetchone()
+        if r:
+            return (list(r.values())[0] if isinstance(r, dict) else r[0]), "key"
+        cur.execute(
+            "SELECT column_name FROM information_schema.key_column_usage WHERE table_schema=%s "
+            "AND table_name='employee_events' AND constraint_name='PRIMARY' ORDER BY ordinal_position LIMIT 1",
+            (s.MYSQL_DB,))
+        r = cur.fetchone()
+        if r:
+            return (list(r.values())[0] if isinstance(r, dict) else r[0]), "key"
+    except Exception:
+        pass
+    return "", "offset"
+
+
+def _read_hr(cursor, n, st):
+    import pymysql
+    import sql_console as s
+    conn = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER, password=s.MYSQL_PASS,
+                           database=s.MYSQL_DB, connect_timeout=8, cursorclass=pymysql.cursors.DictCursor)
+    try:
+        cur = conn.cursor()
+        if not st.get("keycol") and st.get("mode") is None:
+            kc, mode = _mysql_keycol(cur)
+            st["keycol"] = kc
+            st["mode"] = mode
+        if st.get("mode") == "key" and st.get("keycol"):
+            kc = st["keycol"]
+            wm = int(cursor or 0)
+            cur.execute(f"SELECT * FROM employee_events WHERE `{kc}` > %s ORDER BY `{kc}` LIMIT %s", (wm, n))
+            rows = list(cur.fetchall())
+            new_cursor = rows[-1].get(kc, wm) if rows else wm
+        else:  # offset fallback
+            off = int(cursor or 0)
+            cur.execute("SELECT * FROM employee_events LIMIT %s OFFSET %s", (n, off))
+            rows = list(cur.fetchall())
+            new_cursor = off + len(rows)
+        return rows, new_cursor, len(rows) < n
+    finally:
+        conn.close()
+
+
+def _read_supply(cursor, n):
+    from bson import ObjectId
+    import sql_console as s
+    cli = s._mongo_client()
+    try:
+        col = cli[s.MONGO_DB]["events"]
+        q = {"_id": {"$gt": ObjectId(cursor)}} if cursor else {}
+        docs = list(col.find(q).sort("_id", 1).limit(n))
+        new_cursor = str(docs[-1]["_id"]) if docs else cursor
+        for d in docs:
+            d["_id"] = str(d["_id"])
+        return docs, new_cursor, len(docs) < n
+    finally:
+        cli.close()
+
+
+def _read_telemetry(cursor, n):
+    from cassandra.query import SimpleStatement
+    import sql_console as s
+    cluster = s._cass_cluster()
+    try:
+        sess = cluster.connect()
+        stmt = SimpleStatement(f"SELECT * FROM {s.CASS_KS}.device_metrics", fetch_size=n)
+        kwargs = {}
+        if cursor:
+            try:
+                kwargs["paging_state"] = bytes.fromhex(cursor)
+            except Exception:
+                kwargs = {}
+        rs = sess.execute(stmt, **kwargs)
+        rows = [dict(r._asdict()) for r in rs.current_rows]
+        ps = rs.paging_state
+        new_cursor = ps.hex() if ps else None
+        return rows, new_cursor, new_cursor is None
+    finally:
+        cluster.shutdown()
+
+
+# ── business matrix accumulation ────────────────────────────────────────────────
+def _num(v) -> float:
+    try:
+        return float(v)
+    except Exception:
+        return 0.0
+
+
+def _agg(dataset_key: str, rows: list[dict]) -> tuple[int, float]:
+    """Fold a freshly-offloaded chunk into the live business matrix. Returns
+    (order_rows, revenue) for the realtime time-series."""
+    o_rows = 0
+    o_rev = 0.0
+    with _lock:
+        if dataset_key == "orders":
+            b = _business["orders"]
+            for r in rows:
+                amt = _num(r.get("amount") if r.get("amount") is not None else r.get("total_amount"))
+                b["count"] += 1
+                b["revenue"] += amt
+                _bump(b["by_region"], r.get("region"), val=amt)
+                _bump(b["by_status"], r.get("order_status") or r.get("status"))
+                _bump(b["by_channel"], r.get("sales_channel") or r.get("channel"), val=amt)
+                _bump(b["by_currency"], r.get("currency"))
+                o_rows += 1
+                o_rev += amt
+        elif dataset_key == "hr_events":
+            b = _business["hr"]
+            for r in rows:
+                b["count"] += 1
+                _bump(b["by_department"], r.get("department"))
+                _bump(b["by_event"], r.get("event_type") or r.get("event"))
+                _bump(b["by_region"], r.get("region"))
+        elif dataset_key == "supply_events":
+            b = _business["supply"]
+            for r in rows:
+                amt = _num(r.get("amount"))
+                b["count"] += 1
+                b["amount"] += amt
+                _bump(b["by_type"], r.get("type"), val=amt)
+                _bump(b["by_region"], r.get("region"))
+                _bump(b["by_source"], r.get("source"))
+        elif dataset_key == "telemetry":
+            b = _business["telemetry"]
+            for r in rows:
+                b["count"] += 1
+                mt = r.get("metric_type") or r.get("metric_name")
+                if mt:
+                    cur = b["by_metric"].get(str(mt)) or {"count": 0, "sum": 0.0}
+                    cur["count"] += 1
+                    cur["sum"] += _num(r.get("metric_value"))
+                    b["by_metric"][str(mt)] = cur
+    return o_rows, o_rev
+
+
+_READERS = {"orders": _read_orders, "hr_events": _read_hr,
+            "supply_events": _read_supply, "telemetry": _read_telemetry}
+
+
+def _offload_dataset(key: str) -> dict[str, Any]:
+    st = _state["datasets"][key]
+    chunk = int(_state["chunk"])
+    res = {"rows": 0, "bytes": 0, "order_rows": 0, "revenue": 0.0}
+    cur_txt = st["cursor"] if st.get("cursor") not in (None, "") else "<start>"
+    _term(f"$ read {st.get('engine', key)} · {st.get('label', key)}  [cursor={cur_txt} · LIMIT {chunk}]",
+          level="cmd", phase="extract")
+    try:
+        if key == "hr_events":
+            rows, new_cursor, done = _read_hr(st["cursor"], chunk, st)
+        else:
+            rows, new_cursor, done = _READERS[key](st["cursor"], chunk)
+    except Exception as exc:
+        st["error"] = str(exc)[:160]
+        _term(f"  ✗ extract failed: {str(exc)[:140]}", level="err", phase="extract")
+        return res
+    st["error"] = None
+    if not rows:
+        if done:
+            st["backfilled"] = True
+            if key == "telemetry":  # no global order — loop back to keep tailing
+                st["cursor"] = None
+        return res
+    try:
+        obj_key, nbytes = _write_parquet(key, rows)
+    except Exception as exc:
+        st["error"] = f"parquet/s3: {str(exc)[:140]}"
+        _term(f"  ✗ parquet/s3 write failed: {str(exc)[:140]}", level="err", phase="load")
+        return res
+    _term(f"  ← pyarrow.write_table → s3://data/{obj_key}  ({len(rows)} rows · {nbytes / 1024:.1f} KB)",
+          level="ok", phase="load")
+    o_rows, o_rev = _agg(key, rows)
+    with _lock:
+        st["parts"] += 1
+        st["rows"] += len(rows)
+        st["bytes"] += nbytes
+        st["cursor"] = new_cursor
+        st["backfilled"] = bool(done)
+        st["last_ts"] = _now().isoformat()
+        st["last_rows"] = len(rows)
+        st["last_key"] = obj_key
+        if key == "telemetry" and done:
+            st["cursor"] = None
+    res.update({"rows": len(rows), "bytes": nbytes, "order_rows": o_rows, "revenue": o_rev})
+    return res
+
+
+def _source_totals() -> None:
+    """Best-effort backfill targets so the dashboard can show progress %."""
+    import sql_console as s
+    d = _state["datasets"]
+    try:
+        if d["orders"]["total_source"] is None:
+            d["orders"]["total_source"] = s._table_row_count("postgres", "public.sales_orders")
+    except Exception:
+        pass
+    try:
+        if d["hr_events"]["total_source"] is None:
+            d["hr_events"]["total_source"] = s._table_row_count("mysql", "hr.employee_events")
+    except Exception:
+        pass
+    try:
+        if d["supply_events"]["total_source"] is None:
+            d["supply_events"]["total_source"] = s._table_row_count("mongodb", "supplychain.events")
+    except Exception:
+        pass
+
+
+def run_cycle() -> dict[str, Any]:
+    if _state["running_cycle"]:
+        return {"ok": True, "skipped": "cycle already running"}
+    _state["running_cycle"] = True
+    total = 0
+    cbytes = 0
+    order_rows = 0
+    revenue = 0.0
+    try:
+        _term(f"═══ ETL offload cycle {_state['cycles'] + 1} — source DBs → S3 Parquet lake ═══",
+              level="info", phase="cycle")
+        _source_totals()
+        for d in DATASETS:
+            if not _state["enabled"]:
+                break
+            r = _offload_dataset(d["key"])
+            total += r["rows"]
+            cbytes += r["bytes"]
+            order_rows += r["order_rows"]
+            revenue += r["revenue"]
+        now = time.time()
+        _state["cycles"] += 1
+        _state["last_cycle_ts"] = now
+        _state["last_cycle_rows"] = total
+        _state["next_run_ts"] = now + float(_state["interval_s"])
+        if total:
+            minute = int(now // 60) * 60
+            if _state["minute"] and _state["minute"][-1][0] == minute:
+                m, r0, b0 = _state["minute"][-1]
+                _state["minute"][-1] = (minute, r0 + total, b0 + cbytes)
+            else:
+                _state["minute"].append((minute, total, cbytes))
+            point = {"t": _now().strftime("%H:%M:%S"), "rows": total, "bytes": cbytes,
+                     "orders": order_rows, "revenue": round(revenue, 2)}
+            _state["series"].append(point)
+            with _lock:
+                _business["ts"].append(point)
+            _term(f"═══ cycle {_state['cycles']} done — {total:,} rows · {cbytes / 1024:.1f} KB · €{int(revenue):,} revenue ═══",
+                  level="ok", phase="cycle")
+            _feed(f"offloaded {total:,} rows → S3 Parquet · €{int(revenue):,} (cycle {_state['cycles']})")
+        else:
+            _term(f"  cycle {_state['cycles']} — no new rows (all datasets caught up, tailing)",
+                  level="info", phase="cycle")
+    finally:
+        _state["running_cycle"] = False
+    return {"ok": True, "rows": total, "bytes": cbytes, "revenue": round(revenue, 2)}
+
+
+def _loop() -> None:
+    _state["started_at"] = _now().isoformat()
+    time.sleep(12)  # let the API + sources settle
+    while True:
+        try:
+            if _state["enabled"]:
+                run_cycle()
+        except Exception as exc:
+            try:
+                _feed(f"cycle error: {str(exc)[:160]}", level="err")
+            except Exception:
+                pass
+        time.sleep(max(10.0, float(_state["interval_s"])))
+
+
+threading.Thread(target=_loop, daemon=True, name="etl-offload").start()
+
+
+# ── views ───────────────────────────────────────────────────────────────────────
+def _top(d: dict, n: int = 12, value: bool = False) -> list[dict]:
+    if value:
+        items = sorted(d.items(), key=lambda kv: -(kv[1].get("value", 0) if isinstance(kv[1], dict) else kv[1]))
+        return [{"key": k, "count": v.get("count", 0), "value": round(v.get("value", 0.0), 2)} for k, v in items[:n]]
+    items = sorted(d.items(), key=lambda kv: -kv[1])
+    return [{"key": k, "count": v} for k, v in items[:n]]
+
+
+def status_view() -> dict[str, Any]:
+    with _lock:
+        datasets = []
+        tot_parts = tot_rows = tot_bytes = 0
+        for d in DATASETS:
+            st = _state["datasets"][d["key"]]
+            total_src = st.get("total_source")
+            pct = None
+            if total_src and total_src > 0:
+                pct = min(100.0, round(100.0 * st["rows"] / total_src, 1))
+            tot_parts += st["parts"]
+            tot_rows += st["rows"]
+            tot_bytes += st["bytes"]
+            datasets.append({"key": d["key"], **{k: st[k] for k in (
+                "label", "engine", "color", "parts", "rows", "bytes", "backfilled",
+                "total_source", "last_ts", "last_rows", "last_key", "error")},
+                "progress_pct": pct})
+        now_min = int(time.time() // 60) * 60
+        rate = 0
+        for m, r, _b in _state["minute"]:
+            if m >= now_min - 60:
+                rate += r
+        return {
+            "ok": True,
+            "enabled": _state["enabled"],
+            "interval_s": _state["interval_s"],
+            "chunk": _state["chunk"],
+            "running_cycle": _state["running_cycle"],
+            "cycles": _state["cycles"],
+            "started_at": _state["started_at"],
+            "last_cycle_ts": _state["last_cycle_ts"],
+            "last_cycle_rows": _state["last_cycle_rows"],
+            "next_run_ts": _state["next_run_ts"],
+            "totals": {"parts": tot_parts, "rows": tot_rows, "bytes": tot_bytes},
+            "rate_rows_per_min": rate,
+            "datasets": datasets,
+            "series": list(_state["series"]),
+            "feed": list(_state["feed"])[:24],
+        }
+
+
+def business_view() -> dict[str, Any]:
+    with _lock:
+        o = _business["orders"]
+        hr = _business["hr"]
+        sup = _business["supply"]
+        tel = _business["telemetry"]
+        tel_metrics = sorted(
+            ({"key": k, "count": v["count"], "avg": round(v["sum"] / v["count"], 2) if v["count"] else 0}
+             for k, v in tel["by_metric"].items()), key=lambda x: -x["count"])[:12]
+        # combine into a region matrix across datasets
+        regions: dict[str, dict] = {}
+        for k, v in o["by_region"].items():
+            regions.setdefault(k, {})["orders"] = v.get("count", 0)
+            regions[k]["revenue"] = round(v.get("value", 0.0), 2)
+        for k, v in hr["by_region"].items():
+            regions.setdefault(k, {})["hr_events"] = v
+        for k, v in sup["by_region"].items():
+            regions.setdefault(k, {})["supply_events"] = v
+        region_matrix = sorted(
+            ({"region": k, "orders": v.get("orders", 0), "revenue": v.get("revenue", 0),
+              "hr_events": v.get("hr_events", 0), "supply_events": v.get("supply_events", 0)}
+             for k, v in regions.items() if k), key=lambda r: -(r["revenue"] or 0))
+        return {
+            "ok": True,
+            "generated_at": _now().isoformat(),
+            "kpis": {
+                "orders": o["count"], "revenue": round(o["revenue"], 2),
+                "avg_order": round(o["revenue"] / o["count"], 2) if o["count"] else 0,
+                "hr_events": hr["count"], "supply_events": sup["count"],
+                "supply_amount": round(sup["amount"], 2), "telemetry": tel["count"],
+                "rows_total": o["count"] + hr["count"] + sup["count"] + tel["count"],
+            },
+            "orders_by_region": _top(o["by_region"], 12, value=True),
+            "orders_by_status": _top(o["by_status"], 8),
+            "orders_by_channel": _top(o["by_channel"], 8, value=True),
+            "orders_by_currency": _top(o["by_currency"], 6),
+            "hr_by_department": _top(hr["by_department"], 10),
+            "hr_by_event": _top(hr["by_event"], 8),
+            "supply_by_type": _top(sup["by_type"], 10, value=True),
+            "telemetry_by_metric": tel_metrics,
+            "region_matrix": region_matrix,
+            "ts": list(_business["ts"]),
+        }
+
+
+@router.get("/status")
+async def get_status() -> JSONResponse:
+    return JSONResponse(status_view())
+
+
+@router.get("/business")
+async def get_business() -> JSONResponse:
+    return JSONResponse(business_view())
+
+
+@router.post("/run")
+async def post_run() -> JSONResponse:
+    from starlette.concurrency import run_in_threadpool
+    res = await run_in_threadpool(run_cycle)
+    return JSONResponse({**res, "status": status_view()})
+
+
+@router.post("/config")
+async def post_config(body: dict = Body(default={})) -> JSONResponse:
+    if "enabled" in body:
+        _state["enabled"] = bool(body["enabled"])
+    if "interval_s" in body:
+        try:
+            _state["interval_s"] = max(10.0, min(900.0, float(body["interval_s"])))
+        except Exception:
+            pass
+    if "chunk" in body:
+        try:
+            _state["chunk"] = max(200, min(50000, int(body["chunk"])))
+        except Exception:
+            pass
+    _feed(f"config updated · interval={_state['interval_s']}s · chunk={_state['chunk']} · enabled={_state['enabled']}")
+    return JSONResponse({"ok": True, "enabled": _state["enabled"],
+                         "interval_s": _state["interval_s"], "chunk": _state["chunk"]})

api/gpu_config.py

@@ -0,0 +1,304 @@
+"""Runtime GPU / LLM endpoint selection with DB override above env defaults."""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from sqlalchemy import Column, DateTime, String, Text, select
+from sqlalchemy.orm import DeclarativeBase
+
+from db import SessionLocal, engine
+
+GPU_UI_PORT = int(os.getenv("GPU_UI_PORT", "9000"))
+LLM_PORT = int(os.getenv("LLM_PORT", "8001"))
+LLM_PATH = os.getenv("LLM_PATH", "/v1")
+
+ENV_GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
+ENV_GPU_UI_URL = os.getenv("GPU_UI_URL", ENV_GPU_URL)
+ENV_LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
+
+
+class _Base(DeclarativeBase):
+    pass
+
+
+class SystemSetting(_Base):
+    __tablename__ = "system_settings"
+
+    key = Column(String(64), primary_key=True)
+    value = Column(Text, nullable=False, default="")
+    updated_at = Column(DateTime(timezone=True), nullable=True)
+
+
+GPU_PRESETS: list[dict[str, Any]] = [
+    {
+        "id": "gpu-prod",
+        "label": "atc-gpu-prod (VM306)",
+        "vm": "atc-gpu-prod",
+        "vmid": 306,
+        "host": "10.0.10.106",
+        "gpu_ui_port": 9000,
+        "llm_port": 8001,
+        "description": "4× V100 — shared production GPU lab",
+    },
+    {
+        "id": "gpu-dev",
+        "label": "atc-gpu-dev (VM303, legacy)",
+        "vm": "atc-gpu-dev",
+        "vmid": 303,
+        "host": "10.0.20.106",
+        "gpu_ui_port": 9000,
+        "llm_port": 8001,
+        "description": "Legacy dev VM — GPU passthrough removed",
+    },
+    {
+        "id": "gpu-bart",
+        "label": "atc-gpu-bart (VM301)",
+        "vm": "atc-gpu-bart",
+        "vmid": 301,
+        "host": "10.0.11.66",
+        "gpu_ui_port": 9000,
+        "llm_port": 8001,
+        "description": "Bart GPU VM — 2× V100",
+    },
+]
+
+
+def _ensure_table() -> None:
+    SystemSetting.metadata.create_all(engine, tables=[SystemSetting.__table__])
+
+
+def _get_setting(key: str) -> str | None:
+    _ensure_table()
+    with SessionLocal() as db:
+        row = db.get(SystemSetting, key)
+        return row.value if row else None
+
+
+def _set_settings(values: dict[str, str]) -> None:
+    _ensure_table()
+    now = datetime.now(timezone.utc)
+    with SessionLocal() as db:
+        for key, value in values.items():
+            row = db.get(SystemSetting, key)
+            if row:
+                row.value = value
+                row.updated_at = now
+            else:
+                db.add(SystemSetting(key=key, value=value, updated_at=now))
+        db.commit()
+
+
+def _clear_settings(keys: list[str]) -> None:
+    _ensure_table()
+    with SessionLocal() as db:
+        for key in keys:
+            row = db.get(SystemSetting, key)
+            if row:
+                db.delete(row)
+        db.commit()
+
+
+def _build_urls(host: str, gpu_ui_port: int, llm_port: int) -> dict[str, str]:
+    host = host.strip().replace("http://", "").replace("https://", "").split("/")[0]
+    if ":" in host:
+        base_host = host.split(":")[0]
+    else:
+        base_host = host
+    gpu_url = f"http://{base_host}:{gpu_ui_port}"
+    llm_url = f"http://{base_host}:{llm_port}{LLM_PATH}"
+    return {
+        "host": base_host,
+        "gpu_url": gpu_url,
+        "gpu_ui_url": gpu_url,
+        "llm_url": llm_url,
+    }
+
+
+def _env_defaults() -> dict[str, Any]:
+    parsed = urlparse(ENV_GPU_URL)
+    host = parsed.hostname or "10.0.10.106"
+    return {
+        "source": "env",
+        "preset_id": "env",
+        "label": "Environment default",
+        **_build_urls(host, parsed.port or GPU_UI_PORT, LLM_PORT),
+        "env_gpu_url": ENV_GPU_URL,
+        "env_llm_url": ENV_LLM_URL,
+    }
+
+
+def get_gpu_urls() -> dict[str, str]:
+    """Effective GPU/LLM URLs — DB override wins over env."""
+    cfg = get_gpu_config()
+    return {
+        "gpu_url": cfg["gpu_url"],
+        "gpu_ui_url": cfg["gpu_ui_url"],
+        "llm_url": cfg["llm_url"],
+        "host": cfg["host"],
+    }
+
+
+def resolve_gpu_identity(gpu: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Canonical GPU host/VM/URLs for topology, registry links, and presentation."""
+    urls = get_gpu_urls()
+    cfg = get_gpu_config()
+    g = gpu or {}
+    host = str(g.get("ip") or g.get("host") or urls["host"]).strip()
+    preset_id = g.get("preset_id") or cfg.get("preset_id")
+    preset = next((p for p in GPU_PRESETS if p["id"] == preset_id), None)
+    if preset is None:
+        preset = next((p for p in GPU_PRESETS if p["host"] == host), None)
+    if preset is None:
+        preset = GPU_PRESETS[0]
+    ui_url = str(g.get("ui_url") or urls["gpu_ui_url"])
+    llm_url = str(g.get("vllm_url") or urls["llm_url"])
+    return {
+        "host": host,
+        "ip": host,
+        "vm": preset.get("vm") or "atc-gpu-prod",
+        "vmid": preset.get("vmid") or 306,
+        "ui_url": ui_url,
+        "llm_url": llm_url,
+        "preset_id": preset.get("id") or preset_id or "gpu-prod",
+        "label": preset.get("label") or cfg.get("label") or preset.get("vm"),
+    }
+
+
+def get_gpu_config() -> dict[str, Any]:
+    override_host = _get_setting("gpu_host")
+    if not override_host:
+        return _env_defaults()
+
+    preset_id = _get_setting("gpu_preset_id") or "custom"
+    gpu_ui_port = int(_get_setting("gpu_ui_port") or GPU_UI_PORT)
+    llm_port = int(_get_setting("llm_port") or LLM_PORT)
+    urls = _build_urls(override_host, gpu_ui_port, llm_port)
+    preset = next((p for p in GPU_PRESETS if p["id"] == preset_id), None)
+    return {
+        "source": "override",
+        "preset_id": preset_id,
+        "label": preset["label"] if preset else f"Custom ({override_host})",
+        **urls,
+        "env_gpu_url": ENV_GPU_URL,
+        "env_llm_url": ENV_LLM_URL,
+        "updated_at": _get_setting("gpu_updated_at"),
+    }
+
+
+def get_gpu_config_payload() -> dict[str, Any]:
+    cfg = get_gpu_config()
+    return {
+        "active": cfg,
+        "presets": GPU_PRESETS,
+        "defaults": _env_defaults(),
+    }
+
+
+def save_gpu_config(
+    *,
+    preset_id: str | None = None,
+    host: str | None = None,
+    gpu_ui_port: int | None = None,
+    llm_port: int | None = None,
+) -> dict[str, Any]:
+    if preset_id and preset_id != "custom":
+        preset = next((p for p in GPU_PRESETS if p["id"] == preset_id), None)
+        if not preset:
+            raise ValueError(f"Unknown preset: {preset_id}")
+        host = preset["host"]
+        gpu_ui_port = preset.get("gpu_ui_port", GPU_UI_PORT)
+        llm_port = preset.get("llm_port", LLM_PORT)
+    if not host:
+        raise ValueError("host is required for custom GPU target")
+
+    gpu_ui_port = gpu_ui_port or GPU_UI_PORT
+    llm_port = llm_port or LLM_PORT
+    now = datetime.now(timezone.utc).isoformat()
+    _set_settings(
+        {
+            "gpu_host": host.strip(),
+            "gpu_ui_port": str(gpu_ui_port),
+            "llm_port": str(llm_port),
+            "gpu_preset_id": preset_id or "custom",
+            "gpu_updated_at": now,
+        }
+    )
+    return get_gpu_config()
+
+
+def reset_gpu_config() -> dict[str, Any]:
+    _clear_settings(["gpu_host", "gpu_ui_port", "llm_port", "gpu_preset_id", "gpu_updated_at"])
+    return _env_defaults()
+
+
+async def test_gpu_target(
+    host: str | None = None,
+    gpu_ui_port: int | None = None,
+    llm_port: int | None = None,
+    preset_id: str | None = None,
+) -> dict[str, Any]:
+    if preset_id and preset_id != "custom":
+        preset = next((p for p in GPU_PRESETS if p["id"] == preset_id), None)
+        if preset:
+            host = preset["host"]
+            gpu_ui_port = preset.get("gpu_ui_port", GPU_UI_PORT)
+            llm_port = preset.get("llm_port", LLM_PORT)
+    if not host:
+        cfg = get_gpu_config()
+        host = cfg["host"]
+        gpu_ui_port = gpu_ui_port or GPU_UI_PORT
+        llm_port = llm_port or LLM_PORT
+
+    urls = _build_urls(host, gpu_ui_port or GPU_UI_PORT, llm_port or LLM_PORT)
+    result: dict[str, Any] = {
+        "ok": False,
+        "host": urls["host"],
+        "gpu_url": urls["gpu_url"],
+        "llm_url": urls["llm_url"],
+        "metrics_ok": False,
+        "llm_ok": False,
+        "gpu_count": 0,
+        "inference_active": False,
+        "active_model": None,
+        "errors": [],
+    }
+
+    async with httpx.AsyncClient(timeout=8.0) as client:
+        try:
+            mr = await client.get(f"{urls['gpu_url']}/api/gpu/metrics")
+            if mr.status_code == 200:
+                result["metrics_ok"] = True
+                gpus = mr.json().get("current", {}).get("gpus", [])
+                result["gpu_count"] = len(gpus)
+            else:
+                result["errors"].append(f"metrics HTTP {mr.status_code}")
+        except Exception as exc:
+            result["errors"].append(f"metrics: {exc}")
+
+        try:
+            model_r = await client.get(f"{urls['gpu_url']}/api/active-model")
+            if model_r.status_code == 200:
+                md = model_r.json()
+                result["inference_active"] = bool(md.get("inference_active"))
+                result["active_model"] = md.get("name")
+        except Exception as exc:
+            result["errors"].append(f"active-model: {exc}")
+
+        try:
+            lr = await client.get(f"{urls['llm_url']}/models")
+            if lr.status_code == 200:
+                result["llm_ok"] = True
+            else:
+                result["errors"].append(f"llm HTTP {lr.status_code}")
+        except Exception as exc:
+            result["errors"].append(f"llm: {exc}")
+
+    result["ok"] = result["metrics_ok"] and (
+        result["gpu_count"] > 0 or result["inference_active"] or result["llm_ok"]
+    )
+    return result

api/lab_context.py

@@ -17,6 +17,14 @@ from database_inventory import collect_database_inventory
 from node_registry import NODE_REGISTRY
 
 DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
+DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
+
+
+def _dockhand_headers() -> dict[str, str]:
+    if DOCKHAND_API_TOKEN:
+        return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
+    return {}
+
 HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870")
 LAKEHOUSE_HOST = os.getenv("LAKEHOUSE_HOST", "10.0.21.50")
 AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080")
@@ -24,9 +32,14 @@ KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000")
 KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", f"http://{LAKEHOUSE_HOST}:8083")
 TRINO_URL = os.getenv("TRINO_URL", f"http://{LAKEHOUSE_HOST}:8089")
 SPARK_UI_URL = os.getenv("SPARK_UI_URL", f"http://{LAKEHOUSE_HOST}:8080")
-GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
+try:
+    from gpu_config import get_gpu_urls as _get_gpu_urls
+except Exception:
+    _get_gpu_urls = None  # type: ignore
+
+GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
 OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", "http://10.0.20.111:9020")
-YARN_URL = os.getenv("YARN_URL", "http://10.0.21.61:8088")
+YARN_URL = os.getenv("YARN_URL", "http://10.0.21.62:8088")
 
 AGENT_PRIMARY_DOMAIN = {
     "infra-sentinel": "docker",
@@ -110,7 +123,7 @@ async def dockhand_containers(
     await _log(log, "cmd", "fetch", f"$ GET {url}")
     t0 = time.monotonic()
     try:
-        r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}, timeout=8.0)
+        r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}, headers=_dockhand_headers(), timeout=8.0)
         ms = int((time.monotonic() - t0) * 1000)
         r.raise_for_status()
         data = r.json()
@@ -407,11 +420,20 @@ async def collect_objectscale(client: httpx.AsyncClient, log: TerminalLogFn | No
 
 
 async def collect_gpu_metrics(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]:
-    await _log(log, "info", "fetch", "▸ GPU Lab metrics")
-    base = {"ok": False, "host": GPU_URL, "ui_url": GPU_URL}
+    gpu_url = GPU_URL
+    host = GPU_URL
+    if _get_gpu_urls is not None:
+        try:
+            u = _get_gpu_urls()
+            gpu_url = u["gpu_url"]
+            host = u["host"]
+        except Exception:
+            pass
+    await _log(log, "info", "fetch", f"▸ GPU Lab metrics @ {host}")
+    base = {"ok": False, "host": host, "ui_url": gpu_url}
     try:
-        metrics_url = f"{GPU_URL}/api/gpu/metrics"
-        model_url = f"{GPU_URL}/api/active-model"
+        metrics_url = f"{gpu_url}/api/gpu/metrics"
+        model_url = f"{gpu_url}/api/active-model"
         await _log(log, "cmd", "fetch", f"$ GET {metrics_url}")
         await _log(log, "cmd", "fetch", f"$ GET {model_url}")
         t0 = time.monotonic()
@@ -671,6 +693,21 @@ def collect_governance(log: TerminalLogFn | None = None) -> dict[str, Any]:
         "mysql_hr.hr.employee_events -> iceberg.curated_masked.employee_events_masked (PII masked)",
         "hdfs:/data/historical/sales_orders -> iceberg.hadoop.historical_sales_hdfs",
     ]
+    try:
+        from dq_monitor import summary_for_llm as _dq
+        out["data_quality"] = _dq()
+    except Exception as exc:
+        out["data_quality"] = {"error": str(exc)}
+    try:
+        from observability import summary_for_llm as _obs
+        out["observability"] = _obs()
+    except Exception as exc:
+        out["observability"] = {"error": str(exc)}
+    try:
+        from catalog_governance import summary_for_llm as _own
+        out["ownership"] = _own()
+    except Exception as exc:
+        out["ownership"] = {"error": str(exc)}
     return out
 
 
@@ -700,6 +737,27 @@ def _section_governance(g: dict[str, Any]) -> list[str]:
     lines.append("  Lineage:")
     for ln in g.get("lineage") or []:
         lines.append(f"    - {ln}")
+    dq = g.get("data_quality") or {}
+    if isinstance(dq, dict) and "error" not in dq:
+        lines.append(f"  Data quality (continuous, live tables): platform score {dq.get('platform_dq_score')}")
+        for d in dq.get("datasets") or []:
+            iss = f" issues: {', '.join(d['issues'][:3])}" if d.get("issues") else ""
+            lines.append(f"    - {d['key']}: score {d.get('score')}{iss}")
+    own = g.get("ownership") or {}
+    if isinstance(own, dict) and "error" not in own:
+        orph = own.get("orphan_datasets") or []
+        lines.append(f"  Ownership: {len(own.get('owners', {}))} assigned"
+                     + (f", orphans (no owner): {', '.join(orph)}" if orph else ", no orphans"))
+        for k, v in (own.get("owners") or {}).items():
+            if v.get("owner"):
+                lines.append(f"    - {k}: owner={v.get('owner')} steward={v.get('steward') or '—'} tier={v.get('tier') or '—'}")
+    obs = g.get("observability") or {}
+    if isinstance(obs, dict) and "error" not in obs:
+        ac = obs.get("active_alerts") or {}
+        lines.append(f"  Observability alerts: {ac.get('total', 0)} active "
+                     f"(critical={ac.get('critical', 0)}, warning={ac.get('warning', 0)})")
+        for a in (obs.get("alerts") or [])[:5]:
+            lines.append(f"    - [{a['severity']}] {a['dataset']}: {a['message']}")
     return lines
 
 
@@ -750,11 +808,12 @@ async def collect_full_lab_context(
         if gpu_data is None:
             gpu_data = await collect_gpu_metrics(client, log)
 
-        docker_raw, db_raw, lake_raw, cc_raw, hdfs, etl, objectscale = await asyncio.gather(
+        docker_raw, db_raw, lake_raw, cc_raw, gpu_raw, hdfs, etl, objectscale = await asyncio.gather(
             dockhand_containers(client, DOCKHAND_ENVS["docker01"], log),
             dockhand_containers(client, DOCKHAND_ENVS["db02"], log),
             dockhand_containers(client, DOCKHAND_ENVS["lakehouse"], log),
             dockhand_containers(client, DOCKHAND_ENV_COMMAND_CENTER, log),
+            dockhand_containers(client, DOCKHAND_ENVS["gpu_dev"], log),
             collect_hdfs(client, log),
             collect_etl(client, log),
             collect_objectscale(client, log),
@@ -771,6 +830,15 @@ async def collect_full_lab_context(
                 databases["inventory"] = {"error": str(exc)}
         lakehouse = await collect_lakehouse(client, lake_raw, log)
         command_center = await collect_command_center(client, cc_raw, log)
+        if gpu_data is not None and gpu_raw:
+            gpu_running = sum(1 for c in gpu_raw if c.get("state") == "running")
+            gpu_data = {
+                **gpu_data,
+                "dockhand_env": DOCKHAND_ENVS["gpu_dev"],
+                "dockhand_containers": _container_rows(gpu_raw, GPU_URL.replace("http://", "").split(":")[0]),
+                "dockhand_running": gpu_running,
+                "dockhand_total": len(gpu_raw),
+            }
 
     try:
         governance = collect_governance(log)

api/lake_meta.py

@@ -0,0 +1,147 @@
+"""Shared lakehouse metadata + Trino helpers.
+
+Single source of truth for the business datasets the governance / data-quality /
+lineage / observability features operate on, plus a small synchronous Trino
+client. Imported by dq_monitor.py, observability.py, lineage.py and
+catalog_governance.py so every feature reasons about the exact same tables that
+the rest of the Command Center (pii_catalog, etl_offload, trino_federated)
+already exposes.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import httpx
+
+TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
+TRINO_USER = os.getenv("TRINO_USER", "mo")
+
+# Canonical business datasets, aligned with pii_catalog.DATASETS / etl_offload.
+# fqtn  = fully-qualified Trino table name (catalog.schema.table)
+# om_fqn = OpenMetadata FQN best-effort (governance falls back to native store)
+DATASETS: list[dict[str, Any]] = [
+    {"key": "orders", "label": "Sales orders", "engine": "PostgreSQL", "color": "#fbbf24",
+     "catalog": "postgres_sales", "schema": "public", "table": "sales_orders",
+     "fqtn": "postgres_sales.public.sales_orders", "pii_key": "postgres",
+     "om_fqn": "atc_trino.postgres_sales.public.sales_orders",
+     "key_col": "order_id", "ts_col": "order_ts", "amount_col": "amount",
+     "native_count": ("postgres", "public.sales_orders"),
+     "domain": "Sales", "topic": "atc.public.sales_orders"},
+    {"key": "hr", "label": "HR events", "engine": "MySQL", "color": "#60a5fa",
+     "catalog": "mysql_hr", "schema": "hr", "table": "employee_events",
+     "fqtn": "mysql_hr.hr.employee_events", "pii_key": "mysql",
+     "om_fqn": "atc_trino.mysql_hr.hr.employee_events",
+     "key_col": "event_id", "ts_col": "event_ts", "amount_col": "salary_change",
+     "native_count": ("mysql", "hr.employee_events"),
+     "domain": "People", "topic": "atc.hr.employee_events"},
+    {"key": "supply", "label": "Supply chain events", "engine": "MongoDB", "color": "#a78bfa",
+     "catalog": "mongodb_supplychain", "schema": "supplychain", "table": "events",
+     "fqtn": "mongodb_supplychain.supplychain.events", "pii_key": "mongodb",
+     "om_fqn": "atc_trino.mongodb_supplychain.supplychain.events",
+     "key_col": "event_id", "ts_col": "ts", "amount_col": "amount",
+     "native_count": ("mongodb", "supplychain.events"),
+     "domain": "Supply Chain", "topic": "atc.supplychain.events"},
+    {"key": "telemetry", "label": "Device telemetry", "engine": "Cassandra", "color": "#22d3ee",
+     "catalog": "cassandra_telemetry", "schema": "telemetry", "table": "device_metrics",
+     "fqtn": "cassandra_telemetry.telemetry.device_metrics", "pii_key": "cassandra",
+     "om_fqn": "atc_trino.cassandra_telemetry.telemetry.device_metrics",
+     "key_col": "device_id", "ts_col": "metric_ts", "amount_col": "metric_value",
+     "unique_key": False,
+     "domain": "IoT", "topic": "atc.telemetry.device_metrics"},
+    {"key": "curated", "label": "Curated masked (Iceberg)", "engine": "Iceberg / Trino", "color": "#34d399",
+     "catalog": "iceberg", "schema": "curated_masked", "table": "sales_orders_masked",
+     "fqtn": "iceberg.curated_masked.sales_orders_masked", "pii_key": "curated",
+     "om_fqn": "atc_trino.iceberg.curated_masked.sales_orders_masked",
+     "key_col": "order_id", "ts_col": "order_ts", "amount_col": "amount",
+     "domain": "Sales", "curated": True, "topic": None},
+    {"key": "hadoop", "label": "Historical sales (HDFS)", "engine": "Iceberg / HDFS", "color": "#f472b6",
+     "catalog": "iceberg", "schema": "hadoop", "table": "historical_sales_hdfs",
+     "fqtn": "iceberg.hadoop.historical_sales_hdfs", "pii_key": "hadoop",
+     "om_fqn": "atc_trino.iceberg.hadoop.historical_sales_hdfs",
+     "key_col": "order_id", "ts_col": "order_ts", "amount_col": "amount",
+     "domain": "Sales", "curated": True, "topic": None},
+]
+
+DATASET_BY_KEY = {d["key"]: d for d in DATASETS}
+
+# Heuristics for picking the freshness / key column when discovering schema.
+_TS_HINTS = ("_ts", "ts", "updated_at", "created_at", "event_time", "modified", "time")
+_KEY_HINTS = ("_id", "id", "uuid", "key", "pk")
+
+
+def trino(sql: str, timeout: float = 25.0) -> tuple[list[str], list[list[Any]]]:
+    """Run a Trino statement, following nextUri pages. Returns (columns, rows).
+
+    `timeout` is both the per-request timeout AND an overall wall-clock deadline,
+    so a long full-scan (e.g. count(*) on a huge Cassandra table) is aborted and
+    cancelled instead of looping over nextUri pages for minutes and blocking the
+    caller's thread.
+    """
+    import time as _t
+    cols: list[str] = []
+    rows: list[list[Any]] = []
+    deadline = _t.monotonic() + timeout
+    with httpx.Client(timeout=min(timeout, 15.0)) as client:
+        d = client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(),
+                        headers={"X-Trino-User": TRINO_USER}).json()
+        while True:
+            if d.get("error"):
+                raise RuntimeError(d["error"].get("message", "trino error"))
+            c = d.get("columns")
+            if c and not cols:
+                cols = [x["name"] for x in c]
+            rows += d.get("data") or []
+            nxt = d.get("nextUri")
+            if not nxt:
+                break
+            if _t.monotonic() > deadline:
+                try:
+                    client.delete(nxt)  # cancel the running query server-side
+                except Exception:
+                    pass
+                raise TimeoutError(f"trino query exceeded {timeout}s deadline")
+            d = client.get(nxt).json()
+    return cols, rows
+
+
+def trino_scalar(sql: str, timeout: float = 25.0) -> Any:
+    _c, rows = trino(sql, timeout=timeout)
+    if rows and rows[0]:
+        return rows[0][0]
+    return None
+
+
+def discover_columns(ds: dict[str, Any], timeout: float = 12.0) -> list[dict[str, str]]:
+    """[{name, type}] for a dataset's table via information_schema."""
+    sql = (f"SELECT column_name, data_type FROM {ds['catalog']}.information_schema.columns "
+           f"WHERE table_name = '{ds['table']}'")
+    if ds.get("schema"):
+        sql += f" AND table_schema = '{ds['schema']}'"
+    try:
+        _c, rows = trino(sql, timeout=timeout)
+        return [{"name": r[0], "type": r[1]} for r in rows]
+    except Exception:
+        return []
+
+
+def pick_ts_col(columns: list[dict[str, str]], default: str | None = None) -> str | None:
+    names = [c["name"] for c in columns]
+    for c in columns:
+        if "timestamp" in (c.get("type") or "").lower() or "date" in (c.get("type") or "").lower():
+            return c["name"]
+    for n in names:
+        if any(h in n.lower() for h in _TS_HINTS):
+            return n
+    return default if default in names else None
+
+
+def pick_key_col(columns: list[dict[str, str]], default: str | None = None) -> str | None:
+    names = [c["name"] for c in columns]
+    if default in names:
+        return default
+    for n in names:
+        if any(n.lower() == h or n.lower().endswith(h) for h in _KEY_HINTS):
+            return n
+    return names[0] if names else None

api/lineage.py

@@ -0,0 +1,248 @@
+"""Data lineage graph for the Command Center (Disease #6 — Traceability).
+
+Models the real end-to-end pipeline as an explicit, staged node/edge graph:
+
+  source DB → Debezium/Kafka (CDC) → Spark / Kafka Connect → S3 Parquet lake
+            → Iceberg lakehouse (curated_masked / hadoop) → Trino federation
+            → serving (Elasticsearch, RAG/ChromaDB, vLLM, Command Center chat)
+
+Table-backed nodes are enriched with live Trino row counts. For the sales path
+we expose column-level lineage with PII / masking status (reusing pii_catalog),
+so an operator can trace any field from source to the masked curated layer.
+OpenMetadata table lineage is layered in best-effort where the FQN resolves.
+
+Endpoints:
+  GET /api/lineage/graph?dataset=<key>   -> nodes + edges (+ column links)
+  GET /api/lineage/datasets              -> selectable datasets
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+import httpx
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+from lake_meta import DATASETS, DATASET_BY_KEY, trino_scalar
+
+router = APIRouter(prefix="/api/lineage", tags=["lineage"])
+
+OPENMETADATA_URL = os.getenv("OPENMETADATA_URL", "").rstrip("/")
+OPENMETADATA_TOKEN = os.getenv("OPENMETADATA_TOKEN", "")
+
+_cache: dict[str, Any] = {"ts": 0.0, "data": None}
+_TTL = 30.0
+
+# Which source datasets flow through the CDC → lakehouse pipeline.
+_SOURCE_KEYS = ["orders", "hr", "supply", "telemetry"]
+
+
+def _is_active() -> tuple[bool, bool]:
+    gen = arch = False
+    try:
+        from trino_federated import generator_active
+        gen = bool(generator_active())
+    except Exception:
+        pass
+    try:
+        from storage_s3 import archive_active
+        arch = bool(archive_active())
+    except Exception:
+        pass
+    return gen, arch
+
+
+def _obs_counts() -> dict[str, int | None]:
+    """Reuse the row counts the observability monitor already polls in-memory,
+    so the lineage graph never issues its own (potentially slow) count(*)."""
+    try:
+        from observability import _state, _lock
+        with _lock:
+            return {k: v.get("rows") for k, v in _state["datasets"].items()}
+    except Exception:
+        return {}
+
+
+def _row_count(ds_key: str, counts: dict[str, int | None]) -> int | None:
+    return counts.get(ds_key)
+
+
+def _pii_columns(ds_key: str) -> list[dict[str, Any]]:
+    """Column-level detail with PII/masking flags from pii_catalog (best-effort).
+
+    ds_key is a lake_meta key; map it to the pii_catalog key first."""
+    pii_key = (DATASET_BY_KEY.get(ds_key, {}) or {}).get("pii_key", ds_key)
+    try:
+        from pii_catalog import get_pii
+        data = get_pii()
+        for d in data.get("datasets", []):
+            if d.get("key") == pii_key or d.get("node_id") == pii_key:
+                return [{"name": c["name"], "category": c.get("category"),
+                         "masked": bool(c.get("masked")), "pii": True}
+                        for c in d.get("pii_columns", [])]
+    except Exception:
+        pass
+    return []
+
+
+def _om_lineage(fqn: str) -> dict[str, int]:
+    """OpenMetadata table lineage (best-effort, short timeout). Kept out of the
+    hot path by default; only called when LINEAGE_OM_ENRICH=1 is set."""
+    if not OPENMETADATA_URL or not fqn or os.getenv("LINEAGE_OM_ENRICH", "0") != "1":
+        return {}
+    url = f"{OPENMETADATA_URL}/api/v1/lineage/table/name/{fqn}?upstreamDepth=1&downstreamDepth=1"
+    headers = {"Accept": "application/json"}
+    if OPENMETADATA_TOKEN:
+        headers["Authorization"] = f"Bearer {OPENMETADATA_TOKEN}"
+    try:
+        with httpx.Client(timeout=4.0, verify=False) as c:
+            r = c.get(url, headers=headers)
+            if r.status_code != 200:
+                return {}
+            j = r.json()
+            return {"upstream": len(j.get("upstreamEdges") or []),
+                    "downstream": len(j.get("downstreamEdges") or [])}
+    except Exception:
+        return {}
+
+
+def _build() -> dict[str, Any]:
+    gen_active, arch_active = _is_active()
+    counts = _obs_counts()
+    nodes: list[dict[str, Any]] = []
+    edges: list[dict[str, Any]] = []
+    seen: set[str] = set()
+
+    def node(nid: str, label: str, ntype: str, stage: int, **meta: Any) -> None:
+        if nid in seen:
+            return
+        seen.add(nid)
+        nodes.append({"id": nid, "label": label, "type": ntype, "stage": stage, "meta": meta})
+
+    def edge(src: str, dst: str, label: str, kind: str, active: bool = False) -> None:
+        edges.append({"id": f"{src}->{dst}", "source": src, "target": dst,
+                      "label": label, "kind": kind, "active": active})
+
+    # Stage 5/6 shared serving nodes
+    node("trino", "Trino", "engine", 5, engine="Trino", note="Federated SQL across every catalog")
+    node("es", "Elasticsearch", "serving", 6, engine="Elasticsearch", note="Business + infra search indices")
+    node("rag", "ChromaDB (RAG)", "serving", 6, engine="ChromaDB", note="Vector store / embeddings")
+    node("vllm", "vLLM", "serving", 6, engine="vLLM", note="LLM inference (GPU)")
+    node("cc", "Command Center", "serving", 6, engine="React", note="Knowledge Chat & dashboards")
+    edge("rag", "vllm", "context", "serve", active=True)
+    edge("vllm", "cc", "answers", "serve", active=True)
+    edge("trino", "es", "index business data", "serve")
+    edge("es", "cc", "search", "serve")
+    edge("trino", "cc", "dashboards", "serve")
+
+    for key in _SOURCE_KEYS:
+        ds = DATASET_BY_KEY[key]
+        src_id = f"src_{key}"
+        topic_id = f"topic_{key}"
+        lake_id = f"lake_{key}"
+        rows = _row_count(key, counts)
+        cols = _pii_columns(key)
+        node(src_id, f"{ds['engine']}\n{ds['table']}", "source", 0,
+             engine=ds["engine"], table=ds["fqtn"], rows=rows, domain=ds.get("domain"),
+             columns=cols, ts_col=ds.get("ts_col"), key_col=ds.get("key_col"),
+             om=_om_lineage(ds.get("om_fqn", "")))
+
+        # CDC path (Debezium → Kafka topic)
+        if ds.get("topic"):
+            node(topic_id, f"Kafka\n{ds['topic']}", "stream", 1,
+                 engine="Kafka", topic=ds["topic"], note="Debezium CDC topic")
+            edge(src_id, topic_id, "Debezium CDC", "cdc", active=gen_active)
+            node("connect", "Kafka Connect", "stream", 2, engine="Kafka Connect",
+                 note="Debezium source + HDFS/S3 sinks")
+            edge(topic_id, "connect", "consume", "cdc", active=gen_active)
+
+        # ETL offload (direct source → S3 Parquet lake)
+        node(lake_id, f"S3 lake\nlake/{key}", "storage", 3, engine="ObjectScale S3",
+             path=f"s3://data/lake/{key}/", note="Parquet parts (ETL offload)")
+        edge(src_id, lake_id, "ETL offload (Parquet)", "batch", active=arch_active)
+
+    # Spark / Connect → curated + hadoop Iceberg tables
+    node("spark", "Spark", "compute", 2, engine="Spark", note="Transform & mask → curated")
+    edge("connect", "spark", "stream", "transform", active=gen_active)
+
+    cur = DATASET_BY_KEY["curated"]
+    had = DATASET_BY_KEY["hadoop"]
+    node("iceberg_curated", f"Iceberg\n{cur['table']}", "lakehouse", 4, engine="Iceberg",
+         table=cur["fqtn"], rows=_row_count("curated", counts), masked_layer=True,
+         note="PII-masked curated layer", columns=_pii_columns("curated"),
+         om=_om_lineage(cur.get("om_fqn", "")))
+    node("iceberg_hadoop", f"Iceberg/HDFS\n{had['table']}", "lakehouse", 4, engine="Iceberg / HDFS",
+         table=had["fqtn"], rows=_row_count("hadoop", counts), note="Historical sales on HDFS",
+         om=_om_lineage(had.get("om_fqn", "")))
+
+    edge("spark", "iceberg_curated", "mask + write", "transform", active=arch_active)
+    edge("lake_orders", "iceberg_hadoop", "register external", "batch", active=arch_active)
+    edge("src_orders", "iceberg_curated", "curate (masked)", "transform")
+
+    # Lakehouse → Trino
+    for nid in ("iceberg_curated", "iceberg_hadoop"):
+        edge(nid, "trino", "query", "serve")
+    for key in _SOURCE_KEYS:
+        edge(f"src_{key}", "trino", "federate", "serve")
+
+    # Lake → RAG (documents/parquet feeding embeddings is conceptual)
+    edge("iceberg_curated", "rag", "embed (masked-safe)", "serve")
+
+    # Column-level links: source sales_orders → curated masked
+    column_links: list[dict[str, Any]] = []
+    src_cols = {c["name"] for c in _pii_columns("orders")}
+    cur_cols = {c["name"] for c in _pii_columns("curated")}
+    for cn in sorted(src_cols & cur_cols):
+        masked = any(c["name"] == cn and c["masked"] for c in _pii_columns("curated"))
+        column_links.append({"source": "src_orders", "target": "iceberg_curated",
+                             "column": cn, "masked": masked})
+
+    return {
+        "ok": True,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "active": {"generator": gen_active, "archive": arch_active},
+        "stages": ["Sources", "CDC / Stream", "Compute", "Lake storage",
+                   "Lakehouse", "Federation", "Serving"],
+        "nodes": nodes,
+        "edges": edges,
+        "column_links": column_links,
+        "om_connected": bool(OPENMETADATA_URL),
+    }
+
+
+def get_graph(use_cache: bool = True) -> dict[str, Any]:
+    now = time.time()
+    if use_cache and _cache["data"] and now - _cache["ts"] < _TTL:
+        return _cache["data"]
+    data = _build()
+    _cache["data"] = data
+    _cache["ts"] = now
+    return data
+
+
+@router.get("/datasets")
+async def datasets() -> JSONResponse:
+    return JSONResponse({"ok": True, "datasets": [
+        {"key": d["key"], "label": d["label"], "engine": d["engine"],
+         "color": d["color"], "table": d["fqtn"]}
+        for d in DATASETS]})
+
+
+@router.get("/graph")
+async def graph(dataset: str | None = None, refresh: bool = False) -> JSONResponse:
+    from starlette.concurrency import run_in_threadpool
+    data = await run_in_threadpool(get_graph, not refresh)
+    if dataset and dataset in DATASET_BY_KEY:
+        # Filter to nodes reachable from / to the selected source, keep serving spine.
+        keep = {f"src_{dataset}", f"topic_{dataset}", f"lake_{dataset}",
+                "connect", "spark", "iceberg_curated", "iceberg_hadoop",
+                "trino", "es", "rag", "vllm", "cc"}
+        nodes = [n for n in data["nodes"] if n["id"] in keep]
+        node_ids = {n["id"] for n in nodes}
+        edges = [e for e in data["edges"] if e["source"] in node_ids and e["target"] in node_ids]
+        out = {**data, "nodes": nodes, "edges": edges, "focus": dataset}
+        return JSONResponse(out)
+    return JSONResponse(data)

api/main.py

@@ -15,6 +15,7 @@ import httpx
 import redis.asyncio as aioredis
 from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
 from fastapi.middleware.cors import CORSMiddleware
+import auth as cockpit_auth
 from agent_terminal import (
     get_all_terminals,
     get_terminal_lines,
@@ -46,6 +47,8 @@ from hadoop_analytics import router as hadoop_router
 from elasticsearch_api import router as elasticsearch_router
 from sql_console import router as sql_router
 from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
+from agent_activity import agent_activity_loop
+from streaming_ops import connector_autoheal_loop
 from cdc_consumer import router as cdc_router, cdc_consumer_loop
 from movements import router as movements_router
 from movements import MOVEMENT_BY_ID, trigger_and_watch
@@ -54,6 +57,11 @@ from streaming_ops import router as streaming_router
 from spark_workbench import router as spark_workbench_router
 from pii_catalog import router as pii_router
 from trino_federated import router as federated_router
+from etl_offload import router as etl_offload_router
+from lineage import router as lineage_router
+from dq_monitor import router as dq_router
+from observability import router as observability_router
+from catalog_governance import router as governance_router
 from ssh_terminal import ssh_session
 from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
 from node_ops import build_node_detail, probe_node, run_node_probe_task
@@ -70,6 +78,14 @@ from db import SessionLocal, db_health, init_database
 from supervisor import mirror_terminal_line, mirror_to_supervisors
 
 from workload import build_workload_payload
+from gpu_config import (
+    get_gpu_config,
+    get_gpu_config_payload,
+    get_gpu_urls,
+    reset_gpu_config,
+    save_gpu_config,
+    test_gpu_target,
+)
 
 _workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
 _presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
@@ -82,12 +98,130 @@ from sqlalchemy.orm import DeclarativeBase
 
 REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
 DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
-GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
+DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
+
+
+def _dockhand_headers() -> dict[str, str]:
+    if DOCKHAND_API_TOKEN:
+        return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
+    return {}
+
+GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
 GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
-LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1")
+LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
 LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
 LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
 LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))
+# Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget.
+LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096"))
+# Conservative estimate: Llama tokenizers often use ~2.2–2.8 chars/token on English+lab text.
+LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4"))
+LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160"))
+LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256"))
+LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500"))
+
+
+def _estimate_tokens(text: str) -> int:
+    # Slightly inflate so we never underestimate vs vLLM's tokenizer.
+    return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32)
+
+
+def _compact_lab_context(context: str) -> str:
+    """Keep primary section + short per-domain summaries; drop verbose inventory lines."""
+    lines = context.splitlines()
+    out: list[str] = []
+    in_full = False
+    detail = 0
+    max_detail = 4
+    for line in lines:
+        if line.startswith("=== FULL LAB"):
+            in_full = True
+            out.append(line)
+            continue
+        if line.startswith("=== PRIMARY"):
+            in_full = False
+            detail = 0
+            out.append(line)
+            continue
+        if line.startswith("--- "):
+            detail = 0
+            out.append(line)
+            continue
+        # Drop long platform-capabilities essays if present — keep a one-liner marker
+        if line.startswith("=== PLATFORM CAPABILITIES"):
+            out.append(line)
+            out.append("  (see Command Center UI for full feature list)")
+            continue
+        if out and out[-1].startswith("  (see Command Center"):
+            if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
+                pass
+            else:
+                continue
+        # Always keep masking / PII evidence sections in full (demo-critical)
+        if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
+            # flush remaining lines of this section without detail limits by marking
+            out.append(line)
+            continue
+        # Limit bullet detail in general lab dump, but keep masking evidence intact
+        keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING"))
+        if (line.startswith("  - ") or line.startswith("    - ")) and not keep_full:
+            detail += 1
+            if detail > max_detail:
+                if detail == max_detail + 1:
+                    out.append("    …")
+                continue
+        out.append(line)
+    return "\n".join(out)
+
+
+def _truncate_for_llm(context: str, max_chars: int) -> str:
+    context = _compact_lab_context(context)
+    if len(context) <= max_chars:
+        return context
+    # Prefer keeping PRIMARY section; cut FULL LAB first
+    primary_end = context.find("=== FULL LAB")
+    if primary_end > 200:
+        head = context[:primary_end].rstrip()
+        tail_budget = max(400, max_chars - len(head) - 80)
+        tail = context[primary_end: primary_end + tail_budget]
+        trimmed = head + "\n" + tail
+    else:
+        trimmed = context[:max_chars]
+    if len(trimmed) > max_chars:
+        trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0]
+    if len(context) > len(trimmed):
+        trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]"
+    return trimmed
+
+
+def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]:
+    """Fit prompt+completion into the served model length with a safety margin."""
+    user_tok = _estimate_tokens(user_message)
+    rules_tok = _estimate_tokens(system_rules)
+    budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN
+    max_out = min(LLM_MAX_OUTPUT, 256)
+
+    # Absolute char cap first (independent of estimate errors)
+    context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS)
+
+    for _ in range(6):
+        ctx_budget_tok = budget - user_tok - rules_tok - max_out
+        if ctx_budget_tok < 200:
+            max_out = max(64, max_out // 2)
+            continue
+        ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85))
+        fitted = _truncate_for_llm(context, ctx_max_chars)
+        total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out
+        if total <= budget:
+            return fitted, max_out
+        # Still too big — shrink context harder, then output
+        context = fitted
+        LLM_MAX = max(800, int(len(fitted) * 0.7))
+        context = _truncate_for_llm(context, LLM_MAX)
+        max_out = max(64, max_out - 32)
+
+    return _truncate_for_llm(context, 800), 64
+
 
 AGENTS = [
     {
@@ -240,7 +374,11 @@ ZONES = [
 ]
 
 INTENT_KEYWORDS: dict[str, list[str]] = {
-    "data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "],
+    "data-custodian": [
+        "database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ",
+        "pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer",
+        "employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id",
+    ],
     "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
     "hadoop-ranger": [
         "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
@@ -320,10 +458,24 @@ class ApprovalDecision(BaseModel):
 
 def route_agent(message: str) -> str:
     lower = message.lower()
+    # PII / privacy questions always go to Data Custodian (masking demo path)
+    pii_words = (
+        "pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address",
+        "phone", "telefoon", "iban", "address", "adres", "customer name", "employee",
+        "gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig",
+    )
+    if any(w in lower for w in pii_words):
+        return "data-custodian"
     # Storage/data questions default to Hadoop unless clearly about databases
-    if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")):
-        if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")):
+    if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")):
+        if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")):
             return "hadoop-ranger"
+    if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")):
+        return "infra-sentinel"
+    if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")):
+        return "etl-guardian"
+    if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")):
+        return "lakehouse-ops"
     scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
     best = max(scores, key=scores.get)
     if scores[best] == 0:
@@ -331,13 +483,34 @@ def route_agent(message: str) -> str:
     return best
 
 
+
+def _is_pii_question(message: str) -> bool:
+    lower = message.lower()
+    return any(w in lower for w in (
+        "pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone",
+        "iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive",
+        "personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id",
+        "ssn", "bsn", "geboorte", "birth",
+    ))
+
+
+async def _pii_evidence_block(message: str, log: Any | None = None) -> str:
+    """Live policy + samples synced with Data Flow masking toggles."""
+    try:
+        from pii_catalog import build_policy_evidence
+        return build_policy_evidence()
+    except Exception as exc:
+        return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})"
+
+
 async def gather_agent_context(
     agent_id: str,
     status: dict[str, Any],
     log: Any | None = None,
+    message: str | None = None,
 ) -> str:
     """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
-    snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log)
+    snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False)
     snapshot["domains_summary"] = status.get("domains", {})
     ctx = format_context_for_agent(agent_id, snapshot)
     agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
@@ -346,10 +519,24 @@ async def gather_agent_context(
         agent_lines.append(f"  - {a['name']} ({a['id']}){sup}: {a['role']}")
     ctx = ctx + "\n".join(agent_lines)
     try:
-        from platform_context import build_llm_addendum
-        ctx = ctx + "\n\n" + build_llm_addendum()
+        from platform_context import build_masking_section
+        # Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog).
+        ctx = ctx + "\n\n" + build_masking_section(fresh=True)
     except Exception:
-        pass
+        try:
+            from platform_context import build_llm_addendum
+            ctx = ctx + "\n\n" + build_llm_addendum()
+        except Exception:
+            pass
+    if message and _is_pii_question(message):
+        try:
+            evidence = await _pii_evidence_block(message, log=log)
+            ctx = ctx + "\n\n" + evidence
+            if log:
+                await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)")
+        except Exception as exc:
+            if log:
+                await log("warn", "fetch", f"▸ PII evidence skipped: {exc}")
     if log:
         await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
     return ctx
@@ -362,34 +549,44 @@ async def ask_llm(
     log: Any | None = None,
 ) -> str | None:
     agent = next(a for a in AGENTS if a["id"] == agent_id)
-    system = f"""You are {agent['name']}, an autonomous ops agent in the Dell ATC data lab.
-Specialization: {agent['role']}.
-Motto: {agent.get('motto', '')}.
+    # Deterministic PII path: always mirror Data Flow masked vs visible toggles.
+    if _is_pii_question(message):
+        try:
+            from pii_catalog import format_pii_chat_answer
+            answer = format_pii_chat_answer(message)
+            if log:
+                await log("ok", "pii", "▸ Returning Data Flow–synced masking answer (masked + visible)")
+            return answer
+        except Exception as exc:
+            if log:
+                await log("warn", "pii", f"▸ PII answer builder failed: {exc}")
+    rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab.
+Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers.
+If data is missing or DOWN, say so.
 
-You respond on behalf of your domain but have visibility into the FULL lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, and GPU/vLLM.
+PII / masking rules (critical — synced with Data Flow tab):
+- MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present.
+- VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy.
+- Never invent emails, phones, names, IBANs, or addresses that are not in the live samples.
+- If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections.
 
-Rules:
-- Always respond in English.
-- You have full visibility into the entire cluster: all VMs, zones, connectors, GPU, Hadoop, ObjectScale and Command Center.
-- Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
-- Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
-- If something is DOWN or 0 GB, say so honestly.
-- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
-- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
-- Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
-
---- LIVE LAB DATA (primary domain first, then full stack) ---
-{context}
-"""
+--- LIVE LAB DATA ---"""
+    fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message)
+    system = rules + "\n" + fitted_ctx
+    urls = get_gpu_urls()
+    llm_url = urls["llm_url"]
     if log:
-        await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}")
-        await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions")
+        est = _estimate_tokens(system) + _estimate_tokens(message)
+        await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)")
+        if len(context) > len(fitted_ctx):
+            await log("warn", "llm", f"  context trimmed {len(context)} → {len(fitted_ctx)} chars")
+        await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions")
         await log("info", "llm", f"  user: {message[:160]}{'…' if len(message) > 160 else ''}")
     try:
         async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
             t0 = time.monotonic()
             r = await client.post(
-                f"{LLM_URL.rstrip('/')}/chat/completions",
+                f"{llm_url.rstrip('/')}/chat/completions",
                 headers={
                     "Authorization": f"Bearer {LLM_API_KEY}",
                     "Content-Type": "application/json",
@@ -400,7 +597,7 @@ Rules:
                         {"role": "system", "content": system},
                         {"role": "user", "content": message},
                     ],
-                    "max_tokens": 800,
+                    "max_tokens": max_tokens,
                     "temperature": 0.25,
                 },
             )
@@ -415,15 +612,78 @@ Rules:
                 return content
             if log:
                 await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
+    except httpx.HTTPStatusError as exc:
+        detail = exc.response.text[:200] if exc.response is not None else str(exc)
+        if log:
+            await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}")
+        # One hard retry with a minimal context if we blew the window.
+        if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail:
+            tiny = _truncate_for_llm(context, 1200)
+            system2 = rules + "\n" + tiny
+            max2 = 128
+            if log:
+                await log("warn", "llm", f"  retry with tiny context ({len(tiny)} chars, max_tokens={max2})")
+            try:
+                async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
+                    r2 = await client.post(
+                        f"{llm_url.rstrip('/')}/chat/completions",
+                        headers={
+                            "Authorization": f"Bearer {LLM_API_KEY}",
+                            "Content-Type": "application/json",
+                        },
+                        json={
+                            "model": LLM_MODEL,
+                            "messages": [
+                                {"role": "system", "content": system2},
+                                {"role": "user", "content": message},
+                            ],
+                            "max_tokens": max2,
+                            "temperature": 0.25,
+                        },
+                    )
+                    r2.raise_for_status()
+                    content2 = r2.json()["choices"][0]["message"]["content"].strip()
+                    if content2:
+                        if log:
+                            await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars")
+                        return content2
+            except Exception as exc2:
+                if log:
+                    await log("err", "llm", f"✗ vLLM retry failed: {exc2}")
     except Exception as exc:
         if log:
             await log("err", "llm", f"✗ vLLM error: {exc}")
     return None
 
 
-def fallback_answer(agent_id: str, context: str) -> str:
+def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str:
     agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
-    return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}"
+    if user_message and _is_pii_question(user_message):
+        try:
+            from pii_catalog import format_pii_chat_answer
+            return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message)
+        except Exception:
+            marker = "=== PII MASKING EVIDENCE"
+            if marker in context:
+                return (
+                    f"**{agent_name}** — masking policy (synced with Data Flow):\n\n"
+                    + context[context.index(marker):].strip()
+                )
+    preview_lines: list[str] = []
+    for line in context.splitlines():
+        if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")):
+            preview_lines.append(line)
+        if len(preview_lines) >= 14:
+            break
+    hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable."
+    q = f"\n\nYour question: _{user_message[:200]}_" if user_message else ""
+    return (
+        f"**{agent_name}** — I could not get a reply from the GPU LLM "
+        f"(context window or vLLM error).{q}\n\n"
+        "Try a short, specific question "
+        "(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n"
+        f"Quick snapshot:\n{hint}"
+    )
 
 
 async def publish_event(event: dict[str, Any]) -> None:
@@ -473,7 +733,11 @@ def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
 async def dockhand_env_containers(env_id: int) -> list[dict]:
     try:
         async with httpx.AsyncClient(timeout=8.0) as client:
-            r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id})
+            r = await client.get(
+                f"{DOCKHAND_URL}/api/containers",
+                params={"env": env_id},
+                headers=_dockhand_headers(),
+            )
             r.raise_for_status()
             return r.json()
     except Exception:
@@ -490,15 +754,26 @@ async def probe_url(url: str) -> bool:
 
 
 async def collect_gpu() -> dict[str, Any]:
-    host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0]
-    base = {"ok": False, "host": host, "ui_url": GPU_UI_URL}
+    urls = get_gpu_urls()
+    cfg = get_gpu_config()
+    gpu_url = urls["gpu_url"]
+    host = urls["host"]
+    base = {
+        "ok": False,
+        "host": host,
+        "ip": host,
+        "ui_url": urls["gpu_ui_url"],
+        "config_source": cfg.get("source", "env"),
+        "preset_id": cfg.get("preset_id"),
+        "config_label": cfg.get("label"),
+    }
 
     try:
         async with httpx.AsyncClient(timeout=6.0) as client:
             metrics_r, model_r, integration_r = await asyncio.gather(
-                client.get(f"{GPU_URL}/api/gpu/metrics"),
-                client.get(f"{GPU_URL}/api/active-model"),
-                client.get(f"{GPU_URL}/api/integration"),
+                client.get(f"{gpu_url}/api/gpu/metrics"),
+                client.get(f"{gpu_url}/api/active-model"),
+                client.get(f"{gpu_url}/api/integration"),
                 return_exceptions=True,
             )
 
@@ -660,12 +935,12 @@ async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
 
     await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
     status = await collect_status()
-    context = await gather_agent_context(agent_id, status, log=log)
+    context = await gather_agent_context(agent_id, status, log=log, message=message)
 
     answer = await ask_llm(agent_id, message, context, log=log)
     if not answer:
         await log("warn", "llm", "LLM fallback — returning raw context")
-        answer = fallback_answer(agent_id, context)
+        answer = fallback_answer(agent_id, context, message)
 
     if not approval_created:
         proposed = detect_agent_proposed_action(answer, message)
@@ -734,12 +1009,17 @@ async def lifespan(app: FastAPI):
     cdc_task = asyncio.create_task(cdc_consumer_loop())
     etl_task = asyncio.create_task(etl_agent_loop())
     cust_task = asyncio.create_task(custodian_offload_loop())
+    act_task = asyncio.create_task(agent_activity_loop())
+    heal_task = asyncio.create_task(connector_autoheal_loop())
     add_feed("infra-sentinel", "ATC Command Center API online", "info")
     yield
     task.cancel()
     dml_task.cancel()
     cdc_task.cancel()
     etl_task.cancel()
+    cust_task.cancel()
+    act_task.cancel()
+    heal_task.cancel()
     if redis_client:
         await redis_client.close()
 
@@ -759,6 +1039,11 @@ app.include_router(streaming_router)
 app.include_router(spark_workbench_router)
 app.include_router(pii_router)
 app.include_router(federated_router)
+app.include_router(etl_offload_router)
+app.include_router(lineage_router)
+app.include_router(dq_router)
+app.include_router(observability_router)
+app.include_router(governance_router)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -767,6 +1052,10 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+# Authentik OIDC session + API guard
+cockpit_auth.init_auth_middleware(app)
+cockpit_auth.setup_auth(app)
+
 
 @app.get("/api/health")
 async def health():
@@ -966,6 +1255,41 @@ async def get_status():
 async def get_gpu():
     return await collect_gpu()
 
+@app.get("/api/gpu/config")
+async def get_gpu_config_endpoint():
+    return get_gpu_config_payload()
+
+
+@app.post("/api/gpu/config")
+async def post_gpu_config(body: dict[str, Any]):
+    try:
+        saved = save_gpu_config(
+            preset_id=body.get("preset_id"),
+            host=body.get("host"),
+            gpu_ui_port=body.get("gpu_ui_port"),
+            llm_port=body.get("llm_port"),
+        )
+        return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]}
+    except ValueError as exc:
+        return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400)
+
+
+@app.post("/api/gpu/config/test")
+async def post_gpu_config_test(body: dict[str, Any]):
+    return await test_gpu_target(
+        preset_id=body.get("preset_id"),
+        host=body.get("host"),
+        gpu_ui_port=body.get("gpu_ui_port"),
+        llm_port=body.get("llm_port"),
+    )
+
+
+@app.delete("/api/gpu/config")
+async def delete_gpu_config():
+    active = reset_gpu_config()
+    return {"ok": True, "active": active}
+
+
 
 def agent_stats() -> dict[str, dict[str, Any]]:
     stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
@@ -1036,11 +1360,11 @@ async def run_node_ask_task(node_id: str, message: str) -> None:
     await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
     log = make_logger(node_id)
     status = await collect_status()
-    context = await gather_agent_context(agent_id, status, log=log)
+    context = await gather_agent_context(agent_id, status, log=log, message=message)
     node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
     answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
     if not answer:
-        answer = fallback_answer(agent_id, context)
+        answer = fallback_answer(agent_id, context, message)
     await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer")
     await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})
 

api/movements.py

@@ -83,6 +83,15 @@ async def _publish(event: dict[str, Any]) -> None:
         pass
 
 
+async def _term(agent_id: str, text: str, level: str = "info", phase: str = "etl") -> None:
+    """Stream a movement step to the owning agent's terminal."""
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
 async def _airflow_token(client: httpx.AsyncClient) -> str:
     now = time.time()
     if _token_cache["token"] and _token_cache["exp"] > now + 30:
@@ -128,9 +137,12 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     mv = MOVEMENT_BY_ID.get(mid)
     if not mv:
         return {"ok": False, "error": f"unknown movement {mid}"}
+    agent = mv["agent"]
     if mv.get("api"):
         try:
             payload = {**(mv.get("default_conf") or {}), **(conf or {})}
+            await _term(agent, f"$ POST {mv['api']}  # {mv['label']}", level="cmd")
+            await _term(agent, f"  payload={payload}", level="cmd")
             t0 = time.time()
             async with httpx.AsyncClient(timeout=120.0) as client:
                 r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
@@ -145,21 +157,26 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
             _last_runs[mid] = run
             await _publish({"type": "movement", **run})
             lvl = "info" if state == "success" else "err"
+            await _term(agent, f"  ← {state} · {rows if rows is not None else '?'} rows in {dur}s", level="ok" if state == "success" else "err")
             _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
             return {"ok": state == "success", **run}
         except Exception as exc:
             _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
+            await _term(agent, f"  ✗ error {str(exc)[:140]}", level="err")
             _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
             return {"ok": False, "error": str(exc)}
 
     conf = {**(mv.get("default_conf") or {}), **(conf or {})}
-    agent = mv["agent"]
     count_sql = mv.get("count_sql")
+    verb = "autonomously triggered" if autonomous else "triggered"
+    await _term(agent, f"$ airflow dags trigger {mv['dag_id']}  # {mv['label']} ({verb})", level="cmd")
+    await _term(agent, f"  conf={conf}", level="cmd")
     before = await _trino_scalar(count_sql) if count_sql else None
+    if count_sql:
+        await _term(agent, f"  trino: {count_sql} → {before if before is not None else '?'} rows (before)", level="info")
 
     _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "running", "started_at": datetime.now(timezone.utc).isoformat()}
     await _publish({"type": "movement", "movement_id": mid, "state": "running"})
-    verb = "autonomously triggered" if autonomous else "triggered"
     _feed(agent, f"[etl] {mv['label']}: {verb} (conf={conf})", "info")
 
     try:
@@ -182,8 +199,10 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
                 if state in ("success", "failed"):
                     break
             dur = round(time.time() - t0, 1)
+            await _term(agent, f"  ← Airflow run {run_id} finished state={state} in {dur}s", level="ok" if state == "success" else "err")
     except Exception as exc:
         _last_runs[mid] = {**_last_runs[mid], "state": "failed", "error": str(exc)}
+        await _term(agent, f"  ✗ error {str(exc)[:140]}", level="err")
         _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
         return {"ok": False, "error": str(exc)}
 
@@ -201,6 +220,9 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     await _publish({"type": "movement", **run})
     rtxt = f"{rows} rows" if rows is not None else "data"
     lvl = "info" if state == "success" else "err"
+    if count_sql:
+        await _term(agent, f"  trino: count {before if before is not None else '?'} → {after if after is not None else '?'} ({rtxt})", level="info")
+    await _term(agent, f"  ← {mv['label']}: {state} — {rtxt} in {dur}s", level="ok" if state == "success" else "err")
     _feed(agent, f"[etl] {mv['label']}: {state} — {rtxt} in {dur}s", lvl)
     return {"ok": state == "success", **run}
 

api/node_ops.py

@@ -153,6 +153,25 @@ def build_node_detail(node_id: str, snap: dict[str, Any], workload_node: dict |
     wn = workload_node or {}
     agent_id = NODE_AGENT.get(node_id, "infra-sentinel")
 
+    if node_id == "gpu":
+        try:
+            from gpu_config import resolve_gpu_identity
+            gid = resolve_gpu_identity(snap.get("gpu") if isinstance(snap, dict) else None)
+            meta["ip"] = gid["ip"]
+            meta["vm"] = gid["vm"]
+            meta["vmid"] = gid["vmid"]
+            meta["ssh"] = f"ssh root@{gid['ip']}"
+            meta["links"] = [
+                {"label": "GPU Lab UI", "url": gid["ui_url"]},
+                {"label": "vLLM API", "url": gid["llm_url"]},
+            ]
+            meta["endpoints"] = [
+                {"name": "gpu-lab", "host": gid["ip"], "port": "9000", "proto": "http"},
+                {"name": "vllm", "host": gid["ip"], "port": "8001", "proto": "http"},
+            ]
+        except Exception:
+            pass
+
     detail: dict[str, Any] = {
         "id": node_id,
         "agent_id": agent_id,

api/node_registry.py

@@ -183,21 +183,21 @@ NODE_REGISTRY: dict[str, dict[str, Any]] = {
     },
     "gpu": {
         "label": "GPU Lab",
-        "vm": "atc-gpu-dev",
-        "vmid": 303,
+        "vm": "atc-gpu-prod",
+        "vmid": 306,
         "pve": "atc-gpu",
-        "ip": "10.0.20.106",
-        "ssh": "ssh root@10.0.20.106",
+        "ip": "10.0.10.106",
+        "ssh": "ssh root@10.0.10.106",
         "role": "inference",
         "color": "#3fb950",
-        "description": "4× V100 GPU lab. vLLM serves the active model (Llama 3 70B GPTQ) — powers agent reasoning in this Command Center.",
+        "description": "4× V100 GPU lab (VM306). vLLM serves the active model (Llama 3 70B GPTQ) — powers agent reasoning in this Command Center.",
         "links": [
-            {"label": "GPU Lab UI", "url": "http://10.0.20.106:9000"},
-            {"label": "vLLM API", "url": "http://10.0.20.106:8001/v1"},
+            {"label": "GPU Lab UI", "url": "http://10.0.10.106:9000"},
+            {"label": "vLLM API", "url": "http://10.0.10.106:8001/v1"},
         ],
         "endpoints": [
-            {"name": "gpu-lab", "host": "10.0.20.106", "port": "9000", "proto": "http"},
-            {"name": "vllm", "host": "10.0.20.106", "port": "8001", "proto": "http"},
+            {"name": "gpu-lab", "host": "10.0.10.106", "port": "9000", "proto": "http"},
+            {"name": "vllm", "host": "10.0.10.106", "port": "8001", "proto": "http"},
         ],
         "commands": ["gpu metrics", "model status", "vram usage"],
     },

api/observability.py

@@ -0,0 +1,370 @@
+"""Data observability (Disease #6 — Traceability / observability).
+
+A lightweight background monitor that, per business table, tracks over time:
+
+  volume      – row count and its delta between cycles
+  freshness   – age of the newest record
+  schema      – the column set; a change raises a drift alert
+
+It derives alerts (critical/warning/info) for volume drops, stalled ingestion,
+stale data and schema drift, and keeps a rolling time-series per dataset for the
+trend charts in the UI.
+
+Endpoints:
+  GET  /api/observability/metrics   -> per-dataset series + current state
+  GET  /api/observability/alerts    -> active + recent alerts
+  POST /api/observability/run       -> run one probe cycle now
+  POST /api/observability/config    -> {enabled, interval_s, freshness_min}
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+from collections import deque
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse
+
+from lake_meta import DATASETS, trino_scalar, discover_columns
+
+router = APIRouter(prefix="/api/observability", tags=["observability"])
+
+_lock = threading.Lock()
+_state: dict[str, Any] = {
+    "enabled": True,
+    "interval_s": 90.0,
+    "freshness_min": 30.0,
+    "running": False,
+    "cycles": 0,
+    "last_cycle_ts": 0.0,
+    "next_run_ts": 0.0,
+    "started_at": None,
+    "datasets": {
+        d["key"]: {"label": d["label"], "engine": d["engine"], "color": d["color"],
+                   "table": d["fqtn"], "rows": None, "prev_rows": None, "delta": None,
+                   "freshness_age_min": None, "columns": None, "schema_hash": None,
+                   "series": deque(maxlen=80), "deltas": deque(maxlen=20),
+                   "stalled_cycles": 0, "ts": None, "error": None}
+        for d in DATASETS
+    },
+    "alerts_active": {},        # dedup_key -> alert
+    "alerts_history": deque(maxlen=120),
+    "feed": deque(maxlen=40),
+}
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _term(text: str, level: str = "info", phase: str = "observe") -> None:
+    try:
+        from agent_terminal import emit_threadsafe
+        emit_threadsafe("infra-sentinel", text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+def _raise_alert(dataset: str, atype: str, severity: str, message: str) -> None:
+    dedup = f"{dataset}:{atype}"
+    existing = _state["alerts_active"].get(dedup)
+    if existing:
+        existing["count"] += 1
+        existing["last_ts"] = _now().isoformat()
+        existing["message"] = message
+        return
+    alert = {"id": uuid.uuid4().hex[:8], "dataset": dataset, "type": atype,
+             "severity": severity, "message": message, "count": 1,
+             "ts": _now().isoformat(), "last_ts": _now().isoformat(), "resolved": False}
+    _state["alerts_active"][dedup] = alert
+    _state["alerts_history"].appendleft(dict(alert))
+    lvl = "err" if severity == "critical" else ("warn" if severity == "warning" else "info")
+    _term(f"  ⚠ ALERT [{severity}] {dataset}: {message}", level=lvl, phase="alert")
+    _state["feed"].appendleft({"ts": _now().isoformat(), "text": f"[{severity}] {dataset}: {message}", "level": lvl})
+
+
+def _clear_alert(dataset: str, atype: str) -> None:
+    dedup = f"{dataset}:{atype}"
+    a = _state["alerts_active"].pop(dedup, None)
+    if a:
+        a["resolved"] = True
+        a["resolved_ts"] = _now().isoformat()
+        _state["alerts_history"].appendleft({**a, "message": f"resolved: {a['message']}"})
+
+
+def _native_freshness(ds: dict[str, Any]) -> float | None:
+    """Epoch seconds of the newest record, read cheaply via the PK index
+    (ORDER BY <pk> DESC LIMIT 1) on the source DB. Returns None if unavailable."""
+    nc = ds.get("native_count")
+    ts = ds.get("ts_col")
+    key = ds.get("key_col")
+    if not nc or not ts:
+        return None
+    eng = nc[0]
+    try:
+        import sql_console as s
+        if eng == "postgres" and key:
+            import psycopg2
+            conn = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER,
+                                    password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=6)
+            try:
+                cur = conn.cursor()
+                cur.execute("SET statement_timeout = 6000")
+                cur.execute(f'SELECT extract(epoch FROM "{ts}") FROM public."{ds["table"]}" '
+                            f'WHERE "{ts}" IS NOT NULL ORDER BY "{key}" DESC LIMIT 1')
+                row = cur.fetchone()
+                return float(row[0]) if row and row[0] is not None else None
+            finally:
+                conn.close()
+        if eng == "mysql" and key:
+            import pymysql
+            conn = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER,
+                                   password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=6)
+            try:
+                cur = conn.cursor()
+                cur.execute("SET SESSION MAX_EXECUTION_TIME = 6000")
+                cur.execute(f"SELECT UNIX_TIMESTAMP(`{ts}`) FROM `{ds['table']}` "
+                            f"WHERE `{ts}` IS NOT NULL ORDER BY `{key}` DESC LIMIT 1")
+                row = cur.fetchone()
+                return float(row[0]) if row and row[0] is not None else None
+            finally:
+                conn.close()
+        if eng == "mongodb":
+            cli = s._mongo_client()
+            try:
+                doc = list(cli[ds["schema"]][ds["table"]].find({}, {ts: 1, "_id": 0}).sort("_id", -1).limit(1))
+                if doc:
+                    import datetime as _dt
+                    v = doc[0].get(ts)
+                    if isinstance(v, _dt.datetime):
+                        return v.timestamp()
+            finally:
+                cli.close()
+    except Exception:
+        return None
+    return None
+
+
+def _probe(ds: dict[str, Any]) -> None:
+    key = ds["key"]
+    st = _state["datasets"][key]
+    fq = ds["fqtn"]
+
+    # volume — prefer a fast native planner estimate (postgres/mysql/mongo); use
+    # Trino metadata count for Iceberg; treat a slow/failed count as a soft miss.
+    rows = None
+    estimated = False
+    nc = ds.get("native_count")
+    if nc:
+        try:
+            import sql_console as _s
+            rows = _s._table_row_count(nc[0], nc[1])
+            estimated = True
+        except Exception:
+            rows = None
+    if rows is None:
+        try:
+            rows = int(trino_scalar(f"SELECT count(*) FROM {fq}", timeout=30.0) or 0)
+        except Exception as exc:
+            st["error"] = str(exc)[:140]
+            st["count_fails"] = st.get("count_fails", 0) + 1
+            return
+    st["error"] = None
+    st["count_fails"] = 0
+    st["estimated"] = estimated
+
+    prev = st["rows"]
+    st["prev_rows"] = prev
+    st["rows"] = rows
+    delta = (rows - prev) if prev is not None else None
+    st["delta"] = delta
+    st["series"].append({"t": _now().strftime("%H:%M:%S"), "rows": rows, "delta": delta or 0})
+    st["ts"] = _now().isoformat()
+
+    if delta is not None:
+        st["deltas"].append(delta)
+        # Guard against planner-estimate jitter: only alert on a material drop.
+        drop_threshold = max(2000, int(0.02 * (prev or 0)))
+        if delta < -drop_threshold:
+            _raise_alert(key, "volume_drop", "warning",
+                         f"row count dropped by {abs(delta):,} ({prev:,} → {rows:,})")
+        else:
+            _clear_alert(key, "volume_drop")
+        # stalled ingestion: no growth for several cycles on CDC sources. Skip for
+        # estimate-based counts (planner stats update lazily → false stalls).
+        if delta == 0 and not ds.get("curated") and not estimated:
+            st["stalled_cycles"] += 1
+            if st["stalled_cycles"] >= 4:
+                _raise_alert(key, "stalled", "warning",
+                             f"no new rows for {st['stalled_cycles']} cycles")
+        else:
+            st["stalled_cycles"] = 0
+            _clear_alert(key, "stalled")
+        # spike (informational) — only for exact counts
+        pos = [d for d in st["deltas"] if d > 0]
+        if not estimated and pos and delta > (sum(pos) / len(pos)) * 6 and len(pos) >= 4:
+            _raise_alert(key, "spike", "info", f"volume spike +{delta:,} rows this cycle")
+
+    # freshness — cheap PK-indexed newest-row lookup for the CDC sources, with a
+    # short Trino max(ts) fallback for Iceberg tables.
+    if ds.get("ts_col"):
+        epoch = _native_freshness(ds)
+        if epoch is None:
+            try:
+                v = trino_scalar(f'SELECT to_unixtime(max("{ds["ts_col"]}")) FROM {fq}', timeout=10.0)
+                epoch = float(v) if v is not None else None
+            except Exception:
+                epoch = None
+        if epoch:
+            age = max(0.0, (time.time() - float(epoch)) / 60.0)
+            st["freshness_age_min"] = round(age, 1)
+            if age > float(_state["freshness_min"]) and not ds.get("curated"):
+                _raise_alert(key, "stale", "warning",
+                             f"newest record is {age:.0f}m old (> {_state['freshness_min']:.0f}m)")
+            else:
+                _clear_alert(key, "stale")
+
+    # schema drift
+    cols = discover_columns(ds)
+    if cols:
+        names = sorted(c["name"] for c in cols)
+        h = hash(tuple(names))
+        if st["schema_hash"] is not None and h != st["schema_hash"]:
+            old = set(st["columns"] or [])
+            new = set(names)
+            added = sorted(new - old)
+            removed = sorted(old - new)
+            parts = []
+            if added:
+                parts.append(f"+{', '.join(added)}")
+            if removed:
+                parts.append(f"-{', '.join(removed)}")
+            _raise_alert(key, "schema_drift", "critical", "schema changed: " + " ".join(parts))
+        st["columns"] = names
+        st["schema_hash"] = h
+
+
+def run_cycle() -> dict[str, Any]:
+    if _state["running"]:
+        return {"ok": True, "skipped": "already running"}
+    _state["running"] = True
+    try:
+        _term(f"═══ observability sweep {_state['cycles'] + 1} — volume · freshness · schema ═══",
+              level="info", phase="cycle")
+        for ds in DATASETS:
+            if not _state["enabled"]:
+                break
+            try:
+                _probe(ds)
+            except Exception as exc:
+                _state["datasets"][ds["key"]]["error"] = str(exc)[:140]
+        _state["cycles"] += 1
+        _state["last_cycle_ts"] = time.time()
+        _state["next_run_ts"] = time.time() + float(_state["interval_s"])
+        active = len(_state["alerts_active"])
+        _term(f"═══ sweep {_state['cycles']} done — {active} active alert(s) ═══",
+              level=("warn" if active else "ok"), phase="cycle")
+    finally:
+        _state["running"] = False
+    return {"ok": True, "active_alerts": len(_state["alerts_active"])}
+
+
+def _loop() -> None:
+    _state["started_at"] = _now().isoformat()
+    time.sleep(28)
+    while True:
+        try:
+            if _state["enabled"]:
+                run_cycle()
+        except Exception:
+            pass
+        time.sleep(max(30.0, float(_state["interval_s"])))
+
+
+threading.Thread(target=_loop, daemon=True, name="observability").start()
+
+
+def metrics_view() -> dict[str, Any]:
+    with _lock:
+        datasets = []
+        for ds in DATASETS:
+            st = _state["datasets"][ds["key"]]
+            datasets.append({
+                "key": ds["key"], "label": st["label"], "engine": st["engine"],
+                "color": st["color"], "table": st["table"], "rows": st["rows"],
+                "delta": st["delta"], "freshness_age_min": st["freshness_age_min"],
+                "columns": len(st["columns"]) if st["columns"] else None,
+                "stalled_cycles": st["stalled_cycles"], "error": st["error"],
+                "ts": st["ts"], "series": list(st["series"]),
+            })
+        return {
+            "ok": True, "enabled": _state["enabled"], "interval_s": _state["interval_s"],
+            "freshness_min": _state["freshness_min"], "running": _state["running"],
+            "cycles": _state["cycles"], "last_cycle_ts": _state["last_cycle_ts"],
+            "next_run_ts": _state["next_run_ts"],
+            "alert_counts": _alert_counts(),
+            "datasets": datasets, "feed": list(_state["feed"])[:20],
+        }
+
+
+def _alert_counts() -> dict[str, int]:
+    counts = {"critical": 0, "warning": 0, "info": 0}
+    for a in _state["alerts_active"].values():
+        counts[a["severity"]] = counts.get(a["severity"], 0) + 1
+    counts["total"] = len(_state["alerts_active"])
+    return counts
+
+
+def summary_for_llm() -> dict[str, Any]:
+    with _lock:
+        return {
+            "active_alerts": _alert_counts(),
+            "alerts": [{"dataset": a["dataset"], "type": a["type"], "severity": a["severity"],
+                        "message": a["message"]} for a in _state["alerts_active"].values()],
+        }
+
+
+@router.get("/metrics")
+async def get_metrics() -> JSONResponse:
+    return JSONResponse(metrics_view())
+
+
+@router.get("/alerts")
+async def get_alerts() -> JSONResponse:
+    with _lock:
+        return JSONResponse({
+            "ok": True,
+            "counts": _alert_counts(),
+            "active": sorted(_state["alerts_active"].values(),
+                             key=lambda a: {"critical": 0, "warning": 1, "info": 2}.get(a["severity"], 3)),
+            "history": list(_state["alerts_history"])[:60],
+        })
+
+
+@router.post("/run")
+async def post_run() -> JSONResponse:
+    from starlette.concurrency import run_in_threadpool
+    res = await run_in_threadpool(run_cycle)
+    return JSONResponse({**res, "metrics": metrics_view()})
+
+
+@router.post("/config")
+async def post_config(body: dict = Body(default={})) -> JSONResponse:
+    if "enabled" in body:
+        _state["enabled"] = bool(body["enabled"])
+    if "interval_s" in body:
+        try:
+            _state["interval_s"] = max(30.0, min(900.0, float(body["interval_s"])))
+        except Exception:
+            pass
+    if "freshness_min" in body:
+        try:
+            _state["freshness_min"] = max(1.0, min(1440.0, float(body["freshness_min"])))
+        except Exception:
+            pass
+    return JSONResponse({"ok": True, "enabled": _state["enabled"],
+                         "interval_s": _state["interval_s"], "freshness_min": _state["freshness_min"]})

api/pii_catalog.py

@@ -34,6 +34,7 @@ POLICY_PATH = Path(os.getenv("MASKING_POLICY_PATH", "/data/masking_policy.json")
 DEFAULT_MASKED = True
 MASK_TOKEN = "🔒 MASKED (masking policy ON)"
 _policy_cache: dict[str, bool] | None = None
+_policy_mtime: float = -1.0
 
 # Datasets we surface in the PII overlay. node_id matches dataflow.py node ids.
 # om_fqn = OpenMetadata table FQN (service.database.schema.table) for tag lookup.
@@ -77,21 +78,42 @@ DATASET_BY_KEY = {ds["key"]: ds for ds in DATASETS}
 
 
 def _load_policy() -> dict[str, bool]:
-    global _policy_cache
-    if _policy_cache is None:
+    """Load policy from disk; re-read when the file mtime changes (Data Flow toggles)."""
+    global _policy_cache, _policy_mtime
+    try:
+        mtime = POLICY_PATH.stat().st_mtime
+    except Exception:
+        mtime = 0.0
+    if _policy_cache is None or mtime != _policy_mtime:
         try:
             _policy_cache = {k: bool(v) for k, v in json.loads(POLICY_PATH.read_text()).items()}
         except Exception:
-            _policy_cache = {}
+            if _policy_cache is None:
+                _policy_cache = {}
+        _policy_mtime = mtime
     return _policy_cache
 
 
 def _save_policy(p: dict[str, bool]) -> None:
-    global _policy_cache
+    global _policy_cache, _policy_mtime
     _policy_cache = p
     try:
         POLICY_PATH.parent.mkdir(parents=True, exist_ok=True)
         POLICY_PATH.write_text(json.dumps(p, indent=2))
+        _policy_mtime = POLICY_PATH.stat().st_mtime
+    except Exception:
+        _policy_mtime = time.time()
+
+
+def invalidate_pii_caches() -> None:
+    """Force catalog rebuild so chat/Data Flow see the same mask flags immediately."""
+    _cache["data"] = None
+    _cache["ts"] = 0.0
+    try:
+        import trino_federated as tf
+        if hasattr(tf, "_dict_cache"):
+            tf._dict_cache["data"] = None
+            tf._dict_cache["at"] = 0.0
     except Exception:
         pass
 
@@ -325,10 +347,235 @@ async def set_policy(body: MaskPolicyRequest) -> JSONResponse:
     for col in cols:
         p[f"{key}.{col}"] = bool(body.masked)
     _save_policy(p)
-    _cache["data"] = None  # force rebuild so masked flags reflect the new policy
+    invalidate_pii_caches()  # chat + Data Flow must reflect the toggle immediately
     return JSONResponse({"ok": True, "key": key, "columns": cols, "masked": bool(body.masked)})
 
 
+def policy_column_lists(*, use_cache: bool = False) -> dict[str, Any]:
+    """Live masked vs visible PII columns — same source as the Data Flow tab."""
+    catalog = get_pii(use_cache=use_cache)
+    masked: list[dict[str, str]] = []
+    visible: list[dict[str, str]] = []
+    for d in catalog.get("datasets", []):
+        key = d.get("key") or ""
+        label = d.get("label") or key
+        locked = bool(d.get("masked_layer") or d.get("policy_locked"))
+        for c in d.get("pii_columns", []):
+            entry = {
+                "key": key,
+                "label": label,
+                "column": c.get("name") or "",
+                "category": c.get("category") or "PII",
+                "locked": locked,
+            }
+            (masked if c.get("masked") else visible).append(entry)
+    summ = catalog.get("summary") or {}
+    return {
+        "catalog": catalog,
+        "masked": masked,
+        "visible": visible,
+        "summary": summ,
+        "mask_token": MASK_TOKEN,
+    }
+
+
+def _interest_categories(message: str) -> list[str]:
+    lower = (message or "").lower()
+    cat_map = [
+        (("email", "e-mail", "mail"), "EMAIL"),
+        (("phone", "telefoon", "mobile"), "PHONE"),
+        (("name", "naam"), "NAME"),
+        (("iban", "bank", "card"), "FINANCIAL"),
+        (("ssn", "bsn", "national", "passport"), "NATIONAL_ID"),
+        (("address", "adres"), "ADDRESS"),
+        (("birth", "dob", "geboorte"), "DOB"),
+        (("ip",), "IP"),
+    ]
+    out: list[str] = []
+    for words, cat in cat_map:
+        if any(w in lower for w in words):
+            out.append(cat)
+    return out
+
+
+def _sample_rows_for_chat(
+    *,
+    categories: list[str] | None = None,
+    max_datasets: int = 2,
+    rows_per: int = 2,
+) -> list[dict[str, Any]]:
+    """Fetch a few live rows; values already policy-masked."""
+    catalog = get_pii(use_cache=False)
+    prefer = ["mysql", "postgres", "mongodb", "cassandra", "neo4j", "curated"]
+    samples: list[dict[str, Any]] = []
+    for key in prefer:
+        dset = next((d for d in catalog.get("datasets", []) if d.get("key") == key), None)
+        if not dset or not dset.get("pii_columns"):
+            continue
+        ds = DATASET_BY_KEY.get(key)
+        if not ds:
+            continue
+        pii_cols = dset["pii_columns"]
+        if categories:
+            focus = [c for c in pii_cols if c.get("category") in categories]
+            # Always keep one id-like visible column for context when focusing
+            ids = [c for c in pii_cols if c.get("category") == "IDENTIFIER" and not c.get("masked")]
+            pick = (ids[:1] + focus) if focus else pii_cols
+        else:
+            pick = pii_cols
+        if not pick:
+            continue
+        # de-dupe preserving order
+        seen: set[str] = set()
+        select_cols: list[str] = []
+        masked_map: dict[str, bool] = {}
+        for c in pick:
+            name = c["name"]
+            if name in seen:
+                continue
+            seen.add(name)
+            select_cols.append(name)
+            masked_map[name] = bool(c.get("masked"))
+            if len(select_cols) >= 6:
+                break
+        name_col = next((c["name"] for c in pii_cols if c.get("category") == "NAME"), None)
+        try:
+            cols, rows = _lookup_rows(ds, select_cols, name_col, None, rows_per)
+        except Exception:
+            continue
+        if not rows:
+            continue
+        rendered = []
+        for row in rows[:rows_per]:
+            rendered.append({
+                cname: (MASK_TOKEN if masked_map.get(cname) else val)
+                for cname, val in zip(cols, row)
+            })
+        samples.append({
+            "key": key,
+            "label": dset.get("label", key),
+            "table": ds.get("table"),
+            "rows": rendered,
+            "masked_cols": [c for c, m in masked_map.items() if m],
+            "visible_cols": [c for c, m in masked_map.items() if not m],
+        })
+        if len(samples) >= max_datasets:
+            break
+    return samples
+
+
+def build_policy_evidence(*, max_datasets: int = 2, rows_per: int = 2) -> str:
+    """Compact internal evidence for LLM context (not shown raw to users)."""
+    snap = policy_column_lists(use_cache=False)
+    masked = snap["masked"]
+    visible = snap["visible"]
+    summ = snap["summary"]
+    lines = [
+        "=== PII POLICY (Data Flow synced) ===",
+        f"Masked {summ.get('masked_columns', len(masked))}/{summ.get('pii_columns', 0)} · "
+        f"visible {summ.get('unmasked_columns', len(visible))}. Token: {MASK_TOKEN}",
+        "Masked: " + ", ".join(f"{m['key']}.{m['column']}" for m in masked[:25]) + (
+            f" …(+{len(masked)-25})" if len(masked) > 25 else ""
+        ),
+        "Visible: " + (", ".join(f"{v['key']}.{v['column']}" for v in visible[:25]) or "(none)"),
+    ]
+    for s in _sample_rows_for_chat(max_datasets=max_datasets, rows_per=rows_per):
+        lines.append(f"Sample {s['label']}:")
+        for row in s["rows"]:
+            lines.append("  " + " | ".join(f"{k}={v}" for k, v in row.items()))
+    return "\n".join(lines)
+
+
+def format_pii_chat_answer(message: str = "") -> str:
+    """Short personal reply: masked → say masked; visible → show values. Never list field names."""
+    snap = policy_column_lists(use_cache=False)
+    masked = snap["masked"]
+    visible = snap["visible"]
+    lower = (message or "").lower()
+    interest = _interest_categories(message)
+    greeting = any(w in lower for w in ("hi", "hello", "hey", "hallo", "goedemorgen", "goedemiddag"))
+    hi = "Hi! " if greeting else ""
+
+    label = {
+        "EMAIL": "email",
+        "PHONE": "phone number",
+        "NAME": "name",
+        "FINANCIAL": "bank / IBAN details",
+        "NATIONAL_ID": "national ID",
+        "ADDRESS": "address",
+        "DOB": "date of birth",
+        "IP": "IP address",
+    }
+
+    # No specific PII type asked — keep it vague, never enumerate columns
+    if not interest:
+        if any(w in lower for w in ("mask", "pii", "sensitive", "privacy", "personal")):
+            return (
+                f"{hi}Personal data is protected by the masking policy. "
+                f"Ask for something specific (an email, a phone number, a name…) and I'll tell you "
+                f"whether I can share it — or only `{MASK_TOKEN}`."
+            )
+        return (
+            f"{hi}I can't share personal data that's masked. "
+            f"Ask me for an email, phone number, or name if you want to check."
+        )
+
+    topic = ", ".join(label.get(c, c.lower()) for c in interest)
+    interested_masked = [c for c in masked if c["category"] in interest]
+    interested_visible = [c for c in visible if c["category"] in interest]
+
+    # Collect visible sample *values* only (no column names in the reply)
+    values: list[str] = []
+    if interested_visible:
+        samples = _sample_rows_for_chat(categories=interest, max_datasets=2, rows_per=2)
+        for s in samples:
+            for row in s["rows"]:
+                for col in interested_visible:
+                    if col["column"] in row:
+                        val = row[col["column"]]
+                        if val is None or val == "" or val == MASK_TOKEN:
+                            continue
+                        values.append(str(val))
+        # unique, preserve order
+        seen: set[str] = set()
+        uniq: list[str] = []
+        for v in values:
+            if v not in seen:
+                seen.add(v)
+                uniq.append(v)
+        values = uniq[:5]
+
+    # Fully masked for this ask
+    if interested_masked and not interested_visible:
+        return (
+            f"{hi}No — that {topic} is masked (`{MASK_TOKEN}`). "
+            "I can't share it."
+        )
+
+    # Fully visible
+    if interested_visible and not interested_masked:
+        if values:
+            listed = ", ".join(values)
+            return f"{hi}Sure — here's what I can share: {listed}."
+        return f"{hi}That {topic} isn't masked, but I don't have a sample value right now."
+
+    # Mixed: some sources masked, some visible — still don't name columns
+    if interested_visible and interested_masked:
+        if values:
+            listed = ", ".join(values)
+            return (
+                f"{hi}Some of that is masked (`{MASK_TOKEN}`); "
+                f"what I can share: {listed}."
+            )
+        return (
+            f"{hi}Some of that {topic} is masked (`{MASK_TOKEN}`). "
+            "I can't share the protected parts."
+        )
+
+    return f"{hi}I don't have that personal data available."
+
+
+
 def _lookup_rows(ds: dict[str, Any], select: list[str], name_col: str | None,
                  search: str | None, limit: int) -> tuple[list[str], list[list[Any]]]:
     """Fetch rows from the source. Native DB queries (fast, early LIMIT) for

api/platform_context.py

@@ -90,15 +90,15 @@ def build_platform_section() -> str:
     return "\n".join(lines)
 
 
-def build_masking_section() -> str:
+def build_masking_section(fresh: bool = False) -> str:
     """Exact masking policy + strict guidance so the LLM can answer about masked
-    data without ever revealing masked raw values."""
-    lines: list[str] = ["=== DATA MASKING POLICY (enforced) ==="]
+    data without ever revealing masked raw values. Synced with Data Flow toggles."""
+    lines: list[str] = ["=== DATA MASKING POLICY (enforced — synced with Data Flow) ==="]
     masked: list[str] = []
     unmasked: list[str] = []
     try:
         from pii_catalog import get_pii  # type: ignore
-        data = get_pii()
+        data = get_pii(use_cache=not fresh)
         for d in data.get("datasets", []):
             for c in d.get("pii_columns", []):
                 tag = f"{d.get('label')}.{c.get('name')} [{c.get('category')}]"
@@ -112,22 +112,27 @@ def build_masking_section() -> str:
         lines.append(f"(masking catalog unavailable: {exc})")
 
     if masked:
-        lines.append("MASKED columns (raw values are withheld — token 🔒 MASKED):")
+        lines.append("MASKED columns (raw values withheld — token 🔒 MASKED):")
         for m in masked[:40]:
             lines.append(f"  - {m}")
+    else:
+        lines.append("MASKED columns: (none)")
     if unmasked:
-        lines.append("Visible PII columns (operator opted out of masking):")
+        lines.append("VISIBLE columns (operator opted out of masking in Data Flow — real values OK):")
         for u in unmasked[:40]:
             lines.append(f"  - {u}")
+    else:
+        lines.append("VISIBLE columns: (none — all PII masked)")
 
     lines += [
         "",
-        "How to handle masked data when answering:",
-        "  1. NEVER reveal, guess, reconstruct or print the raw value of a MASKED column. If a value comes in as '🔒 MASKED', keep it masked.",
-        "  2. DO still answer helpfully: confirm the column exists and is masked for privacy/governance, and explain why (PII protection policy).",
-        "  3. You MAY use and report non-sensitive aggregates, counts, distributions and derived metrics over masked columns (e.g. 'there are N distinct customers') as long as no individual raw value is exposed.",
-        "  4. Tell the operator they can unmask a specific column from the Data Flow PII overlay if they have the authority, and that the curated/masked Iceberg layer is physically masked and cannot be unmasked.",
-        "  5. Unmasked PII columns may be shown, but flag that they are sensitive.",
+        "How to handle masked vs visible data when answering:",
+        "  1. MASKED: NEVER reveal, guess, or reconstruct raw values. Quote '🔒 MASKED' when present.",
+        "  2. VISIBLE: you MAY show the real sample values and state that the operator made them visible in Data Flow.",
+        "  3. DO still answer helpfully: confirm which columns are masked vs visible from the lists above.",
+        "  4. You MAY use non-sensitive aggregates/counts over masked columns without exposing individuals.",
+        "  5. Curated/masked Iceberg layers are physically masked and cannot be unmasked from the UI.",
+        "  6. Never invent PII that is not in the live samples.",
     ]
     return "\n".join(lines)
 

api/presentation.py

@@ -276,6 +276,17 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
     ))
 
     gpus = gpu.get("gpus") or snap.get("gpu", {}).get("gpus") or []
+    try:
+        from gpu_config import resolve_gpu_identity
+        gpu_id = resolve_gpu_identity(snap.get("gpu") or gpu)
+    except Exception:
+        host = (snap.get("gpu") or gpu).get("host") or "10.0.10.106"
+        gpu_id = {
+            "vm": "atc-gpu-prod",
+            "vmid": 306,
+            "ui_url": f"http://{host}:9000",
+            "llm_url": f"http://{host}:8001/v1",
+        }
     gpu_lines = [
         f"GPU{g['index']}: {g.get('util_gpu', 0):.0f}% util, "
         f"{g.get('memory_used_mib', 0):.0f}/{g.get('memory_total_mib', 0):.0f} MiB"
@@ -284,11 +295,11 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
     slides.append(_slide(
         "gpu",
         "GPU Lab & GenAI",
-        f"{gpu.get('model') or 'vLLM'} on atc-gpu-dev (VM 303)",
+        f"{gpu.get('model') or 'vLLM'} on {gpu_id['vm']} (VM {gpu_id['vmid']})",
         [
             f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}",
-            f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}",
-            "GPU Lab UI: http://10.0.20.106:9000",
+            f"API: {gpu_id['llm_url']}",
+            f"GPU Lab UI: {gpu_id['ui_url']}",
             "Kibana/Elastic: http://10.0.21.46:5601",
             *gpu_lines,
         ],

api/presentation_static.py

@@ -151,7 +151,7 @@ MODERN_DECKS: dict[str, dict[str, Any]] = {
                 "bullets": [
                     "Supervisor + field operators on Command Center",
                     "Each agent sees live workload, GPU, databases, topology",
-                    "LLM: Llama 3 70B GPTQ via vLLM (10.0.20.106:8001)",
+                    "LLM: Llama 3 70B GPTQ via vLLM (10.0.10.106:8001)",
                     "Approval workflow for sensitive operations",
                 ],
             },
@@ -161,7 +161,7 @@ MODERN_DECKS: dict[str, dict[str, Any]] = {
                 "subtitle": "Dell ATC cluster — key IPs",
                 "bullets": [
                     "Command Center VM304: 10.0.21.33 (this dashboard)",
-                    "GPU Lab VM303: 10.0.20.106 — 7× V100, vLLM, model manager",
+                    "GPU Lab VM306: 10.0.10.106 — 4× V100, vLLM, model manager",
                     "DB Vault: 10.0.21.51 · Lakehouse: 10.0.21.50 · Elastic: 10.0.21.46",
                     "Docling UI: http://10.0.21.33:5001/ui/",
                 ],

api/requirements.txt

@@ -16,3 +16,6 @@ python-pptx==1.0.2
 boto3==1.35.99
 paramiko==3.5.0
 aiokafka==0.12.0
+pyarrow==18.1.0
+authlib==1.4.1
+itsdangerous==2.2.0

api/storage_s3.py

@@ -2,7 +2,10 @@
 
 from __future__ import annotations
 
+import json as _json
 import os
+import random as _rnd
+import threading as _threading
 import time
 from collections import deque
 from datetime import datetime, timezone
@@ -34,6 +37,148 @@ _activity: deque[tuple[float, str]] = deque(maxlen=20000)
 _analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
 _ANALYTICS_TTL = 45.0
 
+# ── Pipeline → S3 archiver ────────────────────────────────────────────────────
+# When data is generated, the streaming pipeline must actually land objects in
+# S3 so the dashboard reflects it (Last write / growth / activity). We mirror two
+# real stages straight into the object store:
+#   • Kafka → S3 CDC archive   -> cdc-archive/dt=YYYY-MM-DD/events-*.json (raw NDJSON)
+#   • Spark → S3 curated layer -> curated/sales_orders_masked/dt=…/part-*.json (PII masked)
+# Writes are buffered like a Kafka-Connect S3 sink (flush on size or interval) so
+# we don't create a flood of tiny objects, and force-flushed on a manual burst.
+S3_ARCHIVE_BUCKET = os.getenv("S3_ARCHIVE_BUCKET", "data")
+S3_FLUSH_INTERVAL = float(os.getenv("S3_ARCHIVE_FLUSH_S", "12"))
+S3_FLUSH_SIZE = int(os.getenv("S3_ARCHIVE_FLUSH_SIZE", "400"))
+
+_arch_lock = _threading.Lock()
+_arch_buf: dict[str, list] = {"cdc": [], "curated": []}
+_arch_since: dict[str, float] = {"t": 0.0}
+_last_write: dict[str, Any] = {}
+_arch_stats: dict[str, int] = {"objects": 0, "bytes": 0, "rows": 0}
+
+
+def _iso(v: Any) -> str:
+    return v.isoformat() if hasattr(v, "isoformat") else str(v)
+
+
+def _mask_cust(cid: Any) -> str:
+    h = abs(hash(("cust", cid))) % 0xFFFFFF
+    return f"cust_{h:06x}***"
+
+
+def _order_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
+    cid, pid, region, channel, ots, amt, curr, status = t
+    return {"op": "c", "source": "postgres", "db": "sales", "table": "sales_orders", "ts": ts,
+            "after": {"customer_id": cid, "product_id": pid, "region": region, "sales_channel": channel,
+                      "amount": amt, "currency": curr, "order_status": status, "order_ts": _iso(ots)}}
+
+
+def _order_to_curated(t: tuple, ts: str) -> dict[str, Any]:
+    cid, pid, region, channel, ots, amt, curr, status = t
+    return {"customer_ref": _mask_cust(cid), "product_id": pid, "region": region, "sales_channel": channel,
+            "amount": amt, "currency": curr, "order_status": status, "order_ts": _iso(ots),
+            "ingested_ts": ts, "pii_masked": True}
+
+
+def _hr_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
+    eid, dept, role, region, evt, sal, ets = t
+    return {"op": "c", "source": "mysql", "db": "hr", "table": "employee_events", "ts": ts,
+            "after": {"employee_id": eid, "department": dept, "role_name": role, "region": region,
+                      "event_type": evt, "salary_change": sal, "event_ts": _iso(ets)}}
+
+
+def _supply_to_cdc(d: dict, ts: str) -> dict[str, Any]:
+    return {"op": "c", "source": "mongodb", "db": "supplychain", "table": "events", "ts": ts, "after": dict(d)}
+
+
+def _tel_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
+    dev, mts, mtype, mval, _payload = t
+    return {"op": "c", "source": "cassandra", "db": "telemetry", "table": "device_metrics", "ts": ts,
+            "after": {"device_id": dev, "metric_ts": _iso(mts), "metric_type": mtype, "metric_value": mval}}
+
+
+def _put(s3, bucket: str, key: str, body: bytes, content_type: str) -> None:
+    s3.put_object(Bucket=bucket, Key=key, Body=body, ContentType=content_type)
+    _track("write")
+    _last_write.update({"ts": datetime.now(timezone.utc).isoformat(), "mono": time.time(),
+                        "bucket": bucket, "key": key, "bytes": len(body)})
+    _arch_stats["objects"] += 1
+    _arch_stats["bytes"] += len(body)
+
+
+def _flush_locked() -> list[dict[str, Any]] | None:
+    cdc = _arch_buf["cdc"]
+    cur = _arch_buf["curated"]
+    if not cdc and not cur:
+        return None
+    s3 = _client()
+    now = datetime.now(timezone.utc)
+    day = now.strftime("%Y-%m-%d")
+    ms = int(now.timestamp() * 1000)
+    rid = _rnd.randint(1000, 9999)
+    written: list[dict[str, Any]] = []
+    if cdc:
+        body = ("\n".join(_json.dumps(e, default=str) for e in cdc) + "\n").encode()
+        key = f"cdc-archive/dt={day}/events-{ms}-{rid}.json"
+        _put(s3, S3_ARCHIVE_BUCKET, key, body, "application/x-ndjson")
+        written.append({"stage": "kafka→s3", "key": key, "rows": len(cdc), "bytes": len(body)})
+    if cur:
+        body = ("\n".join(_json.dumps(e, default=str) for e in cur) + "\n").encode()
+        key = f"curated/sales_orders_masked/dt={day}/part-{ms}-{rid}.json"
+        _put(s3, S3_ARCHIVE_BUCKET, key, body, "application/x-ndjson")
+        written.append({"stage": "spark→s3", "key": key, "rows": len(cur), "bytes": len(body)})
+    _arch_buf["cdc"] = []
+    _arch_buf["curated"] = []
+    _arch_since["t"] = time.time()
+    return written
+
+
+def archive_generated_batch(orders_rows=None, hr_rows=None, supply_docs=None, tel_rows=None,
+                            *, force: bool = False) -> list[dict[str, Any]] | dict[str, Any] | None:
+    """Stage a freshly generated batch into S3 (Kafka→S3 CDC archive + Spark→S3
+    curated masked). Buffered; flushes on size/interval or when force=True."""
+    try:
+        with _arch_lock:
+            ts = datetime.now(timezone.utc).isoformat()
+            for t in (orders_rows or []):
+                _arch_buf["cdc"].append(_order_to_cdc(t, ts))
+                _arch_buf["curated"].append(_order_to_curated(t, ts))
+                _arch_stats["rows"] += 1
+            for t in (hr_rows or []):
+                _arch_buf["cdc"].append(_hr_to_cdc(t, ts)); _arch_stats["rows"] += 1
+            for d in (supply_docs or []):
+                _arch_buf["cdc"].append(_supply_to_cdc(d, ts)); _arch_stats["rows"] += 1
+            for t in (tel_rows or []):
+                _arch_buf["cdc"].append(_tel_to_cdc(t, ts)); _arch_stats["rows"] += 1
+            if _arch_since["t"] == 0.0:
+                _arch_since["t"] = time.time()
+            buffered = len(_arch_buf["cdc"]) + len(_arch_buf["curated"])
+            age = time.time() - _arch_since["t"]
+            if force or buffered >= S3_FLUSH_SIZE or age >= S3_FLUSH_INTERVAL:
+                return _flush_locked()
+    except Exception as exc:  # never break the generator on an S3 hiccup
+        return {"error": str(exc)}
+    return None
+
+
+def put_object_bytes(key: str, body: bytes, content_type: str = "application/octet-stream",
+                     bucket: str | None = None) -> dict[str, Any]:
+    """Write raw bytes to S3 (used by the ETL offload agent for Parquet parts).
+    Tracks last-write + activity so the storage dashboard reflects it live."""
+    b = bucket or S3_ARCHIVE_BUCKET
+    s3 = _client()
+    _put(s3, b, key, body, content_type)
+    return {"ok": True, "bucket": b, "key": key, "bytes": len(body)}
+
+
+def archive_active(window_s: float = 25.0) -> bool:
+    """True if the pipeline wrote to S3 recently — drives the kafka→S3 edge pulse."""
+    return (time.time() - float(_last_write.get("mono") or 0.0)) < window_s
+
+
+def archive_info() -> dict[str, Any]:
+    return {"last_write": dict(_last_write) or None, "objects": _arch_stats["objects"],
+            "bytes": _arch_stats["bytes"], "rows": _arch_stats["rows"]}
+
 
 def _client():
     return boto3.client(
@@ -494,5 +639,18 @@ async def analytics(refresh: bool = Query(False)):
             _analytics_cache["ts"] = now
         except Exception as exc:
             return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502)
+
+    # Overlay the live last-write so the dashboard reflects pipeline writes
+    # immediately, without waiting for the (bounded, cached) full rescan.
+    summary = dict(data.get("summary") or {})
+    recent = list(data.get("recent") or [])
+    lw = dict(_last_write)
+    if lw.get("ts"):
+        if not summary.get("newest") or lw["ts"] > summary["newest"]:
+            summary["newest"] = lw["ts"]
+        recent = ([{"modified": lw["ts"], "bucket": lw.get("bucket"), "key": lw.get("key"),
+                    "bytes": lw.get("bytes", 0), "size_human": _human_size(lw.get("bytes", 0))}]
+                  + [r for r in recent if r.get("key") != lw.get("key")])[:15]
     return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(),
-            "activity": _activity_view(), **data}
+            "activity": _activity_view(), **data, "summary": summary, "recent": recent,
+            "archive": archive_info()}

api/streaming_ops.py

@@ -433,3 +433,126 @@ async def resume_kafka_connector(name: str) -> JSONResponse:
         return JSONResponse({"ok": True, "connector": name, "action": "resume"})
     except Exception as exc:
         return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
+
+
+# ── Source CDC re-sync + autonomous self-heal ───────────────────────────────
+# When a source database briefly drops, the Debezium tasks land in FAILED (or
+# stall while still reporting RUNNING) and never recover on their own. The
+# re-sync restarts the connectors so CDC catches up; the auto-heal loop does the
+# same automatically for FAILED tasks.
+SOURCE_CONNECTORS = [c for c in os.getenv(
+    "CDC_SOURCE_CONNECTORS",
+    "postgres-sales-connector,mysql-hr-connector,mongodb-supplychain-connector",
+).split(",") if c.strip()]
+
+
+async def _term(agent_id: str, text: str, level: str = "info", phase: str = "resync") -> None:
+    try:
+        from agent_terminal import terminal_log
+        await terminal_log(agent_id, text, level=level, phase=phase)
+    except Exception:
+        pass
+
+
+async def _connector_status(client: httpx.AsyncClient, name: str) -> dict[str, Any]:
+    try:
+        r = await client.get(f"{KAFKA_CONNECT_URL}/connectors/{name}/status")
+        if r.status_code >= 400:
+            return {"name": name, "state": "MISSING", "tasks": [], "failed": []}
+        st = r.json()
+        tasks = st.get("tasks") or []
+        return {
+            "name": name,
+            "state": (st.get("connector") or {}).get("state"),
+            "tasks": [{"id": t.get("id"), "state": t.get("state")} for t in tasks],
+            "failed": [t.get("id") for t in tasks if t.get("state") == "FAILED"],
+        }
+    except Exception as exc:
+        return {"name": name, "state": "ERROR", "error": str(exc)[:120], "tasks": [], "failed": []}
+
+
+async def _discover_connectors(client: httpx.AsyncClient) -> list[str]:
+    try:
+        r = await client.get(f"{KAFKA_CONNECT_URL}/connectors")
+        if r.status_code < 400 and isinstance(r.json(), list) and r.json():
+            return r.json()
+    except Exception:
+        pass
+    return list(SOURCE_CONNECTORS)
+
+
+async def resync_connectors(force: bool = True, names: list[str] | None = None) -> dict[str, Any]:
+    """force=True → restart every connector incl. all tasks (full re-sync, also
+    recovers stalled-but-RUNNING tasks). force=False → only restart connectors
+    that have a FAILED connector or task (self-heal)."""
+    result: dict[str, Any] = {"ok": True, "restarted": [], "skipped": [], "before": []}
+    async with httpx.AsyncClient(timeout=20.0) as client:
+        targets = names or await _discover_connectors(client)
+        for name in targets:
+            before = await _connector_status(client, name)
+            result["before"].append(before)
+            unhealthy = before.get("state") in ("FAILED", "ERROR", "MISSING") or before.get("failed")
+            if not force and not unhealthy:
+                result["skipped"].append(name)
+                continue
+            try:
+                qs = "includeTasks=true" if force else "includeTasks=true&onlyFailed=true"
+                rr = await client.post(f"{KAFKA_CONNECT_URL}/connectors/{name}/restart?{qs}")
+                if rr.status_code < 400:
+                    result["restarted"].append(name)
+                    await _term("etl-guardian", f"$ kafka-connect restart {name}  ({'full re-sync' if force else 'failed tasks'})", level="cmd")
+                else:
+                    result["ok"] = False
+                    await _term("etl-guardian", f"  ✗ {name}: HTTP {rr.status_code}", level="err")
+            except Exception as exc:
+                result["ok"] = False
+                await _term("etl-guardian", f"  ✗ {name}: {str(exc)[:100]}", level="err")
+    _cache["ts"] = 0
+    return result
+
+
+@router.get("/connectors")
+async def connectors_status() -> JSONResponse:
+    async with httpx.AsyncClient(timeout=12.0) as client:
+        names = await _discover_connectors(client)
+        items = [await _connector_status(client, n) for n in names]
+    healthy = sum(1 for c in items if c.get("state") == "RUNNING" and not c.get("failed"))
+    return JSONResponse({"ok": True, "connect_url": KAFKA_CONNECT_URL,
+                         "healthy": healthy, "total": len(items), "connectors": items})
+
+
+@router.post("/resync")
+async def resync_sources(body: dict[str, Any] = Body(default={})) -> JSONResponse:
+    force = bool(body.get("force", True))
+    _feed("etl-guardian", f"[kafka] Source CDC re-sync requested ({'full' if force else 'failed only'})", "warn")
+    await _term("etl-guardian", "═══ Source CDC re-sync — restarting Debezium connectors ═══", level="info")
+    res = await resync_connectors(force=force)
+    await asyncio.sleep(3)  # let tasks transition
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        after = [await _connector_status(client, c["name"]) for c in res["before"]]
+    res["after"] = after
+    healthy = sum(1 for c in after if c.get("state") == "RUNNING" and not c.get("failed"))
+    res["healthy"] = healthy
+    res["total"] = len(after)
+    await _term("etl-guardian", f"  ← re-sync requested for {len(res['restarted'])} connectors · {healthy}/{len(after)} healthy", level="ok")
+    return JSONResponse(res)
+
+
+async def connector_autoheal_loop() -> None:
+    """Autonomously restart FAILED Debezium tasks so CDC recovers after a source
+    DB outage without operator action."""
+    await asyncio.sleep(40)
+    interval = max(30.0, float(os.getenv("CONNECTOR_AUTOHEAL_SECONDS", "60")))
+    while True:
+        try:
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                for name in await _discover_connectors(client):
+                    st = await _connector_status(client, name)
+                    if st.get("state") in ("FAILED", "ERROR") or st.get("failed"):
+                        await client.post(f"{KAFKA_CONNECT_URL}/connectors/{name}/restart?includeTasks=true&onlyFailed=true")
+                        _feed("etl-guardian", f"[kafka] auto-heal restarted {name}", "warn")
+                        await _term("etl-guardian", f"  ⟳ auto-heal: restarted FAILED task(s) on {name}", level="warn", phase="autoheal")
+            _cache["ts"] = 0
+        except Exception:
+            pass
+        await asyncio.sleep(interval)

api/topology_views.py

@@ -537,13 +537,24 @@ def _build_architecture(
         1, 1, "📓",
     )
     gpu_ok = bool(gpu.get("ok"))
+    try:
+        from gpu_config import resolve_gpu_identity
+        gpu_id = resolve_gpu_identity(gpu)
+    except Exception:
+        host = gpu.get("ip") or gpu.get("host") or "10.0.10.106"
+        gpu_id = {"vm": "atc-gpu-prod", "ip": host}
     ml = _arch_node(
         "cons-ml", "ML / GenAI", "vLLM cluster", C_CON, 70, "#bc8cff", "consumers",
-        "ok" if gpu_ok else "warn", "atc-gpu-dev", "10.0.20.106",
+        "ok" if gpu_ok else "warn", gpu_id["vm"], gpu_id["ip"],
         [gpu.get("active_model") or "offline"],
-        [{"name": gpu.get("active_model") or "vllm", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001"]}],
+        [{"name": gpu.get("active_model") or "vllm", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001", "9000"]}],
         gpu.get("gpu_count", 0) or 0, max(gpu.get("gpu_count", 4) or 4, 1), "🤖",
-        {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1)},
+        {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1),
+         "ui_url": gpu_id.get("ui_url"), "vllm_url": gpu_id.get("llm_url"),
+         "links": [
+             {"label": "GPU Lab UI", "url": gpu_id.get("ui_url") or f"http://{gpu_id['ip']}:9000"},
+             {"label": "vLLM API", "url": gpu_id.get("llm_url") or f"http://{gpu_id['ip']}:8001/v1"},
+         ]},
     )
 
     nodes = [

api/trino_federated.py

@@ -453,25 +453,30 @@ _TEL_INSERT_TPL = ("INSERT INTO {ks}.device_metrics "
 def _gen_orders(n: int):
     rows, by_r, by_s, val = _order_rows(n)
     _gen_pg().cursor().executemany(_PG_INSERT, rows)
-    return by_r, by_s, val
+    return rows, by_r, by_s, val
 
 
 def _gen_hr(n: int):
-    _gen_mysql().cursor().executemany(_MYSQL_INSERT, _hr_rows(n))
+    rows = _hr_rows(n)
+    _gen_mysql().cursor().executemany(_MYSQL_INSERT, rows)
+    return rows
 
 
 def _gen_supply(n: int):
     docs = _supply_docs(n)
     if docs:
-        _gen_mongo()["events"].insert_many(docs)
+        _gen_mongo()["events"].insert_many([dict(d) for d in docs])
+    return docs
 
 
 def _gen_tel(n: int):
     import sql_console as s
     sess = _gen_cass()
     cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
-    for row in _tel_rows(n):
+    rows = _tel_rows(n)
+    for row in rows:
         sess.execute(cql, row)
+    return rows
 
 
 def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any]:
@@ -482,10 +487,15 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
     by_r: dict[str, int] = {}
     by_s: dict[str, int] = {}
     val = 0.0
+    order_built: list = []
+    hr_built: list = []
+    supply_built: list = []
+    tel_built: list = []
     if orders > 0:
         try:
             import psycopg2
             rows, by_r, by_s, val = _order_rows(orders)
+            order_built = rows
             c = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER, password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=8)
             try:
                 c.autocommit = True
@@ -498,9 +508,10 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
     if hr > 0:
         try:
             import pymysql
+            hr_built = _hr_rows(hr)
             c = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER, password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=8, autocommit=True)
             try:
-                c.cursor().executemany(_MYSQL_INSERT, _hr_rows(hr))
+                c.cursor().executemany(_MYSQL_INSERT, hr_built)
                 out["hr_events"] = hr
             finally:
                 c.close()
@@ -508,9 +519,10 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
             pass
     if supply > 0:
         try:
+            supply_built = _supply_docs(supply)
             cli = s._mongo_client()
             try:
-                cli[s.MONGO_DB]["events"].insert_many(_supply_docs(supply))
+                cli[s.MONGO_DB]["events"].insert_many([dict(d) for d in supply_built])
                 out["supply_events"] = supply
             finally:
                 cli.close()
@@ -518,22 +530,36 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
             pass
     if tel > 0:
         try:
+            tel_built = _tel_rows(tel)
             cluster = s._cass_cluster()
             sess = cluster.connect()
             try:
                 cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
-                for row in _tel_rows(tel):
+                for row in tel_built:
                     sess.execute(cql, row)
                 out["telemetry"] = tel
             finally:
                 cluster.shutdown()
         except Exception:
             pass
+    # Land the generated batch in S3 through the pipeline stages (Kafka→S3 CDC
+    # archive + Spark→S3 curated masked) so Object Storage reflects it at once.
+    try:
+        from storage_s3 import archive_generated_batch
+        archive_generated_batch(order_built if out["orders"] else [],
+                                hr_built if out["hr_events"] else [],
+                                supply_built if out["supply_events"] else [],
+                                tel_built if out["telemetry"] else [], force=True)
+    except Exception:
+        pass
     # fold into the live counters + feed so the dashboard reflects it instantly
     with _gen_lock:
         c = _GEN["counts"]
         for k in out:
             c[k] += out[k]
+        _GEN["last_batch"] = {"orders": out["orders"], "hr_events": out["hr_events"],
+                              "supply_events": out["supply_events"], "telemetry": out["telemetry"]}
+        _GEN["last_tick"] = time.time()  # marks recent activity → Data Flow CDC edges pulse
         if out["orders"]:
             _GEN["by_region"] = by_r
             _GEN["by_status"] = by_s
@@ -558,22 +584,32 @@ def _gen_tick():
     by_r: dict[str, int] = {}
     by_s: dict[str, int] = {}
     val = 0.0
+    order_built: list = []
+    hr_built: list = []
+    supply_built: list = []
+    tel_built: list = []
     try:
-        by_r, by_s, val = _gen_orders(no)
+        order_built, by_r, by_s, val = _gen_orders(no)
     except Exception:
         _gen_reset("pg"); no = 0
     try:
-        _gen_hr(nh)
+        hr_built = _gen_hr(nh)
     except Exception:
         _gen_reset("mysql"); nh = 0
     try:
-        _gen_supply(ns)
+        supply_built = _gen_supply(ns)
     except Exception:
         _gen_reset("mongo"); ns = 0
     try:
-        _gen_tel(nt)
+        tel_built = _gen_tel(nt)
     except Exception:
         _gen_reset("cass"); nt = 0
+    # stream the batch into S3 (buffered like a Kafka-Connect S3 sink)
+    try:
+        from storage_s3 import archive_generated_batch
+        archive_generated_batch(order_built, hr_built, supply_built, tel_built, force=False)
+    except Exception:
+        pass
     with _gen_lock:
         c = _GEN["counts"]
         c["orders"] += no
@@ -609,6 +645,13 @@ def _gen_loop():
 threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
 
 
+def generator_active(window_s: float = 12.0) -> bool:
+    """True when the live generator (continuous loop OR a manual 'Generate data'
+    burst) wrote rows very recently. The Data Flow graph uses this to pulse the
+    generator→source edges so generated data is visible flowing into the sources."""
+    return (time.time() - float(_GEN.get("last_tick") or 0.0)) < window_s
+
+
 @router.post("/live/generator")
 async def toggle_generator(body: dict = Body(default={})):
     if "enabled" in body:
@@ -622,6 +665,19 @@ async def toggle_generator(body: dict = Body(default={})):
     return {"ok": True, "enabled": _GEN["enabled"], "interval": _GEN["interval"], "running": _GEN["running"]}
 
 
+@router.post("/live/heartbeat")
+async def live_heartbeat(body: dict = Body(default={})):
+    """Keep-alive for the continuous generator from any watching view (e.g. the
+    Data Flow tab's Run control). active=true → generate; active=false → stop."""
+    active = bool(body.get("active", True))
+    if active:
+        _GEN["enabled"] = True
+        _GEN["last_seen"] = time.time()
+    else:
+        _GEN["last_seen"] = 0.0
+    return {"ok": True, "running": _GEN["running"], "enabled": _GEN["enabled"]}
+
+
 @router.post("/generate")
 async def generate_now(body: dict = Body(default={})):
     """Manual one-shot burst into the source systems (the Data Flow "Generate

api/workload.py

@@ -63,6 +63,18 @@ def build_workload_payload(snap: dict[str, Any]) -> dict[str, Any]:
     gpu = snap.get("gpu", {})
     objectscale = snap.get("objectscale", {})
     command = snap.get("command_center", {})
+    try:
+        from gpu_config import resolve_gpu_identity
+        gpu_id = resolve_gpu_identity(gpu)
+    except Exception:
+        host = gpu.get("ip") or gpu.get("host") or "10.0.10.106"
+        gpu_id = {
+            "vm": "atc-gpu-prod",
+            "ip": host,
+            "ui_url": gpu.get("ui_url") or f"http://{host}:9000",
+            "llm_url": gpu.get("vllm_url") or f"http://{host}:8001/v1",
+            "vmid": 306,
+        }
 
     docker_apps = [_app_row(c) for c in docker.get("containers", [])]
     db_apps = [_app_row(c) for c in databases.get("containers", [])]
@@ -221,6 +233,18 @@ def build_workload_payload(snap: dict[str, Any]) -> dict[str, Any]:
         }
         if extra:
             row.update(extra)
+        if nid == "gpu":
+            row["vm"] = gpu_id["vm"]
+            row["ip"] = gpu_id["ip"]
+            row["vmid"] = gpu_id.get("vmid", row.get("vmid"))
+            row["links"] = [
+                {"label": "GPU Lab UI", "url": gpu_id["ui_url"]},
+                {"label": "vLLM API", "url": gpu_id["llm_url"]},
+            ]
+            row["endpoints"] = [
+                {"name": "gpu-lab", "host": gpu_id["ip"], "port": "9000", "proto": "http"},
+                {"name": "vllm", "host": gpu_id["ip"], "port": "8001", "proto": "http"},
+            ]
         return row
 
     connect_running = 1 if connect_app and connect_app.get("state") == "running" else 0
@@ -247,10 +271,11 @@ def build_workload_payload(snap: dict[str, Any]) -> dict[str, Any]:
         _node("hadoop", "Hadoop HDFS", "atc-hadoop-m01", "10.0.21.61", 50, 52, "#39ff14", "ok" if hdfs_ok else "warn", "parallel",
               zones[-1]["apps"], hadoop.get("live_datanodes", 0), zones[-1]["total"],
               {"hdfs_used_gb": hadoop.get("capacity_used_gb"), "hdfs_total_gb": hadoop.get("capacity_total_gb")}),
-        _node("gpu", "GPU Lab", "atc-gpu-dev", "10.0.20.106", 88, 52, "#76b900", "ok" if gpu.get("ok") else "down", "inference",
-              [{"name": gpu.get("active_model") or "vLLM", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001"]}],
+        _node("gpu", "GPU Lab", gpu_id["vm"], gpu_id["ip"], 88, 52, "#76b900", "ok" if gpu.get("ok") else "down", "inference",
+              [{"name": gpu.get("active_model") or "vLLM", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001", "9000"]}],
               gpu.get("gpu_count", 0), gpu.get("gpu_count", 0) or 4,
-              {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1)}),
+              {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1),
+               "ui_url": gpu_id["ui_url"], "vllm_url": gpu_id["llm_url"]}),
         _node("command", "Command Center", "MCP · VM304", "10.0.21.33", 50, 78, "#00f0ff",
               _level(command.get("running", 0), command.get("total", 1) or 1), "hub",
               command.get("containers") and [_app_row(c) for c in command.get("containers", [])] or [
@@ -267,7 +292,15 @@ def build_workload_payload(snap: dict[str, Any]) -> dict[str, Any]:
         return {"id": eid, "from": src, "to": dst, "label": label, "kind": kind, "active": active}
 
     pipeline_ok = etl.get("airflow_healthy") and len(connectors) > 0 and etl.get("kafka_ui_ok")
+    direct_pipeline_ok = (
+        pipeline_ok
+        and lakehouse.get("trino_ok")
+        and objectscale_ok
+    )
     s3_flow_ok = pipeline_ok and consumer_running and objectscale_ok
+    if not s3_flow_ok and direct_pipeline_ok and not consumer_running:
+        # Dockhand blind (auth/outage) but ETL + Trino + S3 probes healthy
+        s3_flow_ok = True
 
     topology_edges = [
         _edge("e-seed", "airflow", "db", "seed data", "pipeline", bool(etl.get("airflow_healthy"))),

caddy/Caddyfile

@@ -8,6 +8,9 @@
     handle_path /dq/* {
         reverse_proxy dq-api:5010
     }
+    handle /auth/* {
+        reverse_proxy api:3201
+    }
     handle /api/* {
         reverse_proxy api:3201
     }

config/command-center/atc.env.example

@@ -8,11 +8,14 @@ S3_REGION=us-east-1
 
 JUPYTER_TOKEN=choose-a-strong-token
 
-GPU_URL=http://10.0.20.106:9000
-LLM_URL=http://10.0.20.106:8001/v1
+GPU_URL=http://10.0.10.106:9000
+LLM_URL=http://10.0.10.106:8001/v1
 LLM_MODEL=gpt-4o
 LLM_API_KEY=sk-local
 
 LAKEHOUSE_HOST=10.0.21.50
 AIRFLOW_URL=http://10.0.21.55:8080
 KAFKA_UI_URL=http://10.0.21.36:9000
+
+# Dockhand API token (Profile → API tokens in Dockhand UI; prefix dh_)
+DOCKHAND_API_TOKEN=

config/command-center/docker-compose.yml

@@ -32,9 +32,9 @@ services:
       SQLITE_FALLBACK_PATH: /data/atc-agents.db
       DOCLING_URL: http://docling-serve:5001
       PRESENTATIONS_DIR: /data/presentations
-      GPU_URL: http://10.0.20.106:9000
-      GPU_UI_URL: http://10.0.20.106:9000
-      LLM_URL: http://10.0.20.106:8001/v1
+      GPU_URL: http://10.0.10.106:9000
+      GPU_UI_URL: http://10.0.10.106:9000
+      LLM_URL: http://10.0.10.106:8001/v1
       LLM_MODEL: gpt-4o
       LLM_API_KEY: sk-local
       LAKEHOUSE_HOST: 10.0.21.50
@@ -87,7 +87,7 @@ services:
       CHROMA_HOST: chromadb
       CHROMA_PORT: 8000
       DOCLING_URL: http://docling-serve:5001
-      LLM_URL: http://10.0.20.106:8001/v1
+      LLM_URL: http://10.0.10.106:8001/v1
       LLM_MODEL: gpt-4o
       LLM_API_KEY: sk-local
       RAG_DATA_DIR: /data

docker-compose.dockhand.yml

@@ -30,9 +30,9 @@ services:
       DOCKHAND_URL: http://10.0.21.45:8082
       DATABASE_URL: postgresql+psycopg2://atc:atc-agents-pg@postgres:5432/atc_agents
       SQLITE_FALLBACK_PATH: /data/atc-agents.db
-      GPU_URL: http://10.0.20.106:9000
-      GPU_UI_URL: http://10.0.20.106:9000
-      LLM_URL: http://10.0.20.106:8001/v1
+      GPU_URL: http://10.0.10.106:9000
+      GPU_UI_URL: http://10.0.10.106:9000
+      LLM_URL: http://10.0.10.106:8001/v1
       LLM_MODEL: qwen2.5-32b-gptq
       LLM_API_KEY: sk-local
       LAKEHOUSE_HOST: 10.0.21.50

docker-compose.yml

@@ -32,9 +32,9 @@ services:
       SQLITE_FALLBACK_PATH: /data/atc-agents.db
       DOCLING_URL: http://docling-serve:5001
       PRESENTATIONS_DIR: /data/presentations
-      GPU_URL: http://10.0.20.106:9000
-      GPU_UI_URL: http://10.0.20.106:9000
-      LLM_URL: http://10.0.20.106:8001/v1
+      GPU_URL: http://10.0.10.106:9000
+      GPU_UI_URL: http://10.0.10.106:9000
+      LLM_URL: http://10.0.10.106:8001/v1
       LLM_MODEL: gpt-4o
       LLM_API_KEY: sk-local
       LAKEHOUSE_HOST: 10.0.21.50
@@ -44,10 +44,6 @@ services:
       OPENMETADATA_URL: ${OPENMETADATA_URL:-http://10.0.21.47:8585}
       KAFKA_UI_URL: http://10.0.21.36:9000
       HDFS_NN_URL: http://10.0.21.61:9870
-      S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020}
-      S3_ACCESS_KEY: ${S3_ACCESS_KEY:-object_admin1}
-      S3_SECRET_KEY: ${S3_SECRET_KEY}
-      S3_REGION: ${S3_REGION:-us-east-1}
       ELASTICSEARCH_URL: ${ELASTICSEARCH_URL:-https://10.0.21.46:9200}
       KIBANA_URL: ${KIBANA_URL:-http://10.0.21.46:5601}
       ELASTIC_USER: ${ELASTIC_USER:-elastic}
@@ -94,7 +90,7 @@ services:
       CHROMA_HOST: chromadb
       CHROMA_PORT: 8000
       DOCLING_URL: http://docling-serve:5001
-      LLM_URL: http://10.0.20.106:8001/v1
+      LLM_URL: http://10.0.10.106:8001/v1
       LLM_MODEL: gpt-4o
       LLM_API_KEY: sk-local
       RAG_DATA_DIR: /data
@@ -124,10 +120,10 @@ services:
   jupyter:
     image: quay.io/jupyter/scipy-notebook:latest
     restart: unless-stopped
+    env_file:
+      - atc.env
     environment:
       JUPYTER_TOKEN: ${JUPYTER_TOKEN:-atc-jupyter}
-      AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-object_admin1}
-      AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY}
       S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020}
       AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1}
       ELASTICSEARCH_URL: ${ELASTICSEARCH_URL:-https://10.0.21.46:9200}

ui/src/App.tsx

@@ -1,4 +1,6 @@
 import { useRef, useState } from 'react'
+import { useAuth } from './hooks/useAuth'
+import { LoginView } from './components/features/LoginView'
 import { ChevronDown, ChevronUp } from 'lucide-react'
 import { useClock } from './hooks/useClock'
 import { useCommandCenter } from './hooks/useCommandCenter'
@@ -27,6 +29,7 @@ import { resolveInfraNode } from './lib/infraCatalog'
 import { cn } from './lib/utils'
 
 export default function App() {
+  const auth = useAuth()
   const clock = useClock()
   const cc = useCommandCenter()
   const [gpuChatActive, setGpuChatActive] = useState(false)
@@ -53,6 +56,19 @@ export default function App() {
     return 'Lab'
   })()
 
+  if (auth.status === 'loading') {
+    return (
+      <div className="flex h-full min-h-screen items-center justify-center bg-surface text-sm text-foreground-muted">
+        Checking session…
+      </div>
+    )
+  }
+  if (auth.status === 'anon') {
+    return <LoginView />
+  }
+
+  const userLabel = auth.user?.name || auth.user?.preferred_username || auth.user?.email || 'Signed in'
+
   return (
     <div className="flex h-full flex-col overflow-hidden bg-surface">
       <TopBar
@@ -62,6 +78,8 @@ export default function App() {
         agents={cc.agents}
         approvals={cc.approvals}
         onApprovalsClick={openApprovals}
+        userLabel={userLabel}
+        onLogout={() => { window.location.href = '/auth/logout' }}
       />
 
       <div className="flex min-h-0 flex-1 overflow-hidden">

ui/src/components/features/ChangesView.tsx

@@ -1,14 +1,14 @@
-import { useCallback, useEffect, useMemo, useState } from 'react'
-import { Activity, Radio, RefreshCw } from 'lucide-react'
-import { fetchChanges, fetchChangeStats } from '../../lib/api'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import { Activity, Radio, RefreshCw, PlusCircle, Pencil, Trash2, Database, Layers, TrendingUp, Cable } from 'lucide-react'
+import { fetchChanges, fetchChangeStats, resyncSources } from '../../lib/api'
 import type { CdcChange, CdcStats } from '../../types'
 import { cn } from '../../lib/utils'
 
-const OP_STYLE: Record<string, { label: string; cls: string }> = {
-  insert: { label: 'INSERT', cls: 'bg-emerald-500/15 text-emerald-300 border-emerald-500/30' },
-  update: { label: 'UPDATE', cls: 'bg-amber-500/15 text-amber-300 border-amber-500/30' },
-  delete: { label: 'DELETE', cls: 'bg-rose-500/15 text-rose-300 border-rose-500/30' },
-  snapshot: { label: 'SNAPSHOT', cls: 'bg-sky-500/15 text-sky-300 border-sky-500/30' },
+const OP_STYLE: Record<string, { label: string; cls: string; color: string }> = {
+  insert: { label: 'INSERT', cls: 'bg-emerald-500/15 text-emerald-300 border-emerald-500/30', color: '#34d399' },
+  update: { label: 'UPDATE', cls: 'bg-amber-500/15 text-amber-300 border-amber-500/30', color: '#fbbf24' },
+  delete: { label: 'DELETE', cls: 'bg-rose-500/15 text-rose-300 border-rose-500/30', color: '#fb7185' },
+  snapshot: { label: 'SNAPSHOT', cls: 'bg-sky-500/15 text-sky-300 border-sky-500/30', color: '#38bdf8' },
 }
 
 const SOURCE_COLOR: Record<string, string> = {
@@ -21,9 +21,15 @@ const SOURCE_COLOR: Record<string, string> = {
 
 const SOURCES = ['all', 'postgres', 'mysql', 'mongodb', 'cassandra', 'neo4j']
 const OPS = ['all', 'insert', 'update', 'delete']
+const TIME_WINDOWS = [
+  { label: '15 min', minutes: 15 },
+  { label: '1 hour', minutes: 60 },
+  { label: '6 hours', minutes: 360 },
+  { label: '24 hours', minutes: 1440 },
+] as const
 
 function opOf(o: string) {
-  return OP_STYLE[o] || { label: o.toUpperCase(), cls: 'bg-slate-500/15 text-slate-300 border-slate-500/30' }
+  return OP_STYLE[o] || { label: o.toUpperCase(), cls: 'bg-slate-500/15 text-slate-300 border-slate-500/30', color: '#94a3b8' }
 }
 
 function timeAgo(ts: string) {
@@ -34,6 +40,147 @@ function timeAgo(ts: string) {
   return `${Math.floor(d / 3600000)}h ago`
 }
 
+// Smoothly animates a number toward its target so counters tick up nicely.
+function useTween(target: number, ms = 700) {
+  const [val, setVal] = useState(target)
+  const from = useRef(target)
+  const start = useRef(0)
+  const raf = useRef(0)
+  useEffect(() => {
+    from.current = val
+    start.current = performance.now()
+    const step = (now: number) => {
+      const t = Math.min(1, (now - start.current) / ms)
+      const eased = 1 - Math.pow(1 - t, 3)
+      setVal(from.current + (target - from.current) * eased)
+      if (t < 1) raf.current = requestAnimationFrame(step)
+    }
+    raf.current = requestAnimationFrame(step)
+    return () => cancelAnimationFrame(raf.current)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [target, ms])
+  return val
+}
+
+function KpiCard({ label, value, accent, icon: Icon, sub }: {
+  label: string; value: number; accent: string; icon: typeof PlusCircle; sub?: string
+}) {
+  const v = useTween(value)
+  return (
+    <div className="relative overflow-hidden rounded-lg border border-border/60 bg-surface-raised p-3">
+      <div className="absolute -right-3 -top-3 h-14 w-14 rounded-full opacity-[0.12] blur-xl" style={{ background: accent }} />
+      <div className="flex items-center gap-1.5 text-[9px] uppercase tracking-wide text-foreground-faint">
+        <Icon className="h-3 w-3" style={{ color: accent }} /> {label}
+      </div>
+      <div className="mt-1 text-2xl font-semibold tabular-nums text-foreground" style={{ textShadow: `0 0 18px ${accent}22` }}>
+        {Math.round(v).toLocaleString()}
+      </div>
+      {sub && <div className="text-[10px] text-foreground-muted">{sub}</div>}
+    </div>
+  )
+}
+
+// SVG donut for the operation mix.
+function Donut({ segments, total }: { segments: { label: string; value: number; color: string }[]; total: number }) {
+  const R = 42
+  const C = 2 * Math.PI * R
+  let offset = 0
+  return (
+    <div className="flex items-center gap-4">
+      <svg viewBox="0 0 110 110" className="h-28 w-28 shrink-0 -rotate-90">
+        <circle cx="55" cy="55" r={R} fill="none" stroke="rgba(148,163,184,0.12)" strokeWidth="13" />
+        {total > 0 && segments.map((s) => {
+          const frac = s.value / total
+          const dash = frac * C
+          const el = (
+            <circle key={s.label} cx="55" cy="55" r={R} fill="none" stroke={s.color} strokeWidth="13"
+              strokeDasharray={`${dash} ${C - dash}`} strokeDashoffset={-offset} strokeLinecap="butt"
+              style={{ transition: 'stroke-dasharray .6s ease, stroke-dashoffset .6s ease' }} />
+          )
+          offset += dash
+          return el
+        })}
+        <g className="rotate-90" style={{ transformOrigin: '55px 55px' }}>
+          <text x="55" y="51" textAnchor="middle" className="fill-foreground text-[16px] font-semibold tabular-nums">{total.toLocaleString()}</text>
+          <text x="55" y="65" textAnchor="middle" className="fill-foreground-faint text-[7px] uppercase tracking-wider">changes</text>
+        </g>
+      </svg>
+      <div className="flex-1 space-y-1.5">
+        {segments.map((s) => (
+          <div key={s.label} className="flex items-center gap-2 text-[11px]">
+            <span className="h-2.5 w-2.5 rounded-sm" style={{ background: s.color }} />
+            <span className="capitalize text-foreground-muted">{s.label}</span>
+            <span className="ml-auto font-mono text-foreground">{s.value.toLocaleString()}</span>
+            <span className="w-9 text-right font-mono text-foreground-faint">{total ? Math.round((s.value / total) * 100) : 0}%</span>
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+
+// Smooth area chart of per-minute change volume.
+function VolumeArea({ buckets }: { buckets: { t: string; n: number }[] }) {
+  const w = 600
+  const h = 120
+  const pad = 6
+  const data = buckets.length ? buckets : [{ t: '', n: 0 }]
+  const max = Math.max(1, ...data.map((b) => b.n))
+  const stepX = data.length > 1 ? (w - pad * 2) / (data.length - 1) : 0
+  const pts = data.map((b, i) => {
+    const x = pad + i * stepX
+    const y = h - pad - (b.n / max) * (h - pad * 2)
+    return [x, y] as const
+  })
+  const line = pts.map(([x, y], i) => `${i ? 'L' : 'M'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
+  const area = `${line} L${pts[pts.length - 1][0].toFixed(1)},${h - pad} L${pts[0][0].toFixed(1)},${h - pad} Z`
+  const last = data[data.length - 1]
+  return (
+    <div>
+      <div className="relative">
+        <svg viewBox={`0 0 ${w} ${h}`} preserveAspectRatio="none" className="h-28 w-full">
+          <defs>
+            <linearGradient id="cdcvol" x1="0" y1="0" x2="0" y2="1">
+              <stop offset="0%" stopColor="#38bdf8" stopOpacity="0.45" />
+              <stop offset="100%" stopColor="#38bdf8" stopOpacity="0" />
+            </linearGradient>
+          </defs>
+          {[0.25, 0.5, 0.75].map((g) => (
+            <line key={g} x1={pad} x2={w - pad} y1={pad + g * (h - pad * 2)} y2={pad + g * (h - pad * 2)} stroke="rgba(148,163,184,0.08)" strokeWidth="1" />
+          ))}
+          {buckets.length > 0 && <path d={area} fill="url(#cdcvol)" />}
+          {buckets.length > 0 && <path d={line} fill="none" stroke="#38bdf8" strokeWidth="2" vectorEffect="non-scaling-stroke" />}
+          {buckets.length > 0 && (
+            <circle cx={pts[pts.length - 1][0]} cy={pts[pts.length - 1][1]} r="3.5" fill="#38bdf8">
+              <animate attributeName="r" values="3.5;6;3.5" dur="1.6s" repeatCount="indefinite" />
+            </circle>
+          )}
+        </svg>
+        <div className="pointer-events-none absolute left-1.5 top-1 text-[9px] font-mono text-foreground-faint">{max}/min</div>
+      </div>
+      <div className="mt-1 flex justify-between text-[9px] font-mono text-foreground-faint">
+        <span>{data[0]?.t || '—'}</span>
+        <span className="text-docker">now · {last?.n ?? 0}/min</span>
+      </div>
+      {buckets.length === 0 && <div className="mt-1 text-center text-[10px] text-foreground-faint">No changes in the window yet…</div>}
+    </div>
+  )
+}
+
+function BarRow({ label, value, max, color }: { label: string; value: number; max: number; color: string }) {
+  return (
+    <div className="flex items-center gap-2">
+      <span className="flex w-24 shrink-0 items-center gap-1.5 text-[11px] capitalize text-foreground-muted">
+        <span className="h-2 w-2 shrink-0 rounded-full" style={{ background: color }} /> {label}
+      </span>
+      <div className="h-2.5 flex-1 overflow-hidden rounded-full bg-surface">
+        <div className="h-full rounded-full transition-all duration-500" style={{ width: `${Math.max(3, (value / max) * 100)}%`, background: color }} />
+      </div>
+      <span className="w-12 shrink-0 text-right font-mono text-[11px] text-foreground">{value.toLocaleString()}</span>
+    </div>
+  )
+}
+
 function Diff({ change }: { change: CdcChange }) {
   const keys = useMemo(() => {
     const set = new Set<string>()
@@ -82,19 +229,81 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
   const [op, setOp] = useState('all')
   const [expanded, setExpanded] = useState<string | null>(null)
   const [connected, setConnected] = useState(false)
+  const [flash, setFlash] = useState(false)
+  const [resyncing, setResyncing] = useState(false)
+  const [resyncMsg, setResyncMsg] = useState<string | null>(null)
+  // Live overlay: CDC events counted straight off the WebSocket stream since the
+  // last server stats snapshot. The top KPIs/charts = authoritative server stats
+  // (refreshed every 2.5s) + this overlay, so they move in lock-step with the
+  // bottom feed instead of lagging behind it.
+  const emptyOverlay = { total: 0, by_op: {} as Record<string, number>, by_source: {} as Record<string, number>, by_table: {} as Record<string, number> }
+  const [overlay, setOverlay] = useState(emptyOverlay)
+  const lastSeenId = useRef<string | null>(null)
+  const primed = useRef(false)
+
+  const applyStats = useCallback((s: CdcStats | null) => {
+    if (!s) return
+    setStats(s)
+    setOverlay({ total: 0, by_op: {}, by_source: {}, by_table: {} })  // server is now authoritative
+  }, [])
 
   const load = useCallback(async () => {
-    const [c, s] = await Promise.all([fetchChanges({ limit: 150 }), fetchChangeStats(15)])
+    const [c, s] = await Promise.all([fetchChanges({ limit: 200 }), fetchChangeStats(15)])
     setSeed(c.changes)
     setConnected(c.connected)
-    if (s) setStats(s)
-  }, [])
+    applyStats(s)
+  }, [applyStats])
+
+  const doResync = useCallback(async () => {
+    setResyncing(true)
+    setResyncMsg('Restarting Debezium connectors…')
+    try {
+      const res = await resyncSources(true)
+      if (res) {
+        setResyncMsg(`Re-synced · ${res.healthy ?? 0}/${res.total ?? 0} connectors healthy`)
+        setTimeout(() => load(), 2500)
+      } else {
+        setResyncMsg('Re-sync failed — check ETL Guardian terminal')
+      }
+    } catch {
+      setResyncMsg('Re-sync failed — check ETL Guardian terminal')
+    } finally {
+      setResyncing(false)
+      setTimeout(() => setResyncMsg(null), 7000)
+    }
+  }, [load])
 
   useEffect(() => {
     load()
-    const iv = setInterval(() => fetchChangeStats(15).then((s) => s && setStats(s)), 5000)
+    const iv = setInterval(() => fetchChangeStats(15).then((s) => applyStats(s)), 2500)
     return () => clearInterval(iv)
-  }, [load])
+  }, [load, applyStats])
+
+  // Fold freshly-arrived WS changes into the overlay → instant top-of-page update.
+  useEffect(() => {
+    if (!liveChanges.length) return
+    if (!primed.current) {
+      primed.current = true
+      lastSeenId.current = liveChanges[0].id
+      return
+    }
+    const idx = liveChanges.findIndex((c) => c.id === lastSeenId.current)
+    const fresh = idx === -1 ? liveChanges : liveChanges.slice(0, idx)
+    if (!fresh.length) return
+    lastSeenId.current = liveChanges[0].id
+    setOverlay((o) => {
+      const next = { total: o.total + fresh.length, by_op: { ...o.by_op }, by_source: { ...o.by_source }, by_table: { ...o.by_table } }
+      for (const c of fresh) {
+        next.by_op[c.op] = (next.by_op[c.op] || 0) + 1
+        next.by_source[c.source] = (next.by_source[c.source] || 0) + 1
+        next.by_table[c.table] = (next.by_table[c.table] || 0) + 1
+      }
+      return next
+    })
+    setFlash(true)
+    const id = setTimeout(() => setFlash(false), 800)
+    return () => clearTimeout(id)
+  }, [liveChanges])
 
   // Merge live (WS) with seeded backlog, dedupe by id, newest first.
   const merged = useMemo(() => {
@@ -109,78 +318,150 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
     [merged, source, op],
   )
 
-  const maxBucket = Math.max(1, ...(stats?.buckets || []).map((b) => b.n))
+  const opVal = (k: string) => (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0)
+  const inserts = opVal('insert')
+  const updates = opVal('update')
+  const deletes = opVal('delete')
+  const total = (stats?.total ?? 0) + overlay.total
+  const perMin = total / Math.max(1, stats?.window_minutes ?? 15)
+
+  const opSegments = useMemo(() => (
+    ['insert', 'update', 'delete', 'snapshot']
+      .map((k) => ({ label: k, value: (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0), color: opOf(k).color }))
+      .filter((s) => s.value > 0)
+  ), [stats?.by_op, overlay])
+
+  const sourceRows = useMemo(() => {
+    const m: Record<string, number> = { ...(stats?.by_source || {}) }
+    for (const [k, v] of Object.entries(overlay.by_source)) m[k] = (m[k] || 0) + v
+    return Object.entries(m).sort((a, b) => b[1] - a[1])
+  }, [stats?.by_source, overlay])
+  const maxSource = Math.max(1, ...sourceRows.map(([, v]) => v))
+
+  const tableRows = useMemo(() => {
+    const m: Record<string, number> = { ...(stats?.by_table || {}) }
+    for (const [k, v] of Object.entries(overlay.by_table)) m[k] = (m[k] || 0) + v
+    return Object.entries(m).sort((a, b) => b[1] - a[1]).slice(0, 7)
+  }, [stats?.by_table, overlay])
+  const maxTable = Math.max(1, ...tableRows.map(([, v]) => v))
+
+  // Volume chart: bump the current-minute bar with the live overlay so the curve
+  // visibly rises as changes stream in.
+  const liveBuckets = useMemo(() => {
+    const b = (stats?.buckets || []).map((x) => ({ ...x }))
+    if (overlay.total) {
+      if (b.length) b[b.length - 1] = { ...b[b.length - 1], n: b[b.length - 1].n + overlay.total }
+      else b.push({ t: 'now', n: overlay.total })
+    }
+    return b
+  }, [stats?.buckets, overlay.total])
 
   return (
-    <div className="flex h-full min-h-0 flex-col gap-3">
+    <div className="flex h-full min-h-0 flex-col gap-3 overflow-y-auto scrollbar-thin pr-1">
       {/* Header */}
-      <div className="flex items-center justify-between">
+      <div className="flex shrink-0 items-center justify-between">
         <div>
           <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
-            <Activity className="h-4 w-4 text-docker" /> Live Changes · CDC Stream
+            <Activity className={cn('h-4 w-4 text-docker', flash && 'animate-pulse')} /> New &amp; Changed Data
           </h1>
           <p className="text-[11px] text-foreground-muted">
-            Real-time Debezium change data capture from all source databases via Kafka
+            Live Debezium change data capture — every insert, update &amp; delete across all source databases, streamed via Kafka
           </p>
         </div>
         <div className="flex items-center gap-3">
+          <div className="flex items-center gap-1 rounded border border-border/60 p-0.5">
+            {TIME_WINDOWS.map((w) => (
+              <button
+                key={w.minutes}
+                type="button"
+                onClick={() => setWindowMin(w.minutes)}
+                className={cn(
+                  'rounded px-2 py-0.5 text-[10px]',
+                  windowMin === w.minutes
+                    ? 'bg-docker/20 text-docker'
+                    : 'text-foreground-muted hover:text-foreground',
+                )}
+              >
+                {w.label}
+              </button>
+            ))}
+          </div>
           <span className={cn('flex items-center gap-1.5 rounded-full border px-2.5 py-1 text-[10px] font-medium',
             connected ? 'border-emerald-500/40 bg-emerald-500/10 text-emerald-300' : 'border-rose-500/40 bg-rose-500/10 text-rose-300')}>
             <Radio className={cn('h-3 w-3', connected && 'animate-pulse')} /> {connected ? 'STREAMING' : 'OFFLINE'}
           </span>
+          {resyncMsg && (
+            <span className="hidden text-[10px] text-amber-300/90 md:inline">{resyncMsg}</span>
+          )}
+          <button
+            type="button"
+            onClick={doResync}
+            disabled={resyncing}
+            title="Restart all Debezium source connectors so CDC catches up after a database outage"
+            className="flex items-center gap-1 rounded border border-amber-500/40 bg-amber-500/10 px-2 py-1 text-[10px] font-medium text-amber-300 hover:bg-amber-500/20 disabled:opacity-60"
+          >
+            <Cable className={cn('h-3 w-3', resyncing && 'animate-spin')} /> {resyncing ? 'Re-syncing…' : 'Re-sync sources'}
+          </button>
           <button type="button" onClick={load} className="flex items-center gap-1 rounded border border-border/60 px-2 py-1 text-[10px] text-foreground-muted hover:text-docker">
             <RefreshCw className="h-3 w-3" /> Refresh
           </button>
         </div>
       </div>
 
-      {/* Stat cards */}
+      {/* KPI row */}
       <div className="grid shrink-0 grid-cols-2 gap-2 lg:grid-cols-4">
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">Changes / 15 min</div>
-          <div className="text-xl font-semibold text-foreground">{stats?.total ?? 0}</div>
-        </div>
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">Total consumed</div>
-          <div className="text-xl font-semibold text-foreground">{stats?.consumed ?? 0}</div>
-        </div>
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">By operation</div>
-          <div className="mt-1 flex flex-wrap gap-1">
-            {Object.entries(stats?.by_op || {}).map(([k, v]) => (
-              <span key={k} className={cn('rounded border px-1.5 py-0.5 text-[9px]', opOf(k).cls)}>{opOf(k).label} {v}</span>
-            ))}
-            {!Object.keys(stats?.by_op || {}).length && <span className="text-[10px] text-foreground-faint">—</span>}
+        <KpiCard label="New records" value={inserts} accent="#34d399" icon={PlusCircle} sub="inserts · last 15m" />
+        <KpiCard label="Updates" value={updates} accent="#fbbf24" icon={Pencil} sub="modified rows · 15m" />
+        <KpiCard label="Deletes" value={deletes} accent="#fb7185" icon={Trash2} sub="removed rows · 15m" />
+        <KpiCard label="Throughput" value={Math.round(perMin)} accent="#38bdf8" icon={TrendingUp} sub={`changes/min · ${(stats?.consumed ?? 0).toLocaleString()} total consumed`} />
+      </div>
+
+      {/* Volume + operation mix */}
+      <div className="grid shrink-0 grid-cols-1 gap-2 lg:grid-cols-3">
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3 lg:col-span-2">
+          <div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <TrendingUp className="h-3 w-3" /> Change volume — last 15 minutes
           </div>
+          <VolumeArea buckets={liveBuckets} />
         </div>
         <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">By source</div>
-          <div className="mt-1 flex flex-wrap gap-1">
-            {Object.entries(stats?.by_source || {}).map(([k, v]) => (
-              <span key={k} className="flex items-center gap-1 rounded border border-border/60 px-1.5 py-0.5 text-[9px] text-foreground-muted">
-                <span className="h-2 w-2 rounded-full" style={{ background: SOURCE_COLOR[k] || '#94a3b8' }} />{k} {v}
-              </span>
-            ))}
-            {!Object.keys(stats?.by_source || {}).length && <span className="text-[10px] text-foreground-faint">—</span>}
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Layers className="h-3 w-3" /> Operation mix
           </div>
+          {opSegments.length ? <Donut segments={opSegments} total={total} /> : (
+            <div className="flex h-28 items-center justify-center text-[10px] text-foreground-faint">No changes yet…</div>
+          )}
         </div>
       </div>
 
-      {/* Volume sparkbars */}
-      <div className="shrink-0 rounded-lg border border-border/60 bg-surface-raised p-3">
-        <div className="mb-1.5 text-[9px] uppercase tracking-wide text-foreground-faint">Change volume per minute (last 15m)</div>
-        <div className="flex h-16 items-end gap-0.5">
-          {(stats?.buckets || []).map((b) => (
-            <div key={b.t} className="group relative flex-1" title={`${b.t}: ${b.n}`}>
-              <div className="w-full rounded-t bg-docker/70 transition-all group-hover:bg-docker" style={{ height: `${Math.max(4, (b.n / maxBucket) * 100)}%` }} />
-            </div>
-          ))}
-          {!(stats?.buckets || []).length && <div className="text-[10px] text-foreground-faint">No changes in the window yet…</div>}
+      {/* By system + top tables */}
+      <div className="grid shrink-0 grid-cols-1 gap-2 lg:grid-cols-2">
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Database className="h-3 w-3" /> New &amp; changed by system
+          </div>
+          <div className="space-y-2">
+            {sourceRows.length ? sourceRows.map(([k, v]) => (
+              <BarRow key={k} label={k} value={v} max={maxSource} color={SOURCE_COLOR[k] || '#94a3b8'} />
+            )) : <div className="text-[10px] text-foreground-faint">No source activity in the window…</div>}
+          </div>
+        </div>
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Layers className="h-3 w-3" /> Most active tables / collections
+          </div>
+          <div className="space-y-2">
+            {tableRows.length ? tableRows.map(([k, v]) => (
+              <BarRow key={k} label={k} value={v} max={maxTable} color="#818cf8" />
+            )) : <div className="text-[10px] text-foreground-faint">No table activity yet…</div>}
+          </div>
         </div>
       </div>
 
       {/* Filters */}
       <div className="flex shrink-0 flex-wrap items-center gap-2">
+        <span className="text-[10px] font-semibold uppercase tracking-wide text-foreground-muted">Latest changes</span>
+        <span className="mx-1 h-3 w-px bg-border/60" />
         <span className="text-[9px] uppercase tracking-wide text-foreground-faint">Source</span>
         {SOURCES.map((s) => (
           <button key={s} type="button" onClick={() => setSource(s)}
@@ -200,10 +481,10 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
       </div>
 
       {/* Live list */}
-      <div className="min-h-0 flex-1 overflow-y-auto scrollbar-thin rounded-lg border border-border/60 bg-surface-raised">
+      <div className="min-h-[180px] rounded-lg border border-border/60 bg-surface-raised">
         {filtered.length === 0 && (
           <div className="p-6 text-center text-[11px] text-foreground-faint">
-            Waiting for changes… trigger data generation or agent DML to see live CDC events.
+            Waiting for changes… trigger data generation (Data Flow → Generate data) or agent DML to see live CDC events.
           </div>
         )}
         {filtered.map((c) => (

ui/src/components/features/DataExplorerView.tsx

@@ -16,12 +16,18 @@ import {
   HardDrive,
   ShieldCheck,
   Radio,
+  GitBranch,
+  Lock,
 } from 'lucide-react'
 import { cn } from '../../lib/utils'
 import { TrinoFederationView, type SubTab } from './TrinoFederationView'
 import { LiveDashboard } from './LiveDashboard'
+import { LineageView } from './LineageView'
+import { GovernanceOwnershipView } from './GovernanceOwnershipView'
+import { GovernanceAccessView } from './GovernanceAccessView'
+import { ObservabilityView } from './ObservabilityView'
 
-type ExplorerTab = 'business' | 'live' | SubTab
+type ExplorerTab = 'business' | 'live' | SubTab | 'lineage' | 'ownership' | 'access' | 'observability'
 
 const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string; live?: boolean }[] = [
   { id: 'business', label: 'Business Overview', icon: BarChart3, hint: 'Customers, orders, workforce, supply chain & telemetry across every source' },
@@ -29,6 +35,10 @@ const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string;
   { id: 'federated', label: 'Federated (Trino)', icon: Network, hint: 'One SQL across all 5 databases + region scorecard joined live' },
   { id: 'lake', label: 'Hadoop Lake', icon: HardDrive, hint: 'All business data mirrored as external Iceberg tables on HDFS' },
   { id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck, hint: 'Every table & column with PII / masking status — exactly what the assistant sees' },
+  { id: 'lineage', label: 'Lineage', icon: GitBranch, hint: 'End-to-end data lineage with column-level PII tracing from source to curated layer' },
+  { id: 'ownership', label: 'Ownership', icon: UserCog, hint: 'Data owners, stewards, tiers & business glossary — accountability per dataset' },
+  { id: 'access', label: 'Access & Policies', icon: Lock, hint: 'Governance posture: PII masking, ownership, live DQ & alerts vs each data contract' },
+  { id: 'observability', label: 'Observability', icon: Activity, hint: 'Volume, freshness & schema-drift monitoring with live alerts across every table' },
 ]
 
 type Bucket = { key: string; count: number; value?: number }
@@ -295,6 +305,12 @@ export function DataExplorerView() {
       {/* Trino federation / lake / dictionary tabs */}
       {(view === 'federated' || view === 'lake' || view === 'dictionary') && <TrinoFederationView embedded activeTab={view} />}
 
+      {/* Governance / lineage / observability sub-tabs */}
+      {view === 'lineage' && <div className="flex min-h-0 flex-1 flex-col"><LineageView /></div>}
+      {view === 'ownership' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceOwnershipView /></div>}
+      {view === 'access' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceAccessView /></div>}
+      {view === 'observability' && <div className="flex min-h-0 flex-1 flex-col"><ObservabilityView /></div>}
+
       {/* ───────── BUSINESS OVERVIEW ───────── */}
       {view === 'business' && (
       <>

ui/src/components/features/DataFlowView.tsx

@@ -206,10 +206,43 @@ export function DataFlowView() {
   }, [genRows, load])
 
   const flowMode = (graph as unknown as { flow?: string })?.flow ?? 'running'
+
+  const heartbeat = useCallback((active: boolean) => {
+    return fetch('/api/federated/live/heartbeat', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ active }),
+    }).catch(() => {})
+  }, [])
+
   const onFlow = useCallback(async (action: 'pause' | 'resume' | 'stop') => {
     await setStreamingFlow(action)
-    setTimeout(() => load(true), 300)
-  }, [load])
+    // The master pulse also drives the live data generator: Run starts the
+    // stream into the sources, Pause/Stop halts it.
+    await heartbeat(action === 'resume')
+    if (action === 'resume') {
+      try {
+        const r = await fetch('/api/federated/generate', {
+          method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ rows: 800 }),
+        })
+        const d = await r.json()
+        const i = d?.inserted || {}
+        setGenToast(`Generator running — seeded +${(i.orders ?? 0).toLocaleString()} orders, +${(i.telemetry ?? 0).toLocaleString()} telemetry, +${(i.hr_events ?? 0).toLocaleString()} HR, +${(i.supply_events ?? 0).toLocaleString()} supply; now streaming every few seconds via CDC`)
+        setTimeout(() => setGenToast(null), 7000)
+      } catch { /* ignore */ }
+    }
+    setTimeout(() => load(true), 400)
+  }, [load, heartbeat])
+
+  // While the flow is running and this tab is open, keep the generator alive
+  // (it stops a few seconds after you pause/stop or leave the tab).
+  useEffect(() => {
+    if (flowMode !== 'running') {
+      heartbeat(false)
+      return
+    }
+    heartbeat(true)
+    const t = setInterval(() => heartbeat(true), 5000)
+    return () => clearInterval(t)
+  }, [flowMode, heartbeat])
 
   const [maskBusy, setMaskBusy] = useState<string | null>(null)
   const onToggleMask = useCallback(async (key: string, column: string, masked: boolean) => {
@@ -334,7 +367,7 @@ export function DataFlowView() {
             >
               <FileCode2 className="h-3 w-3" /> Scripts
             </button>
-            <div className="inline-flex items-center gap-0.5 rounded border border-border p-0.5" title="Master pulse control">
+            <div className="inline-flex items-center gap-0.5 rounded border border-border p-0.5" title="Master pulse + live generator: Run streams data into the sources, Pause/Stop halts it">
               <button type="button" onClick={() => onFlow('resume')}
                 className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'running' ? 'bg-emerald-500/25 text-emerald-200' : 'text-foreground-muted hover:text-foreground')}>
                 <Play className="h-3 w-3" /> Run

ui/src/components/features/DataQualityView.tsx

@@ -11,9 +11,11 @@ import {
   Table2,
   Upload,
   XCircle,
+  Gauge,
 } from 'lucide-react'
 import { cn } from '../../lib/utils'
 import { subTabActive, subTabIdle } from '../../lib/tabActive'
+import { DqMonitoringPanel } from './DqMonitoringPanel'
 
 type Dimension = {
   id: string
@@ -167,13 +169,13 @@ type ReportSummary = {
   columns: number
 }
 
-type Tab = 'assess' | 'docling' | 'reports'
+type Tab = 'assess' | 'docling' | 'reports' | 'monitoring'
 
 const SCORE_COLOR = (s: number) => (s >= 80 ? 'text-success' : s >= 60 ? 'text-warning' : 'text-danger')
 const BAR_COLOR = (s: number) => (s >= 80 ? 'bg-success' : s >= 60 ? 'bg-warning' : 'bg-danger')
 
 export function DataQualityView() {
-  const [tab, setTab] = useState<Tab>('assess')
+  const [tab, setTab] = useState<Tab>('monitoring')
   const [caps, setCaps] = useState<Capabilities | null>(null)
   const [loading, setLoading] = useState(false)
   const [assess, setAssess] = useState<AssessResult | null>(null)
@@ -252,6 +254,7 @@ export function DataQualityView() {
   }
 
   const tabs: { id: Tab; label: string; icon: typeof FileSearch }[] = [
+    { id: 'monitoring', label: 'Live Monitoring', icon: Gauge },
     { id: 'assess', label: 'Maturity Assessment', icon: FileSearch },
     { id: 'docling', label: 'Docling Parser', icon: FileText },
     { id: 'reports', label: 'Reports', icon: CheckCircle2 },
@@ -312,6 +315,8 @@ export function DataQualityView() {
           </div>
         )}
 
+        {tab === 'monitoring' && <DqMonitoringPanel />}
+
         {tab === 'assess' && (
           <div className="space-y-5">
             <UploadZone

ui/src/components/features/DqMonitoringPanel.tsx

@@ -0,0 +1,173 @@
+import { useCallback, useEffect, useState } from 'react'
+import { Activity, RefreshCw, Loader2, Play, Gauge, AlertTriangle, Database } from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type Trend = { t: string; score: number }
+type Card = {
+  key: string; label: string; engine: string; color: string; table: string; domain?: string
+  score: number | null; score_color: string; dimensions: Record<string, number>
+  volume: number | null; volume_delta: number | null; freshness_age_min: number | null
+  issues: string[]; columns?: number; worst_columns?: { name: string; completeness: number; nulls: number }[]
+  trend: Trend[]; pending?: boolean; error?: string
+}
+type Resp = {
+  ok: boolean; enabled: boolean; running: boolean; cycles: number; sample: number
+  platform_score: number | null; dimension_averages: Record<string, number>; cards: Card[]
+  feed: { ts: string; text: string; level: string }[]
+}
+
+const DIMS = ['completeness', 'uniqueness', 'validity', 'freshness']
+
+function ScoreRing({ score, color }: { score: number | null; color: string }) {
+  const r = 26
+  const c = 2 * Math.PI * r
+  const pct = score == null ? 0 : score / 100
+  return (
+    <div className="relative h-16 w-16 shrink-0">
+      <svg viewBox="0 0 64 64" className="h-16 w-16 -rotate-90">
+        <circle cx="32" cy="32" r={r} fill="none" stroke="currentColor" strokeWidth="6" className="text-surface-overlay" />
+        <circle cx="32" cy="32" r={r} fill="none" stroke={color} strokeWidth="6" strokeLinecap="round"
+          strokeDasharray={`${pct * c} ${c}`} />
+      </svg>
+      <div className="absolute inset-0 flex items-center justify-center">
+        <span className="text-[13px] font-bold text-foreground">{score == null ? '—' : Math.round(score)}</span>
+      </div>
+    </div>
+  )
+}
+
+function TrendLine({ data, color }: { data: Trend[]; color: string }) {
+  const pts = data.slice(-30)
+  if (pts.length < 2) return <div className="h-8" />
+  const w = 200
+  const h = 32
+  const step = w / (pts.length - 1)
+  const line = pts.map((p, i) => `${i === 0 ? 'M' : 'L'}${(i * step).toFixed(1)},${(h - (p.score / 100) * (h - 4) - 2).toFixed(1)}`).join(' ')
+  return (
+    <svg viewBox={`0 0 ${w} ${h}`} className="h-8 w-full" preserveAspectRatio="none">
+      <path d={line} fill="none" stroke={color} strokeWidth="1.5" />
+    </svg>
+  )
+}
+
+export function DqMonitoringPanel() {
+  const [data, setData] = useState<Resp | null>(null)
+  const [loading, setLoading] = useState(true)
+  const [running, setRunning] = useState(false)
+
+  const load = useCallback(async () => {
+    try {
+      const r = await fetch('/api/dq/scorecards')
+      if (r.ok) setData(await r.json())
+    } catch { /* */ } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => { load() }, [load])
+  useEffect(() => { const t = setInterval(load, 10000); return () => clearInterval(t) }, [load])
+
+  const runNow = async () => {
+    setRunning(true)
+    try {
+      const r = await fetch('/api/dq/run', { method: 'POST' })
+      if (r.ok) { const j = await r.json(); if (j.scorecards) setData(j.scorecards) }
+    } catch { /* */ } finally {
+      setRunning(false)
+    }
+  }
+
+  const dimColor = (v: number) => (v >= 90 ? '#34d399' : v >= 75 ? '#fbbf24' : v >= 50 ? '#fb923c' : '#f87171')
+
+  return (
+    <div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto">
+      {/* header */}
+      <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4 xl:grid-cols-6">
+        <div className="panel col-span-2 flex items-center gap-3 px-3 py-2.5">
+          <ScoreRing score={data?.platform_score ?? null} color={data?.platform_score != null && data.platform_score >= 80 ? '#34d399' : '#fbbf24'} />
+          <div>
+            <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">Platform DQ score</p>
+            <p className="text-2xl font-bold leading-tight text-foreground">{data?.platform_score ?? '—'}</p>
+            <p className="text-[9px] text-foreground-faint">{data?.cycles ?? 0} cycles · live via Trino</p>
+          </div>
+        </div>
+        {DIMS.map((dim) => {
+          const v = data?.dimension_averages?.[dim]
+          return (
+            <div key={dim} className="panel flex flex-col justify-center px-3 py-2.5">
+              <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{dim}</p>
+              <p className="text-lg font-bold leading-tight" style={{ color: v != null ? dimColor(v) : undefined }}>{v ?? '—'}</p>
+            </div>
+          )
+        })}
+      </div>
+
+      <div className="flex shrink-0 items-center gap-2 px-1">
+        <span className="text-[10px] text-foreground-faint">
+          Continuous quality checks on live tables (completeness · uniqueness/dedup · validity · freshness){data?.running && ' · running…'}
+        </span>
+        <div className="ml-auto flex items-center gap-2">
+          <button type="button" onClick={runNow} disabled={running || data?.running} className="inline-flex items-center gap-1.5 rounded-md border border-docker/40 bg-docker/10 px-2.5 py-1 text-[10px] font-medium text-docker hover:bg-docker/20 disabled:opacity-60">
+            {running ? <Loader2 className="h-3 w-3 animate-spin" /> : <Play className="h-3 w-3" />} Run now
+          </button>
+          <button type="button" onClick={load} className="inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
+            {loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
+          </button>
+        </div>
+      </div>
+
+      {/* scorecards */}
+      <div className="grid gap-2 pb-2 lg:grid-cols-2">
+        {(data?.cards || []).map((c) => (
+          <div key={c.key} className="panel p-3">
+            <div className="flex items-start gap-3">
+              <ScoreRing score={c.score} color={c.score_color} />
+              <div className="min-w-0 flex-1">
+                <div className="flex items-center gap-1.5">
+                  <Database className="h-3.5 w-3.5" style={{ color: c.color }} />
+                  <span className="text-[12px] font-semibold text-foreground">{c.label}</span>
+                  <span className="ml-auto text-[8px] text-foreground-faint">{c.engine}</span>
+                </div>
+                <p className="truncate font-mono text-[8px] text-foreground-faint">{c.table}</p>
+                {c.pending ? (
+                  <p className="mt-2 text-[10px] text-foreground-faint">Awaiting first cycle…</p>
+                ) : c.error ? (
+                  <p className="mt-2 flex items-center gap-1 text-[10px] text-rose-400"><AlertTriangle className="h-3 w-3" /> {c.error}</p>
+                ) : (
+                  <div className="mt-1.5 space-y-1">
+                    {DIMS.filter((d) => c.dimensions[d] != null).map((d) => (
+                      <div key={d} className="flex items-center gap-2 text-[9px]">
+                        <span className="w-20 shrink-0 capitalize text-foreground-muted">{d}</span>
+                        <div className="relative h-2 flex-1 overflow-hidden rounded bg-surface-overlay">
+                          <div className="h-full rounded" style={{ width: `${c.dimensions[d]}%`, backgroundColor: dimColor(c.dimensions[d]) }} />
+                        </div>
+                        <span className="w-8 shrink-0 text-right font-mono text-foreground">{c.dimensions[d]}</span>
+                      </div>
+                    ))}
+                  </div>
+                )}
+              </div>
+            </div>
+            <div className="mt-2 flex items-center justify-between gap-2">
+              <div className="flex flex-wrap items-center gap-2 text-[9px] text-foreground-muted">
+                {c.volume != null && <span className="flex items-center gap-1"><Gauge className="h-3 w-3" /> {c.volume.toLocaleString()} rows</span>}
+                {c.volume_delta != null && c.volume_delta !== 0 && (
+                  <span className={c.volume_delta > 0 ? 'text-emerald-400' : 'text-rose-400'}>{c.volume_delta > 0 ? '+' : ''}{c.volume_delta.toLocaleString()}</span>
+                )}
+                {c.freshness_age_min != null && <span><Activity className="mr-1 inline h-3 w-3" />{c.freshness_age_min.toFixed(0)}m</span>}
+              </div>
+              <div className="w-1/3"><TrendLine data={c.trend} color={c.score_color} /></div>
+            </div>
+            {c.issues && c.issues.length > 0 && (
+              <div className="mt-1.5 flex flex-wrap gap-1">
+                {c.issues.map((iss, i) => (
+                  <span key={i} className="rounded bg-amber-500/15 px-1.5 py-0.5 text-[8px] text-amber-300">{iss}</span>
+                ))}
+              </div>
+            )}
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}

ui/src/components/features/GovernanceAccessView.tsx

@@ -0,0 +1,117 @@
+import { useCallback, useEffect, useState } from 'react'
+import { ShieldCheck, RefreshCw, Loader2, Lock, Unlock, Check, X, FileCheck2, AlertTriangle } from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type Check = { name: string; ok: boolean; value: unknown; target: unknown }
+type Pii = { pii_count: number; all_masked: boolean; masked: number; unmasked: number }
+type Alert = { type: string; severity: string; message: string }
+type Posture = {
+  key: string; label: string; engine: string; color: string; table: string
+  owner?: string | null; steward?: string | null; tier?: string | null
+  pii: Pii; dq_score?: number | null; issues: string[]; alerts: Alert[]
+  contract: Record<string, number>; checks: Check[]; compliant: boolean
+}
+type Resp = { ok: boolean; datasets: Posture[]; summary: { total: number; compliant: number; non_compliant: number } }
+
+export function GovernanceAccessView() {
+  const [data, setData] = useState<Resp | null>(null)
+  const [loading, setLoading] = useState(true)
+
+  const load = useCallback(async () => {
+    setLoading(true)
+    try {
+      const r = await fetch('/api/governance/posture')
+      if (r.ok) setData(await r.json())
+    } catch { /* */ } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => { load() }, [load])
+  useEffect(() => { const t = setInterval(load, 12000); return () => clearInterval(t) }, [load])
+
+  const sev = (s: string) => (s === 'critical' ? 'text-rose-400' : s === 'warning' ? 'text-amber-400' : 'text-sky-400')
+
+  return (
+    <div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto">
+      <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
+        <Kpi icon={FileCheck2} label="Compliant datasets" value={data ? `${data.summary.compliant}/${data.summary.total}` : '—'}
+          accent={data && data.summary.non_compliant === 0 ? '#34d399' : '#fbbf24'} />
+        <Kpi icon={AlertTriangle} label="Non-compliant" value={data ? String(data.summary.non_compliant) : '—'}
+          accent={data && data.summary.non_compliant ? '#f87171' : '#34d399'} />
+        <Kpi icon={Lock} label="Masking policy" value="Enforced" accent="#60a5fa" />
+        <Kpi icon={ShieldCheck} label="Contracts" value="Active" accent="#a78bfa" />
+      </div>
+
+      <div className="flex shrink-0 items-center px-1">
+        <span className="text-[10px] text-foreground-faint">Governance posture = ownership + PII masking + live DQ + observability alerts vs each data contract</span>
+        <button type="button" onClick={load} className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
+          {loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
+        </button>
+      </div>
+
+      <div className="grid gap-2 pb-2 lg:grid-cols-2">
+        {(data?.datasets || []).map((d) => (
+          <div key={d.key} className={cn('panel p-3', !d.compliant && 'ring-1 ring-rose-500/30')}>
+            <div className="mb-2 flex items-center gap-2">
+              <span className="h-2.5 w-2.5 rounded-full" style={{ backgroundColor: d.color }} />
+              <span className="text-[12px] font-semibold text-foreground">{d.label}</span>
+              <span className={cn('ml-auto inline-flex items-center gap-1 rounded-full px-2 py-0.5 text-[9px] font-medium',
+                d.compliant ? 'bg-emerald-500/15 text-emerald-300' : 'bg-rose-500/15 text-rose-300')}>
+                {d.compliant ? <Check className="h-3 w-3" /> : <X className="h-3 w-3" />} {d.compliant ? 'Compliant' : 'Action needed'}
+              </span>
+            </div>
+
+            <div className="grid grid-cols-2 gap-1.5">
+              {d.checks.map((c) => (
+                <div key={c.name} className="flex items-center gap-1.5 rounded bg-surface-overlay px-2 py-1 text-[10px]">
+                  {c.ok ? <Check className="h-3 w-3 shrink-0 text-emerald-400" /> : <X className="h-3 w-3 shrink-0 text-rose-400" />}
+                  <span className="flex-1 truncate text-foreground-muted">{c.name}</span>
+                  <span className={cn('font-mono', c.ok ? 'text-foreground' : 'text-rose-300')}>{String(c.value)}</span>
+                </div>
+              ))}
+            </div>
+
+            <div className="mt-2 flex flex-wrap items-center gap-2 text-[9px] text-foreground-muted">
+              <span>Owner: <span className="text-foreground">{d.owner || '—'}</span></span>
+              <span>· Tier: {d.tier || '—'}</span>
+              <span className="flex items-center gap-1">·
+                {d.pii.unmasked === 0
+                  ? <><Lock className="h-3 w-3 text-emerald-400" /> {d.pii.masked}/{d.pii.pii_count} PII masked</>
+                  : <><Unlock className="h-3 w-3 text-amber-400" /> {d.pii.unmasked} PII visible</>}
+              </span>
+            </div>
+
+            {d.alerts.length > 0 && (
+              <div className="mt-2 space-y-1 border-t border-border/50 pt-2">
+                {d.alerts.map((a, i) => (
+                  <p key={i} className={cn('flex items-center gap-1 text-[9px]', sev(a.severity))}>
+                    <AlertTriangle className="h-3 w-3" /> {a.message}
+                  </p>
+                ))}
+              </div>
+            )}
+
+            <p className="mt-2 text-[8px] text-foreground-faint">
+              Contract: DQ ≥ {d.contract.min_score} · completeness ≥ {d.contract.min_completeness}% · freshness ≤ {d.contract.min_freshness_min}m · crit alerts ≤ {d.contract.max_critical_alerts}
+            </p>
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+
+function Kpi({ icon: Icon, label, value, accent }: { icon: typeof ShieldCheck; label: string; value: string; accent: string }) {
+  return (
+    <div className="panel flex items-center gap-3 px-3 py-2.5">
+      <div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
+        <Icon className="h-4 w-4" />
+      </div>
+      <div className="min-w-0">
+        <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
+        <p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
+      </div>
+    </div>
+  )
+}

ui/src/components/features/GovernanceOwnershipView.tsx

@@ -0,0 +1,246 @@
+import { useCallback, useEffect, useState } from 'react'
+import { UserCircle, RefreshCw, Loader2, AlertTriangle, BookOpen, ShieldCheck, Check, X } from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type Pii = { pii_count: number; all_masked: boolean; masked: number; unmasked: number }
+type DsRow = {
+  key: string; label: string; engine: string; color: string; table: string
+  domain?: string; owner?: string | null; steward?: string | null; team?: string | null
+  tier?: string | null; classification?: string | null; updated_at?: string | null
+  orphan: boolean; pii: Pii
+}
+type Summary = { total: number; orphans: number; stewarded: number; owned: number }
+type GUser = { id: string | null; name: string; display: string; type: string }
+type Term = { name: string; description: string; domain?: string; related?: string[] }
+
+const TIERS = ['Tier1', 'Tier2', 'Tier3']
+const CLASSES = ['Public', 'Internal', 'Confidential', 'Restricted']
+
+export function GovernanceOwnershipView() {
+  const [rows, setRows] = useState<DsRow[]>([])
+  const [summary, setSummary] = useState<Summary | null>(null)
+  const [users, setUsers] = useState<GUser[]>([])
+  const [glossary, setGlossary] = useState<Term[]>([])
+  const [glossarySource, setGlossarySource] = useState('')
+  const [loading, setLoading] = useState(true)
+  const [omConnected, setOmConnected] = useState(false)
+  const [editKey, setEditKey] = useState<string | null>(null)
+  const [form, setForm] = useState<{ owner: string; steward: string; team: string; tier: string; classification: string }>(
+    { owner: '', steward: '', team: '', tier: '', classification: '' },
+  )
+  const [saving, setSaving] = useState(false)
+
+  const load = useCallback(async () => {
+    setLoading(true)
+    try {
+      const [d, u, g] = await Promise.all([
+        fetch('/api/governance/datasets').then((r) => r.json()),
+        fetch('/api/governance/users').then((r) => r.json()),
+        fetch('/api/governance/glossary').then((r) => r.json()),
+      ])
+      setRows(d.datasets || [])
+      setSummary(d.summary || null)
+      setOmConnected(!!d.om_connected)
+      setUsers(u.users || [])
+      setGlossary(g.terms || [])
+      setGlossarySource(g.source || '')
+    } catch { /* */ } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => { load() }, [load])
+
+  const openEdit = (r: DsRow) => {
+    setEditKey(r.key)
+    setForm({ owner: r.owner || '', steward: r.steward || '', team: r.team || '', tier: r.tier || '', classification: r.classification || '' })
+  }
+
+  const save = async () => {
+    if (!editKey) return
+    setSaving(true)
+    try {
+      await fetch('/api/governance/assign', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ key: editKey, ...form }),
+      })
+      setEditKey(null)
+      await load()
+    } catch { /* */ } finally {
+      setSaving(false)
+    }
+  }
+
+  const people = users.filter((u) => u.type === 'user')
+  const teams = users.filter((u) => u.type === 'team')
+
+  return (
+    <div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto">
+      {/* KPIs */}
+      <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
+        <Kpi icon={UserCircle} label="Owned" value={summary ? `${summary.owned}/${summary.total}` : '—'} accent="#34d399" />
+        <Kpi icon={AlertTriangle} label="Orphan datasets" value={summary ? String(summary.orphans) : '—'} accent={summary && summary.orphans ? '#f87171' : '#34d399'} />
+        <Kpi icon={ShieldCheck} label="Stewarded" value={summary ? String(summary.stewarded) : '—'} accent="#60a5fa" />
+        <Kpi icon={BookOpen} label="Glossary terms" value={String(glossary.length)} accent="#a78bfa" />
+      </div>
+
+      <div className="flex shrink-0 items-center gap-2 px-1">
+        <span className="text-[10px] text-foreground-faint">
+          OpenMetadata {omConnected ? 'connected' : 'offline'} · assignments stored locally{omConnected ? ' + synced to OM' : ''}
+        </span>
+        <button type="button" onClick={load} className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
+          {loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
+        </button>
+      </div>
+
+      {/* ownership matrix */}
+      <div className="panel min-h-0 shrink-0 overflow-x-auto p-0">
+        <table className="w-full text-[10px]">
+          <thead>
+            <tr className="border-b border-border text-left text-foreground-faint">
+              <th className="px-3 py-2 font-semibold">Dataset</th>
+              <th className="px-3 py-2 font-semibold">Owner</th>
+              <th className="px-3 py-2 font-semibold">Steward</th>
+              <th className="px-3 py-2 font-semibold">Team</th>
+              <th className="px-3 py-2 font-semibold">Tier</th>
+              <th className="px-3 py-2 font-semibold">PII</th>
+              <th className="px-3 py-2 font-semibold" />
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((r) => (
+              <tr key={r.key} className={cn('border-b border-border/50', r.orphan && 'bg-rose-500/5')}>
+                <td className="px-3 py-2">
+                  <div className="flex items-center gap-1.5">
+                    <span className="h-2 w-2 rounded-full" style={{ backgroundColor: r.color }} />
+                    <span className="font-medium text-foreground">{r.label}</span>
+                  </div>
+                  <span className="font-mono text-[8px] text-foreground-faint">{r.table}</span>
+                </td>
+                <td className="px-3 py-2">
+                  {r.owner ? <span className="text-foreground">{r.owner}</span>
+                    : <span className="flex items-center gap-1 text-rose-400"><AlertTriangle className="h-3 w-3" /> unassigned</span>}
+                </td>
+                <td className="px-3 py-2 text-foreground-muted">{r.steward || '—'}</td>
+                <td className="px-3 py-2 text-foreground-muted">{r.team || '—'}</td>
+                <td className="px-3 py-2">
+                  {r.tier ? <span className="rounded bg-surface-overlay px-1.5 py-0.5 text-foreground-muted">{r.tier}</span> : '—'}
+                </td>
+                <td className="px-3 py-2">
+                  {r.pii.pii_count > 0 ? (
+                    <span className={cn('rounded px-1.5 py-0.5', r.pii.unmasked === 0 ? 'bg-emerald-500/15 text-emerald-300' : 'bg-amber-500/15 text-amber-300')}>
+                      {r.pii.masked}/{r.pii.pii_count} masked
+                    </span>
+                  ) : <span className="text-foreground-faint">none</span>}
+                </td>
+                <td className="px-3 py-2 text-right">
+                  <button type="button" onClick={() => openEdit(r)} className="rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
+                    Assign
+                  </button>
+                </td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </div>
+
+      {/* glossary */}
+      <h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">
+        Business glossary {glossarySource && <span className="text-foreground-faint">· {glossarySource}</span>}
+      </h2>
+      <div className="grid shrink-0 gap-2 pb-2 md:grid-cols-2 lg:grid-cols-3">
+        {glossary.map((t) => (
+          <div key={t.name} className="panel p-2.5">
+            <div className="flex items-center gap-1.5">
+              <BookOpen className="h-3.5 w-3.5 text-docker" />
+              <span className="text-[11px] font-semibold text-foreground">{t.name}</span>
+              {t.domain && <span className="ml-auto rounded bg-surface-overlay px-1.5 py-0.5 text-[8px] text-foreground-faint">{t.domain}</span>}
+            </div>
+            <p className="mt-1 text-[10px] text-foreground-muted">{t.description}</p>
+            {t.related && t.related.length > 0 && (
+              <div className="mt-1.5 flex flex-wrap gap-1">
+                {t.related.map((rl) => <span key={rl} className="rounded bg-surface-overlay px-1.5 py-0.5 font-mono text-[8px] text-foreground-faint">{rl}</span>)}
+              </div>
+            )}
+          </div>
+        ))}
+      </div>
+
+      {/* assign modal */}
+      {editKey && (
+        <div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50 p-4" onClick={() => setEditKey(null)}>
+          <div className="panel w-full max-w-md p-4" onClick={(e) => e.stopPropagation()}>
+            <div className="mb-3 flex items-center justify-between">
+              <h3 className="text-[13px] font-semibold text-foreground">Assign ownership — {rows.find((r) => r.key === editKey)?.label}</h3>
+              <button type="button" onClick={() => setEditKey(null)} className="text-foreground-muted hover:text-foreground"><X className="h-4 w-4" /></button>
+            </div>
+            <div className="space-y-2.5">
+              <Field label="Owner">
+                <select value={form.owner} onChange={(e) => setForm({ ...form, owner: e.target.value })} className="w-full rounded border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground">
+                  <option value="">— unassigned —</option>
+                  {people.map((u) => <option key={u.name} value={u.display}>{u.display}</option>)}
+                </select>
+              </Field>
+              <Field label="Steward">
+                <select value={form.steward} onChange={(e) => setForm({ ...form, steward: e.target.value })} className="w-full rounded border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground">
+                  <option value="">— none —</option>
+                  {people.map((u) => <option key={u.name} value={u.display}>{u.display}</option>)}
+                </select>
+              </Field>
+              <Field label="Team">
+                <select value={form.team} onChange={(e) => setForm({ ...form, team: e.target.value })} className="w-full rounded border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground">
+                  <option value="">— none —</option>
+                  {teams.map((u) => <option key={u.name} value={u.display}>{u.display}</option>)}
+                </select>
+              </Field>
+              <div className="grid grid-cols-2 gap-2">
+                <Field label="Tier">
+                  <select value={form.tier} onChange={(e) => setForm({ ...form, tier: e.target.value })} className="w-full rounded border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground">
+                    <option value="">—</option>
+                    {TIERS.map((t) => <option key={t} value={t}>{t}</option>)}
+                  </select>
+                </Field>
+                <Field label="Classification">
+                  <select value={form.classification} onChange={(e) => setForm({ ...form, classification: e.target.value })} className="w-full rounded border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground">
+                    <option value="">—</option>
+                    {CLASSES.map((c) => <option key={c} value={c}>{c}</option>)}
+                  </select>
+                </Field>
+              </div>
+            </div>
+            <div className="mt-4 flex justify-end gap-2">
+              <button type="button" onClick={() => setEditKey(null)} className="rounded border border-border px-3 py-1.5 text-[11px] text-foreground-muted hover:bg-surface-overlay">Cancel</button>
+              <button type="button" onClick={save} disabled={saving} className="inline-flex items-center gap-1.5 rounded bg-docker px-3 py-1.5 text-[11px] font-medium text-white hover:bg-docker/90 disabled:opacity-60">
+                {saving ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Check className="h-3.5 w-3.5" />} Save
+              </button>
+            </div>
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+function Kpi({ icon: Icon, label, value, accent }: { icon: typeof UserCircle; label: string; value: string; accent: string }) {
+  return (
+    <div className="panel flex items-center gap-3 px-3 py-2.5">
+      <div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
+        <Icon className="h-4 w-4" />
+      </div>
+      <div className="min-w-0">
+        <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
+        <p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
+      </div>
+    </div>
+  )
+}
+
+function Field({ label, children }: { label: string; children: React.ReactNode }) {
+  return (
+    <div>
+      <label className="mb-1 block text-[9px] font-semibold uppercase tracking-wider text-foreground-faint">{label}</label>
+      {children}
+    </div>
+  )
+}

ui/src/components/features/GpuMatrixPanel.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useMemo, useState } from 'react'
-import { Activity, ChevronDown, ChevronUp, Cpu, ExternalLink, Zap } from 'lucide-react'
-import { fetchGpu } from '../../lib/api'
-import type { GpuDevice, GpuStatus } from '../../types'
+import { Activity, ChevronDown, ChevronUp, Cpu, ExternalLink, Settings2, Zap } from 'lucide-react'
+import { fetchGpu, fetchGpuConfig, resetGpuConfig, saveGpuConfig, testGpuConfig } from '../../lib/api'
+import type { GpuConfigPayload, GpuConfigTestResult, GpuDevice, GpuStatus } from '../../types'
 import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
 import { cn } from '../../lib/utils'
 
@@ -53,6 +53,13 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
   const [localGpu, setLocalGpu] = useState<GpuStatus | null>(gpu)
   const [lastPoll, setLastPoll] = useState<Date | null>(null)
   const [expanded, setExpanded] = useState(false)
+  const [showSettings, setShowSettings] = useState(false)
+  const [gpuConfig, setGpuConfig] = useState<GpuConfigPayload | null>(null)
+  const [selectedPreset, setSelectedPreset] = useState('gpu-prod')
+  const [customHost, setCustomHost] = useState('')
+  const [configBusy, setConfigBusy] = useState(false)
+  const [testResult, setTestResult] = useState<GpuConfigTestResult | null>(null)
+  const [configMsg, setConfigMsg] = useState<string | null>(null)
 
   useEffect(() => {
     setLocalGpu(gpu)
@@ -62,6 +69,19 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
     if (boost) setExpanded(true)
   }, [boost])
 
+  useEffect(() => {
+    if (!showSettings) return
+    fetchGpuConfig().then((cfg) => {
+      if (!cfg) return
+      setGpuConfig(cfg)
+      const active = cfg.active
+      setSelectedPreset(active.preset_id === 'env' ? 'gpu-prod' : active.preset_id)
+      if (active.preset_id === 'custom' || active.source === 'override') {
+        setCustomHost(active.host)
+      }
+    })
+  }, [showSettings])
+
   useEffect(() => {
     const poll = async () => {
       const g = await fetchGpu()
@@ -76,6 +96,49 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
     return () => clearInterval(iv)
   }, [boost])
 
+  const handleTestTarget = async () => {
+    setConfigBusy(true)
+    setTestResult(null)
+    setConfigMsg(null)
+    const body =
+      selectedPreset === 'custom'
+        ? { preset_id: 'custom', host: customHost.trim() }
+        : { preset_id: selectedPreset }
+    const result = await testGpuConfig(body)
+    setTestResult(result)
+    setConfigBusy(false)
+  }
+
+  const handleSaveTarget = async () => {
+    setConfigBusy(true)
+    setConfigMsg(null)
+    const body =
+      selectedPreset === 'custom'
+        ? { preset_id: 'custom', host: customHost.trim() }
+        : { preset_id: selectedPreset }
+    const res = await saveGpuConfig(body)
+    setConfigBusy(false)
+    if (res?.active) {
+      setConfigMsg(`Saved → ${res.active.label}`)
+      const g = await fetchGpu()
+      if (g) setLocalGpu(g)
+    } else {
+      setConfigMsg(res?.detail || 'Save failed')
+    }
+  }
+
+  const handleResetTarget = async () => {
+    setConfigBusy(true)
+    await resetGpuConfig()
+    setSelectedPreset('gpu-prod')
+    setCustomHost('')
+    setTestResult(null)
+    setConfigMsg('Reset to environment default')
+    const g = await fetchGpu()
+    if (g) setLocalGpu(g)
+    setConfigBusy(false)
+  }
+
   const g = localGpu
   const devices = g?.gpus || []
   const inferenceOn = g?.ok && g.inference_active
@@ -102,7 +165,31 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
         <h2 className="flex items-center gap-1.5 text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">
           <Cpu className="h-3 w-3" /> GPU Matrix
         </h2>
-        <p className="mt-1 text-[9px] text-foreground-faint">GPU Lab offline</p>
+        <p className="mt-1 text-[9px] text-foreground-faint">
+          GPU Lab offline{g?.host ? ` · ${g.ip || g.host}` : ''}
+        </p>
+        <button
+          type="button"
+          onClick={() => setShowSettings((v) => !v)}
+          className="mt-1 flex items-center gap-1 text-[8px] text-docker hover:underline"
+        >
+          <Settings2 className="h-2.5 w-2.5" /> GPU target
+        </button>
+        {showSettings && gpuConfig && (
+          <GpuTargetSettings
+            gpuConfig={gpuConfig}
+            selectedPreset={selectedPreset}
+            customHost={customHost}
+            configBusy={configBusy}
+            testResult={testResult}
+            configMsg={configMsg}
+            onPresetChange={setSelectedPreset}
+            onCustomHostChange={setCustomHost}
+            onTest={handleTestTarget}
+            onSave={handleSaveTarget}
+            onReset={handleResetTarget}
+          />
+        )}
       </section>
     )
   }
@@ -130,6 +217,17 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
           )}
         </button>
         <div className="flex shrink-0 items-center gap-0.5">
+          <button
+            type="button"
+            onClick={() => setShowSettings((v) => !v)}
+            className={cn(
+              'rounded p-1 hover:bg-surface-overlay',
+              showSettings ? 'text-docker' : 'text-foreground-muted hover:text-foreground',
+            )}
+            title="GPU target settings"
+          >
+            <Settings2 className="h-3 w-3" />
+          </button>
           {g.ui_url && (
             <a
               href={g.ui_url}
@@ -207,11 +305,121 @@ export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props)
           </div>
 
           <p className="font-mono text-[7px] text-foreground-faint">
-            {g.gpu_count ?? devices.length}× V100 · {g.host} · poll {boost ? '1s' : '3s'}
+            {g.gpu_count ?? devices.length}× V100 · {g.ip || g.host}
+            {g.config_label && g.config_source === 'override' ? ` · ${g.config_label}` : ''}
+            {' · poll '}{boost ? '1s' : '3s'}
             {lastPoll && ` · ${lastPoll.toLocaleTimeString()}`}
           </p>
+
+          {showSettings && gpuConfig && (
+            <GpuTargetSettings
+              gpuConfig={gpuConfig}
+              selectedPreset={selectedPreset}
+              customHost={customHost}
+              configBusy={configBusy}
+              testResult={testResult}
+              configMsg={configMsg}
+              onPresetChange={setSelectedPreset}
+              onCustomHostChange={setCustomHost}
+              onTest={handleTestTarget}
+              onSave={handleSaveTarget}
+              onReset={handleResetTarget}
+            />
+          )}
         </div>
       )}
     </section>
   )
 }
+
+type GpuTargetSettingsProps = {
+  gpuConfig: GpuConfigPayload
+  selectedPreset: string
+  customHost: string
+  configBusy: boolean
+  testResult: GpuConfigTestResult | null
+  configMsg: string | null
+  onPresetChange: (id: string) => void
+  onCustomHostChange: (host: string) => void
+  onTest: () => void
+  onSave: () => void
+  onReset: () => void
+}
+
+function GpuTargetSettings({
+  gpuConfig,
+  selectedPreset,
+  customHost,
+  configBusy,
+  testResult,
+  configMsg,
+  onPresetChange,
+  onCustomHostChange,
+  onTest,
+  onSave,
+  onReset,
+}: GpuTargetSettingsProps) {
+  return (
+    <div className="mt-1.5 rounded border border-border/80 bg-surface-overlay/40 p-2 space-y-1.5">
+      <p className="text-[8px] font-semibold uppercase tracking-wider text-foreground-faint">GPU Target</p>
+      <select
+        value={selectedPreset}
+        onChange={(e) => onPresetChange(e.target.value)}
+        className="w-full rounded border border-border bg-surface px-1.5 py-1 font-mono text-[9px] text-foreground"
+      >
+        {gpuConfig.presets.map((p) => (
+          <option key={p.id} value={p.id}>
+            {p.label} ({p.host})
+          </option>
+        ))}
+        <option value="custom">Custom IP…</option>
+      </select>
+      {selectedPreset === 'custom' && (
+        <input
+          type="text"
+          value={customHost}
+          onChange={(e) => onCustomHostChange(e.target.value)}
+          placeholder="10.0.x.x"
+          className="w-full rounded border border-border bg-surface px-1.5 py-1 font-mono text-[9px] text-foreground"
+        />
+      )}
+      <div className="flex flex-wrap gap-1">
+        <button
+          type="button"
+          disabled={configBusy}
+          onClick={onTest}
+          className="rounded border border-border px-2 py-0.5 font-mono text-[8px] text-foreground-muted hover:bg-surface-overlay disabled:opacity-50"
+        >
+          Test
+        </button>
+        <button
+          type="button"
+          disabled={configBusy}
+          onClick={onSave}
+          className="rounded border border-docker/40 bg-docker/10 px-2 py-0.5 font-mono text-[8px] text-docker hover:bg-docker/20 disabled:opacity-50"
+        >
+          Save
+        </button>
+        <button
+          type="button"
+          disabled={configBusy}
+          onClick={onReset}
+          className="rounded border border-border px-2 py-0.5 font-mono text-[8px] text-foreground-faint hover:bg-surface-overlay disabled:opacity-50"
+        >
+          Reset
+        </button>
+      </div>
+      {testResult && (
+        <p className={cn('font-mono text-[8px]', testResult.ok ? 'text-success' : 'text-warning')}>
+          {testResult.ok
+            ? `OK · ${testResult.gpu_count} GPU(s) · ${testResult.active_model || 'no model'}`
+            : `Failed · ${testResult.errors.join('; ') || 'unreachable'}`}
+        </p>
+      )}
+      {configMsg && <p className="font-mono text-[8px] text-foreground-muted">{configMsg}</p>}
+      <p className="font-mono text-[7px] text-foreground-faint">
+        Active: {gpuConfig.active.label} ({gpuConfig.active.host})
+      </p>
+    </div>
+  )
+}

ui/src/components/features/LineageView.tsx

@@ -0,0 +1,309 @@
+import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
+import { GitBranch, RefreshCw, Loader2, Database, Cpu, HardDrive, Layers, Network, Sparkles, Lock, ShieldCheck } from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type LCol = { name: string; category?: string; masked: boolean; pii?: boolean }
+type LNode = {
+  id: string
+  label: string
+  type: string
+  stage: number
+  meta: {
+    engine?: string
+    table?: string
+    rows?: number | null
+    note?: string
+    columns?: LCol[]
+    masked_layer?: boolean
+    domain?: string
+    topic?: string
+    path?: string
+    om?: { upstream?: number; downstream?: number }
+  }
+}
+type LEdge = { id: string; source: string; target: string; label: string; kind: string; active: boolean }
+type Graph = {
+  ok: boolean
+  generated_at: string
+  active: { generator: boolean; archive: boolean }
+  stages: string[]
+  nodes: LNode[]
+  edges: LEdge[]
+  column_links: { source: string; target: string; column: string; masked: boolean }[]
+  om_connected: boolean
+}
+type DsOpt = { key: string; label: string; engine: string; color: string; table: string }
+
+const TYPE_ICON: Record<string, typeof Database> = {
+  source: Database,
+  stream: Network,
+  compute: Cpu,
+  storage: HardDrive,
+  lakehouse: Layers,
+  engine: Network,
+  serving: Sparkles,
+}
+const KIND_COLOR: Record<string, string> = {
+  cdc: '#f472b6',
+  batch: '#fbbf24',
+  transform: '#a78bfa',
+  serve: '#38bdf8',
+}
+
+function fmtRows(n?: number | null) {
+  if (n == null) return null
+  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`
+  if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`
+  return String(n)
+}
+
+export function LineageView() {
+  const [graph, setGraph] = useState<Graph | null>(null)
+  const [datasets, setDatasets] = useState<DsOpt[]>([])
+  const [focus, setFocus] = useState<string>('')
+  const [loading, setLoading] = useState(true)
+  const [selected, setSelected] = useState<string | null>(null)
+
+  const wrapRef = useRef<HTMLDivElement>(null)
+  const nodeRefs = useRef<Map<string, HTMLDivElement>>(new Map())
+  const [coords, setCoords] = useState<Map<string, { x: number; y: number; w: number; h: number }>>(new Map())
+
+  const load = useCallback(async () => {
+    setLoading(true)
+    try {
+      const url = focus ? `/api/lineage/graph?dataset=${focus}` : '/api/lineage/graph'
+      const r = await fetch(url)
+      if (r.ok) setGraph(await r.json())
+    } catch { /* */ } finally {
+      setLoading(false)
+    }
+  }, [focus])
+
+  useEffect(() => {
+    fetch('/api/lineage/datasets').then((r) => r.json()).then((d) => setDatasets(d.datasets || [])).catch(() => {})
+  }, [])
+  useEffect(() => { load() }, [load])
+  useEffect(() => {
+    const t = setInterval(load, 15000)
+    return () => clearInterval(t)
+  }, [load])
+
+  // measure node positions for edge drawing
+  useLayoutEffect(() => {
+    if (!wrapRef.current || !graph) return
+    const measure = () => {
+      const wrap = wrapRef.current
+      if (!wrap) return
+      const base = wrap.getBoundingClientRect()
+      const next = new Map<string, { x: number; y: number; w: number; h: number }>()
+      nodeRefs.current.forEach((el, id) => {
+        const r = el.getBoundingClientRect()
+        next.set(id, { x: r.left - base.left + wrap.scrollLeft, y: r.top - base.top + wrap.scrollTop, w: r.width, h: r.height })
+      })
+      setCoords(next)
+    }
+    measure()
+    const ro = new ResizeObserver(measure)
+    if (wrapRef.current) ro.observe(wrapRef.current)
+    nodeRefs.current.forEach((el) => ro.observe(el))
+    return () => ro.disconnect()
+  }, [graph])
+
+  const stages = graph?.stages || []
+  const byStage: Record<number, LNode[]> = {}
+  ;(graph?.nodes || []).forEach((n) => { (byStage[n.stage] = byStage[n.stage] || []).push(n) })
+
+  const selNode = graph?.nodes.find((n) => n.id === selected) || null
+  const connectedEdges = new Set(
+    (graph?.edges || []).filter((e) => !selected || e.source === selected || e.target === selected).map((e) => e.id),
+  )
+
+  return (
+    <div className="flex h-full min-h-0 flex-col gap-2">
+      {/* controls */}
+      <div className="flex shrink-0 flex-wrap items-center gap-2 px-1">
+        <span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Trace dataset</span>
+        <button
+          type="button"
+          onClick={() => setFocus('')}
+          className={cn('rounded-full border px-2.5 py-1 text-[10px] font-medium', focus === '' ? 'border-docker bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:bg-surface-overlay')}
+        >
+          Full platform
+        </button>
+        {datasets.map((d) => (
+          <button
+            key={d.key}
+            type="button"
+            onClick={() => { setFocus(d.key); setSelected(null) }}
+            className={cn('rounded-full border px-2.5 py-1 text-[10px] font-medium', focus === d.key ? 'border-docker bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:bg-surface-overlay')}
+          >
+            {d.label}
+          </button>
+        ))}
+        <div className="ml-auto flex items-center gap-2">
+          {graph?.active && (
+            <span className="flex items-center gap-1 text-[10px] text-foreground-muted">
+              <span className={cn('h-2 w-2 rounded-full', graph.active.generator ? 'animate-pulse bg-emerald-400' : 'bg-foreground-faint/40')} /> CDC
+              <span className={cn('ml-1 h-2 w-2 rounded-full', graph.active.archive ? 'animate-pulse bg-amber-400' : 'bg-foreground-faint/40')} /> ETL
+            </span>
+          )}
+          <button type="button" onClick={load} className="inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
+            {loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
+          </button>
+        </div>
+      </div>
+
+      <div className="flex min-h-0 flex-1 gap-2">
+        {/* graph */}
+        <div ref={wrapRef} className="panel scrollbar-thin relative min-h-0 flex-1 overflow-auto p-4">
+          {/* edges overlay */}
+          <svg className="pointer-events-none absolute inset-0 h-full w-full" style={{ minWidth: '100%', minHeight: '100%' }}>
+            <defs>
+              <marker id="lin-arrow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
+                <path d="M0,0 L6,3 L0,6 Z" fill="#64748b" />
+              </marker>
+            </defs>
+            {(graph?.edges || []).map((e) => {
+              const a = coords.get(e.source)
+              const b = coords.get(e.target)
+              if (!a || !b) return null
+              const x1 = a.x + a.w
+              const y1 = a.y + a.h / 2
+              const x2 = b.x
+              const y2 = b.y + b.h / 2
+              const dx = Math.max(40, Math.abs(x2 - x1) / 2)
+              const path = `M${x1},${y1} C${x1 + dx},${y1} ${x2 - dx},${y2} ${x2},${y2}`
+              const color = KIND_COLOR[e.kind] || '#64748b'
+              const dim = selected && !connectedEdges.has(e.id)
+              return (
+                <g key={e.id} opacity={dim ? 0.12 : 1}>
+                  <path d={path} fill="none" stroke={color} strokeWidth={e.active ? 2.5 : 1.5}
+                    strokeDasharray={e.active ? '6 5' : undefined} markerEnd="url(#lin-arrow)">
+                    {e.active && (
+                      <animate attributeName="stroke-dashoffset" from="22" to="0" dur="0.8s" repeatCount="indefinite" />
+                    )}
+                  </path>
+                </g>
+              )
+            })}
+          </svg>
+
+          {/* stage columns */}
+          <div className="relative flex gap-6" style={{ minWidth: 'max-content' }}>
+            {stages.map((label, si) => (
+              <div key={si} className="flex w-[150px] shrink-0 flex-col gap-3">
+                <div className="text-center text-[9px] font-semibold uppercase tracking-wider text-foreground-faint">{label}</div>
+                {(byStage[si] || []).map((n) => {
+                  const Icon = TYPE_ICON[n.type] || Database
+                  const isSel = selected === n.id
+                  const rows = fmtRows(n.meta.rows)
+                  return (
+                    <div
+                      key={n.id}
+                      ref={(el) => { if (el) nodeRefs.current.set(n.id, el); else nodeRefs.current.delete(n.id) }}
+                      onClick={() => setSelected(isSel ? null : n.id)}
+                      className={cn(
+                        'relative z-10 cursor-pointer rounded-lg border bg-surface-raised p-2 transition-all',
+                        isSel ? 'border-docker ring-1 ring-docker/40' : 'border-border hover:border-docker/50',
+                      )}
+                    >
+                      <div className="flex items-center gap-1.5">
+                        <Icon className="h-3.5 w-3.5 shrink-0 text-docker" />
+                        <span className="truncate text-[10px] font-semibold text-foreground" title={n.label}>
+                          {n.label.split('\n')[0]}
+                        </span>
+                        {n.meta.masked_layer && <Lock className="ml-auto h-3 w-3 shrink-0 text-emerald-400" />}
+                      </div>
+                      {n.label.includes('\n') && (
+                        <p className="mt-0.5 truncate font-mono text-[8px] text-foreground-muted" title={n.label.split('\n')[1]}>
+                          {n.label.split('\n')[1]}
+                        </p>
+                      )}
+                      <div className="mt-1 flex items-center gap-1.5 text-[8px] text-foreground-faint">
+                        {rows && <span className="rounded bg-surface-overlay px-1 py-0.5 font-mono text-foreground-muted">{rows} rows</span>}
+                        {n.meta.columns && n.meta.columns.length > 0 && (
+                          <span className="rounded bg-rose-500/15 px-1 py-0.5 text-rose-300">{n.meta.columns.length} PII</span>
+                        )}
+                      </div>
+                    </div>
+                  )
+                })}
+              </div>
+            ))}
+          </div>
+        </div>
+
+        {/* detail */}
+        <div className="panel scrollbar-thin w-[260px] shrink-0 overflow-y-auto p-3">
+          {!selNode ? (
+            <div className="flex h-full flex-col items-center justify-center text-center text-foreground-faint">
+              <GitBranch className="mb-2 h-7 w-7" />
+              <p className="text-[11px]">Click any node to inspect its schema, row count and column-level PII lineage.</p>
+              {graph && (
+                <p className="mt-3 text-[9px]">
+                  {graph.om_connected ? 'OpenMetadata lineage layered in.' : 'OpenMetadata not connected.'}
+                </p>
+              )}
+            </div>
+          ) : (
+            <div className="space-y-2">
+              <h3 className="text-[12px] font-semibold text-foreground">{selNode.label.split('\n')[0]}</h3>
+              {selNode.meta.table && <p className="break-all font-mono text-[9px] text-docker">{selNode.meta.table}</p>}
+              <div className="grid grid-cols-2 gap-1.5 text-[9px]">
+                {selNode.meta.engine && <Info label="Engine" value={selNode.meta.engine} />}
+                {selNode.meta.rows != null && <Info label="Rows" value={fmtRows(selNode.meta.rows) || '—'} />}
+                {selNode.meta.domain && <Info label="Domain" value={selNode.meta.domain} />}
+                {selNode.meta.topic && <Info label="Topic" value={selNode.meta.topic} />}
+                {selNode.meta.om && (selNode.meta.om.upstream != null) && (
+                  <Info label="OM lineage" value={`↑${selNode.meta.om.upstream} ↓${selNode.meta.om.downstream}`} />
+                )}
+              </div>
+              {selNode.meta.note && <p className="text-[10px] text-foreground-muted">{selNode.meta.note}</p>}
+              {selNode.meta.columns && selNode.meta.columns.length > 0 && (
+                <div>
+                  <p className="mb-1 mt-2 flex items-center gap-1 text-[10px] font-semibold text-foreground">
+                    <ShieldCheck className="h-3 w-3 text-emerald-400" /> PII columns
+                  </p>
+                  <div className="space-y-1">
+                    {selNode.meta.columns.map((c) => (
+                      <div key={c.name} className="flex items-center gap-1.5 rounded bg-surface-overlay px-1.5 py-1 text-[9px]">
+                        <span className="flex-1 truncate font-mono text-foreground-muted" title={c.name}>{c.name}</span>
+                        {c.category && <span className="text-foreground-faint">{c.category}</span>}
+                        {c.masked
+                          ? <span className="flex items-center gap-0.5 text-emerald-400"><Lock className="h-2.5 w-2.5" /> masked</span>
+                          : <span className="text-amber-400">visible</span>}
+                      </div>
+                    ))}
+                  </div>
+                </div>
+              )}
+              {graph?.column_links && graph.column_links.length > 0 && (selNode.id === 'src_orders' || selNode.id === 'iceberg_curated') && (
+                <div>
+                  <p className="mb-1 mt-2 text-[10px] font-semibold text-foreground">Column lineage → curated</p>
+                  <div className="space-y-1">
+                    {graph.column_links.map((l) => (
+                      <div key={l.column} className="flex items-center gap-1 text-[9px]">
+                        <span className="flex-1 truncate font-mono text-foreground-muted">{l.column}</span>
+                        <span className="text-foreground-faint">→</span>
+                        {l.masked ? <Lock className="h-2.5 w-2.5 text-emerald-400" /> : <span className="text-amber-400">visible</span>}
+                      </div>
+                    ))}
+                  </div>
+                </div>
+              )}
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+function Info({ label, value }: { label: string; value: string }) {
+  return (
+    <div className="rounded bg-surface-overlay px-1.5 py-1">
+      <p className="text-[8px] uppercase tracking-wider text-foreground-faint">{label}</p>
+      <p className="truncate text-[10px] text-foreground" title={value}>{value}</p>
+    </div>
+  )
+}

ui/src/components/features/LoginView.tsx

@@ -0,0 +1,47 @@
+import { Box, LogIn } from 'lucide-react'
+import { useMemo } from 'react'
+
+export function LoginView() {
+  const error = useMemo(() => {
+    try {
+      const p = new URLSearchParams(window.location.search)
+      const err = p.get('error')
+      if (!err) return null
+      if (err === 'login_failed') return 'Sign-in failed — try again.'
+      return err
+    } catch {
+      return null
+    }
+  }, [])
+
+  return (
+    <div className="flex h-full min-h-screen flex-col items-center justify-center bg-surface px-4">
+      <div className="w-full max-w-md border border-border bg-surface-raised/90 p-8 shadow-panel backdrop-blur-sm">
+        <div className="mb-6 flex items-center gap-3">
+          <div className="flex h-10 w-10 items-center justify-center rounded-lg bg-gradient-to-br from-docker to-blue-600 shadow-docker">
+            <Box className="h-5 w-5 text-white" />
+          </div>
+          <div>
+            <p className="text-[10px] uppercase tracking-wider text-foreground-muted">ATC Lab</p>
+            <h1 className="text-lg font-semibold text-foreground">Data & AI Command Center</h1>
+          </div>
+        </div>
+        <p className="mb-6 text-sm text-foreground-muted">
+          Sign in with Authentik to open the ops glass. Lab environment — not an official Dell product.
+        </p>
+        <a
+          href="/auth/login"
+          className="inline-flex w-full items-center justify-center gap-2 rounded-md bg-docker px-4 py-2.5 text-sm font-semibold text-white hover:opacity-90"
+        >
+          <LogIn className="h-4 w-4" />
+          Continue with Authentik
+        </a>
+        {error ? (
+          <p className="mt-4 text-sm text-warning" role="alert">
+            {error}
+          </p>
+        ) : null}
+      </div>
+    </div>
+  )
+}

ui/src/components/features/ObservabilityView.tsx

@@ -0,0 +1,141 @@
+import { useCallback, useEffect, useState } from 'react'
+import { Activity, RefreshCw, Loader2, AlertTriangle, Clock, Database, TrendingUp, TrendingDown } from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type Point = { t: string; rows: number; delta: number }
+type DsMetric = {
+  key: string; label: string; engine: string; color: string; table: string
+  rows: number | null; delta: number | null; freshness_age_min: number | null
+  columns: number | null; stalled_cycles: number; error: string | null; ts: string | null
+  series: Point[]
+}
+type Alert = { id: string; dataset: string; type: string; severity: string; message: string; count: number; ts: string; last_ts: string }
+type Metrics = {
+  ok: boolean; enabled: boolean; cycles: number; freshness_min: number
+  alert_counts: { critical: number; warning: number; info: number; total: number }
+  datasets: DsMetric[]
+}
+
+function Spark({ data, color }: { data: Point[]; color: string }) {
+  const pts = data.slice(-40)
+  if (pts.length < 2) return <div className="h-10 w-full" />
+  const w = 240
+  const h = 40
+  const vals = pts.map((p) => p.rows)
+  const max = Math.max(...vals)
+  const min = Math.min(...vals)
+  const range = max - min || 1
+  const step = w / (pts.length - 1)
+  const line = pts.map((p, i) => `${i === 0 ? 'M' : 'L'}${(i * step).toFixed(1)},${(h - ((p.rows - min) / range) * (h - 6) - 3).toFixed(1)}`).join(' ')
+  return (
+    <svg viewBox={`0 0 ${w} ${h}`} className="h-10 w-full" preserveAspectRatio="none">
+      <path d={`${line} L${w},${h} L0,${h} Z`} fill={color} fillOpacity="0.12" />
+      <path d={line} fill="none" stroke={color} strokeWidth="1.5" />
+    </svg>
+  )
+}
+
+export function ObservabilityView() {
+  const [metrics, setMetrics] = useState<Metrics | null>(null)
+  const [alerts, setAlerts] = useState<Alert[]>([])
+  const [loading, setLoading] = useState(true)
+
+  const load = useCallback(async () => {
+    setLoading(true)
+    try {
+      const [m, a] = await Promise.all([
+        fetch('/api/observability/metrics').then((r) => r.json()),
+        fetch('/api/observability/alerts').then((r) => r.json()),
+      ])
+      setMetrics(m)
+      setAlerts(a.active || [])
+    } catch { /* */ } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => { load() }, [load])
+  useEffect(() => { const t = setInterval(load, 8000); return () => clearInterval(t) }, [load])
+
+  const ac = metrics?.alert_counts
+  const sevBorder = (s: string) => (s === 'critical' ? 'border-rose-500/40 bg-rose-500/5' : s === 'warning' ? 'border-amber-500/40 bg-amber-500/5' : 'border-sky-500/40 bg-sky-500/5')
+  const sevText = (s: string) => (s === 'critical' ? 'text-rose-400' : s === 'warning' ? 'text-amber-400' : 'text-sky-400')
+
+  return (
+    <div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto">
+      <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
+        <Kpi icon={AlertTriangle} label="Critical" value={String(ac?.critical ?? 0)} accent={ac?.critical ? '#f87171' : '#34d399'} />
+        <Kpi icon={AlertTriangle} label="Warnings" value={String(ac?.warning ?? 0)} accent={ac?.warning ? '#fbbf24' : '#34d399'} />
+        <Kpi icon={Activity} label="Sweeps" value={String(metrics?.cycles ?? 0)} accent="#60a5fa" />
+        <Kpi icon={Clock} label="Freshness SLA" value={`${metrics?.freshness_min ?? '—'}m`} accent="#a78bfa" />
+      </div>
+
+      <div className="flex shrink-0 items-center px-1">
+        <span className="text-[10px] text-foreground-faint">Tracking volume, freshness &amp; schema drift across every business table</span>
+        <button type="button" onClick={load} className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
+          {loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
+        </button>
+      </div>
+
+      {/* active alerts */}
+      {alerts.length > 0 && (
+        <div className="shrink-0 space-y-1.5">
+          {alerts.map((a) => (
+            <div key={a.id} className={cn('flex items-center gap-2 rounded-md border px-3 py-2 text-[10px]', sevBorder(a.severity))}>
+              <AlertTriangle className={cn('h-3.5 w-3.5 shrink-0', sevText(a.severity))} />
+              <span className="font-semibold text-foreground">{a.dataset}</span>
+              <span className="text-foreground-muted">{a.message}</span>
+              <span className={cn('ml-auto rounded px-1.5 py-0.5 text-[8px] uppercase', sevText(a.severity))}>{a.type}</span>
+              {a.count > 1 && <span className="rounded bg-surface-overlay px-1.5 py-0.5 text-[8px] text-foreground-faint">×{a.count}</span>}
+            </div>
+          ))}
+        </div>
+      )}
+
+      {/* per-dataset volume + freshness */}
+      <div className="grid gap-2 pb-2 lg:grid-cols-2 xl:grid-cols-3">
+        {(metrics?.datasets || []).map((d) => {
+          const stale = d.freshness_age_min != null && d.freshness_age_min > (metrics?.freshness_min ?? 30)
+          return (
+            <div key={d.key} className="panel p-3">
+              <div className="mb-1 flex items-center gap-1.5">
+                <Database className="h-3.5 w-3.5" style={{ color: d.color }} />
+                <span className="text-[11px] font-semibold text-foreground">{d.label}</span>
+                {d.delta != null && d.delta !== 0 && (
+                  <span className={cn('ml-auto flex items-center gap-0.5 text-[9px]', d.delta > 0 ? 'text-emerald-400' : 'text-rose-400')}>
+                    {d.delta > 0 ? <TrendingUp className="h-3 w-3" /> : <TrendingDown className="h-3 w-3" />}
+                    {d.delta > 0 ? '+' : ''}{d.delta.toLocaleString()}
+                  </span>
+                )}
+              </div>
+              <Spark data={d.series} color={d.color} />
+              <div className="mt-1.5 flex flex-wrap items-center gap-x-3 gap-y-1 text-[9px] text-foreground-muted">
+                <span>Rows: <span className="font-mono text-foreground">{d.rows?.toLocaleString() ?? '—'}</span></span>
+                <span className={cn('flex items-center gap-1', stale && 'text-amber-400')}>
+                  <Clock className="h-3 w-3" /> {d.freshness_age_min != null ? `${d.freshness_age_min.toFixed(0)}m old` : 'n/a'}
+                </span>
+                {d.columns != null && <span>{d.columns} cols</span>}
+                {d.stalled_cycles > 0 && <span className="text-amber-400">stalled ×{d.stalled_cycles}</span>}
+                {d.error && <span className="text-rose-400">err</span>}
+              </div>
+            </div>
+          )
+        })}
+      </div>
+    </div>
+  )
+}
+
+function Kpi({ icon: Icon, label, value, accent }: { icon: typeof Activity; label: string; value: string; accent: string }) {
+  return (
+    <div className="panel flex items-center gap-3 px-3 py-2.5">
+      <div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
+        <Icon className="h-4 w-4" />
+      </div>
+      <div className="min-w-0">
+        <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
+        <p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
+      </div>
+    </div>
+  )
+}

ui/src/components/features/StorageView.tsx

@@ -184,6 +184,118 @@ function Panel({ title, icon: Icon, children, className, right }: { title: strin
 
 /* ── Main ────────────────────────────────────────────────────────────────── */
 
+type EtlDataset = {
+  key: string; label: string; engine: string; color: string; parts: number; rows: number; bytes: number
+  backfilled: boolean; total_source: number | null; last_ts: string | null; last_rows: number
+  last_key: string | null; error: string | null; progress_pct: number | null
+}
+type EtlStatus = {
+  ok: boolean; enabled: boolean; interval_s: number; chunk: number; running_cycle: boolean; cycles: number
+  last_cycle_rows: number; totals: { parts: number; rows: number; bytes: number }; rate_rows_per_min: number
+  datasets: EtlDataset[]; series: { t: string; rows: number; bytes: number; orders: number; revenue: number }[]
+  feed: { ts: string; text: string; level: string }[]
+}
+
+function EtlIngestPanel() {
+  const [etl, setEtl] = useState<EtlStatus | null>(null)
+  const [busy, setBusy] = useState(false)
+  const load = useCallback(async () => {
+    try { const r = await fetch('/api/etl/status'); if (r.ok) setEtl(await r.json()) } catch { /* */ }
+  }, [])
+  useEffect(() => { load(); const t = setInterval(load, 5000); return () => clearInterval(t) }, [load])
+  const runNow = async () => {
+    setBusy(true)
+    try { await fetch('/api/etl/run', { method: 'POST' }) } catch { /* */ }
+    setTimeout(() => { load(); setBusy(false) }, 900)
+  }
+  const cfg = async (body: Record<string, unknown>) => {
+    await fetch('/api/etl/config', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
+    load()
+  }
+  const ds = etl?.datasets || []
+  return (
+    <Panel
+      title="Lakehouse ETL · source databases → S3 Parquet (realtime offload)"
+      icon={Boxes}
+      right={
+        <span className="flex items-center gap-2 text-[9px]">
+          <span className={cn('inline-flex items-center gap-1 rounded-full border px-2 py-0.5',
+            etl?.running_cycle ? 'border-emerald-500/40 bg-emerald-500/10 text-emerald-300'
+              : etl?.enabled ? 'border-sky-500/40 bg-sky-500/10 text-sky-300'
+                : 'border-border text-foreground-faint')}>
+            <span className={cn('h-1.5 w-1.5 rounded-full', etl?.running_cycle ? 'animate-ping bg-emerald-400' : etl?.enabled ? 'bg-sky-400' : 'bg-foreground-faint')} />
+            {etl?.running_cycle ? 'OFFLOADING' : etl?.enabled ? 'STREAMING' : 'PAUSED'}
+          </span>
+          <span className="text-foreground-faint">{etl?.cycles ?? 0} cycles · {fmtNum(etl?.rate_rows_per_min || 0)} rows/min</span>
+        </span>
+      }
+    >
+      <div className="mb-2 flex flex-wrap items-center gap-2 text-[10px]">
+        <span className="text-foreground-muted">A background ETL agent pulls small chunks from every source and writes partitioned Parquet to <span className="font-mono text-docker">s3://data/lake/</span> every</span>
+        <select value={etl?.interval_s ?? 60} onChange={(e) => cfg({ interval_s: Number(e.target.value) })}
+          className="rounded border border-border bg-surface px-1.5 py-0.5 font-mono text-foreground">
+          {[30, 60, 120, 300, 600].map((v) => <option key={v} value={v}>{v >= 60 ? `${v / 60} min` : `${v}s`}</option>)}
+        </select>
+        <button type="button" onClick={() => cfg({ enabled: !etl?.enabled })}
+          className={cn('rounded border px-2 py-0.5', etl?.enabled ? 'border-amber-500/40 text-amber-300' : 'border-emerald-500/40 text-emerald-300')}>
+          {etl?.enabled ? 'Pause' : 'Resume'}
+        </button>
+        <button type="button" onClick={runNow} disabled={busy}
+          className="inline-flex items-center gap-1 rounded border border-docker/40 bg-docker/10 px-2 py-0.5 text-docker disabled:opacity-50">
+          {busy ? <Loader2 className="h-3 w-3 animate-spin" /> : <Activity className="h-3 w-3" />} Offload now
+        </button>
+        <span className="ml-auto font-mono text-foreground-faint">
+          {fmtNum(etl?.totals.parts || 0)} parts · {fmtNum(etl?.totals.rows || 0)} rows · {fmtBytes(etl?.totals.bytes || 0)}
+        </span>
+      </div>
+
+      <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
+        {ds.map((d) => (
+          <div key={d.key} className="rounded-lg border border-border/60 bg-surface-overlay/40 p-2.5">
+            <div className="flex items-center justify-between">
+              <span className="flex items-center gap-1.5 text-[10px] font-semibold text-foreground">
+                <span className="h-2 w-2 rounded-full" style={{ background: d.color }} /> {d.label}
+              </span>
+              {d.backfilled
+                ? <span className="rounded bg-emerald-500/15 px-1 py-0.5 text-[8px] font-medium text-emerald-300">TAILING</span>
+                : <span className="rounded bg-sky-500/15 px-1 py-0.5 text-[8px] font-medium text-sky-300">BACKFILL</span>}
+            </div>
+            <p className="mt-0.5 text-[8px] uppercase tracking-wide text-foreground-faint">{d.engine}</p>
+            <p className="mt-1 font-mono text-base font-bold leading-none text-foreground">{fmtNum(d.rows)}</p>
+            <p className="text-[9px] text-foreground-faint">rows · {fmtNum(d.parts)} parts · {fmtBytes(d.bytes)}</p>
+            <div className="mt-1.5 h-1.5 overflow-hidden rounded-full bg-surface">
+              <div className="h-full rounded-full transition-all" style={{ width: `${d.progress_pct ?? (d.backfilled ? 100 : 3)}%`, background: d.color }} />
+            </div>
+            <p className="mt-0.5 flex justify-between text-[8px] text-foreground-faint">
+              <span>{d.progress_pct != null ? `${d.progress_pct}% of ${fmtNum(d.total_source || 0)}` : 'streaming'}</span>
+              {d.last_rows ? <span className="text-emerald-400">+{fmtNum(d.last_rows)}</span> : null}
+            </p>
+            {d.error && <p className="mt-0.5 truncate text-[8px] text-danger" title={d.error}>{d.error}</p>}
+          </div>
+        ))}
+      </div>
+
+      <div className="mt-2 grid grid-cols-1 gap-2 lg:grid-cols-3">
+        <div className="lg:col-span-2">
+          <p className="mb-1 text-[9px] uppercase tracking-wide text-foreground-faint">Rows offloaded per cycle (realtime)</p>
+          <Sparkline values={(etl?.series || []).map((p) => p.rows)} color="#34d399" />
+        </div>
+        <div>
+          <p className="mb-1 text-[9px] uppercase tracking-wide text-foreground-faint">ETL agent activity</p>
+          <div className="max-h-[78px] space-y-0.5 overflow-y-auto scrollbar-thin">
+            {(etl?.feed || []).slice(0, 6).map((f, i) => (
+              <p key={i} className="truncate text-[9px] text-foreground-muted" title={f.text}>
+                <span className="text-foreground-faint">{f.ts.slice(11, 19)}</span> {f.text}
+              </p>
+            ))}
+            {!etl?.feed?.length && <Empty label="Warming up…" />}
+          </div>
+        </div>
+      </div>
+    </Panel>
+  )
+}
+
 export function StorageView() {
   const [tab, setTab] = useState<'overview' | 'browser'>('overview')
   const [an, setAn] = useState<Analytics | null>(null)
@@ -207,7 +319,7 @@ export function StorageView() {
 
   useEffect(() => {
     loadAnalytics()
-    const t = setInterval(() => loadAnalytics(), 30000)
+    const t = setInterval(() => loadAnalytics(), 12000)
     return () => clearInterval(t)
   }, [loadAnalytics])
 
@@ -264,6 +376,9 @@ export function StorageView() {
                 <Kpi icon={Clock} label="Last write" value={s?.newest ? new Date(s.newest).toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }) : '—'} accent="#22d3ee" />
               </div>
 
+              {/* Realtime ETL offload (source → S3 Parquet) */}
+              <EtlIngestPanel />
+
               {/* Growth + bucket distribution */}
               <div className="grid grid-cols-1 gap-3 xl:grid-cols-3">
                 <Panel title="Data growth (cumulative size · daily ingest)" icon={TrendingUp} className="xl:col-span-2"

ui/src/components/features/TrinoFederationView.tsx

@@ -117,6 +117,31 @@ function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; la
   )
 }
 
+function MiniArea({ values, color = '#34d399', label }: { values: number[]; color?: string; label?: string }) {
+  const w = 280, h = 46, pad = 3
+  const d = values.length ? values : [0]
+  const max = Math.max(1, ...d)
+  const step = d.length > 1 ? (w - pad * 2) / (d.length - 1) : 0
+  const pts = d.map((v, i) => [pad + i * step, h - pad - (v / max) * (h - pad * 2)] as const)
+  const line = pts.map(([x, y], i) => `${i ? 'L' : 'M'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
+  const area = `${line} L${pts[pts.length - 1][0].toFixed(1)},${h - pad} L${pts[0][0].toFixed(1)},${h - pad} Z`
+  return (
+    <div>
+      {label && <p className="mb-0.5 text-[9px] uppercase tracking-wide text-foreground-faint">{label}</p>}
+      <svg viewBox={`0 0 ${w} ${h}`} preserveAspectRatio="none" className="h-11 w-full">
+        <defs>
+          <linearGradient id={`ma-${color}`} x1="0" y1="0" x2="0" y2="1">
+            <stop offset="0%" stopColor={color} stopOpacity="0.4" /><stop offset="100%" stopColor={color} stopOpacity="0" />
+          </linearGradient>
+        </defs>
+        {values.length > 0 && <path d={area} fill={`url(#ma-${color})`} />}
+        {values.length > 0 && <path d={line} fill="none" stroke={color} strokeWidth="1.5" vectorEffect="non-scaling-stroke" />}
+        {values.length > 0 && <circle cx={pts[pts.length - 1][0]} cy={pts[pts.length - 1][1]} r="2.5" fill={color}><animate attributeName="r" values="2.5;5;2.5" dur="1.6s" repeatCount="indefinite" /></circle>}
+      </svg>
+    </div>
+  )
+}
+
 export function TrinoFederationView({ embedded = false, activeTab }: { embedded?: boolean; activeTab?: SubTab } = {}) {
   const [tabState, setTab] = useState<SubTab>('federated')
   const tab = embedded ? activeTab ?? 'federated' : tabState
@@ -124,9 +149,26 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
   const [marquee, setMarquee] = useState<any>(null)
   const [lake, setLake] = useState<any>(null)
   const [dict, setDict] = useState<any>(null)
+  const [biz, setBiz] = useState<any>(null)
   const [loading, setLoading] = useState(false)
   const [matRunning, setMatRunning] = useState(false)
 
+  const loadBiz = useCallback(async () => {
+    try {
+      const r = await fetch('/api/etl/business')
+      if (r.ok) setBiz(await r.json())
+    } catch { /* */ }
+  }, [])
+
+  // Poll the live federated business model (built from the ETL lakehouse offload)
+  // while the federated tab is open, so the graphs move with newly generated data.
+  useEffect(() => {
+    if (tab !== 'federated') return
+    loadBiz()
+    const t = setInterval(loadBiz, 5000)
+    return () => clearInterval(t)
+  }, [tab, loadBiz])
+
   const loadFederated = useCallback(async () => {
     setLoading(true)
     try {
@@ -198,6 +240,66 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
       {/* ───────── FEDERATED ───────── */}
       {tab === 'federated' && (
         <>
+          {/* ───────── REALTIME FEDERATED BUSINESS MODEL ───────── */}
+          <Panel
+            title="Realtime federated business model"
+            subtitle={biz?.generated_at ? `updated ${new Date(biz.generated_at).toLocaleTimeString()}` : 'live'}
+            icon={Activity}
+          >
+            <p className="mb-2 flex items-center gap-1.5 text-[10px] text-foreground-muted">
+              <span className="h-1.5 w-1.5 animate-ping rounded-full bg-emerald-400" />
+              Business matrices built continuously from the lakehouse offload across <span className="text-docker">all five data points</span> — orders, HR, supply &amp; telemetry — and they move as new data is generated &amp; streamed to S3.
+            </p>
+            <div className="mb-2 grid grid-cols-2 gap-2 md:grid-cols-3 xl:grid-cols-6">
+              <Kpi icon={ShoppingCart} label="Orders analyzed" value={fmtNum(biz?.kpis?.orders)} accent="#fbbf24" />
+              <Kpi icon={DollarSign} label="Revenue" value={fmtMoney(biz?.kpis?.revenue)} sub={`avg ${fmtMoney(biz?.kpis?.avg_order)}`} accent="#34d399" />
+              <Kpi icon={Users} label="HR events" value={fmtNum(biz?.kpis?.hr_events)} accent="#60a5fa" />
+              <Kpi icon={Boxes} label="Supply events" value={fmtNum(biz?.kpis?.supply_events)} sub={fmtMoney(biz?.kpis?.supply_amount)} accent="#a78bfa" />
+              <Kpi icon={Activity} label="Telemetry pts" value={fmtNum(biz?.kpis?.telemetry)} accent="#22d3ee" />
+              <Kpi icon={Layers} label="Rows in model" value={fmtNum(biz?.kpis?.rows_total)} sub="federated" accent="#dd00a1" />
+            </div>
+            <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
+              <MiniArea label="Orders ingested / cycle" values={(biz?.ts || []).map((p: any) => p.orders)} color="#fbbf24" />
+              <MiniArea label="Revenue / cycle (€)" values={(biz?.ts || []).map((p: any) => p.revenue)} color="#34d399" />
+            </div>
+          </Panel>
+
+          <div className="grid shrink-0 gap-2 lg:grid-cols-2">
+            <Panel title="Revenue by region" subtitle="live" icon={DollarSign}><BarsH data={biz?.orders_by_region} valueKind="money" colorByIndex /></Panel>
+            <Panel title="Region matrix — orders · revenue · HR · supply" icon={Network}>
+              {biz?.region_matrix?.length ? (
+                <div className="overflow-x-auto">
+                  <table className="w-full text-[10px]">
+                    <thead><tr className="border-b border-border text-left text-foreground-muted">
+                      <th className="py-1 pr-3">Region</th><th className="py-1 pr-3 text-right">Orders</th>
+                      <th className="py-1 pr-3 text-right">Revenue</th><th className="py-1 pr-3 text-right">HR</th><th className="py-1 pr-3 text-right">Supply</th>
+                    </tr></thead>
+                    <tbody>
+                      {biz.region_matrix.map((r: any, i: number) => (
+                        <tr key={i} className="border-b border-border/40">
+                          <td className="py-1 pr-3 font-medium text-foreground">{r.region}</td>
+                          <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.orders)}</td>
+                          <td className="py-1 pr-3 text-right font-mono text-emerald-400">{fmtMoney(r.revenue)}</td>
+                          <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.hr_events)}</td>
+                          <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.supply_events)}</td>
+                        </tr>
+                      ))}
+                    </tbody>
+                  </table>
+                </div>
+              ) : <p className="py-4 text-center text-[10px] text-foreground-faint">Building the model from the lakehouse offload…</p>}
+            </Panel>
+            <div className="grid gap-2 sm:grid-cols-2">
+              <Panel title="Orders by status"><Donut data={biz?.orders_by_status} /></Panel>
+              <Panel title="Revenue by channel"><BarsH data={biz?.orders_by_channel} valueKind="money" colorByIndex /></Panel>
+            </div>
+            <Panel title="HR events by department" icon={Users}><BarsH data={biz?.hr_by_department} colorByIndex /></Panel>
+            <Panel title="Supply value by type" icon={Boxes}><BarsH data={biz?.supply_by_type} valueKind="money" colorByIndex /></Panel>
+            <Panel title="Telemetry — avg value by metric" icon={Activity}>
+              <BarsH data={(biz?.telemetry_by_metric || []).map((x: any) => ({ key: x.key, count: x.count, value: x.avg }))} valueKind="num" />
+            </Panel>
+          </div>
+
           <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
             <Kpi icon={Layers} label="Federated catalogs" value={String(catalogs?.count ?? '—')} sub="one Trino engine" accent="#dd00a1" />
             <Kpi icon={ShoppingCart} label="Orders" value={fmtNum(totals.orders)} sub="PostgreSQL (live)" accent="#fbbf24" />

ui/src/components/layout/TopBar.tsx

@@ -1,4 +1,4 @@
-import { Activity, Bot, Box, Clock, ShieldAlert } from 'lucide-react'
+import { Activity, Bot, Box, Clock, LogOut, ShieldAlert } from 'lucide-react'
 import type { Agent, Approval, StatusData, WorkloadData } from '../../types'
 import { Badge } from '../ui/Badge'
 import { ThemeToggle } from './ThemeToggle'
@@ -11,9 +11,11 @@ type Props = {
   agents: Agent[]
   approvals: Approval[]
   onApprovalsClick: () => void
+  userLabel?: string
+  onLogout?: () => void
 }
 
-export function TopBar({ clock, workload, agents, approvals, onApprovalsClick }: Props) {
+export function TopBar({ clock, workload, agents, approvals, onApprovalsClick, userLabel, onLogout }: Props) {
   const pipelineOk = workload?.totals?.pipeline_active ?? false
   const running = workload?.totals?.apps_running ?? 0
   const total = workload?.totals?.apps_total ?? 0
@@ -53,6 +55,22 @@ export function TopBar({ clock, workload, agents, approvals, onApprovalsClick }:
       </div>
 
       <div className="flex items-center gap-2">
+        {userLabel ? (
+          <span className="hidden max-w-[10rem] truncate text-[11px] text-foreground-muted sm:inline" title={userLabel}>
+            {userLabel}
+          </span>
+        ) : null}
+        {onLogout ? (
+          <button
+            type="button"
+            onClick={onLogout}
+            className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[10px] text-foreground-muted hover:bg-surface hover:text-foreground"
+            title="Logout"
+          >
+            <LogOut className="h-3 w-3" />
+            Logout
+          </button>
+        ) : null}
         <ThemeToggle />
         <div className="flex items-center gap-1.5 font-mono text-[10px] text-foreground-muted">
           <Clock className="h-3 w-3" />

ui/src/hooks/useAuth.ts

@@ -0,0 +1,42 @@
+import { useCallback, useEffect, useState } from 'react'
+
+export type AuthUser = {
+  user?: string
+  email?: string
+  name?: string
+  preferred_username?: string
+  auth_enabled?: boolean
+}
+
+type AuthState =
+  | { status: 'loading'; user: null }
+  | { status: 'anon'; user: null }
+  | { status: 'authed'; user: AuthUser }
+
+export function useAuth(): AuthState & { refresh: () => Promise<void> } {
+  const [state, setState] = useState<AuthState>({ status: 'loading', user: null })
+
+  const refresh = useCallback(async () => {
+    try {
+      const r = await fetch('/api/auth/me', { credentials: 'same-origin' })
+      if (r.status === 401) {
+        setState({ status: 'anon', user: null })
+        return
+      }
+      if (!r.ok) {
+        setState({ status: 'anon', user: null })
+        return
+      }
+      const me = (await r.json()) as AuthUser
+      setState({ status: 'authed', user: me })
+    } catch {
+      setState({ status: 'anon', user: null })
+    }
+  }, [])
+
+  useEffect(() => {
+    void refresh()
+  }, [refresh])
+
+  return { ...state, refresh }
+}

ui/src/hooks/useCommandCenter.ts

@@ -129,7 +129,7 @@ export function useCommandCenter() {
       if (msg.type === 'terminal') appendTerminal(msg.line)
       if (msg.type === 'terminal_history' && msg.terminals) setTerminals(msg.terminals)
       if (msg.type === 'feed') setFeed((prev) => [msg.entry, ...prev].slice(0, 100))
-      if (msg.type === 'cdc_change' && msg.entry) setChanges((prev) => [msg.entry, ...prev].slice(0, 400))
+      if (msg.type === 'cdc_change' && msg.entry) setChanges((prev) => [msg.entry, ...prev].slice(0, 800))
       if (msg.type === 'agent_dispatch') {
         setSelectedAgentId(msg.agent_id)
         setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'walk', zone: msg.zone } }))
@@ -313,7 +313,22 @@ export function useCommandCenter() {
   const inspectorLines = useMemo(() => {
     if (!terminalSubjectId) return []
     const probeId = resolveProbeId(terminalSubjectId)
-    return terminals[probeId] || terminals[terminalSubjectId] || terminals[selectedAgentId || ''] || []
+    const nodeLines = terminals[probeId] || terminals[terminalSubjectId] || []
+    const agentLines = selectedAgentId ? terminals[selectedAgentId] || [] : []
+    // When an agent is selected we also show its own autonomous ops stream
+    // (DML/ETL/probes) merged chronologically with the node probe output, so
+    // the terminal reflects what the agent is doing in the background — not just
+    // a one-shot probe.
+    if (!agentLines.length) return nodeLines
+    if (!nodeLines.length) return agentLines
+    const seen = new Set<string>()
+    const merged = [...nodeLines, ...agentLines].filter((l) => {
+      if (seen.has(l.id)) return false
+      seen.add(l.id)
+      return true
+    })
+    merged.sort((a, b) => ((a.ts || '') < (b.ts || '') ? -1 : (a.ts || '') > (b.ts || '') ? 1 : 0))
+    return merged.slice(-300)
   }, [terminalSubjectId, terminals, selectedAgentId])
 
   const focusApprovals = useCallback(() => {

ui/src/lib/api.ts

@@ -6,6 +6,8 @@ import type {
   CdcStats,
   DataflowGraph,
   FeedEntry,
+  GpuConfigPayload,
+  GpuConfigTestResult,
   GpuStatus,
   Movement,
   PiiDataset,
@@ -21,7 +23,7 @@ async function fetchJson<T>(url: string, timeoutMs = 10000): Promise<T | null> {
   const ctrl = new AbortController()
   const timer = setTimeout(() => ctrl.abort(), timeoutMs)
   try {
-    const r = await fetch(url, { signal: ctrl.signal })
+    const r = await fetch(url, { signal: ctrl.signal, credentials: 'same-origin' })
     if (!r.ok) return null
     return (await r.json()) as T
   } catch {
@@ -62,6 +64,54 @@ export async function fetchGpu(): Promise<GpuStatus | null> {
   return fetchJson<GpuStatus>('/api/gpu', 8000)
 }
 
+export async function fetchGpuConfig(): Promise<GpuConfigPayload | null> {
+  return fetchJson<GpuConfigPayload>('/api/gpu/config', 8000)
+}
+
+export async function saveGpuConfig(body: {
+  preset_id?: string
+  host?: string
+  gpu_ui_port?: number
+  llm_port?: number
+}) {
+  const r = await fetch('/api/gpu/config', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  })
+  return r.json()
+}
+
+export async function testGpuConfig(body: {
+  preset_id?: string
+  host?: string
+  gpu_ui_port?: number
+  llm_port?: number
+}): Promise<GpuConfigTestResult | null> {
+  const ctrl = new AbortController()
+  const timer = setTimeout(() => ctrl.abort(), 12000)
+  try {
+    const r = await fetch('/api/gpu/config/test', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+      signal: ctrl.signal,
+    })
+    if (!r.ok) return null
+    return (await r.json()) as GpuConfigTestResult
+  } catch {
+    return null
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+export async function resetGpuConfig() {
+  const r = await fetch('/api/gpu/config', { method: 'DELETE' , credentials: 'same-origin' })
+  return r.json()
+}
+
+
 export async function fetchTerminals(): Promise<Record<string, TerminalLine[]>> {
   const j = await fetchJson<{ terminals?: Record<string, TerminalLine[]> }>('/api/terminals', 8000)
   return j?.terminals || {}
@@ -77,7 +127,7 @@ export async function fetchNodeDetail(nodeId: string) {
 }
 
 export function probeNode(nodeId: string) {
-  return fetch(`/api/nodes/${nodeId}/probe`, { method: 'POST' })
+  return fetch(`/api/nodes/${nodeId}/probe`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export function askNode(nodeId: string, message: string) {
@@ -100,21 +150,34 @@ export async function fetchPresentation(): Promise<PresentationData | null> {
   return fetchJson<PresentationData>('/api/presentation', 60000)
 }
 
-export async function fetchChanges(opts: { source?: string; op?: string; limit?: number } = {}) {
+export async function fetchChanges(opts: { source?: string; op?: string; limit?: number; minutes?: number } = {}) {
   const p = new URLSearchParams()
   if (opts.source) p.set('source', opts.source)
   if (opts.op) p.set('op', opts.op)
-  p.set('limit', String(opts.limit ?? 150))
-  const j = await fetchJson<{ changes?: CdcChange[]; connected?: boolean; consumed?: number }>(
+  p.set('limit', String(opts.limit ?? 200))
+  if (opts.minutes) p.set('minutes', String(opts.minutes))
+  const j = await fetchJson<{ changes?: CdcChange[]; connected?: boolean; consumed?: number; last_ts?: string | null }>(
     `/api/changes?${p.toString()}`, 8000,
   )
-  return { changes: j?.changes || [], connected: !!j?.connected, consumed: j?.consumed || 0 }
+  return { changes: j?.changes || [], connected: !!j?.connected, consumed: j?.consumed || 0, last_ts: j?.last_ts }
 }
 
 export async function fetchChangeStats(minutes = 15): Promise<CdcStats | null> {
   return fetchJson<CdcStats>(`/api/changes/stats?minutes=${minutes}`, 8000)
 }
 
+export type ConnectorState = { name: string; state?: string; failed?: number[] }
+export type ResyncResult = { ok: boolean; restarted?: string[]; healthy?: number; total?: number; after?: ConnectorState[] }
+
+export async function resyncSources(force = true): Promise<ResyncResult | null> {
+  const r = await fetch('/api/pipeline/streaming/resync', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ force }),
+  })
+  return r.ok ? ((await r.json()) as ResyncResult) : null
+}
+
 export async function fetchAgentOpsStatus() {
   return fetchJson<Record<string, unknown>>('/api/agent-ops/status', 8000)
 }
@@ -195,19 +258,19 @@ export function triggerStreamingJob(jobId: string, conf?: Record<string, unknown
 }
 
 export function restartKafkaConnector(name: string) {
-  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/restart`, { method: 'POST' })
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/restart`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export function pauseKafkaConnector(name: string) {
-  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/pause`, { method: 'POST' })
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/pause`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export function resumeKafkaConnector(name: string) {
-  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/resume`, { method: 'POST' })
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/resume`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export function triggerStreamingPipeline(pipelineId: string) {
-  return fetch(`/api/pipeline/streaming/pipeline/${encodeURIComponent(pipelineId)}`, { method: 'POST' })
+  return fetch(`/api/pipeline/streaming/pipeline/${encodeURIComponent(pipelineId)}`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export function exportHdfsToKafka(body?: { path?: string; topic?: string; limit?: number }) {
@@ -219,7 +282,7 @@ export function exportHdfsToKafka(body?: { path?: string; topic?: string; limit?
 }
 
 export function setStreamingFlow(action: 'pause' | 'resume' | 'stop') {
-  return fetch(`/api/pipeline/streaming/flow/${action}`, { method: 'POST' })
+  return fetch(`/api/pipeline/streaming/flow/${action}`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 // ── Spark Workbench ──────────────────────────────────────────────
@@ -259,7 +322,7 @@ export async function fetchSparkRun(runId: string) {
 }
 
 export function cancelSparkRun(runId: string) {
-  return fetch(`/api/spark/run/${runId}/cancel`, { method: 'POST' })
+  return fetch(`/api/spark/run/${runId}/cancel`, { method: 'POST' , credentials: 'same-origin' })
 }
 
 export async function fetchSparkLive() {

ui/src/lib/infraCatalog.ts

@@ -158,17 +158,17 @@ export const INFRA_CATALOG: InfraNode[] = [
   {
     id: 'gpu',
     label: 'GPU Lab',
-    vm: 'atc-gpu-dev',
-    ip: '10.0.20.106',
+    vm: 'atc-gpu-prod',
+    ip: '10.0.10.106',
     zone: 'gpu',
     agentId: 'infra-sentinel',
     icon: Sparkles,
     accent: '#3fb950',
     description: 'vLLM inference — Llama 3 70B on 4× V100',
-    ssh: 'ssh root@10.0.20.106',
+    ssh: 'ssh root@10.0.10.106',
     apps: [
-      { label: 'GPU Lab UI', url: 'http://10.0.20.106:9000', port: '9000' },
-      { label: 'vLLM API', url: 'http://10.0.20.106:8001/v1', port: '8001' },
+      { label: 'GPU Lab UI', url: 'http://10.0.10.106:9000', port: '9000' },
+      { label: 'vLLM API', url: 'http://10.0.10.106:8001/v1', port: '8001' },
     ],
     topoIds: ['llm', 'cons-ml'],
   },

ui/src/types.ts

@@ -62,12 +62,19 @@ export type CdcStats = {
   ok: boolean
   window_minutes: number
   total: number
+  inserts?: number
+  updates?: number
+  deletes?: number
+  rate_per_min?: number
   by_source: Record<string, number>
   by_op: Record<string, number>
   by_table: Record<string, number>
   buckets: { t: string; n: number }[]
   connected: boolean
   consumed: number
+  buffered?: number
+  buffer_cap?: number
+  last_ts?: string | null
 }
 
 export type PiiColumn = { name: string; category: string; masked: boolean; policy_locked?: boolean }
@@ -217,15 +224,63 @@ export type GpuDevice = {
 export type GpuStatus = {
   ok: boolean
   host: string
+  ip?: string
   ui_url: string
   inference_active?: boolean
   active_model?: string | null
   vllm_url?: string | null
   gpu_count?: number
   gpus?: GpuDevice[]
+  config_source?: string
+  preset_id?: string
+  config_label?: string
   error?: string
 }
 
+export type GpuPreset = {
+  id: string
+  label: string
+  vm: string
+  vmid: number
+  host: string
+  gpu_ui_port: number
+  llm_port: number
+  description: string
+}
+
+export type GpuTargetConfig = {
+  source: string
+  preset_id: string
+  label: string
+  host: string
+  gpu_url: string
+  gpu_ui_url: string
+  llm_url: string
+  env_gpu_url?: string
+  env_llm_url?: string
+  updated_at?: string
+}
+
+export type GpuConfigPayload = {
+  active: GpuTargetConfig
+  presets: GpuPreset[]
+  defaults: GpuTargetConfig
+}
+
+export type GpuConfigTestResult = {
+  ok: boolean
+  host: string
+  gpu_url: string
+  llm_url: string
+  metrics_ok: boolean
+  llm_ok: boolean
+  gpu_count: number
+  inference_active: boolean
+  active_model: string | null
+  errors: string[]
+}
+
+
 export type AgentState = 'idle' | 'walk' | 'fetch' | 'return'
 
 export type AgentAnim = {