Add HDFS browser, SSH terminal, presentation editor, lab health panel

- HDFS WebHDFS file browser (api/hdfs_api.py + HdfsView)
- In-browser SSH terminal via paramiko WebSocket bridge (api/ssh_terminal.py + SshTerminal, xterm.js)
- Presentation deck editor (text + image upload) and CRUD endpoints
- Collapsible GPU matrix + new LabHealthPanel in SideNav
- Topology fixes (edge alignment, Hadoop node, compact nodes)
- nginx ws timeout bump for long-lived SSH sessions
This commit is contained in:
mo
2026-06-26 00:47:49 +00:00
parent fefe3016ad
commit 46b51a891c
21 changed files with 1957 additions and 328 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py .
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py .
RUN mkdir -p /data
ENV DATABASE_URL=sqlite:////data/atc-agents.db
EXPOSE 3201
+197
View File
@@ -0,0 +1,197 @@
"""HDFS (WebHDFS) file browser API for Command Center — mirrors the S3 browser."""
from __future__ import annotations
import os
from datetime import datetime, timezone
from typing import Any
from urllib.parse import quote
import httpx
from fastapi import APIRouter, Query
from fastapi.responses import JSONResponse, StreamingResponse
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
HDFS_USER = os.getenv("HDFS_USER", "hdfs")
WEBHDFS = f"{HDFS_NN_URL}/webhdfs/v1"
router = APIRouter(prefix="/api/storage/hdfs", tags=["hdfs"])
def _human_size(n: int | float | None) -> str:
if not n:
return "0 B"
n = float(n)
for unit in ("B", "KB", "MB", "GB", "TB"):
if n < 1024:
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
n /= 1024
return f"{n:.1f} PB"
def _perm_to_rwx(perm: str, is_dir: bool) -> str:
try:
bits = int(str(perm), 8)
except ValueError:
return ("d" if is_dir else "-") + "---------"
chars = ["d" if is_dir else "-"]
for shift in (6, 3, 0):
triplet = (bits >> shift) & 0o7
chars.append("r" if triplet & 4 else "-")
chars.append("w" if triplet & 2 else "-")
chars.append("x" if triplet & 1 else "-")
return "".join(chars)
def _ms_to_iso(ms: int | None) -> str:
if not ms:
return ""
try:
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat()
except Exception:
return ""
def _norm(path: str) -> str:
p = (path or "/").strip()
if not p.startswith("/"):
p = "/" + p
if len(p) > 1 and p.endswith("/"):
p = p.rstrip("/")
return p or "/"
def _encode(path: str) -> str:
# keep the slash structure, encode each segment
return "/".join(quote(seg, safe="") for seg in path.split("/"))
@router.get("/health")
async def hdfs_health() -> dict[str, Any]:
try:
async with httpx.AsyncClient(timeout=6.0) as client:
jmx = f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState"
r = await client.get(jmx)
beans = (r.json().get("beans") or [{}])[0]
cap = beans.get("CapacityTotal") or 0
used = beans.get("CapacityUsed") or 0
return {
"ok": True,
"namenode": HDFS_NN_URL,
"user": HDFS_USER,
"live_datanodes": beans.get("NumLiveDataNodes"),
"dead_datanodes": beans.get("NumDeadDataNodes"),
"capacity_total_human": _human_size(cap),
"capacity_used_human": _human_size(used),
"capacity_used_pct": round(used / cap * 100, 1) if cap else 0,
}
except Exception as exc:
return {"ok": False, "namenode": HDFS_NN_URL, "error": str(exc)}
@router.get("/list")
async def hdfs_list(path: str = Query("/")) -> JSONResponse:
p = _norm(path)
url = f"{WEBHDFS}{_encode(p)}"
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(url, params={"op": "LISTSTATUS", "user.name": HDFS_USER})
if r.status_code >= 400:
try:
msg = r.json().get("RemoteException", {}).get("message", r.text[:200])
except Exception:
msg = r.text[:200]
return JSONResponse({"ok": False, "path": p, "error": msg}, status_code=200)
statuses = (r.json().get("FileStatuses") or {}).get("FileStatus") or []
except Exception as exc:
return JSONResponse({"ok": False, "path": p, "error": str(exc)}, status_code=200)
folders: list[dict[str, Any]] = []
files: list[dict[str, Any]] = []
for s in statuses:
name = s.get("pathSuffix") or ""
is_dir = s.get("type") == "DIRECTORY"
full = (p.rstrip("/") + "/" + name) if name else p
entry = {
"name": name,
"path": full,
"type": "directory" if is_dir else "file",
"size": s.get("length", 0),
"size_human": _human_size(s.get("length", 0)),
"modified": _ms_to_iso(s.get("modificationTime")),
"owner": s.get("owner"),
"group": s.get("group"),
"perms": _perm_to_rwx(s.get("permission", "000"), is_dir),
"replication": s.get("replication"),
}
(folders if is_dir else files).append(entry)
folders.sort(key=lambda e: e["name"].lower())
files.sort(key=lambda e: e["name"].lower())
return JSONResponse({"ok": True, "path": p, "folders": folders, "files": files})
@router.get("/summary")
async def hdfs_summary(path: str = Query("/")) -> dict[str, Any]:
p = _norm(path)
url = f"{WEBHDFS}{_encode(p)}"
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(url, params={"op": "GETCONTENTSUMMARY", "user.name": HDFS_USER})
cs = r.json().get("ContentSummary") or {}
return {
"ok": True,
"path": p,
"directories": cs.get("directoryCount"),
"files": cs.get("fileCount"),
"length": cs.get("length"),
"length_human": _human_size(cs.get("length")),
"space_consumed": cs.get("spaceConsumed"),
"space_consumed_human": _human_size(cs.get("spaceConsumed")),
}
except Exception as exc:
return {"ok": False, "path": p, "error": str(exc)}
@router.get("/download")
async def hdfs_download(path: str = Query(...)):
p = _norm(path)
url = f"{WEBHDFS}{_encode(p)}"
fname = p.rsplit("/", 1)[-1] or "download"
async def stream():
async with httpx.AsyncClient(timeout=None, follow_redirects=True, verify=False) as client:
async with client.stream("GET", url, params={"op": "OPEN", "user.name": HDFS_USER}) as resp:
if resp.status_code >= 400:
return
async for chunk in resp.aiter_bytes(chunk_size=65536):
yield chunk
return StreamingResponse(
stream(),
media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{fname}"'},
)
@router.get("/preview")
async def hdfs_preview(path: str = Query(...), max_bytes: int = Query(65536)) -> dict[str, Any]:
"""Return the first chunk of a file as text (for quick inspection)."""
p = _norm(path)
url = f"{WEBHDFS}{_encode(p)}"
cap = max(1024, min(max_bytes, 512 * 1024))
try:
async with httpx.AsyncClient(timeout=20.0, follow_redirects=True, verify=False) as client:
r = await client.get(url, params={"op": "OPEN", "user.name": HDFS_USER, "length": cap})
if r.status_code >= 400:
return {"ok": False, "path": p, "error": f"HTTP {r.status_code}"}
raw = r.content[:cap]
try:
text = raw.decode("utf-8")
binary = False
except UnicodeDecodeError:
text = raw[:4096].decode("latin-1", errors="replace")
binary = True
return {"ok": True, "path": p, "binary": binary, "truncated": len(r.content) >= cap, "text": text}
except Exception as exc:
return {"ok": False, "path": p, "error": str(exc)}
+80 -2
View File
@@ -13,7 +13,7 @@ from typing import Any
import httpx
import redis.asyncio as aioredis
from fastapi import FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from agent_terminal import (
get_all_terminals,
@@ -25,11 +25,22 @@ from agent_terminal import (
)
from lab_context import collect_full_lab_context, format_context_for_agent
from presentation import build_presentation_payload, render_presentation_html
from presentation_upload import get_deck, list_decks, save_upload
from presentation_upload import (
create_deck,
delete_deck,
get_asset_path,
get_deck,
list_decks,
save_deck,
save_image,
save_upload,
)
from presentation_static import get_static_deck, list_static_decks
from storage_s3 import router as storage_s3_router
from hdfs_api import router as hdfs_router
from elasticsearch_api import router as elasticsearch_router
from sql_console import router as sql_router
from ssh_terminal import ssh_session
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
from node_ops import build_node_detail, probe_node, run_node_probe_task
from approval_service import (
@@ -707,6 +718,7 @@ async def lifespan(app: FastAPI):
app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
app.include_router(storage_s3_router)
app.include_router(hdfs_router)
app.include_router(elasticsearch_router)
app.include_router(sql_router)
app.add_middleware(
@@ -814,6 +826,67 @@ async def upload_presentation(file: UploadFile = File(...)):
deck = await save_upload(file.filename or "upload.pptx", content)
return {"ok": True, "deck": deck}
@app.post("/api/presentation/decks/new")
async def create_presentation_deck(body: dict[str, Any] | None = Body(default=None)):
body = body or {}
title = str(body.get("title") or "Untitled deck")
seed_id = body.get("from")
slides = None
if seed_id:
src = get_static_deck(seed_id) or get_deck(seed_id)
if seed_id == "live":
src = await get_presentation_data()
if src and src.get("slides"):
slides = [
{
"id": s.get("id") or f"slide-{i}",
"title": s.get("title") or f"Slide {i}",
"subtitle": s.get("subtitle") or "",
"bullets": list(s.get("bullets") or []),
"image": s.get("image") or "",
"kind": s.get("kind") or "narrative",
}
for i, s in enumerate(src["slides"], start=1)
]
if not title or title == "Untitled deck":
title = f"{src.get('title', 'Deck')} (copy)"
deck = create_deck(title, slides)
return {"ok": True, "deck": deck}
@app.put("/api/presentation/decks/{deck_id}")
async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
deck = save_deck(deck_id, body)
if not deck:
return {"error": "deck not found or not editable"}
return {"ok": True, "deck": deck}
@app.delete("/api/presentation/decks/{deck_id}")
async def remove_presentation_deck(deck_id: str):
return {"ok": delete_deck(deck_id)}
@app.post("/api/presentation/decks/{deck_id}/image")
async def upload_presentation_image(deck_id: str, file: UploadFile = File(...)):
content = await file.read()
if len(content) > 12 * 1024 * 1024:
return {"error": "image too large (max 12MB)"}
result = save_image(deck_id, file.filename or "image.png", content)
if not result:
return {"error": "deck not found"}
return {"ok": True, **result}
@app.get("/api/presentation/decks/{deck_id}/assets/{name}")
async def get_presentation_asset(deck_id: str, name: str):
from fastapi.responses import FileResponse, Response
path = get_asset_path(deck_id, name)
if not path:
return Response(status_code=404)
return FileResponse(str(path))
@app.get("/api/workload")
async def get_workload(fast: bool = True):
return await collect_workload(fast=fast, use_cache=True)
@@ -1004,6 +1077,11 @@ async def post_prompt(body: PromptRequest):
return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"}
@app.websocket("/api/ws/ssh")
async def ws_ssh(websocket: WebSocket):
await ssh_session(websocket)
@app.websocket("/api/ws/ops")
async def ws_ops(websocket: WebSocket):
await websocket.accept()
-1
View File
@@ -135,7 +135,6 @@ async def probe_node(node_id: str) -> dict[str, Any]:
await _log(node_id, "info", "shell", " WebSocket: /api/ws/ops")
elif node_id in ("mo-commander", "bart-commander", "mcp-coordinator", "network-watcher"):
from node_registry import NODE_REGISTRY
meta = NODE_REGISTRY[node_id]
await _log(node_id, "ok", "shell", f"{meta['label']} online — monitoring all agent comms")
await _log(node_id, "info", "shell", meta.get("description", ""))
+3 -1
View File
@@ -245,6 +245,7 @@ header .meta{{font-size:.75rem;opacity:.7}}
.slide ul{{list-style:none;font-size:1.15rem;line-height:1.9}}
.slide li::before{{content:"";color:#60a5fa}}
.slide.hero h2{{font-size:3.2rem}}
.slide img.slide-img{{max-height:42vh;max-width:100%;margin-top:1.5rem;border-radius:10px;border:1px solid rgba(96,165,250,.25);box-shadow:0 8px 30px rgba(0,0,0,.4);object-fit:contain}}
nav{{display:flex;gap:.5rem;padding:1rem 2rem;border-top:1px solid rgba(96,165,250,.2);align-items:center}}
nav button{{background:#1e3a5f;border:1px solid rgba(96,165,250,.3);color:#e8f1ff;padding:.5rem 1rem;border-radius:6px;cursor:pointer}}
nav button:hover{{background:#234876}}
@@ -275,7 +276,8 @@ slides.forEach((s,idx)=>{{
const el=document.createElement("section");
el.className="slide"+(s.kind==="hero"?" hero":"")+(idx===0?" active":"");
const bullets=(s.bullets||[]).map(b=>"<li>"+b+"</li>").join("");
el.innerHTML="<h2>"+s.title+"</h2><h3>"+(s.subtitle||"")+"</h3><ul>"+bullets+"</ul>";
const img=s.image?'<img class="slide-img" src="'+s.image+'" alt=""/>':"";
el.innerHTML="<h2>"+s.title+"</h2><h3>"+(s.subtitle||"")+"</h3><ul>"+bullets+"</ul>"+img;
container.appendChild(el);
const d=document.createElement("button");
d.className="dot"+(idx===0?" active":"");
+106
View File
@@ -163,3 +163,109 @@ def get_deck(deck_id: str) -> dict[str, Any] | None:
if not path.exists():
return None
return json.loads(path.read_text())
def _blank_slide(idx: int = 1) -> dict[str, Any]:
return {
"id": f"slide-{idx}",
"title": f"Slide {idx}",
"subtitle": "",
"bullets": ["Double-click to edit this point"],
"image": "",
"kind": "narrative",
}
def create_deck(title: str = "Untitled deck", slides: list[dict[str, Any]] | None = None) -> dict[str, Any]:
"""Create a new editable deck (optionally seeded from existing slides)."""
_ensure_dir()
deck_id = str(uuid.uuid4())[:8]
deck_dir = PRESENTATIONS_DIR / deck_id
(deck_dir / "assets").mkdir(parents=True, exist_ok=True)
seeded = [dict(s) for s in (slides or [])] or [_blank_slide(1)]
payload = {
"id": deck_id,
"filename": "",
"source": "editor",
"editable": True,
"ts": datetime.now(timezone.utc).isoformat(),
"title": (title or "Untitled deck").strip()[:120] or "Untitled deck",
"subtitle": "Editable deck",
"slide_count": len(seeded),
"slides": seeded,
}
(deck_dir / "meta.json").write_text(json.dumps(payload, indent=2, default=str))
(deck_dir / "deck.json").write_text(json.dumps(payload, default=str))
return payload
def save_deck(deck_id: str, body: dict[str, Any]) -> dict[str, Any] | None:
"""Overwrite an existing editable deck's content. Returns None if missing."""
deck_dir = PRESENTATIONS_DIR / deck_id
meta_path = deck_dir / "meta.json"
if not meta_path.exists():
return None
existing = json.loads(meta_path.read_text())
incoming_slides = body.get("slides")
clean_slides: list[dict[str, Any]] = []
for i, s in enumerate(incoming_slides or [], start=1):
if not isinstance(s, dict):
continue
bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
clean_slides.append({
"id": str(s.get("id") or f"slide-{i}"),
"title": str(s.get("title") or f"Slide {i}")[:200],
"subtitle": str(s.get("subtitle") or "")[:300],
"bullets": bullets,
"image": str(s.get("image") or "")[:300],
"kind": str(s.get("kind") or "narrative")[:40],
})
if not clean_slides:
clean_slides = [_blank_slide(1)]
existing.update({
"title": str(body.get("title") or existing.get("title") or "Untitled deck")[:120],
"subtitle": str(body.get("subtitle") or existing.get("subtitle") or "")[:300],
"editable": True,
"source": existing.get("source") or "editor",
"ts": datetime.now(timezone.utc).isoformat(),
"slides": clean_slides,
"slide_count": len(clean_slides),
})
meta_path.write_text(json.dumps(existing, indent=2, default=str))
(deck_dir / "deck.json").write_text(json.dumps(existing, default=str))
return existing
def delete_deck(deck_id: str) -> bool:
import shutil
deck_dir = PRESENTATIONS_DIR / deck_id
if not deck_dir.exists():
return False
shutil.rmtree(deck_dir, ignore_errors=True)
return True
def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | None:
"""Store an image in the deck's assets folder; return its served URL."""
deck_dir = PRESENTATIONS_DIR / deck_id
if not deck_dir.exists():
return None
assets = deck_dir / "assets"
assets.mkdir(parents=True, exist_ok=True)
ext = ""
if "." in filename:
ext = "." + _safe_name(filename.rsplit(".", 1)[-1].lower())[:8]
name = f"{uuid.uuid4().hex[:12]}{ext}"
(assets / name).write_bytes(content)
return {"name": name, "url": f"/api/presentation/decks/{deck_id}/assets/{name}"}
def get_asset_path(deck_id: str, name: str) -> Path | None:
safe = _safe_name(name)
path = PRESENTATIONS_DIR / deck_id / "assets" / safe
if not path.exists() or not path.is_file():
return None
return path
+1
View File
@@ -14,3 +14,4 @@ cassandra-driver==3.29.2
neo4j==5.26.0
python-pptx==1.0.2
boto3==1.35.99
paramiko==3.5.0
+146
View File
@@ -0,0 +1,146 @@
"""Interactive SSH terminal bridge over WebSocket (paramiko PTY <-> xterm.js)."""
from __future__ import annotations
import asyncio
import json
from typing import Any
import paramiko
from fastapi import WebSocket, WebSocketDisconnect
async def _send(ws: WebSocket, type_: str, **kw: Any) -> None:
try:
await ws.send_text(json.dumps({"type": type_, **kw}))
except Exception:
pass
def _open_ssh(cfg: dict[str, Any]) -> tuple[paramiko.SSHClient, paramiko.Channel]:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
connect_kwargs: dict[str, Any] = {
"hostname": cfg["host"],
"port": int(cfg.get("port") or 22),
"username": cfg.get("username") or "root",
"timeout": 12,
"banner_timeout": 12,
"auth_timeout": 12,
"look_for_keys": False,
"allow_agent": False,
}
password = cfg.get("password")
key_data = cfg.get("private_key")
if key_data:
from io import StringIO
pkey = None
for loader in (paramiko.Ed25519Key, paramiko.RSAKey, paramiko.ECDSAKey):
try:
pkey = loader.from_private_key(StringIO(key_data), password=password or None)
break
except Exception:
continue
if pkey is None:
raise ValueError("Unsupported or invalid private key")
connect_kwargs["pkey"] = pkey
elif password:
connect_kwargs["password"] = password
else:
raise ValueError("No password or private key provided")
client.connect(**connect_kwargs)
cols = int(cfg.get("cols") or 120)
rows = int(cfg.get("rows") or 32)
chan = client.invoke_shell(term="xterm-256color", width=cols, height=rows)
chan.settimeout(0.0)
return client, chan
async def ssh_session(ws: WebSocket) -> None:
await ws.accept()
client: paramiko.SSHClient | None = None
chan: paramiko.Channel | None = None
loop = asyncio.get_event_loop()
try:
# First message must be the connect config
first = await ws.receive_text()
cfg = json.loads(first)
if cfg.get("type") != "connect":
await _send(ws, "error", message="Expected connect message")
await ws.close()
return
await _send(ws, "status", message=f"Connecting to {cfg.get('username','root')}@{cfg.get('host')}:{cfg.get('port',22)}")
try:
client, chan = await loop.run_in_executor(None, _open_ssh, cfg)
except paramiko.AuthenticationException:
await _send(ws, "error", message="Authentication failed — check username/password")
await ws.close()
return
except Exception as exc:
await _send(ws, "error", message=f"Connection failed: {exc}")
await ws.close()
return
await _send(ws, "connected", message="connected")
async def pump_out() -> None:
assert chan is not None
while True:
if chan.closed or chan.exit_status_ready() and not chan.recv_ready():
if not chan.recv_ready():
break
if chan.recv_ready():
try:
data = await loop.run_in_executor(None, chan.recv, 65536)
except Exception:
break
if not data:
break
await ws.send_text(json.dumps({"type": "data", "data": data.decode("utf-8", errors="replace")}))
else:
await asyncio.sleep(0.02)
await _send(ws, "closed", message="Session closed")
out_task = asyncio.create_task(pump_out())
try:
while True:
msg = await ws.receive_text()
try:
parsed = json.loads(msg)
except json.JSONDecodeError:
parsed = {"type": "data", "data": msg}
mtype = parsed.get("type")
if mtype == "data" and chan and not chan.closed:
chan.send(parsed.get("data", ""))
elif mtype == "resize" and chan and not chan.closed:
try:
chan.resize_pty(width=int(parsed.get("cols", 120)), height=int(parsed.get("rows", 32)))
except Exception:
pass
elif mtype == "disconnect":
break
finally:
out_task.cancel()
except WebSocketDisconnect:
pass
except Exception as exc:
await _send(ws, "error", message=str(exc))
finally:
try:
if chan is not None:
chan.close()
except Exception:
pass
try:
if client is not None:
client.close()
except Exception:
pass
try:
await ws.close()
except Exception:
pass