Add iDRAC Console wall, power control, and proxy embeds.

Operators can power nodes via OME jobs and view multiple live iDRAC
consoles in-panel (4/8 wall) through a same-origin reverse proxy;
Present decks updated for the new remote-ops story.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
root
2026-07-19 00:16:19 +02:00
parent 795707e47b
commit 03e207b46d
9 changed files with 2066 additions and 31 deletions
+516 -2
View File
@@ -1,4 +1,5 @@
import urllib.parse
import asyncio
import json
import re
@@ -14,9 +15,9 @@ from typing import Any
import httpx
import asyncssh
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, Response
from fastapi.responses import FileResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from pydantic_settings import BaseSettings
@@ -3133,6 +3134,514 @@ async def device_detail(device_id: int):
return detail
# OME JobService powerState values (Dell set_power_state.py)
OME_POWER_ACTIONS = {
"on": ("2", "Power On"),
"cycle": ("5", "Power Cycle"),
"force_off": ("8", "Power Off Non-Graceful"),
"off": ("12", "Power Off Graceful"),
}
class PowerActionIn(BaseModel):
action: str # on | off | cycle | force_off
confirm: bool = False
def _fleet_device(device_id: int) -> dict | None:
return next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
@app.post("/api/devices/{device_id}/power")
async def device_power_control(device_id: int, payload: PowerActionIn):
"""Power control via OME JobService DeviceAction_Task (iDRAC through OME)."""
action = (payload.action or "").strip().lower()
if action not in OME_POWER_ACTIONS:
raise HTTPException(400, f"Invalid action. Use one of: {', '.join(OME_POWER_ACTIONS)}")
if action in ("off", "force_off", "cycle") and not payload.confirm:
raise HTTPException(400, "confirm=true required for off/cycle actions")
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
if node.get("source") == "inventory" or (isinstance(device_id, int) and device_id < 0):
raise HTTPException(400, "Inventory-only hosts cannot be powered via OME")
if not (node.get("is_server") or node.get("is_idrac") or node.get("type") == 1000):
raise HTTPException(400, "Power control is only supported for servers / iDRAC endpoints")
power_state, label = OME_POWER_ACTIONS[action]
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
body = {
"Id": 0,
"JobName": f"Cockpit {label}",
"JobDescription": f"OME Cockpit power control · {node.get('name')} · {node.get('service_tag')}",
"State": "Enabled",
"Schedule": "startnow",
"JobType": {"Name": "DeviceAction_Task"},
"Targets": [
{
"Id": int(device_id),
"Data": "",
"TargetType": {"Id": 1000, "Name": "DEVICE"},
}
],
"Params": [
{"Key": "override", "Value": "true"},
{"Key": "powerState", "Value": str(power_state)},
{"Key": "operationName", "Value": "POWER_CONTROL"},
{"Key": "deviceTypes", "Value": "1000"},
],
}
r = await client.post(
f"{base}/api/JobService/Jobs",
headers={**headers, "Content-Type": "application/json"},
json=body,
)
if r.status_code >= 400:
raise HTTPException(
r.status_code,
f"OME power job rejected: {(r.text or '')[:300]}",
)
job = r.json() if r.content else {}
job_id = job.get("Id")
log.info(
"OME power %s for device %s (%s) → job %s",
action,
device_id,
node.get("service_tag"),
job_id,
)
return {
"ok": True,
"action": action,
"label": label,
"device_id": device_id,
"name": node.get("name"),
"service_tag": node.get("service_tag"),
"job_id": job_id,
"note": "OME accepted the job. Power state updates on the next fleet poll.",
}
finally:
await ome_session_delete(client, base, headers, sid)
@app.get("/api/devices/{device_id}/idrac-console")
async def device_idrac_console(device_id: int, request: Request):
"""Return iDRAC web / HTML5 console URLs — prefer same-origin proxy embed."""
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
ip = node.get("idrac_ip") or node.get("ip")
if not ip:
raise HTTPException(400, "No iDRAC / management IP for this device")
base = f"https://{ip}"
# Same-origin reverse proxy so X-Frame-Options / CSP cannot block the iframe
embed = f"/api/idrac-proxy/{device_id}/restgui/start.html?console"
urls = {
"web": base + "/",
"html5": base + "/console",
"restgui": base + "/restgui/start.html?console",
"viewer": base + "/virtualconsole",
"embed": str(request.base_url).rstrip("/") + embed,
"embed_path": embed,
}
return {
"device_id": device_id,
"name": node.get("name"),
"service_tag": node.get("service_tag"),
"ip": ip,
"powered_on": node.get("powered_on"),
"connected": node.get("connected"),
"urls": urls,
"primary": embed,
"note": (
"Console is proxied through Cockpit (same-origin) so it can run in this panel. "
"Log in with your iDRAC credentials. Popout still opens the iDRAC directly."
),
}
_IDRAC_HOP_HEADERS = {
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailers",
"transfer-encoding",
"upgrade",
"content-length",
"content-encoding",
"content-security-policy",
"content-security-policy-report-only",
"x-frame-options",
"x-xss-protection",
"strict-transport-security",
}
def _idrac_proxy_prefix(device_id: int) -> str:
return f"/api/idrac-proxy/{device_id}"
def _idrac_resolve(device_id: int) -> tuple[dict, str]:
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
# Prefer explicit iDRAC management IP; never treat bare OS inventory as iDRAC
ip = node.get("idrac_ip")
if not ip and node.get("is_idrac"):
ip = node.get("ip")
if not ip:
st = (node.get("service_tag") or "").strip()
if st:
sibling = next(
(
d
for d in (STATE.get("devices") or [])
if d.get("is_idrac")
and (d.get("service_tag") or "").strip().upper() == st.upper()
and (d.get("idrac_ip") or d.get("ip"))
),
None,
)
if sibling:
node = sibling
ip = sibling.get("idrac_ip") or sibling.get("ip")
if not ip:
raise HTTPException(
400,
"No iDRAC management IP for this device — select an iDRAC endpoint (OOB), not the OS host",
)
try:
ipaddress.ip_address(ip)
except ValueError as e:
raise HTTPException(400, "Invalid management IP") from e
return node, ip
def _idrac_rewrite_location(value: str, device_id: int, idrac_ip: str) -> str:
prefix = _idrac_proxy_prefix(device_id)
if not value:
return value
if value.startswith("/"):
return prefix + value
try:
u = urllib.parse.urlparse(value)
except Exception:
return value
host = (u.hostname or "").lower()
if host == idrac_ip.lower() or host.endswith(".dell-atc.lan") or host.endswith(".internal."):
path = u.path or "/"
return prefix + path + (("?" + u.query) if u.query else "") + (("#" + u.fragment) if u.fragment else "")
return value
def _idrac_rewrite_cookie(value: str, device_id: int) -> str:
"""Point cookies at the proxy path; drop Domain/Secure so HTTP cockpit can store them."""
prefix = _idrac_proxy_prefix(device_id)
parts = []
for part in value.split(";"):
p = part.strip()
pl = p.lower()
if pl.startswith("domain="):
continue
if pl == "secure":
continue
if pl.startswith("path="):
parts.append(f"Path={prefix}/")
continue
parts.append(p)
if not any(p.lower().startswith("path=") for p in parts):
parts.append(f"Path={prefix}/")
return "; ".join(parts)
def _idrac_inject_bootstrap(html: bytes, device_id: int, idrac_ip: str) -> bytes:
"""Rewrite absolute paths + patch fetch/XHR/WebSocket so the SPA stays on the proxy."""
prefix = _idrac_proxy_prefix(device_id)
try:
text = html.decode("utf-8")
except UnicodeDecodeError:
text = html.decode("latin-1")
# Absolute root paths in markup
for attr in ("href", "src", "action"):
text = re.sub(
rf'({attr}\s*=\s*["\'])/(?!/)',
rf"\1{prefix}/",
text,
flags=re.I,
)
text = text.replace(f"https://{idrac_ip}/", f"{prefix}/")
text = text.replace(f"http://{idrac_ip}/", f"{prefix}/")
boot = f"""<script>(function(){{
var PREFIX={json.dumps(prefix)};
var IDRAC_IP={json.dumps(idrac_ip)};
function abs(u){{
if(!u || typeof u!=="string") return u;
try{{
if(u.indexOf("blob:")===0 || u.indexOf("data:")===0 || u.indexOf("mailto:")===0) return u;
var x=new URL(u, location.href);
var h=(x.hostname||"").toLowerCase();
if(h===IDRAC_IP.toLowerCase() || (h && h.indexOf("dell-atc.lan")!==-1)){{
return location.origin + PREFIX + (x.pathname||"/") + x.search + x.hash;
}}
if(x.origin===location.origin && x.pathname.indexOf(PREFIX)!==0){{
return location.origin + PREFIX + (x.pathname||"/") + x.search + x.hash;
}}
return u;
}}catch(e){{ return u; }}
}}
function wsAbs(u){{
var a=abs(u);
if(typeof a!=="string") return u;
if(location.protocol==="http:" && a.indexOf("wss://"+location.host)===0)
return "ws://"+location.host+a.slice(("wss://"+location.host).length);
if(location.protocol==="https:" && a.indexOf("ws://"+location.host)===0)
return "wss://"+location.host+a.slice(("ws://"+location.host).length);
return a;
}}
var _f=window.fetch;
window.fetch=function(input, init){{
if(typeof input==="string") input=abs(input);
else if(input && typeof Request!=="undefined" && input instanceof Request)
input=new Request(abs(input.url), input);
return _f.call(this, input, init);
}};
var _o=XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open=function(method, url){{
var args=Array.prototype.slice.call(arguments, 2);
return _o.apply(this, [method, abs(url)].concat(args));
}};
var _WS=window.WebSocket;
window.WebSocket=function(url, protocols){{
var u=wsAbs(url);
return protocols===undefined ? new _WS(u) : new _WS(u, protocols);
}};
window.WebSocket.prototype=_WS.prototype;
window.WebSocket.CONNECTING=_WS.CONNECTING;
window.WebSocket.OPEN=_WS.OPEN;
window.WebSocket.CLOSING=_WS.CLOSING;
window.WebSocket.CLOSED=_WS.CLOSED;
}})();</script>"""
if re.search(r"<head[^>]*>", text, flags=re.I):
text = re.sub(r"(<head[^>]*>)", r"\1" + boot, text, count=1, flags=re.I)
else:
text = boot + text
return text.encode("utf-8")
def _idrac_rewrite_css(css: bytes, device_id: int, idrac_ip: str) -> bytes:
prefix = _idrac_proxy_prefix(device_id)
try:
text = css.decode("utf-8")
except UnicodeDecodeError:
text = css.decode("latin-1")
text = re.sub(r"url\((['\"]?)/", rf"url(\1{prefix}/", text)
text = text.replace(f"https://{idrac_ip}/", f"{prefix}/")
return text.encode("utf-8")
async def _idrac_proxy_http(device_id: int, path: str, request: Request):
import gzip
_, idrac_ip = _idrac_resolve(device_id)
prefix = _idrac_proxy_prefix(device_id)
rel = path or ""
qs = request.url.query
target = f"https://{idrac_ip}/{rel}" + (f"?{qs}" if qs else "")
# Forward headers; force gzip so iDRAC serves .js/.css (precompressed only)
fwd: dict[str, str] = {}
for k, v in request.headers.items():
lk = k.lower()
if lk in ("host", "content-length", "connection", "accept-encoding"):
continue
if lk == "referer" and v:
# Map our proxy referer back to iDRAC origin when possible
v = v.replace(str(request.base_url).rstrip("/") + prefix, f"https://{idrac_ip}")
v = v.replace(prefix, f"https://{idrac_ip}")
fwd[k] = v
fwd["Host"] = idrac_ip
fwd["Accept-Encoding"] = "gzip, deflate"
# Avoid long-lived upstream hangs
body = await request.body()
try:
async with httpx.AsyncClient(verify=False, timeout=120.0, follow_redirects=False) as client:
upstream = await client.request(
request.method,
target,
headers=fwd,
content=body if body else None,
)
except httpx.RequestError as e:
accept = (request.headers.get("accept") or "").lower()
dest = (request.headers.get("sec-fetch-dest") or "").lower()
if "text/html" in accept or dest == "iframe" or request.method == "GET":
err = (
str(e)
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
)
html = f"""<!DOCTYPE html><html><head><meta charset="utf-8"><title>iDRAC unreachable</title>
<style>
body{{margin:0;font:14px/1.45 system-ui,sans-serif;background:#0a0e14;color:#e8f0f8;
display:grid;place-items:center;min-height:100vh;padding:1.5rem}}
.card{{max-width:28rem;border:1px solid rgba(232,122,42,.45);border-radius:12px;padding:1.25rem 1.4rem;
background:rgba(20,12,8,.9)}}
h1{{margin:0 0 .4rem;font-size:1.05rem;color:#ffb06a}}
p{{margin:.35rem 0;color:#9eb4c8}}
code{{color:#fff;font-size:.85em}}
</style></head><body><div class="card">
<h1>iDRAC not reachable via proxy</h1>
<p>Target <code>{idrac_ip}</code> did not accept a connection from Cockpit.</p>
<p>Use an <strong>iDRAC / OOB</strong> endpoint (VLAN 40/41), not the OS host IP.</p>
<p style="font-size:.78rem;opacity:.75">{err}</p>
</div></body></html>"""
return Response(content=html, status_code=502, media_type="text/html")
raise HTTPException(502, f"iDRAC proxy unreachable: {e}") from e
raw = upstream.content
enc = (upstream.headers.get("content-encoding") or "").lower()
if "gzip" in enc and raw:
try:
raw = gzip.decompress(raw)
except Exception:
pass
elif "deflate" in enc and raw:
try:
import zlib
raw = zlib.decompress(raw)
except Exception:
try:
import zlib
raw = zlib.decompress(raw, -zlib.MAX_WBITS)
except Exception:
pass
ctype = (upstream.headers.get("content-type") or "").lower()
if "text/html" in ctype and raw:
raw = _idrac_inject_bootstrap(raw, device_id, idrac_ip)
elif "text/css" in ctype and raw:
raw = _idrac_rewrite_css(raw, device_id, idrac_ip)
elif ("javascript" in ctype or "ecmascript" in ctype) and raw:
# Light touch: rewrite absolute iDRAC URLs embedded as strings
try:
t = raw.decode("utf-8")
except UnicodeDecodeError:
t = raw.decode("latin-1")
t = t.replace(f"https://{idrac_ip}", prefix)
t = t.replace(f"wss://{idrac_ip}", f"ws://{request.url.hostname}:{request.url.port or (443 if request.url.scheme == 'https' else 80)}{prefix}")
raw = t.encode("utf-8")
out_headers: list[tuple[str, str]] = []
for k, v in upstream.headers.multi_items():
lk = k.lower()
if lk in _IDRAC_HOP_HEADERS:
continue
if lk == "location":
out_headers.append((k, _idrac_rewrite_location(v, device_id, idrac_ip)))
continue
if lk == "set-cookie":
out_headers.append((k, _idrac_rewrite_cookie(v, device_id)))
continue
out_headers.append((k, v))
# Allow embedding in Cockpit UI
out_headers.append(("Content-Security-Policy", "frame-ancestors *"))
out_headers.append(("Cache-Control", "no-store"))
resp = Response(content=raw, status_code=upstream.status_code)
for k, v in out_headers:
resp.headers.append(k, v)
if not any(k.lower() == "content-type" for k, _ in out_headers):
ct = upstream.headers.get("content-type")
if ct:
resp.headers["content-type"] = ct
return resp
@app.api_route(
"/api/idrac-proxy/{device_id}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],
)
@app.api_route(
"/api/idrac-proxy/{device_id}/{path:path}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],
)
async def idrac_http_proxy(device_id: int, request: Request, path: str = ""):
return await _idrac_proxy_http(device_id, path, request)
@app.websocket("/api/idrac-proxy/{device_id}/{path:path}")
async def idrac_ws_proxy(ws: WebSocket, device_id: int, path: str):
"""Bridge browser WebSocket ↔ iDRAC (virtual console / live sessions)."""
import ssl
import websockets
_, idrac_ip = _idrac_resolve(device_id)
await ws.accept()
qs = ws.scope.get("query_string", b"").decode()
target = f"wss://{idrac_ip}/{path}" + (f"?{qs}" if qs else "")
sub = ws.scope.get("subprotocols") or []
ssl_ctx = ssl._create_unverified_context()
try:
async with websockets.connect(
target,
ssl=ssl_ctx,
subprotocols=list(sub) if sub else None,
open_timeout=30,
max_size=8 * 1024 * 1024,
) as upstream:
async def client_to_upstream():
while True:
msg = await ws.receive()
if msg.get("type") == "websocket.disconnect":
break
if "text" in msg and msg["text"] is not None:
await upstream.send(msg["text"])
elif "bytes" in msg and msg["bytes"] is not None:
await upstream.send(msg["bytes"])
async def upstream_to_client():
async for message in upstream:
if isinstance(message, (bytes, bytearray)):
await ws.send_bytes(message)
else:
await ws.send_text(message)
_done, pending = await asyncio.wait(
[
asyncio.create_task(client_to_upstream()),
asyncio.create_task(upstream_to_client()),
],
return_when=asyncio.FIRST_COMPLETED,
)
for t in pending:
t.cancel()
except HTTPException:
try:
await ws.close()
except Exception:
pass
raise
except Exception as e:
log.warning("iDRAC WS proxy error device=%s: %s", device_id, e)
try:
await ws.close(code=1011)
except Exception:
pass
@app.get("/api/devices/{device_id}/expansion")
async def device_expansion(device_id: int, force: bool = False):
if force and device_id in DETAIL_CACHE:
@@ -5949,6 +6458,11 @@ async def network_js():
return FileResponse(STATIC_DIR / "network.js", media_type="application/javascript")
@app.get("/console.js")
async def console_js():
return FileResponse(STATIC_DIR / "console.js", media_type="application/javascript")
@app.get("/ssh.js")
async def ssh_js():
return FileResponse(STATIC_DIR / "ssh.js", media_type="application/javascript")