ATC/ome-cockpit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
03e207b46d4a575ea7e89bb937e56b1ee78bc18d
ome-cockpit/api/main.py
T

6609 lines
260 KiB
Python
Raw Normal View History

Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
Add iDRAC Console wall, power control, and proxy embeds.
2026-07-19 00:16:19 +02:00
import urllib.parse
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
import asyncio
import json
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
import re
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
import os
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
import socket
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
import sqlite3
import ipaddress
import logging
import time
from collections import defaultdict
from pathlib import Path
from typing import Any
import httpx
import asyncssh
Add iDRAC Console wall, power control, and proxy embeds.
2026-07-19 00:16:19 +02:00
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Request
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
from fastapi.middleware.cors import CORSMiddleware
Add iDRAC Console wall, power control, and proxy embeds.
2026-07-19 00:16:19 +02:00
from fastapi.responses import FileResponse, Response, StreamingResponse
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
from pydantic import BaseModel, Field
from pydantic_settings import BaseSettings
logging.basicConfig(level=logging.INFO)
log = logging.getLogger("ome-cockpit")
class Settings(BaseSettings):
ome_url: str = "https://cov-omeprod01.dell-atc.lan"
ome_user: str = "admin"
ome_password: str = ""
poll_interval: float = 15.0
openwebui_url: str = "http://atc-portal01.dell-atc.lan:3080"
cors_origins: str = "*"
power_fetch_limit: int = 40
gpu_metrics_url: str = "http://10.0.10.106:9110"
vllm_url: str = "http://10.0.10.106:8000/v1"
vllm_model: str = "llama3-70b-gptq"
vllm_max_model_len: int = 4096
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
vllm_max_tokens: int = 1100
chat_system_chars: int = 12000
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
openwebui_email: str = ""
openwebui_password: str = ""
cockpit_data: str = "/data"
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
reports_cache_ttl: float = 600.0
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
class Config:
env_file = ".env"
settings = Settings()
app = FastAPI(title="OME Cockpit API", version="1.2.0")
app.add_middleware(
CORSMiddleware,
allow_origins=[o.strip() for o in settings.cors_origins.split(",")],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
STATE: dict[str, Any] = {
"updated_at": 0,
"ome": {},
"summary": {},
"subnets": [],
"devices": [],
"groups": [],
"models": [],
"alerts": [],
"events": [],
"context": {},
"gpu": {},
"pulse": 0,
"openwebui_url": settings.openwebui_url,
}
CLIENTS: set[WebSocket] = set()
SSH_SESSIONS: dict[str, dict] = {}
SSH_MAX_SESSIONS = 25
SSH_MAX_PER_CLIENT = 3
_ssh_sem: asyncio.Semaphore | None = None
_lock = asyncio.Lock()
DETAIL_CACHE: dict[int, dict] = {}
PREV_DEVICES: dict[int, dict] = {}
EVENT_FEED: list[dict] = []
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
REPORT_CACHE: dict[str, Any] = {
"warranties": None,
"warranties_ts": 0.0,
"compliance": None,
"compliance_ts": 0.0,
"baselines": None,
"baselines_ts": 0.0,
"catalogs": None,
"catalogs_ts": 0.0,
"report_defs": None,
"report_defs_ts": 0.0,
"jobs": None,
"jobs_ts": 0.0,
}
async def ome_session(client: httpx.AsyncClient) -> tuple[str, dict, str]:
"""Create an OME API session; returns (base, headers, session_id)."""
base = settings.ome_url.rstrip("/")
r = await client.post(
f"{base}/api/SessionService/Sessions",
json={
"UserName": settings.ome_user,
"Password": settings.ome_password,
"SessionType": "API",
},
)
r.raise_for_status()
token = r.headers.get("X-Auth-Token")
sid = str((r.json() or {}).get("Id") or "")
headers = {"X-Auth-Token": token, "Accept": "application/json"}
return base, headers, sid
async def ome_session_delete(client: httpx.AsyncClient, base: str, headers: dict, sid: str) -> None:
if not sid:
return
try:
await client.delete(f"{base}/api/SessionService/Sessions('{sid}')", headers=headers)
except Exception:
pass
def _cache_get(key: str) -> Any | None:
ts = float(REPORT_CACHE.get(f"{key}_ts") or 0)
if REPORT_CACHE.get(key) is not None and time.time() - ts < settings.reports_cache_ttl:
return REPORT_CACHE.get(key)
return None
def _cache_set(key: str, value: Any) -> Any:
REPORT_CACHE[key] = value
REPORT_CACHE[f"{key}_ts"] = time.time()
return value
def csv_escape(val: Any) -> str:
s = "" if val is None else str(val)
if any(c in s for c in (",", '"', "\n", "\r")):
return '"' + s.replace('"', '""') + '"'
return s
def rows_to_csv(rows: list[dict], columns: list[str] | None = None) -> str:
if not rows and not columns:
return ""
cols = columns or list(rows[0].keys())
lines = [",".join(cols)]
for row in rows:
lines.append(",".join(csv_escape(row.get(c)) for c in cols))
return "\n".join(lines) + "\n"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
# ---------------------------------------------------------------------------
# Dell PowerEdge expansion / memory population catalog (from Dell Tech Guides
# + Installation & Service Manuals / KB 000141539). Used for empty-slot maps,
# max capacity, and best-practice guidance — not a live scrape.
# ---------------------------------------------------------------------------
_DELL_LINKS = {
# Verified live PDFs / product docs (checked 2026-07-17: %PDF or HTTP 200 HTML).
"spec_r740xd": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740xd-spec-sheet.pdf",
"spec_r740": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740-spec-sheet.pdf",
"tech_r740": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R740_R740xd_Technical_Guide.pdf",
"own_r740xd": "https://dl.dell.com/topicspdf/poweredge-r740xd_owners-manual_en-us.pdf",
"docs_r740xd": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740xd/docs",
"docs_r740": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740/docs",
"spec_r640": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r640-spec-sheet.pdf",
"own_r640": "https://dl.dell.com/topicspdf/poweredge-r640_owners-manual_en-us.pdf",
"docs_r640": "https://www.dell.com/support/product-details/en-us/product/poweredge-r640/docs",
"spec_r440": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r440-spec-sheet.pdf",
"docs_r440": "https://www.dell.com/support/product-details/en-us/product/poweredge-r440/docs",
"spec_r6415": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R6415_Spec_Sheet_EN.pdf",
"ref_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_reference-guide_en-us.pdf",
"own_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_owners-manual_en-us.pdf",
"docs_r6415": "https://www.dell.com/support/product-details/en-us/product/poweredge-r6415/docs",
"spec_r7425": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge-R7425-Spec-Sheet.pdf",
"own_r7425": "https://dl.dell.com/topicspdf/poweredge-r7425_owners-manual_en-us.pdf",
"docs_r7425": "https://www.dell.com/support/product-details/en-us/product/poweredge-r7425/docs",
"spec_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-spec-sheet.pdf",
"tech_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-technical-guide.pdf",
"docs_r660": "https://www.dell.com/support/product-details/en-us/product/poweredge-r660/docs",
"spec_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-spec-sheet.pdf",
"tech_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-technical-guide.pdf",
"docs_xr5610": "https://www.dell.com/support/product-details/en-us/product/poweredge-xr5610/docs",
"support_home": "https://www.dell.com/support/home",
}
# 14G dual-socket channel map (R640/R740/R740xd family)
_CH14 = {
"A": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5},
"B": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5},
}
# Dell datacenter rack catalog — visual U heights + face styles for rack designer
DELL_RACK_CATALOG: list[dict[str, Any]] = [
# --- PowerEdge 1U ---
{"sku": "PE-R440", "name": "PowerEdge R440", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r440"]},
{"sku": "PE-R640", "name": "PowerEdge R640", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r640"]},
{"sku": "PE-R650", "name": "PowerEdge R650", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650"]},
{"sku": "PE-R650xs", "name": "PowerEdge R650xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650xs"]},
{"sku": "PE-R660", "name": "PowerEdge R660", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660"]},
{"sku": "PE-R660xs", "name": "PowerEdge R660xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660xs"]},
{"sku": "PE-R6615", "name": "PowerEdge R6615", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6615"]},
{"sku": "PE-R6415", "name": "PowerEdge R6415", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6415"]},
{"sku": "PE-R6515", "name": "PowerEdge R6515", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6515"]},
{"sku": "PE-R6525", "name": "PowerEdge R6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6525"]},
{"sku": "PE-R6715", "name": "PowerEdge R6715", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6715"]},
{"sku": "PE-R670", "name": "PowerEdge R670", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r670"]},
{"sku": "PE-XR5610", "name": "PowerEdge XR5610", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["xr5610"]},
{"sku": "PE-C6520", "name": "PowerEdge C6520", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["c6520"]},
{"sku": "PE-C6525", "name": "PowerEdge C6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c6525"]},
{"sku": "PE-C4140", "name": "PowerEdge C4140", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c4140"]},
# --- PowerEdge 2U ---
{"sku": "PE-R740", "name": "PowerEdge R740", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740"]},
{"sku": "PE-R740xd", "name": "PowerEdge R740xd", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd"]},
{"sku": "PE-R740xd2", "name": "PowerEdge R740xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd2"]},
{"sku": "PE-R7425", "name": "PowerEdge R7425", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7425"]},
{"sku": "PE-R750", "name": "PowerEdge R750", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750"]},
{"sku": "PE-R750xa", "name": "PowerEdge R750xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750xa"]},
{"sku": "PE-R7515", "name": "PowerEdge R7515", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7515"]},
{"sku": "PE-R7525", "name": "PowerEdge R7525", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7525"]},
{"sku": "PE-R760", "name": "PowerEdge R760", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760"]},
{"sku": "PE-R760xd2", "name": "PowerEdge R760xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xd2"]},
{"sku": "PE-R760xa", "name": "PowerEdge R760xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xa"]},
{"sku": "PE-R770", "name": "PowerEdge R770", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r770"]},
{"sku": "PE-R840", "name": "PowerEdge R840", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r840"]},
{"sku": "PE-R940", "name": "PowerEdge R940", "family": "PowerEdge", "category": "server", "u_height": 3, "face": "server_4u", "match": ["r940"]},
{"sku": "PE-R940xa", "name": "PowerEdge R940xa", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["r940xa"]},
{"sku": "PE-R340", "name": "PowerEdge R340", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r340"]},
{"sku": "PE-R450", "name": "PowerEdge R450", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r450"]},
{"sku": "PE-R550", "name": "PowerEdge R550", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r550"]},
{"sku": "PE-T640", "name": "PowerEdge T640", "family": "PowerEdge", "category": "server", "u_height": 5, "face": "server_4u", "match": ["t640"]},
{"sku": "PE-DSS8440", "name": "DSS8440", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["dss8440"]},
{"sku": "PE-XE9680", "name": "PowerEdge XE9680", "family": "PowerEdge", "category": "server", "u_height": 6, "face": "server_4u", "match": ["xe9680"]},
{"sku": "PE-XE8640", "name": "PowerEdge XE8640", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["xe8640"]},
# --- MX / modular ---
{"sku": "PE-MX7000", "name": "PowerEdge MX7000", "family": "PowerEdge MX", "category": "chassis", "u_height": 7, "face": "chassis_mx", "match": ["mx7000"]},
{"sku": "PE-MX740c", "name": "PowerEdge MX740c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx740c"]},
{"sku": "PE-MX750c", "name": "PowerEdge MX750c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx750c"]},
{"sku": "PE-M640", "name": "PowerEdge M640", "family": "PowerEdge", "category": "blade", "u_height": 0, "face": "blade", "match": ["m640"]},
# --- Networking / PowerSwitch / Force10 ---
{"sku": "PS-S4048", "name": "PowerSwitch S4048-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4048", "force10"]},
{"sku": "PS-S4148", "name": "PowerSwitch S4148-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4148"]},
{"sku": "PS-S5248", "name": "PowerSwitch S5248F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5248"]},
{"sku": "PS-S5296", "name": "PowerSwitch S5296F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5296"]},
{"sku": "PS-N3248", "name": "PowerSwitch N3248TE-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3248"]},
{"sku": "PS-Z9264", "name": "PowerSwitch Z9264F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 2, "face": "switch_2u", "match": ["z9264"]},
{"sku": "PS-Z9432", "name": "PowerSwitch Z9432F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9432"]},
{"sku": "PC-5548", "name": "PowerConnect 5548", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 5548", "pc5548"]},
{"sku": "PC-6248", "name": "PowerConnect 6248", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 6248", "pc6248"]},
{"sku": "PC-7048", "name": "PowerConnect 7048", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7048", "pc7048"]},
{"sku": "PC-8024", "name": "PowerConnect 8024F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8024", "pc8024"]},
{"sku": "PC-8132", "name": "PowerConnect 8132F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8132", "pc8132"]},
{"sku": "MX-9116n", "name": "MX9116n Fabric Switching Engine", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx9116"]},
{"sku": "MX-5108n", "name": "MX5108n Ethernet Switch", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx5108"]},
# --- Storage ---
{"sku": "ME-4012", "name": "PowerVault ME4012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4012"]},
{"sku": "ME-4024", "name": "PowerVault ME4024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4024"]},
{"sku": "ME-5012", "name": "PowerVault ME5012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5012"]},
{"sku": "ME-5024", "name": "PowerVault ME5024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5024"]},
{"sku": "SC-AllFlash", "name": "SC Series Array", "family": "PowerStore/SC", "category": "storage", "u_height": 3, "face": "storage_3u", "match": ["sc "]},
{"sku": "PST-5000T", "name": "PowerStore 5000T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 5000"]},
{"sku": "PST-1200T", "name": "PowerStore 1200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 1200"]},
{"sku": "PST-3200T", "name": "PowerStore 3200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 3200"]},
{"sku": "DD-6900", "name": "PowerProtect DD6900", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd6900"]},
{"sku": "DD-9400", "name": "PowerProtect DD9400", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd9400"]},
{"sku": "UNITY-380", "name": "Unity XT 380", "family": "Unity", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["unity"]},
# --- VxRail appliances ---
{"sku": "VX-E460F", "name": "VxRail E460F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["e460f", "vxrail e"]},
{"sku": "VX-V470F", "name": "VxRail V470F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["v470f"]},
{"sku": "VX-P570F", "name": "VxRail P570F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["p570f"]},
{"sku": "VX-V670F", "name": "VxRail V670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["v670f"]},
{"sku": "VX-P670F", "name": "VxRail P670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["p670f"]},
{"sku": "VX-S570", "name": "VxRail S570", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["s570"]},
# --- Networking extras ---
{"sku": "PS-S5232", "name": "PowerSwitch S5232F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5232"]},
{"sku": "PS-S5212", "name": "PowerSwitch S5212F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5212"]},
{"sku": "PS-N3224", "name": "PowerSwitch N3224T-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3224"]},
{"sku": "PS-Z9100", "name": "PowerSwitch Z9100-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9100"]},
{"sku": "PC-7024", "name": "PowerConnect 7024", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7024"]},
{"sku": "PC-8100", "name": "PowerConnect 8100", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8100"]},
# --- PDU / misc ---
{"sku": "PDU-Geist-0U", "name": "Geist Rack PDU (0U vertical)", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["geist", "mg03", "mgs3"]},
{"sku": "PDU-1U", "name": "Rack PDU 1U horizontal", "family": "PDU", "category": "pdu", "u_height": 1, "face": "pdu_1u", "match": []},
{"sku": "PDU-APC-0U", "name": "APC Rack PDU 0U", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["apc"]},
{"sku": "PANEL-1U", "name": "Blanking panel 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []},
{"sku": "PANEL-2U", "name": "Blanking panel 2U", "family": "Accessory", "category": "blank", "u_height": 2, "face": "blank_2u", "match": []},
{"sku": "CABLE-MGMT-1U", "name": "Cable management arm 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []},
{"sku": "KVM-1U", "name": "Rack KVM 1U", "family": "Accessory", "category": "custom", "u_height": 1, "face": "custom", "match": ["kvm"]},
{"sku": "CUSTOM", "name": "Custom device", "family": "Custom", "category": "custom", "u_height": 1, "face": "custom", "match": []},
]
FACE_IMAGE_MAP: dict[str, str] = {
"server_1u": "/assets/dell/face-server-1u.jpg?v=bezel2",
"server_2u": "/assets/dell/face-server-2u.jpg?v=bezel2",
"server_4u": "/assets/dell/face-server-4u.jpg?v=bezel2",
"switch_1u": "/assets/dell/face-switch-1u.jpg?v=bezel2",
"switch_2u": "/assets/dell/face-switch-1u.jpg?v=bezel2",
"storage_2u": "/assets/dell/face-storage-2u.jpg?v=bezel2",
"storage_3u": "/assets/dell/face-storage-2u.jpg?v=bezel2",
"chassis_mx": "/assets/dell/face-chassis-mx.jpg?v=bezel2",
"pdu_1u": "/assets/dell/face-pdu-1u.jpg?v=bezel2",
"pdu_0u": "/assets/dell/face-pdu-1u.jpg?v=bezel2",
"blank_1u": "/assets/dell/face-blank-1u.jpg?v=bezel2",
"blank_2u": "/assets/dell/face-blank-1u.jpg?v=bezel2",
"blade": "/assets/dell/face-chassis-mx.jpg?v=bezel2",
"fabric": "/assets/dell/face-switch-1u.jpg?v=bezel2",
"custom": "/assets/dell/face-server-1u.jpg?v=bezel2",
}
# Official Dell product shots by SKU (processed to dark background to match faces)
SKU_IMAGE_MAP: dict[str, str] = {
# Prefer shared dark face art for visual consistency across all PowerEdge 1U/2U
}
def catalog_image_for(item: dict) -> str:
sku = item.get("sku") or ""
if sku in SKU_IMAGE_MAP:
return SKU_IMAGE_MAP[sku]
face = item.get("face") or "custom"
return FACE_IMAGE_MAP.get(face, FACE_IMAGE_MAP["custom"])
def enrich_catalog_images() -> None:
for item in DELL_RACK_CATALOG:
item["image"] = catalog_image_for(item)
enrich_catalog_images()
def match_rack_catalog(model: str | None, role: str | None = None) -> dict | None:
m = (model or "").lower()
best = None
for item in DELL_RACK_CATALOG:
for token in item.get("match") or []:
if token and token in m:
best = item
break
if best:
break
if best:
return best
if role == "switch":
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PS-S4048"), None)
if role == "storage":
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "ME-4024"), None)
if role == "chassis":
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-MX7000"), None)
if role == "pdu":
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PDU-Geist-0U"), None)
if role == "server":
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-R740"), None)
return None
MODEL_EXPANSION: dict[str, dict[str, Any]] = {
"PowerEdge R740xd": {
"generation": "14G",
"dimm_slots": 24,
"dimms_per_cpu": 12,
"channels_per_cpu": 6,
"max_memory_tb": 3.0,
"max_memory_note": "Up to 3 TB with LRDIMM (+ DCPMM configs); typical RDIMM max lower per CPU SKU",
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 8,
"pcie_note": "Up to 8× PCIe 3.0 via riser options; slot count depends on riser kit + CPU count",
"drive_bays_max": 24,
"drive_note": "Chassis-dependent: up to 24× 2.5\" (or mixed 3.5\"/NVMe mid-tray options on xd)",
"cpu_sockets": 2,
"best_practices": [
"Populate identical DIMMs across all six channels per CPU (6 or 12 DIMMs/CPU) for best bandwidth.",
"White release-tab sockets first, then black (channel rank 1 then rank 2).",
"Do not mix RDIMM and LRDIMM in the same system.",
"Unbalanced channel population reduces memory bandwidth / can lower effective speed.",
"Performance Optimized (4 DIMMs/CPU): slots 1,2,4,5 — (8 DIMMs/CPU): 1,2,4,5,7,8,10,11 (Dell KB 000141539).",
],
"performance_notes": [
"Peak memory bandwidth needs one matched DIMM in every channel (6 per CPU) before adding the second rank (slots 7–12).",
"Mixing capacities: place largest DIMMs in the first populated sockets of each channel.",
"DCPMM: up to 6 PMem + 6 DRAM per CPU; full 12-slot population recommended for App Direct / Memory Mode.",
],
"docs": [
{"title": "R740xd Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740xd"], "kind": "spec"},
{"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"},
{"title": "R740xd Owner's Manual (PDF)", "url": _DELL_LINKS["own_r740xd"], "kind": "manual"},
{"title": "Dell Support · R740xd docs", "url": _DELL_LINKS["docs_r740xd"], "kind": "support"},
],
"channel_map": _CH14,
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
},
"PowerEdge R740": {
"generation": "14G",
"dimm_slots": 24,
"dimms_per_cpu": 12,
"channels_per_cpu": 6,
"max_memory_tb": 3.0,
"max_memory_note": "Up to 3 TB with LRDIMM/DCPMM depending on CPU",
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 8,
"pcie_note": "Up to 8× PCIe 3.0 (riser dependent)",
"drive_bays_max": 16,
"drive_note": "Up to 16× 2.5\" or 8× 3.5\" depending on chassis kit",
"cpu_sockets": 2,
"best_practices": [
"Same 14G rules as R740xd: balance channels; white tabs before black.",
"Performance Optimized 4/8 DIMM/CPU populations per Dell KB 000141539.",
],
"performance_notes": [
"Populate 6 identical DIMMs per CPU before doubling up channels for max throughput.",
],
"docs": [
{"title": "R740 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740"], "kind": "spec"},
{"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"},
{"title": "Dell Support · R740 docs", "url": _DELL_LINKS["docs_r740"], "kind": "support"},
],
"channel_map": _CH14,
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
},
"PowerEdge R640": {
"generation": "14G",
"dimm_slots": 24,
"dimms_per_cpu": 12,
"channels_per_cpu": 6,
"max_memory_tb": 3.0,
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "DCPMM/PMem"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 3,
"pcie_note": "1U: up to 3× PCIe 3.0 (+ rNDC)",
"drive_bays_max": 10,
"drive_note": "Up to 10× 2.5\" (NVMe options vary)",
"cpu_sockets": 2,
"best_practices": [
"14G dual-socket population: identical DIMMs across 6 channels per CPU.",
"White-tab sockets first; avoid RDIMM+LRDIMM mix.",
],
"performance_notes": [
"1U thermal/airflow limits denser GPU/NVMe kits — verify riser/backplane BOM.",
],
"docs": [
{"title": "R640 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r640"], "kind": "spec"},
{"title": "R640 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r640"], "kind": "manual"},
{"title": "Dell Support · R640 docs", "url": _DELL_LINKS["docs_r640"], "kind": "support"},
],
"channel_map": _CH14,
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
},
"PowerEdge R440": {
"generation": "14G",
"dimm_slots": 16,
"dimms_per_cpu": 8,
"channels_per_cpu": 6,
"max_memory_tb": 0.512,
"memory_types": ["DDR4 RDIMM"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 3,
"drive_bays_max": 10,
"cpu_sockets": 2,
"best_practices": ["Populate channels evenly; match DIMM size/speed/rank per Dell 14G guidance."],
"performance_notes": ["Entry 14G — keep DIMM speeds matched to CPU QPI/UPI memory controller limits."],
"docs": [
{"title": "R440 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r440"], "kind": "spec"},
{"title": "Dell Support · R440 docs", "url": _DELL_LINKS["docs_r440"], "kind": "support"},
],
"slot_labels": [f"A{i}" for i in range(1, 9)] + [f"B{i}" for i in range(1, 9)],
},
"PowerEdge R6415": {
"generation": "14G-AMD",
"dimm_slots": 16,
"dimms_per_cpu": 16,
"channels_per_cpu": 8,
"max_memory_tb": 1.0,
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 3,
"drive_bays_max": 10,
"cpu_sockets": 1,
"best_practices": [
"AMD EPYC Naples/Rome 14G: for 4 DIMMs use slots 1,3,5,7 (Dell KB 000141539).",
"Populate evenly across memory channels for Infinity Fabric bandwidth.",
],
"performance_notes": [
"EPYC memory performance scales strongly with channels populated — prefer 8 identical DIMMs when possible.",
],
"docs": [
{"title": "R6415 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r6415"], "kind": "spec"},
{"title": "R6415 Tech Specs / Reference (PDF)", "url": _DELL_LINKS["ref_r6415"], "kind": "tech"},
{"title": "R6415 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r6415"], "kind": "manual"},
{"title": "Dell Support · R6415 docs", "url": _DELL_LINKS["docs_r6415"], "kind": "support"},
],
"slot_labels": [f"A{i}" for i in range(1, 17)],
},
"PowerEdge R7425": {
"generation": "14G-AMD",
"dimm_slots": 32,
"dimms_per_cpu": 16,
"channels_per_cpu": 8,
"max_memory_tb": 2.0,
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"],
"max_dimm_speed_mts": 2666,
"pcie_slots_max": 8,
"drive_bays_max": 24,
"cpu_sockets": 2,
"best_practices": [
"Dual EPYC: mirror CPU1 and CPU2 DIMM populations.",
"4 DIMMs/CPU → slots 1,3,5,7 per Dell AMD 14G guidance.",
],
"performance_notes": ["Balance both sockets identically to avoid NUMA imbalance."],
"docs": [
{"title": "R7425 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r7425"], "kind": "spec"},
{"title": "R7425 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r7425"], "kind": "manual"},
{"title": "Dell Support · R7425 docs", "url": _DELL_LINKS["docs_r7425"], "kind": "support"},
],
"slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)],
},
"PowerEdge R660": {
"generation": "16G",
"dimm_slots": 32,
"dimms_per_cpu": 16,
"channels_per_cpu": 8,
"max_memory_tb": 8.0,
"memory_types": ["DDR5 RDIMM"],
"max_dimm_speed_mts": 4800,
"pcie_slots_max": 3,
"drive_bays_max": 10,
"cpu_sockets": 2,
"best_practices": [
"16G DDR5: populate all memory channels evenly; follow white/black tab order in the ISM for the SKU.",
"Use matched DDR5 RDIMM kits; mixing ranks/speeds can downclock the entire domain.",
],
"performance_notes": [
"Sapphire Rapids / Emerald Rapids memory bandwidth benefits strongly from full channel population.",
],
"docs": [
{"title": "R660 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r660"], "kind": "spec"},
{"title": "R660 Technical Guide (PDF)", "url": _DELL_LINKS["tech_r660"], "kind": "tech"},
{"title": "Dell Support · R660 docs", "url": _DELL_LINKS["docs_r660"], "kind": "support"},
],
"slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)],
},
"PowerEdge XR5610": {
"generation": "16G-XR",
"dimm_slots": 8,
"dimms_per_cpu": 8,
"channels_per_cpu": 8,
"max_memory_tb": 1.0,
"max_memory_note": "Dell Spec Sheet: 8× DDR5 RDIMM, up to 1 TB, up to 5600 MT/s (CPU/SKU dependent)",
"memory_types": ["DDR5 RDIMM"],
"max_dimm_speed_mts": 5600,
"pcie_slots_max": 2,
"pcie_note": "Spec Sheet: up to 2× PCIe Gen5 x16 (+ OCP 3.0) — riser/GPU option dependent",
"drive_bays_max": 4,
"drive_note": "Spec Sheet: up to 4× 2.5\" SAS/SATA/NVMe front; BOSS-N1 M.2 is separate boot — not front-bay count",
"cpu_sockets": 1,
"best_practices": [
"Populate DDR5 channels evenly with matched RDIMMs.",
"Confirm exact DIMM map in the XR5610 Technical Guide / ISM for this BOM.",
],
"performance_notes": [
"Single-socket edge platform: memory bandwidth scales with channels populated.",
],
"docs": [
{"title": "XR5610 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_xr5610"], "kind": "spec"},
{"title": "XR5610 Technical Guide (PDF)", "url": _DELL_LINKS["tech_xr5610"], "kind": "tech"},
{"title": "Dell Support · XR5610 docs", "url": _DELL_LINKS["docs_xr5610"], "kind": "support"},
],
"slot_labels": [f"A{i}" for i in range(1, 9)],
},
}
def _normalize_model_key(model: str | None) -> str:
m = (model or "").strip()
if not m:
return ""
if m in MODEL_EXPANSION:
return m
# fuzzy family match
for key in MODEL_EXPANSION:
if key.lower() in m.lower() or m.lower() in key.lower():
return key
# strip suffixes like "vSAN Ready Node"
for key in MODEL_EXPANSION:
short = key.replace("PowerEdge ", "")
if short.lower() in m.lower():
return key
return m
def _parse_dimm_socket(name: str | None) -> tuple[str | None, int | None]:
import re
s = name or ""
m = re.search(r"(?:DIMM\.Socket\.|DIMM\s*)([A-D])(\d{1,2})", s, re.I)
if not m:
m = re.search(r"\b([A-D])(\d{1,2})\b", s)
if not m:
return None, None
return m.group(1).upper(), int(m.group(2))
def _disk_size_gb(raw: Any) -> float | None:
"""OME serverArrayDisks.Size is already capacity in GB (e.g. '446.62')."""
if raw is None or raw == "":
return None
try:
return round(float(str(raw).strip()), 2)
except Exception:
return None
def _mem_size_gb(raw: Any) -> float | None:
"""OME serverMemoryDevices.Size is capacity in MB."""
if raw is None or raw == "":
return None
try:
return round(float(raw) / 1024.0, 1)
except Exception:
return None
def build_expansion_report(model: str | None, inventory: dict) -> dict:
"""Split live OME facts (current) from Dell catalog max expansion (recommendations)."""
key = _normalize_model_key(model)
catalog_known = bool(MODEL_EXPANSION.get(key))
spec = dict(MODEL_EXPANSION.get(key) or {})
if not catalog_known:
spec = {
"generation": None,
"dimm_slots": None,
"max_memory_tb": None,
"pcie_slots_max": None,
"drive_bays_max": None,
"cpu_sockets": None,
"best_practices": [
"No Cockpit catalog entry for this model — verify DIMM/CPU/drive maxima in the Dell Tech Specs / ISM for the exact Service Tag BOM.",
],
"performance_notes": [],
"docs": [{"title": "Dell Support home", "url": _DELL_LINKS["support_home"]}],
"slot_labels": [],
"channel_map": {},
"memory_types": [],
"max_memory_note": "Maximum not asserted — catalog unknown for this model.",
"drive_note": "Bay maximum not asserted — catalog unknown for this model.",
"pcie_note": "PCIe maximum not asserted — catalog unknown for this model.",
}
mem_raw = inventory.get("serverMemoryDevices") or []
cpu_raw = inventory.get("serverProcessors") or []
disk_raw = inventory.get("serverArrayDisks") or []
cards_raw = inventory.get("serverDeviceCards") or []
raid_raw = inventory.get("serverRaidControllers") or []
os_raw = inventory.get("serverOperatingSystems") or []
dimm_modules: list[dict] = []
for dimm in mem_raw:
bank, num = _parse_dimm_socket(
dimm.get("Name") or dimm.get("InstanceId") or dimm.get("DeviceDescription")
)
if not bank or not num:
continue
label = f"{bank}{num}"
size_gb = _mem_size_gb(dimm.get("Size"))
dimm_modules.append(
{
"slot": label,
"cpu_bank": bank,
"slot_num": num,
"size_gb": size_gb,
"size_mb": dimm.get("Size"),
"rated_mts": dimm.get("Speed"),
"operating_mts": dimm.get("CurrentOperatingSpeed"),
"rank": dimm.get("Rank"),
"type": dimm.get("TypeDetails") or dimm.get("MemoryType") or "DDR",
"manufacturer": dimm.get("Manufacturer"),
"part_number": dimm.get("PartNumber"),
"serial": dimm.get("SerialNumber"),
"status": dimm.get("Status"),
}
)
dimm_modules.sort(key=lambda s: (s.get("cpu_bank") or "", s.get("slot_num") or 0))
mem_total_gb = round(sum(s.get("size_gb") or 0 for s in dimm_modules), 1)
rated_speeds = sorted({s["rated_mts"] for s in dimm_modules if s.get("rated_mts") is not None})
op_speeds = sorted({s["operating_mts"] for s in dimm_modules if s.get("operating_mts") is not None})
cpus: list[dict] = []
for p in cpu_raw:
cpus.append(
{
"slot": p.get("SlotNumber") or p.get("InstanceId"),
"model": p.get("ModelName") or p.get("BrandName"),
"cores": p.get("NumberOfEnabledCores") or p.get("NumberOfCores"),
"threads": p.get("NumberOfEnabledThreads") or p.get("NumberOfThreads"),
"current_mhz": p.get("CurrentSpeed"),
"max_mhz": p.get("MaxSpeed"),
"status": p.get("Status"),
}
)
cpu_cores_total = sum(int(c.get("cores") or 0) for c in cpus)
disks: list[dict] = []
for d in disk_raw:
disks.append(
{
"bay": d.get("SlotNumber"),
"model": d.get("ModelNumber"),
"vendor": d.get("VendorName"),
"media": d.get("MediaType"),
"size_gb": _disk_size_gb(d.get("Size")),
"bus": d.get("BusType"),
"status": d.get("StatusString") or d.get("Status"),
"serial": d.get("SerialNumber"),
"enclosure": d.get("EnclosureId"),
"form_factor": d.get("FormFactor"),
}
)
disks.sort(key=lambda x: (str(x.get("bus") or ""), x.get("bay") is None, x.get("bay") or 0))
disk_total_gb = round(sum(d.get("size_gb") or 0 for d in disks), 2)
adapters: list[dict] = []
for c in cards_raw:
adapters.append(
{
"slot": str(c.get("SlotNumber") or ""),
"description": c.get("Description"),
"manufacturer": c.get("Manufacturer"),
"slot_type": c.get("SlotType"),
"width": c.get("DatabusWidth"),
}
)
for r in raid_raw:
adapters.append(
{
"slot": str(r.get("PciSlot") or r.get("Fqdd") or "RAID"),
"description": r.get("Name") or r.get("DeviceDescription"),
"manufacturer": "Dell PERC",
"slot_type": "RAID",
"firmware": r.get("FirmwareVersion"),
"cache_mb": r.get("CacheSizeInMb"),
}
)
os_info = []
for o in os_raw:
os_info.append(
{
"os_name": (o.get("OsName") or "").strip() or None,
"os_version": (o.get("OsVersion") or "").strip() or None,
"hostname": (o.get("Hostname") or "").strip() or None,
}
)
current = {
"source": "OME DeviceService inventory (live)",
"memory": {
"installed_gb": mem_total_gb,
"dimm_count": len(dimm_modules),
"modules": dimm_modules,
"rated_mts": rated_speeds,
"operating_mts": op_speeds,
"banks_seen": sorted({m["cpu_bank"] for m in dimm_modules if m.get("cpu_bank")}),
},
"cpu": {
"sockets_populated": len(cpus),
"cores_total": cpu_cores_total,
"processors": cpus,
},
"disks": {
"count": len(disks),
"capacity_gb": disk_total_gb,
"items": disks,
},
"adapters": adapters,
"os": os_info,
}
populated_map = {m["slot"]: m for m in dimm_modules}
labels = list(spec.get("slot_labels") or [])
slot_map: list[dict] = []
for label in labels:
bank = label[0]
num = int(label[1:] or 0)
ch_map = (spec.get("channel_map") or {}).get(bank) or {}
if label in populated_map:
row = dict(populated_map[label])
row["populated"] = True
row["channel"] = ch_map.get(num)
row["rank_color"] = "white" if num <= 6 else "black"
row["speed_mts"] = row.get("operating_mts") or row.get("rated_mts")
slot_map.append(row)
else:
slot_map.append(
{
"slot": label,
"cpu_bank": bank,
"slot_num": num,
"channel": ch_map.get(num),
"rank_color": "white" if num <= 6 else "black",
"populated": False,
}
)
for label, row in populated_map.items():
if label not in labels:
extra = dict(row)
extra["populated"] = True
extra["speed_mts"] = extra.get("operating_mts") or extra.get("rated_mts")
slot_map.append(extra)
empty_n = sum(1 for s in slot_map if not s.get("populated")) if labels else None
dimm_slots_catalog = spec.get("dimm_slots")
max_tb = spec.get("max_memory_tb")
headroom_gb = None
if max_tb is not None:
try:
headroom_gb = round(float(max_tb) * 1024 - mem_total_gb, 1)
except Exception:
headroom_gb = None
drive_max = spec.get("drive_bays_max")
free_bays = (int(drive_max) - len(disks)) if catalog_known and isinstance(drive_max, int) else None
cpu_max = spec.get("cpu_sockets")
free_cpu = (int(cpu_max) - len(cpus)) if catalog_known and isinstance(cpu_max, int) else None
findings: list[dict] = []
if catalog_known and labels:
by_bank: dict[str, list] = {}
for s in slot_map:
by_bank.setdefault(s.get("cpu_bank") or "?", []).append(s)
for bank, bank_slots in by_bank.items():
ch_counts: dict[int, int] = {}
for s in bank_slots:
if s.get("populated") and s.get("channel") is not None:
ch_counts[int(s["channel"])] = ch_counts.get(int(s["channel"]), 0) + 1
if ch_counts and spec.get("channels_per_cpu"):
vals = list(ch_counts.values())
if max(vals) - min(vals) > 0 or len(ch_counts) < int(spec["channels_per_cpu"]):
findings.append(
{
"severity": "warning",
"code": "unbalanced_channels",
"text": f"Bank {bank}: channel population {dict(sorted(ch_counts.items()))} — Dell guidance prefers even channel fill for bandwidth.",
}
)
if len(op_speeds) > 1:
findings.append(
{
"severity": "warning",
"code": "mixed_operating_speeds",
"text": f"OME reports mixed operating speeds {op_speeds} MT/s — controller usually clocks all DIMMs to the lowest.",
}
)
if rated_speeds and op_speeds and max(op_speeds) < max(rated_speeds):
findings.append(
{
"severity": "info",
"code": "downclocked",
"text": f"DIMMs rated {rated_speeds} MT/s, operating at {op_speeds} MT/s (CPU/population limited).",
}
)
expansion = {
"source": "Dell model catalog in Cockpit (not live measurement)" if catalog_known else "Catalog unknown",
"catalog_known": catalog_known,
"catalog_model": key if catalog_known else None,
"generation": spec.get("generation"),
"memory": {
"max_capacity_tb": max_tb,
"max_capacity_gb": round(float(max_tb) * 1024, 1) if max_tb is not None else None,
"dimm_slots_max": dimm_slots_catalog,
"dimms_installed": len(dimm_modules),
"empty_slots_estimate": empty_n if catalog_known else None,
"headroom_gb": headroom_gb if catalog_known else None,
"supported_types": spec.get("memory_types") or [],
"max_rated_mts": spec.get("max_dimm_speed_mts"),
"note": spec.get("max_memory_note")
or (
f"Catalog max {max_tb} TB / {dimm_slots_catalog} DIMM slots"
if catalog_known
else "No catalog max — do not treat empty map as measured free slots."
),
"slot_map": slot_map if catalog_known else [],
},
"cpu": {
"sockets_max": cpu_max,
"sockets_populated": len(cpus),
"sockets_free_estimate": free_cpu,
"note": (
f"Chassis supports up to {cpu_max} CPU socket(s); OME sees {len(cpus)} populated."
if catalog_known and cpu_max is not None
else "CPU socket maximum not in Cockpit catalog for this model."
),
},
"disks": {
"bays_max": drive_max if catalog_known else None,
"drives_installed": len(disks),
"bays_free_estimate": free_bays,
"note": spec.get("drive_note")
or (
"Bay maximum from Dell chassis options — verify BOM; M.2/BOSS may not count as front bays."
if catalog_known
else "Drive bay maximum unknown for this model."
),
},
"pcie": {
"slots_max": spec.get("pcie_slots_max") if catalog_known else None,
"note": spec.get("pcie_note")
or ("PCIe max depends on riser kit." if catalog_known else "PCIe maximum unknown for this model."),
},
"best_practices": spec.get("best_practices") or [],
"performance_notes": spec.get("performance_notes") or [],
"docs": spec.get("docs") or [],
}
return {
"model": model,
"catalog_key": key if catalog_known else None,
"generic": not catalog_known,
"current": current,
"expansion": expansion,
"findings": findings,
"spec": {
"generation": expansion.get("generation"),
"dimm_slots": expansion["memory"].get("dimm_slots_max"),
"max_memory_tb": expansion["memory"].get("max_capacity_tb"),
"max_memory_note": expansion["memory"].get("note"),
"memory_types": expansion["memory"].get("supported_types"),
"max_dimm_speed_mts": expansion["memory"].get("max_rated_mts"),
"pcie_slots_max": expansion["pcie"].get("slots_max"),
"pcie_note": expansion["pcie"].get("note"),
"drive_bays_max": expansion["disks"].get("bays_max"),
"drive_note": expansion["disks"].get("note"),
"cpu_sockets": expansion["cpu"].get("sockets_max"),
"best_practices": expansion.get("best_practices") or [],
"performance_notes": expansion.get("performance_notes") or [],
"docs": expansion.get("docs") or [],
},
"memory": {
"total_gb": mem_total_gb,
"populated": len(dimm_modules),
"empty": empty_n,
"headroom_gb": headroom_gb,
"slots": slot_map if slot_map else [{**m, "populated": True, "speed_mts": m.get("operating_mts") or m.get("rated_mts")} for m in dimm_modules],
"banks": current["memory"]["banks_seen"],
},
"processors": cpus,
"disks": {
"installed": len(disks),
"max_bays": drive_max if catalog_known else None,
"free_bays_estimate": free_bays,
"items": [{**d, "slot": d.get("bay"), "name": d.get("model")} for d in disks],
},
"pcie": {
"max_slots": expansion["pcie"].get("slots_max"),
"items": adapters,
"note": expansion["pcie"].get("note"),
},
}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
def subnet_of(ip: str | None) -> str:
if not ip or ip.count(".") != 3:
return "unknown"
try:
return str(ipaddress.ip_network(f"{ip}/24", strict=False))
except Exception:
parts = ip.split(".")
return f"{parts[0]}.{parts[1]}.{parts[2]}.0/24"
def pick_ip(device: dict) -> str | None:
for m in device.get("DeviceManagement") or []:
addr = m.get("NetworkAddress")
if addr and addr.count(".") == 3:
return addr
return None
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def pick_management(device: dict) -> dict:
"""Extract iDRAC/management IP plus OS hostname hints from OME DeviceManagement."""
mgmt = (device.get("DeviceManagement") or [{}])[0] or {}
idrac_ip = None
for m in device.get("DeviceManagement") or []:
addr = m.get("NetworkAddress")
if addr and str(addr).count(".") == 3:
idrac_ip = addr
mgmt = m
break
os_hostname = (mgmt.get("InstrumentationName") or "").strip() or None
dns_name = (mgmt.get("DnsName") or "").strip() or None
return {
"idrac_ip": idrac_ip,
"os_hostname": os_hostname,
"mgmt_dns_name": dns_name,
"mac": mgmt.get("MacAddress"),
}
async def resolve_host_ips(hostname: str | None) -> list[str]:
if not hostname or hostname.count(".") == 3:
return []
loop = asyncio.get_running_loop()
try:
infos = await loop.getaddrinfo(hostname, None, family=socket.AF_INET)
return sorted({i[4][0] for i in infos})
except Exception:
return []
async def enrich_rdp_targets(devices: list[dict]) -> None:
"""DNS-resolve OS hostnames from OME InstrumentationName → RDP candidate IPs."""
hostnames: set[str] = set()
for n in devices:
h = (n.get("os_hostname") or "").strip()
if h and h.count(".") != 3:
hostnames.add(h)
# also try short name
if "." in h:
hostnames.add(h.split(".", 1)[0])
resolved: dict[str, list[str]] = {}
sem = asyncio.Semaphore(20)
async def one(h: str):
async with sem:
resolved[h] = await resolve_host_ips(h)
if hostnames:
await asyncio.gather(*[one(h) for h in hostnames])
for n in devices:
idrac = n.get("idrac_ip") or n.get("ip")
h = (n.get("os_hostname") or "").strip()
ips: list[str] = []
seen: set[str] = set()
for candidate in (h, h.split(".", 1)[0] if h and "." in h else None):
if not candidate:
continue
for ip in resolved.get(candidate) or []:
if ip not in seen:
seen.add(ip)
ips.append(ip)
# Prefer OS IPs that differ from iDRAC
os_ips = [ip for ip in ips if ip != idrac]
rdp_ips = os_ips or ips
n["rdp_ips"] = rdp_ips
n["rdp_host"] = rdp_ips[0] if rdp_ips else None
n["os_ips"] = os_ips
# Windows hint from name / later OS inventory
blob = f"{n.get('name') or ''} {n.get('os_hostname') or ''} {n.get('os_name') or ''}".lower()
n["is_windows"] = any(k in blob for k in ("windows", "hyperv", "hyper-v", "-hv", "hv0")) or bool(
n.get("os_name") and "windows" in str(n.get("os_name")).lower()
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
async def fetch_power(client: httpx.AsyncClient, base: str, headers: dict, device_id: int) -> dict:
try:
r = await client.get(f"{base}/api/DeviceService/Devices({device_id})/Power", headers=headers)
if r.status_code != 200:
return {}
p = r.json()
def num(key):
v = p.get(key)
try:
return float(v) if v not in (None, "") else None
except Exception:
return None
return {
"watts": num("power"),
"avg_watts": num("avgPower"),
"peak_watts": num("peakPower"),
"min_watts": num("minimumPower"),
"energy_kwh": num("systemEnergyConsumption"),
"power_unit": p.get("powerUnit") or "watt",
}
except Exception:
return {}
async def ome_fetch() -> dict:
base = settings.ome_url.rstrip("/")
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
r = await client.post(
f"{base}/api/SessionService/Sessions",
json={
"UserName": settings.ome_user,
"Password": settings.ome_password,
"SessionType": "API",
},
)
r.raise_for_status()
token = r.headers.get("X-Auth-Token")
sid = r.json().get("Id")
headers = {"X-Auth-Token": token, "Accept": "application/json"}
try:
info = (await client.get(f"{base}/api/ApplicationService/Info", headers=headers)).json()
devices_resp = (
await client.get(f"{base}/api/DeviceService/Devices?$top=5000", headers=headers)
).json()
groups_resp = (
await client.get(f"{base}/api/GroupService/Groups?$top=200", headers=headers)
).json()
alerts_raw = []
alerts_total = None
try:
alerts_resp = (
await client.get(
f"{base}/api/AlertService/Alerts?$top=200",
headers=headers,
)
).json()
alerts_raw = alerts_resp.get("value") or []
alerts_total = alerts_resp.get("@odata.count")
except Exception as e:
log.warning("OME alerts fetch failed: %s", e)
devices_raw = devices_resp.get("value") or []
# power for connected servers first
power_targets = [
d.get("Id")
for d in devices_raw
if d.get("Type") == 1000 and d.get("ConnectionState") and d.get("Id")
][: settings.power_fetch_limit]
# also a few offline servers for comparison
offline_ids = [
d.get("Id")
for d in devices_raw
if d.get("Type") == 1000 and not d.get("ConnectionState") and d.get("Id")
][:5]
power_targets = list(dict.fromkeys(power_targets + offline_ids))
sem = asyncio.Semaphore(8)
async def limited(did):
async with sem:
return did, await fetch_power(client, base, headers, did)
power_map = {}
if power_targets:
results = await asyncio.gather(*[limited(i) for i in power_targets])
power_map = {i: p for i, p in results if p}
finally:
if sid:
try:
await client.delete(
f"{base}/api/SessionService/Sessions('{sid}')",
headers=headers,
)
except Exception:
pass
devices = []
model_counts: dict[str, int] = defaultdict(int)
idrac_count = server_count = connected = powered = 0
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
switch_count = chassis_count = pdu_count = storage_count = 0
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
total_watts = 0.0
watts_samples = 0
status_counts: dict[str, int] = defaultdict(int)
for d in devices_raw:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
mgmt = pick_management(d)
ip = mgmt.get("idrac_ip") or pick_ip(d)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
subnet = subnet_of(ip)
dtype = d.get("Type")
sub = d.get("SubDeviceType") or ""
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
model = d.get("Model") or "Unknown"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
is_server = dtype == 1000
is_idrac = sub == "iDRAC"
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
is_switch = dtype == 7000 or "switch" in (sub or "").lower() or "switch" in model.lower()
is_chassis = dtype == 2000 or "chassis" in (sub or "").lower()
is_pdu = dtype == 100
is_storage = dtype == 5000 or "storage" in (sub or "").lower()
if is_switch:
role = "switch"
elif is_chassis:
role = "chassis"
elif is_pdu:
role = "pdu"
elif is_storage:
role = "storage"
elif is_server:
role = "server"
else:
role = "other"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
conn = bool(d.get("ConnectionState"))
power_state = d.get("PowerState")
status = str(d.get("Status") or "")
status_counts[status] += 1
if is_server:
model_counts[model] += 1
if is_idrac:
idrac_count += 1
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if is_switch:
switch_count += 1
if is_chassis:
chassis_count += 1
if is_pdu:
pdu_count += 1
if is_storage:
storage_count += 1
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
if is_server:
server_count += 1
if conn:
connected += 1
if power_state == 17:
powered += 1
pwr = power_map.get(d.get("Id")) or {}
watts = pwr.get("watts")
if watts is not None:
total_watts += watts
watts_samples += 1
node = {
"id": d.get("Id"),
"name": d.get("DeviceName") or f"device-{d.get('Id')}",
"model": model,
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
"service_tag": d.get("DeviceServiceTag")
or d.get("Identifier")
or d.get("ChassisServiceTag"),
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"type": dtype,
"sub_type": sub,
"connected": conn,
"power_state": power_state,
"powered_on": power_state == 17,
"status": status,
"ip": ip,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"idrac_ip": mgmt.get("idrac_ip") or ip,
"os_hostname": mgmt.get("os_hostname"),
"mgmt_dns_name": mgmt.get("mgmt_dns_name"),
"rdp_host": None,
"rdp_ips": [],
"os_ips": [],
"is_windows": False,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"subnet": subnet,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"role": role,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"is_server": is_server,
"is_idrac": is_idrac,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"is_switch": is_switch,
"is_chassis": is_chassis,
"is_pdu": is_pdu,
"is_storage": is_storage,
"chassis_service_tag": d.get("ChassisServiceTag"),
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"idrac_url": f"https://{ip}" if ip else None,
"watts": watts,
"avg_watts": pwr.get("avg_watts"),
"peak_watts": pwr.get("peak_watts"),
"energy_kwh": pwr.get("energy_kwh"),
"last_status_time": d.get("LastStatusTime"),
"last_inventory_time": d.get("LastInventoryTime"),
}
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
node["map_cidrs"] = [subnet] if subnet and subnet != "unknown" else []
node["extra_ips"] = []
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
devices.append(node)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
await enrich_rdp_targets(devices)
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
sync_endpoints_from_fleet(devices)
apply_network_endpoints(devices)
subnets = build_subnet_summaries(devices)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
groups = [
{"id": g.get("Id"), "name": g.get("Name")}
for g in (groups_resp.get("value") or [])
if g.get("Name")
][:60]
models = [
{"name": m, "count": c}
for m, c in sorted(model_counts.items(), key=lambda x: -x[1])
]
# --- realtime context: fleet deltas + alerts ---
global PREV_DEVICES, EVENT_FEED
now = time.time()
new_events: list[dict] = []
first_poll = not PREV_DEVICES
current_ids: set[int] = set()
def _role(n: dict) -> str:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if n.get("is_switch"):
return "switch"
if n.get("is_chassis"):
return "chassis"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
if n.get("is_idrac"):
return "iDRAC"
if n.get("is_server"):
return "server"
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if n.get("is_storage"):
return "storage"
if n.get("is_pdu"):
return "PDU"
return n.get("role") or "device"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
for n in devices:
did = n.get("id")
if did is None:
continue
current_ids.add(did)
prev = PREV_DEVICES.get(did)
if not prev:
# First poll only seeds baseline; later polls raise new-node alerts
if not first_poll and (n.get("is_server") or n.get("is_idrac")):
role = _role(n)
new_events.append({
"ts": now,
"kind": "device_new",
"severity": "critical",
"device_id": did,
"title": f"New {role} discovered",
"text": (
f"{n.get('name')} · {n.get('ip') or 'no IP'} · "
f"{n.get('model') or 'unknown model'} · {n.get('subnet') or 'unknown subnet'}"
),
"notify": True,
"role": role,
"ip": n.get("ip"),
"model": n.get("model"),
"subnet": n.get("subnet"),
"name": n.get("name"),
})
continue
if bool(prev.get("connected")) != bool(n.get("connected")):
new_events.append({
"ts": now,
"kind": "connection",
"severity": "info" if n.get("connected") else "warning",
"device_id": did,
"title": n.get("name"),
"text": "Connected" if n.get("connected") else "Went offline",
})
if bool(prev.get("powered_on")) != bool(n.get("powered_on")):
new_events.append({
"ts": now,
"kind": "power_state",
"severity": "info" if n.get("powered_on") else "warning",
"device_id": did,
"title": n.get("name"),
"text": "Powered on" if n.get("powered_on") else "Powered off",
})
pw, pp = n.get("watts"), prev.get("watts")
if pw is not None and pp is not None and abs(pw - pp) >= 40:
new_events.append({
"ts": now,
"kind": "power_watt",
"severity": "info",
"device_id": did,
"title": n.get("name"),
"text": f"Power {round(pp)}W → {round(pw)}W",
})
if str(prev.get("status")) != str(n.get("status")):
new_events.append({
"ts": now,
"kind": "status",
"severity": "warning",
"device_id": did,
"title": n.get("name"),
"text": f"Status {prev.get('status')} → {n.get('status')}",
})
if not first_poll:
for did, prev in PREV_DEVICES.items():
if did in current_ids:
continue
if not (prev.get("is_server") or prev.get("is_idrac")):
continue
role = _role(prev)
new_events.append({
"ts": now,
"kind": "device_removed",
"severity": "warning",
"device_id": did,
"title": f"{role} left inventory",
"text": (
f"{prev.get('name')} · {prev.get('ip') or 'no IP'} · "
f"{prev.get('model') or 'unknown model'}"
),
"notify": True,
"role": role,
"name": prev.get("name"),
"ip": prev.get("ip"),
"model": prev.get("model"),
})
PREV_DEVICES = {n["id"]: n for n in devices if n.get("id") is not None}
if new_events:
EVENT_FEED = (new_events + EVENT_FEED)[:120]
sev_map = {"Critical": 0, "Warning": 0, "Normal": 0, "Info": 0, "Unknown": 0}
alerts = []
for a in alerts_raw:
sev = a.get("SeverityName") or "Unknown"
if sev not in sev_map:
sev_map[sev] = 0
sev_map[sev] = sev_map.get(sev, 0) + 1
alerts.append({
"id": a.get("Id"),
"severity": sev,
"device_id": a.get("AlertDeviceId") or a.get("AlertEntityId"),
"device": a.get("AlertDeviceName") or a.get("AlertEntityName"),
"ip": a.get("AlertDeviceIpAddress"),
"category": a.get("CategoryName"),
"subcategory": a.get("SubCategoryName"),
"message": a.get("Message"),
"message_id": a.get("AlertMessageId"),
"status": a.get("StatusName"),
"time": a.get("TimeStamp"),
"action": a.get("RecommendedAction"),
})
hottest = sorted(
[n for n in devices if n.get("watts") is not None],
key=lambda x: x.get("watts") or 0,
reverse=True,
)[:5]
recent_status = sorted(
[n for n in devices if n.get("last_status_time")],
key=lambda x: str(x.get("last_status_time")),
reverse=True,
)[:5]
context = {
"refreshed_at": now,
"poll_seconds": settings.poll_interval,
"alerts_total": alerts_total,
"alert_severity": sev_map,
"events_new": len(new_events),
"notifications": [e for e in new_events if e.get("notify")],
"hottest": [
{"id": n["id"], "name": n["name"], "watts": n["watts"], "subnet": n["subnet"]}
for n in hottest
],
"recent_status": [
{
"id": n["id"],
"name": n["name"],
"time": n.get("last_status_time"),
"connected": n.get("connected"),
"status": n.get("status"),
}
for n in recent_status
],
"focus_hint": (
f"{connected} connected · {round(total_watts)} W sampled · "
f"{sev_map.get('Critical', 0)} critical / {sev_map.get('Warning', 0)} warning alerts in feed"
),
}
return {
"ome": {
"name": info.get("Name") or "OpenManage Enterprise",
"version": info.get("Version"),
"build": info.get("BuildNumber"),
"url": base,
"fqdn": "cov-omeprod01.dell-atc.lan",
"console_url": f"{base}/management-console/ome/",
},
"summary": {
"total": len(devices),
"idracs": idrac_count,
"servers": server_count,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"switches": switch_count,
"chassis": chassis_count,
"pdus": pdu_count,
"storage": storage_count,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"connected": connected,
"powered_on": powered,
"offline": max(0, server_count - connected),
"status_counts": dict(status_counts),
"total_watts": round(total_watts, 1),
"avg_node_watts": round(total_watts / watts_samples, 1) if watts_samples else None,
"power_samples": watts_samples,
"alerts_critical": sev_map.get("Critical", 0),
"alerts_warning": sev_map.get("Warning", 0),
"alerts_total": alerts_total,
},
"subnets": subnets,
"devices": devices,
"groups": groups,
"models": models,
"alerts": alerts,
"events": EVENT_FEED[:40],
"context": context,
"openwebui_url": settings.openwebui_url,
"updated_at": now,
}
async def broadcast(payload: dict):
dead = []
for ws in list(CLIENTS):
try:
await ws.send_json(payload)
except Exception:
dead.append(ws)
for ws in dead:
CLIENTS.discard(ws)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def _seed_atc_racks() -> None:
"""Idempotent seed: ATC1 (8×42U) + ATC2 (6×42U)."""
now = time.time()
with _db() as conn:
n = conn.execute("SELECT COUNT(*) AS c FROM racks").fetchone()["c"]
if n > 0:
return
conn.execute(
"INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
("ATC1", "ATC1", 1),
)
conn.execute(
"INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
("ATC2", "ATC2", 2),
)
for i in range(1, 9):
conn.execute(
"""
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
VALUES (?, ?, 42, ?, ?, ?)
""",
("ATC1", f"ATC1-R{i:02d}", i, now, now),
)
for i in range(1, 7):
conn.execute(
"""
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
VALUES (?, ?, 42, ?, ?, ?)
""",
("ATC2", f"ATC2-R{i:02d}", i, now, now),
)
log.info("Seeded ATC racks: ATC1×8 + ATC2×6 (42U)")
def build_network_fabric(devices: list[dict] | None = None) -> dict:
"""Inferred switch↔server / chassis↔blade edges (no LLDP)."""
devices = list(devices if devices is not None else (STATE.get("devices") or []))
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
edges: list[dict] = []
seen: set[tuple] = set()
def add_edge(a: int, b: int, kind: str):
if a is None or b is None or a == b:
return
key = (min(a, b), max(a, b), kind)
if key in seen:
return
seen.add(key)
da, db = by_id.get(a) or {}, by_id.get(b) or {}
edges.append(
{
"from_id": a,
"to_id": b,
"kind": kind,
"confidence": "inferred",
"note": (
"management subnet"
if kind == "subnet"
else "MX chassis / blade"
),
"live": bool(da.get("connected") and db.get("connected")),
}
)
switches = [d for d in devices if d.get("is_switch")]
for sw in switches:
cidr = sw.get("subnet") or "unknown"
if cidr == "unknown":
continue
for d in devices:
if d.get("id") == sw.get("id"):
continue
if d.get("subnet") != cidr:
continue
if d.get("is_switch") or d.get("is_pdu"):
continue
# Prefer linking servers/storage/chassis endpoints
if d.get("is_server") or d.get("is_storage") or d.get("is_chassis") or d.get("is_idrac"):
add_edge(sw["id"], d["id"], "subnet")
chassis_list = [d for d in devices if d.get("is_chassis")]
for ch in chassis_list:
cst = (ch.get("service_tag") or "").upper()
for d in devices:
if not d.get("is_server"):
continue
model = (d.get("model") or "").upper()
blade_cst = (d.get("chassis_service_tag") or "").upper()
if cst and blade_cst and blade_cst == cst:
add_edge(ch["id"], d["id"], "chassis")
elif "MX" in model and "MX7000" not in model:
# fallback: all MX blades to MX chassis in same environment
add_edge(ch["id"], d["id"], "chassis")
return {
"edges": edges,
"note": "Edges inferred from management subnet / MX chassis — not LLDP.",
"switches": [
{
"id": s.get("id"),
"name": s.get("name"),
"service_tag": s.get("service_tag"),
"ip": s.get("ip") or s.get("idrac_ip"),
"subnet": s.get("subnet"),
"connected": s.get("connected"),
"model": s.get("model"),
"connected_device_ids": [
e["to_id"] if e["from_id"] == s.get("id") else e["from_id"]
for e in edges
if e["kind"] == "subnet"
and (e["from_id"] == s.get("id") or e["to_id"] == s.get("id"))
],
}
for s in switches
],
"summary": {
"devices": len(devices),
"edges": len(edges),
"switches": len(switches),
"chassis": sum(1 for d in devices if d.get("is_chassis")),
},
}
def _default_u_height(device: dict | None) -> int:
if not device:
return 1
cat = match_rack_catalog(device.get("model"), device.get("role"))
if cat is not None:
uh = int(cat.get("u_height") or 0)
return uh if uh > 0 else 1 # 0U vertical PDUs place as 1U marker unless side=left/right
if device.get("is_chassis"):
return 7
if device.get("is_storage"):
return 2
if device.get("is_switch"):
return 1
return 1
def _rack_overlap(
conn: sqlite3.Connection,
rack_id: int,
u_start: int,
u_height: int,
side: str = "front",
exclude_item_id: int | None = None,
exclude_device_id: int | None = None,
) -> bool:
if side in ("left", "right"):
return False # side PDUs do not consume front/rear U map
u_end = u_start + max(u_height, 1) - 1
rows = conn.execute(
"SELECT id, device_id, u_start, u_height, side FROM rack_items WHERE rack_id=?",
(rack_id,),
).fetchall()
for r in rows:
if exclude_item_id is not None and r["id"] == exclude_item_id:
continue
if exclude_device_id is not None and r["device_id"] == exclude_device_id:
continue
rside = (r["side"] or "front")
if rside in ("left", "right"):
continue
if rside != side:
continue
a, b = r["u_start"], r["u_start"] + r["u_height"] - 1
if not (u_end < a or u_start > b):
return True
return False
def _enrich_rack_item(item: dict, devices: dict) -> dict:
out = dict(item)
dev = devices.get(item.get("device_id")) if item.get("device_id") is not None else None
sku = item.get("catalog_sku")
cat = None
if sku:
cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None)
if not cat and dev:
cat = match_rack_catalog(dev.get("model"), dev.get("role"))
if cat:
out["catalog"] = {k: cat[k] for k in ("sku", "name", "family", "category", "u_height", "face", "image") if k in cat}
out["face"] = cat.get("face")
out["image"] = cat.get("image") or catalog_image_for(cat)
out["category"] = cat.get("category")
else:
out["catalog"] = None
out["face"] = "custom"
out["image"] = FACE_IMAGE_MAP["custom"]
out["category"] = (dev or {}).get("role") or "custom"
if dev:
out["device"] = {
"id": dev.get("id"),
"name": dev.get("name"),
"model": dev.get("model"),
"service_tag": dev.get("service_tag"),
"role": dev.get("role"),
"ip": dev.get("ip") or dev.get("idrac_ip"),
"connected": dev.get("connected"),
"is_switch": dev.get("is_switch"),
"is_server": dev.get("is_server"),
"is_storage": dev.get("is_storage"),
"is_chassis": dev.get("is_chassis"),
"is_pdu": dev.get("is_pdu"),
}
out["label"] = item.get("label") or dev.get("name")
out["model"] = dev.get("model")
else:
out["device"] = None
out["label"] = item.get("label") or ((cat or {}).get("name") if cat else "Custom")
out["model"] = (cat or {}).get("name")
return out
def _racks_payload() -> dict:
_migrate_rack_items()
devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None}
with _db() as conn:
sites = [dict(r) for r in conn.execute("SELECT * FROM sites ORDER BY sort_order").fetchall()]
racks = [dict(r) for r in conn.execute("SELECT * FROM racks ORDER BY site_id, sort_order").fetchall()]
items = [dict(r) for r in conn.execute("SELECT * FROM rack_items ORDER BY u_start DESC").fetchall()]
placed_ids = {i["device_id"] for i in items if i.get("device_id") is not None}
by_site: dict[str, list] = {}
for rk in racks:
card = dict(rk)
card["items"] = []
card["placements"] = [] # backward compat alias
for it in items:
if it["rack_id"] != rk["id"]:
continue
enriched = _enrich_rack_item(it, devices)
card["items"].append(enriched)
# compat shape for older UI bits
card["placements"].append(
{
"device_id": it.get("device_id"),
"rack_id": it["rack_id"],
"u_start": it["u_start"],
"u_height": it["u_height"],
"item_id": it["id"],
"side": it.get("side") or "front",
"face": enriched.get("face"),
"catalog": enriched.get("catalog"),
"label": enriched.get("label"),
"device": enriched.get("device") or {
"id": it.get("device_id"),
"name": enriched.get("label"),
"model": enriched.get("model"),
"role": enriched.get("category"),
"service_tag": None,
},
}
)
by_site.setdefault(rk["site_id"], []).append(card)
site_out = [{**s, "racks": by_site.get(s["id"], [])} for s in sites]
unplaced = []
for d in STATE.get("devices") or []:
if d.get("id") in placed_ids:
continue
if not (d.get("is_server") or d.get("is_switch") or d.get("is_chassis") or d.get("is_storage") or d.get("is_pdu")):
continue
cat = match_rack_catalog(d.get("model"), d.get("role"))
unplaced.append(
{
"id": d.get("id"),
"name": d.get("name"),
"model": d.get("model"),
"service_tag": d.get("service_tag"),
"role": d.get("role"),
"ip": d.get("ip") or d.get("idrac_ip"),
"connected": d.get("connected"),
"default_u_height": _default_u_height(d),
"catalog_sku": (cat or {}).get("sku"),
"face": (cat or {}).get("face") or "custom",
"image": (cat or {}).get("image") or catalog_image_for(cat or {"face": "custom"}),
"is_switch": d.get("is_switch"),
"is_server": d.get("is_server"),
"is_storage": d.get("is_storage"),
"is_chassis": d.get("is_chassis"),
"is_pdu": d.get("is_pdu"),
}
)
unplaced.sort(key=lambda x: ((x.get("role") or ""), (x.get("name") or "").lower()))
return {
"sites": site_out,
"items": items,
"placements": items,
"unplaced": unplaced,
"catalog": DELL_RACK_CATALOG,
}
def _switch_port_count(device: dict | None) -> int:
"""Default front-panel port count from model (OME has no LLDP ports here)."""
model = ((device or {}).get("model") or "").lower()
if "s4048" in model:
return 48
if "s5248" in model or "s5296" in model:
return 48
if "s4148" in model:
return 48
if "mx9116" in model:
return 16
if "mx5108" in model:
return 8
return 48
def _port_label(port_num: int, total: int = 48) -> str:
return f"Eth1/{port_num}"
def build_switch_portmap(switch_id: int) -> dict:
devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None}
sw = devices.get(switch_id)
if not sw or not sw.get("is_switch"):
# still allow if type 7000 missing flag somehow
sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None)
if not sw:
raise HTTPException(404, "Switch not found in fleet")
nports = _switch_port_count(sw)
with _db() as conn:
rows = conn.execute(
"SELECT * FROM port_links WHERE switch_id=? ORDER BY switch_port",
(switch_id,),
).fetchall()
links = [dict(r) for r in rows]
by_port = {int(l["switch_port"]): l for l in links}
ports = []
for i in range(1, nports + 1):
link = by_port.get(i)
peer = devices.get(link["device_id"]) if link else None
ports.append(
{
"port": i,
"label": _port_label(i, nports),
"wired": bool(link),
"link": (
{
"id": link["id"],
"device_id": link["device_id"],
"device_port": link.get("device_port") or "",
"note": link.get("note") or "",
"device": {
"id": peer.get("id"),
"name": peer.get("name"),
"service_tag": peer.get("service_tag"),
"ip": peer.get("ip") or peer.get("idrac_ip"),
"model": peer.get("model"),
"role": peer.get("role"),
"connected": peer.get("connected"),
}
if peer
else None,
}
if link
else None
),
}
)
return {
"switch": {
"id": sw.get("id"),
"name": sw.get("name"),
"model": sw.get("model"),
"service_tag": sw.get("service_tag"),
"ip": sw.get("ip") or sw.get("idrac_ip"),
"subnet": sw.get("subnet"),
"connected": sw.get("connected"),
"port_count": nports,
},
"ports": ports,
"wired_count": sum(1 for p in ports if p["wired"]),
"note": "Port wiring is manual (OME has no LLDP port map for this switch). Map switch port → server NIC/port.",
}
def _seed_atc_vlans() -> None:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"""Seed known ATC lab VLANs/subnets (editable). Ensures new catalog entries are added."""
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
now = time.time()
defaults = [
(40, "iDRAC / OOB A", "10.0.40.0/24", "Out-of-band management (has S4048)", "ATC1", "#00a8e8", 1),
(41, "iDRAC / OOB B", "10.0.41.0/24", "Out-of-band management", "ATC1", "#3dffe0", 2),
(42, "iDRAC / OOB C", "10.0.42.0/24", "Out-of-band management", "ATC2", "#7c5cff", 3),
(10, "Production / OS", "10.0.10.0/24", "Hypervisor & Windows OS / RDP", "ATC1", "#ff9a3c", 4),
(11, "Lab / OS", "10.0.11.0/24", "Lab OS network", "ATC2", "#ff5c5c", 5),
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
(20, "Shared services", "10.0.20.0/24", "Mgmt / demo / shared ATC services", "ATC1", "#a3e635", 6),
(120, "Secondary OS", "10.0.120.0/24", "Secondary OS / RDP path", "ATC1", "#f0d060", 7),
(90, "FDE cluster / AI", "10.0.90.0/24", "FDE platform · DNS · HAProxy · OPNsense · Proxmox (dell-fde.lan)", "FDE", "#38bdf8", 8),
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
]
with _db() as conn:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
existing = {
int(r["vlan_id"])
for r in conn.execute("SELECT vlan_id FROM vlans WHERE vlan_id IS NOT NULL").fetchall()
if r["vlan_id"] is not None
}
added = 0
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
for vlan_id, name, cidr, purpose, site, color, so in defaults:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
if vlan_id in existing:
continue
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
conn.execute(
"""
INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(vlan_id, name, cidr, purpose, site, color, so, now, now),
)
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
added += 1
if added:
log.info("Seeded ATC VLAN catalog (+%s new, %s known defaults)", added, len(defaults))
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
def _synthetic_device_id(ip: str) -> int:
"""Stable negative id for inventory-only hosts (safe for JS Number)."""
try:
a, b, c, d = (int(x) for x in ip.split("."))
return -(a * 1_000_000 + b * 10_000 + c * 100 + d)
except Exception:
return -abs(hash(ip)) % 1_000_000_000
def _vlan_id_for_ip(ip: str) -> int | None:
try:
return int(ip.split(".")[2])
except Exception:
return None
def _ensure_network_endpoints_schema(conn) -> None:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS network_endpoints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hostname TEXT,
ip TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'server',
kind TEXT NOT NULL DEFAULT 'physical',
device_id INTEGER,
vlan_id INTEGER,
switch_id INTEGER,
switch_port INTEGER,
model TEXT,
note TEXT,
user_note TEXT,
created_at REAL NOT NULL,
updated_at REAL NOT NULL
)
"""
)
cols = {r[1] for r in conn.execute("PRAGMA table_info(network_endpoints)").fetchall()}
if "user_note" not in cols:
conn.execute("ALTER TABLE network_endpoints ADD COLUMN user_note TEXT")
def _upsert_network_endpoint(
*,
hostname: str | None,
ip: str,
role: str = "server",
kind: str = "host",
device_id: int | None = None,
vlan_id: int | None = None,
switch_id: int | None = None,
switch_port: int | None = None,
model: str | None = None,
note: str | None = None,
overwrite_identity: bool = False,
clear_device_id: bool = False,
) -> None:
"""Insert/update inventory row. Never clobber user_note. Soft-update identity fields."""
ip = (ip or "").strip()
if not ip or ip.count(".") != 3:
return
now = time.time()
vlan_id = vlan_id if vlan_id is not None else _vlan_id_for_ip(ip)
with _db() as conn:
_ensure_network_endpoints_schema(conn)
row = conn.execute("SELECT * FROM network_endpoints WHERE ip=?", (ip,)).fetchone()
if not row:
conn.execute(
"""
INSERT INTO network_endpoints(
hostname, ip, role, kind, device_id, vlan_id, switch_id, switch_port,
model, note, user_note, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
""",
(
hostname,
ip,
role or "server",
kind or "host",
device_id,
vlan_id,
switch_id,
switch_port,
model,
note,
now,
now,
),
)
return
# Update carefully
sets = ["updated_at=?"]
vals: list = [now]
if overwrite_identity or not row["hostname"]:
if hostname:
sets.append("hostname=?")
vals.append(hostname)
if overwrite_identity or not row["model"]:
if model:
sets.append("model=?")
vals.append(model)
if overwrite_identity or not row["note"]:
if note:
sets.append("note=?")
vals.append(note)
if overwrite_identity and note is not None:
sets.append("note=?")
vals.append(note)
if overwrite_identity and model is not None:
if "model=?" not in sets:
sets.append("model=?")
vals.append(model)
if overwrite_identity and hostname is not None:
if "hostname=?" not in sets:
sets.append("hostname=?")
vals.append(hostname)
if role:
sets.append("role=?")
vals.append(role)
if kind:
sets.append("kind=?")
vals.append(kind)
if vlan_id is not None:
sets.append("vlan_id=?")
vals.append(vlan_id)
if clear_device_id:
sets.append("device_id=NULL")
elif device_id is not None and (overwrite_identity or row["device_id"] is None):
sets.append("device_id=?")
vals.append(device_id)
if switch_id is not None and (overwrite_identity or row["switch_id"] is None):
sets.append("switch_id=?")
vals.append(switch_id)
if switch_port is not None and (overwrite_identity or row["switch_port"] is None):
sets.append("switch_port=?")
vals.append(switch_port)
vals.append(ip)
conn.execute(f"UPDATE network_endpoints SET {', '.join(sets)} WHERE ip=?", vals)
def _known_host_inventory() -> list[dict]:
"""Static + discovered lab hosts (OS / FDE / shared services)."""
return [
# VLAN 90 — FDE (physical Proxmox nodes are R640 per ops; OME idrac-proxmox R740xd is a different box)
{"hostname": "router.dell-fde.lan", "ip": "10.0.90.1", "role": "gateway", "kind": "appliance", "vlan_id": 90, "model": None, "note": "FDE gateway"},
{"hostname": "ns.dell-fde.lan", "ip": "10.0.90.2", "role": "dns", "kind": "vm", "vlan_id": 90, "note": "FDE DNS"},
{"hostname": "haproxy.dell-fde.lan", "ip": "10.0.90.3", "role": "lb", "kind": "vm", "vlan_id": 90, "note": "FDE HAProxy"},
{"hostname": "caddy.dell-fde.lan", "ip": "10.0.90.4", "role": "proxy", "kind": "vm", "vlan_id": 90, "note": "FDE Caddy"},
{"hostname": "opnsense01.dell-fde.lan", "ip": "10.0.90.5", "role": "firewall", "kind": "vm", "vlan_id": 90, "note": "FDE OPNsense 1"},
{"hostname": "opnsense02.dell-fde.lan", "ip": "10.0.90.6", "role": "firewall", "kind": "vm", "vlan_id": 90, "note": "FDE OPNsense 2"},
{
"hostname": "proxmox01.dell-fde.lan",
"ip": "10.0.90.21",
"role": "server",
"kind": "physical",
"vlan_id": 90,
"model": "PowerEdge R640",
"device_id": None,
"switch_id": 51950,
"note": "FDE Proxmox node 1 · R640 (ops). Candidate iDRAC: C90XK53/B90XK53 on VLAN41 — not the R740xd idrac-proxmox.",
"overwrite_identity": True,
"clear_device_id": True,
},
{
"hostname": "proxmox02.dell-fde.lan",
"ip": "10.0.90.22",
"role": "server",
"kind": "physical",
"vlan_id": 90,
"model": "PowerEdge R640",
"device_id": None,
"switch_id": 51950,
"note": "FDE Proxmox node 2 · R640 (ops). Candidate iDRAC: C90XK53/B90XK53 on VLAN41.",
"overwrite_identity": True,
"clear_device_id": True,
},
{"hostname": "fde-dns.fde-dns.lan", "ip": "10.0.90.41", "role": "dns", "kind": "vm", "vlan_id": 90},
{"hostname": "dns-server.fde-dns.lan", "ip": "10.0.90.42", "role": "dns", "kind": "vm", "vlan_id": 90},
{"hostname": "db01.dell-fde.lan", "ip": "10.0.90.181", "role": "database", "kind": "vm", "vlan_id": 90},
# VLAN 10 — Production OS
{"hostname": "cop-hv01.dell-atc.lan", "ip": "10.0.10.13", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
{"hostname": "cop-dc01.dell-atc.lan", "ip": "10.0.10.15", "role": "dc", "kind": "vm", "vlan_id": 10},
{"hostname": "cop-dc02.dell-atc.lan", "ip": "10.0.10.16", "role": "dc", "kind": "vm", "vlan_id": 10},
{"hostname": "cop-vsan-vdi01.dell-atc.lan", "ip": "10.0.10.41", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
{"hostname": "cop-vsan-vdi02.dell-atc.lan", "ip": "10.0.10.42", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
{"hostname": "cop-vsan-vdi03.dell-atc.lan", "ip": "10.0.10.43", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
{"hostname": "cop-vsan-vdi04.dell-atc.lan", "ip": "10.0.10.44", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
{"hostname": "pve01.dell-atc.dell.nl", "ip": "10.0.10.65", "role": "server", "kind": "physical", "vlan_id": 10, "note": "Proxmox VE (ATC)"},
{"hostname": "atc-gpu-prod.dell-atc.lan", "ip": "10.0.10.106", "role": "gpu", "kind": "physical", "vlan_id": 10},
# VLAN 20 — Shared services
{"hostname": "cov-dc03.dell-atc.lan", "ip": "10.0.20.15", "role": "dc", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-omedemo01.dell-atc.lan", "ip": "10.0.20.22", "role": "ome", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-cloudiq01.dell-atc.lan", "ip": "10.0.20.23", "role": "cloudiq", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-scg01.dell-atc.lan", "ip": "10.0.20.31", "role": "scg", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-vcsa-vsan01.dell-atc.lan", "ip": "10.0.20.40", "role": "vcenter", "kind": "vm", "vlan_id": 20},
{"hostname": "photon-machine.dell-atc.lan", "ip": "10.0.20.102", "role": "host", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-grafana.dell-atc.lan", "ip": "10.0.20.103", "role": "monitoring", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-mgt01.dell-atc.lan", "ip": "10.0.20.104", "role": "mgmt", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-dataflow01.dell-atc.lan", "ip": "10.0.20.105", "role": "dataflow", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-gpu-dev.dell-atc.lan", "ip": "10.0.20.106", "role": "gpu", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-spearmint.dell-atc.lan", "ip": "10.0.20.108", "role": "app", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-file01.dell-atc.lan", "ip": "10.0.20.109", "role": "files", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-nas.dell-atc.lan", "ip": "10.0.20.110", "role": "nas", "kind": "appliance", "vlan_id": 20},
{"hostname": "atc-objectscale.dell-atc.lan", "ip": "10.0.20.111", "role": "object", "kind": "appliance", "vlan_id": 20},
{"hostname": "atc-db01.dell-atc.lan", "ip": "10.0.20.112", "role": "database", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-backup.dell-atc.lan", "ip": "10.0.20.113", "role": "backup", "kind": "vm", "vlan_id": 20},
{"hostname": "opnsense03.dell-atc.lan", "ip": "10.0.20.114", "role": "firewall", "kind": "vm", "vlan_id": 20},
{"hostname": "atc-git.dell-atc.lan", "ip": "10.0.20.118", "role": "git", "kind": "vm", "vlan_id": 20},
{"hostname": "cvd-jody01.dell-atc.lan", "ip": "10.0.20.119", "role": "desktop", "kind": "vm", "vlan_id": 20},
{"hostname": "emj-wilcor01.dell-atc.lan", "ip": "10.0.20.133", "role": "desktop", "kind": "vm", "vlan_id": 20},
{"hostname": "cov-wac01.dell-atc.lan", "ip": "10.0.20.147", "role": "wac", "kind": "vm", "vlan_id": 20},
{"hostname": "CVP-MGMT01.dell-atc.lan", "ip": "10.0.20.164", "role": "mgmt", "kind": "vm", "vlan_id": 20},
]
def _seed_network_endpoints() -> None:
"""Ensure schema + known host inventory + correct FDE Proxmox identity."""
with _db() as conn:
_ensure_network_endpoints_schema(conn)
for h in _known_host_inventory():
_upsert_network_endpoint(
hostname=h.get("hostname"),
ip=h["ip"],
role=h.get("role") or "server",
kind=h.get("kind") or "host",
device_id=h.get("device_id"),
vlan_id=h.get("vlan_id"),
switch_id=h.get("switch_id"),
switch_port=h.get("switch_port"),
model=h.get("model"),
note=h.get("note"),
overwrite_identity=bool(h.get("overwrite_identity")),
clear_device_id=bool(h.get("clear_device_id")),
)
def sync_endpoints_from_fleet(devices: list[dict] | None = None) -> int:
"""Upsert every OME fleet mgmt IP into network_endpoints (enables notes on all systems)."""
_seed_network_endpoints()
devices = devices if devices is not None else (STATE.get("devices") or [])
n = 0
for d in devices:
if d.get("source") == "inventory":
continue
did = d.get("id")
if not isinstance(did, int) or did <= 0:
continue
ip = d.get("idrac_ip") or d.get("ip")
if not ip:
continue
role = d.get("role") or "server"
kind = "switch" if d.get("is_switch") else ("pdu" if d.get("is_pdu") else ("chassis" if d.get("is_chassis") else "physical"))
_upsert_network_endpoint(
hostname=d.get("name"),
ip=ip,
role=role,
kind=kind,
device_id=did,
vlan_id=_vlan_id_for_ip(ip),
model=d.get("model"),
note=f"OME {d.get('service_tag') or did}",
)
n += 1
# Also register secondary OS IPs from rdp/os/extra
for sip in (d.get("rdp_ips") or []) + (d.get("os_ips") or []) + (d.get("extra_ips") or []):
if not sip or sip == ip:
continue
_upsert_network_endpoint(
hostname=d.get("cluster_hostname") or d.get("os_hostname") or d.get("name"),
ip=sip,
role=role,
kind="os",
device_id=did,
vlan_id=_vlan_id_for_ip(sip),
model=d.get("model"),
note=f"OS/secondary IP for OME {d.get('service_tag') or did}",
)
n += 1
return n
def _load_network_endpoints() -> list[dict]:
_seed_network_endpoints()
with _db() as conn:
_ensure_network_endpoints_schema(conn)
return [dict(r) for r in conn.execute("SELECT * FROM network_endpoints ORDER BY ip").fetchall()]
def _load_vlan_catalog_by_cidr() -> dict[str, dict]:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_seed_atc_vlans()
with _db() as conn:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
rows = [dict(r) for r in conn.execute("SELECT * FROM vlans").fetchall()]
out = {}
for r in rows:
cidr = (r.get("cidr") or "").strip()
if cidr:
out[cidr] = r
return out
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
def _device_ips(d: dict) -> list[str]:
out: list[str] = []
for k in ("ip", "idrac_ip"):
v = d.get(k)
if v and v not in out:
out.append(v)
for key in ("rdp_ips", "os_ips", "extra_ips"):
for v in d.get(key) or []:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if v and v not in out:
out.append(v)
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
return out
def _refresh_map_cidrs(d: dict) -> None:
cidrs: list[str] = []
seen: set[str] = set()
primary = d.get("subnet")
if primary and primary != "unknown" and primary not in seen:
seen.add(primary)
cidrs.append(primary)
for ip in _device_ips(d):
c = subnet_of(ip)
if c != "unknown" and c not in seen:
seen.add(c)
cidrs.append(c)
d["map_cidrs"] = cidrs
def apply_network_endpoints(devices: list[dict]) -> None:
"""Attach inventory IPs to OME devices and inject synthetic hosts for unlinked endpoints."""
endpoints = _load_network_endpoints()
if not endpoints:
for d in devices:
_refresh_map_cidrs(d)
return
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
# Index existing IPs on devices to avoid duplicate synthetics for OME mgmt IPs
device_ips: set[str] = set()
for d in devices:
for ip in _device_ips(d):
device_ips.add(ip)
existing_ids = set(by_id)
used_synth: set[int] = set()
for ep in endpoints:
ip = (ep.get("ip") or "").strip()
if not ip or ip.count(".") != 3:
continue
hostname = (ep.get("hostname") or ip).strip()
did = ep.get("device_id")
user_note = ep.get("user_note") or ""
sys_note = ep.get("note") or ""
if did is not None and did in by_id:
d = by_id[did]
extras = list(d.get("extra_ips") or [])
# Only treat as extra if not already primary mgmt IP
if ip not in extras and ip != d.get("ip") and ip != d.get("idrac_ip"):
extras.append(ip)
d["extra_ips"] = extras
if ip != d.get("ip") and ip != d.get("idrac_ip"):
os_ips = list(d.get("os_ips") or [])
if ip not in os_ips:
os_ips.append(ip)
d["os_ips"] = os_ips
rdp = list(d.get("rdp_ips") or [])
if ip not in rdp:
rdp.append(ip)
d["rdp_ips"] = rdp
d["cluster_hostname"] = hostname
d["endpoint_id"] = ep.get("id")
d["endpoint_note"] = sys_note
d["user_note"] = user_note
if ep.get("model") and (d.get("source") == "inventory" or not d.get("model")):
pass # keep OME model authoritative for linked devices
if ep.get("switch_id") is not None:
d["endpoint_switch_id"] = ep.get("switch_id")
d["endpoint_switch_port"] = ep.get("switch_port")
_refresh_map_cidrs(d)
continue
# Skip synthetic if this IP already belongs to an OME device in the fleet
if ip in device_ips:
# Still attach notes onto the matching device if we can find it
for d in devices:
if ip in _device_ips(d) or d.get("ip") == ip or d.get("idrac_ip") == ip:
d["endpoint_id"] = ep.get("id")
d["endpoint_note"] = sys_note
d["user_note"] = user_note
break
continue
sid = _synthetic_device_id(ip)
if sid in existing_ids or sid in used_synth:
continue
used_synth.add(sid)
cidr = subnet_of(ip)
role = ep.get("role") or "server"
devices.append(
{
"id": sid,
"name": hostname,
"model": ep.get("model") or "Inventory host",
"service_tag": None,
"type": 1000,
"sub_type": "inventory",
"connected": True,
"power_state": None,
"powered_on": None,
"status": "inventory",
"ip": ip,
"idrac_ip": None,
"os_hostname": hostname,
"mgmt_dns_name": hostname,
"rdp_host": ip,
"rdp_ips": [ip],
"os_ips": [ip],
"extra_ips": [ip],
"is_windows": False,
"subnet": cidr,
"map_cidrs": [cidr],
"role": role,
"is_server": True,
"is_idrac": False,
"is_switch": role == "switch",
"is_chassis": False,
"is_pdu": role == "pdu",
"is_storage": role == "storage",
"chassis_service_tag": None,
"idrac_url": None,
"watts": None,
"source": "inventory",
"endpoint_id": ep.get("id"),
"endpoint_note": sys_note,
"user_note": user_note,
"endpoint_switch_id": ep.get("switch_id"),
"endpoint_switch_port": ep.get("switch_port"),
"cluster_hostname": hostname,
}
)
for d in devices:
_refresh_map_cidrs(d)
def build_subnet_summaries(devices: list[dict]) -> list[dict]:
"""Aggregate devices by map_cidrs (mgmt + secondary/inventory IPs) and attach VLAN catalog labels."""
catalog = _load_vlan_catalog_by_cidr()
buckets: dict[str, list[dict]] = defaultdict(list)
for d in devices:
cidrs = d.get("map_cidrs") or ([d.get("subnet")] if d.get("subnet") else [])
for c in cidrs:
if not c or c == "unknown":
continue
buckets[c].append(d)
for cidr in catalog:
buckets.setdefault(cidr, [])
subnets = []
for cidr, members in sorted(buckets.items(), key=lambda x: (-len(x[1]), x[0])):
seen: set = set()
uniq = []
for m in members:
mid = m.get("id")
if mid in seen:
continue
seen.add(mid)
uniq.append(m)
ids = [m["id"] for m in uniq if m.get("id") is not None]
sub_watts = sum(n["watts"] or 0 for n in uniq if n.get("watts") is not None)
switches = [n for n in uniq if n.get("is_switch")]
meta = catalog.get(cidr) or {}
subnets.append(
{
"cidr": cidr,
"count": len(uniq),
"connected": sum(1 for n in uniq if n.get("connected")),
"watts": round(sub_watts, 1) if sub_watts else None,
"device_ids": ids,
"has_switch": bool(switches),
"switches": [
{
"id": s["id"],
"name": s["name"],
"service_tag": s.get("service_tag"),
"ip": s.get("ip"),
}
for s in switches
],
"vlan_id": meta.get("vlan_id"),
"vlan_name": meta.get("name"),
"vlan_color": meta.get("color"),
"site_id": meta.get("site_id"),
}
)
return subnets
def _all_port_links() -> list[dict]:
with _db() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS port_links (
id INTEGER PRIMARY KEY AUTOINCREMENT,
switch_id INTEGER NOT NULL,
switch_port INTEGER NOT NULL,
device_id INTEGER NOT NULL,
device_port TEXT,
note TEXT,
updated_at REAL NOT NULL,
UNIQUE(switch_id, switch_port)
)
"""
)
return [dict(r) for r in conn.execute("SELECT * FROM port_links").fetchall()]
_NETWORK_INV_TS = 0.0
def ensure_network_inventory_in_state(force: bool = False) -> None:
"""Merge inventory endpoints into live fleet STATE (safe between OME polls)."""
global _NETWORK_INV_TS
now = time.time()
if not force and STATE.get("devices") and (now - _NETWORK_INV_TS) < 8:
return
devices = list(STATE.get("devices") or [])
sync_endpoints_from_fleet(devices)
devices = [d for d in devices if d.get("source") != "inventory"]
apply_network_endpoints(devices)
STATE["devices"] = devices
STATE["subnets"] = build_subnet_summaries(devices)
_NETWORK_INV_TS = now
def build_vlan_map() -> dict:
"""VLANs from catalog + membership from fleet IPs + inventory endpoints; switch attachment matrix."""
_seed_atc_vlans()
_seed_network_endpoints()
_migrate_rack_items()
ensure_network_inventory_in_state()
devices = list(STATE.get("devices") or [])
with _db() as conn:
rows = [dict(r) for r in conn.execute("SELECT * FROM vlans ORDER BY sort_order, vlan_id").fetchall()]
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def in_cidr(ip: str, cidr: str) -> bool:
try:
return ipaddress.ip_address(ip) in ipaddress.ip_network(cidr, strict=False)
except Exception:
return False
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
switches_by_id = {d.get("id"): d for d in devices if d.get("is_switch")}
links = _all_port_links()
links_by_device: dict[int, list[dict]] = defaultdict(list)
for link in links:
did = link.get("device_id")
if did is None:
continue
sw = switches_by_id.get(link.get("switch_id")) or {}
links_by_device[int(did)].append(
{
"switch_id": link.get("switch_id"),
"switch_name": sw.get("name") or f"switch-{link.get('switch_id')}",
"switch_ip": sw.get("ip") or sw.get("idrac_ip"),
"switch_port": link.get("switch_port"),
"device_port": link.get("device_port") or "",
"note": link.get("note") or "",
}
)
for d in devices:
sid = d.get("endpoint_switch_id")
if sid is None:
continue
did = d.get("id")
if did is None:
continue
already = {(x["switch_id"], x.get("switch_port")) for x in links_by_device.get(did, [])}
port = d.get("endpoint_switch_port")
key = (sid, port)
if key in already:
continue
sw = switches_by_id.get(sid) or {}
links_by_device[int(did)].append(
{
"switch_id": sid,
"switch_name": sw.get("name") or f"switch-{sid}",
"switch_ip": sw.get("ip") or sw.get("idrac_ip"),
"switch_port": port,
"device_port": "",
"note": "from inventory endpoint",
"inferred": True,
}
)
eps_by_ip = {e.get("ip"): e for e in _load_network_endpoints() if e.get("ip")}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
vlans = []
for v in rows:
members = []
for d in devices:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
matched = [ip for ip in _device_ips(d) if in_cidr(ip, v["cidr"])]
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if not matched:
continue
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
mid = d.get("id")
display = d.get("cluster_hostname") or d.get("name")
ep = None
for ip in matched:
if ip in eps_by_ip:
ep = eps_by_ip[ip]
break
model = d.get("model")
if d.get("source") == "inventory" and ep and ep.get("model"):
model = ep.get("model")
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
members.append(
{
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"id": mid,
"name": display,
"ome_name": d.get("name"),
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"service_tag": d.get("service_tag"),
"role": d.get("role"),
"connected": d.get("connected"),
"ips": matched,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"model": model,
"source": d.get("source") or "ome",
"switch_ports": links_by_device.get(mid, []),
"endpoint_id": d.get("endpoint_id") or (ep or {}).get("id"),
"endpoint_note": d.get("endpoint_note") or (ep or {}).get("note") or "",
"user_note": d.get("user_note") or (ep or {}).get("user_note") or "",
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
}
)
members.sort(key=lambda m: (m.get("name") or "").lower())
vlans.append(
{
**v,
"member_count": len(members),
"connected_count": sum(1 for m in members if m.get("connected")),
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"members": members[:250],
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
}
)
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
matrix = []
for d in devices:
if not (d.get("is_server") or d.get("source") == "inventory"):
continue
if d.get("is_switch") or d.get("is_pdu"):
continue
ips = _device_ips(d)
member_vlans = []
for v in rows:
matched = [ip for ip in ips if in_cidr(ip, v["cidr"])]
if matched:
member_vlans.append(
{
"vlan_id": v.get("vlan_id"),
"name": v.get("name"),
"cidr": v.get("cidr"),
"color": v.get("color"),
"ips": matched,
}
)
if not member_vlans and not links_by_device.get(d.get("id")):
continue
matrix.append(
{
"id": d.get("id"),
"name": d.get("cluster_hostname") or d.get("name"),
"ome_name": d.get("name"),
"service_tag": d.get("service_tag"),
"model": d.get("model"),
"role": d.get("role"),
"connected": d.get("connected"),
"source": d.get("source") or "ome",
"ips": ips,
"vlans": member_vlans,
"switch_ports": links_by_device.get(d.get("id"), []),
"endpoint_id": d.get("endpoint_id"),
"endpoint_note": d.get("endpoint_note") or "",
"user_note": d.get("user_note") or "",
}
)
matrix.sort(key=lambda m: (m.get("name") or "").lower())
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
seen = {v["cidr"] for v in rows}
discovered = []
buckets: dict[str, int] = {}
for d in devices:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
for ip in _device_ips(d):
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if ip.count(".") != 3:
continue
cidr = ".".join(ip.split(".")[:3]) + ".0/24"
if cidr.startswith("127."):
continue
buckets[cidr] = buckets.get(cidr, 0) + 1
for cidr, cnt in sorted(buckets.items(), key=lambda x: -x[1]):
if cidr not in seen:
discovered.append({"cidr": cidr, "device_ips": cnt, "in_catalog": False})
return {
"vlans": vlans,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"attachment_matrix": matrix,
"port_links": [
{
"switch_id": l.get("switch_id"),
"switch_port": l.get("switch_port"),
"device_id": l.get("device_id"),
"device_port": l.get("device_port"),
"note": l.get("note"),
}
for l in links
],
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"discovered_subnets": discovered,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"note": "VLAN catalog + OME IPs + inventory endpoints. Switch ports from Fabric wiring and endpoint hints.",
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
}
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def _migrate_rack_items() -> None:
"""Copy legacy rack_placements into flexible rack_items once."""
with _db() as conn:
n = conn.execute("SELECT COUNT(*) AS c FROM rack_items").fetchone()["c"]
if n > 0:
return
try:
rows = conn.execute("SELECT * FROM rack_placements").fetchall()
except Exception:
return
now = time.time()
for r in rows:
conn.execute(
"""
INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at)
VALUES (?, ?, NULL, NULL, ?, ?, 'front', NULL, ?)
""",
(r["rack_id"], r["device_id"], r["u_start"], r["u_height"], now),
)
if rows:
log.info("Migrated %s rack_placements → rack_items", len(rows))
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
OPS_USERS = [
{
"id": "jody",
"name": "Jody van Dongen",
"role": "ATC Datacenter Admin",
"team": "admin",
"email": "jody.van.dongen@dell.com",
"focus": "OME fleet · racks · warranty / compliance",
},
{
"id": "laurens",
"name": "Laurens Rammers",
"role": "ATC Datacenter Admin",
"team": "admin",
"email": "laurens.rammers@dell.com",
"focus": "Datacenter ops · handoffs · escalation",
},
{
"id": "mo",
"name": "Mohamed El Kadi",
"role": "Data Forward Deployed Engineer",
"team": "fde",
"email": "mohamed.el.kadi@dell.com",
"focus": "OME Cockpit · OpenManage AI · AI workloads on FDE cluster",
},
{
"id": "bart",
"name": "Bart Sjerps",
"role": "Data Forward Deployed Engineer",
"team": "fde",
"email": "bart.sjerps@dell.com",
"focus": "FDE cluster · AI workload deployment with Mo",
},
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
]
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
# Back-compat alias used by older ticket endpoints / UI
ADMINS = OPS_USERS
def _ops_user(uid: str) -> dict | None:
for u in OPS_USERS:
if u["id"] == uid:
return u
return None
def _default_assignee(created_by: str) -> str:
"""Admins hand off to each other; FDE hand off to an admin by default."""
if created_by == "jody":
return "laurens"
if created_by == "laurens":
return "jody"
if created_by in ("mo", "bart"):
return "jody"
return "jody"
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
DATA_DIR = Path(settings.cockpit_data)
DB_PATH = DATA_DIR / "ops.db"
def _db():
DATA_DIR.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA foreign_keys=ON")
return conn
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def _backup_ops_db(reason: str = "manual") -> Path | None:
"""Snapshot ops.db so rebuilds / wipes do not silently lose ticket history."""
try:
if not DB_PATH.exists() or DB_PATH.stat().st_size < 100:
return None
bak_dir = DATA_DIR / "backups"
bak_dir.mkdir(parents=True, exist_ok=True)
stamp = time.strftime("%Y%m%d-%H%M%S")
dest = bak_dir / f"ops-{stamp}-{reason}-{int(time.time()*1000)%100000}.db"
import shutil
shutil.copy2(DB_PATH, dest)
# keep last 20 backups
olds = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True)
for stale in olds[20:]:
try:
stale.unlink()
except Exception:
pass
return dest
except Exception as e:
log.warning("ops.db backup failed: %s", e)
return None
def _restore_ops_db_if_empty() -> bool:
"""If tickets table is empty but a backup exists, restore the newest backup with tickets."""
try:
with _db() as conn:
n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
if n > 0:
return False
bak_dir = DATA_DIR / "backups"
if not bak_dir.exists():
return False
candidates = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True)
for cand in candidates:
try:
c2 = sqlite3.connect(cand)
c2.row_factory = sqlite3.Row
cnt = c2.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
c2.close()
if cnt <= 0:
continue
import shutil
# preserve empty current as .empty
if DB_PATH.exists():
shutil.copy2(DB_PATH, bak_dir / f"ops-empty-before-restore-{int(time.time())}.db")
shutil.copy2(cand, DB_PATH)
log.warning("Restored ops.db from backup %s (%s tickets)", cand.name, cnt)
return True
except Exception as e:
log.debug("skip backup %s: %s", cand, e)
return False
except Exception as e:
log.warning("ops.db restore check failed: %s", e)
return False
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
def init_db():
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
DATA_DIR.mkdir(parents=True, exist_ok=True)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
with _db() as conn:
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS tickets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
body TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'open',
priority TEXT NOT NULL DEFAULT 'normal',
created_by TEXT NOT NULL,
assignee TEXT,
accepted_by TEXT,
created_at REAL NOT NULL,
updated_at REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS ticket_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ticket_id INTEGER NOT NULL REFERENCES tickets(id) ON DELETE CASCADE,
author TEXT NOT NULL,
body TEXT NOT NULL,
created_at REAL NOT NULL
);
"""
)
cols = {r[1] for r in conn.execute("PRAGMA table_info(tickets)").fetchall()}
if "accepted_by" not in cols:
conn.execute("ALTER TABLE tickets ADD COLUMN accepted_by TEXT")
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS sites (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
sort_order INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS racks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_id TEXT NOT NULL REFERENCES sites(id),
name TEXT NOT NULL,
units INTEGER NOT NULL DEFAULT 42,
sort_order INTEGER NOT NULL,
created_at REAL NOT NULL,
updated_at REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS rack_placements (
device_id INTEGER PRIMARY KEY,
rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE,
u_start INTEGER NOT NULL,
u_height INTEGER NOT NULL,
updated_at REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS rack_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE,
device_id INTEGER,
catalog_sku TEXT,
label TEXT,
u_start INTEGER NOT NULL,
u_height INTEGER NOT NULL,
side TEXT NOT NULL DEFAULT 'front',
notes TEXT,
updated_at REAL NOT NULL
);
CREATE TABLE IF NOT EXISTS port_links (
id INTEGER PRIMARY KEY AUTOINCREMENT,
switch_id INTEGER NOT NULL,
switch_port INTEGER NOT NULL,
device_id INTEGER NOT NULL,
device_port TEXT,
note TEXT,
updated_at REAL NOT NULL,
UNIQUE(switch_id, switch_port)
);
CREATE TABLE IF NOT EXISTS vlans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
vlan_id INTEGER,
name TEXT NOT NULL,
cidr TEXT NOT NULL,
purpose TEXT,
site_id TEXT,
color TEXT,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at REAL NOT NULL,
updated_at REAL NOT NULL
);
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
CREATE TABLE IF NOT EXISTS network_endpoints (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hostname TEXT,
ip TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'server',
kind TEXT NOT NULL DEFAULT 'physical',
device_id INTEGER,
vlan_id INTEGER,
switch_id INTEGER,
switch_port INTEGER,
model TEXT,
note TEXT,
created_at REAL NOT NULL,
updated_at REAL NOT NULL
);
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"""
)
_restore_ops_db_if_empty()
_seed_atc_racks()
_seed_atc_vlans()
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
_seed_network_endpoints()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
n = 0
try:
with _db() as conn:
n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
except Exception:
pass
log.info("Ops tickets DB ready at %s (%s tickets)", DB_PATH, n)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
async def fetch_gpu() -> dict:
url = settings.gpu_metrics_url.rstrip("/") + "/api/gpu"
try:
async with httpx.AsyncClient(timeout=4.0) as client:
r = await client.get(url)
if r.status_code == 200:
return r.json()
except Exception as e:
log.warning("GPU metrics fetch failed: %s", e)
return {
"host": "atc-gpu-prod",
"gpus": [],
"error": "unreachable",
"updated_at": time.time(),
"model": settings.vllm_model,
}
async def poll_loop():
while True:
try:
data = await ome_fetch()
gpu = await fetch_gpu()
data["gpu"] = gpu
async with _lock:
STATE.update(data)
STATE["pulse"] = int(STATE.get("pulse") or 0) + 1
await broadcast({"type": "snapshot", "data": {**STATE}})
log.info(
"OME refresh: %s devices, %s connected, %s W sampled, GPU util avg %s",
STATE["summary"].get("total"),
STATE["summary"].get("connected"),
STATE["summary"].get("total_watts"),
(gpu.get("summary") or {}).get("avg_util"),
)
except Exception as e:
log.exception("OME poll failed: %s", e)
await broadcast({"type": "error", "message": str(e)})
await asyncio.sleep(settings.poll_interval)
async def gpu_loop():
"""Faster GPU pulse so matrix feels live during inference."""
while True:
try:
gpu = await fetch_gpu()
async with _lock:
STATE["gpu"] = gpu
await broadcast({"type": "gpu", "data": gpu})
except Exception as e:
log.warning("gpu loop: %s", e)
await asyncio.sleep(2.0)
@app.on_event("startup")
async def startup():
init_db()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("startup")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
asyncio.create_task(poll_loop())
asyncio.create_task(gpu_loop())
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
asyncio.create_task(warm_reports_cache())
async def warm_reports_cache():
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"""Background warm of warranty/compliance/memory for chat + inspector."""
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
await asyncio.sleep(8)
try:
await fetch_warranties()
await fetch_compliance()
log.info(
"Reports cache warmed: warranties=%s compliance_outdated=%s",
(REPORT_CACHE.get("warranties") or {}).get("count"),
((REPORT_CACHE.get("compliance") or {}).get("summary") or {}).get("outdated_devices"),
)
except Exception as e:
log.warning("Reports cache warm failed: %s", e)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
# Wait until fleet snapshot exists, then warm memory inventory for chat tools
for _ in range(40):
if STATE.get("devices"):
break
await asyncio.sleep(2)
try:
mem = await fetch_fleet_memory(force=False)
log.info(
"Memory cache warmed: %s/%s servers",
mem.get("with_memory"),
mem.get("count"),
)
except Exception as e:
log.warning("Memory cache warm failed: %s", e)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/api/health")
async def health():
return {
"status": "ok",
"updated_at": STATE.get("updated_at"),
"pulse": STATE.get("pulse"),
}
@app.get("/api/fleet")
async def fleet():
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
ensure_network_inventory_in_state()
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
return {**STATE}
@app.get("/api/devices/{device_id}")
async def device_detail(device_id: int):
# lightweight detail: return fleet node + optional inventory slices
node = next((d for d in STATE.get("devices") or [] if d.get("id") == device_id), None)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
if device_id in DETAIL_CACHE and time.time() - DETAIL_CACHE[device_id].get("_ts", 0) < 300:
return DETAIL_CACHE[device_id]
base = settings.ome_url.rstrip("/")
detail = {"device": node, "inventory": {}, "power": {}}
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
r = await client.post(
f"{base}/api/SessionService/Sessions",
json={
"UserName": settings.ome_user,
"Password": settings.ome_password,
"SessionType": "API",
},
)
r.raise_for_status()
token = r.headers.get("X-Auth-Token")
sid = r.json().get("Id")
headers = {"X-Auth-Token": token, "Accept": "application/json"}
try:
detail["power"] = await fetch_power(client, base, headers, device_id)
# Discover all inventory types OME has for this device (app/firmware landscape)
inv_types: list[str] = []
try:
tr = await client.get(
f"{base}/api/DeviceService/Devices({device_id})/InventoryTypes",
headers=headers,
)
if tr.status_code == 200:
inv_types = list((tr.json() or {}).get("InventoryTypes") or [])
except Exception:
inv_types = []
# Ensure core + software landscape types are always attempted
for required in (
"serverProcessors",
"serverMemoryDevices",
"serverArrayDisks",
"serverPowerSupplies",
"serverNetworkInterfaces",
"deviceLocation",
"deviceManagement",
"subsystemRollupStatus",
"serverOperatingSystems",
"deviceSoftware",
"deviceLicense",
"deviceFru",
"deviceCapabilities",
"serverRaidControllers",
"serverDeviceCards",
"deviceBaseboards",
"serverDellVideos",
"serverFcCards",
"serverVirtualFlashes",
"serverStorageEnclosures",
"serverSupportedPowerStates",
"serverBiosSystemProfileSettings",
"deviceInventory",
):
if required not in inv_types:
inv_types.append(required)
detail["inventory_types"] = inv_types
for inv_type in inv_types:
try:
ir = await client.get(
f"{base}/api/DeviceService/Devices({device_id})/InventoryDetails('{inv_type}')",
headers=headers,
)
if ir.status_code == 200:
info = ir.json().get("InventoryInfo") or []
if info:
detail["inventory"][inv_type] = info
except Exception:
pass
# Normalized application / firmware landscape view
software = detail["inventory"].get("deviceSoftware") or []
os_info = detail["inventory"].get("serverOperatingSystems") or []
mgmt = detail["inventory"].get("deviceManagement") or []
licenses = detail["inventory"].get("deviceLicense") or []
def _stype(s: dict) -> str:
return str(s.get("SoftwareType") or "").strip().upper()
fw_types = {"BIOS", "FRMW", "FIRMWARE", "IDRAC", "USC", "BMC", "CPLD", "DCSM"}
drv_types = {"DRVR", "DRIVER", "DRV"}
firmware = [s for s in software if _stype(s) in fw_types or "FW" in _stype(s)]
drivers = [s for s in software if _stype(s) in drv_types or "DRIVER" in _stype(s)]
fw_ids = {id(s) for s in firmware}
drv_ids = {id(s) for s in drivers}
applications = [s for s in software if id(s) not in fw_ids and id(s) not in drv_ids]
detail["landscape"] = {
"os": os_info,
"software": software,
"management": mgmt,
"licenses": licenses,
"firmware": firmware,
"drivers": drivers,
"applications": applications,
}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
detail["expansion"] = build_expansion_report(node.get("model"), detail.get("inventory") or {})
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
finally:
if sid:
try:
await client.delete(
f"{base}/api/SessionService/Sessions('{sid}')",
headers=headers,
)
except Exception:
pass
detail["_ts"] = time.time()
DETAIL_CACHE[device_id] = detail
return detail
Add iDRAC Console wall, power control, and proxy embeds.
2026-07-19 00:16:19 +02:00
# OME JobService powerState values (Dell set_power_state.py)
OME_POWER_ACTIONS = {
"on": ("2", "Power On"),
"cycle": ("5", "Power Cycle"),
"force_off": ("8", "Power Off Non-Graceful"),
"off": ("12", "Power Off Graceful"),
}
class PowerActionIn(BaseModel):
action: str # on | off | cycle | force_off
confirm: bool = False
def _fleet_device(device_id: int) -> dict | None:
return next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
@app.post("/api/devices/{device_id}/power")
async def device_power_control(device_id: int, payload: PowerActionIn):
"""Power control via OME JobService DeviceAction_Task (iDRAC through OME)."""
action = (payload.action or "").strip().lower()
if action not in OME_POWER_ACTIONS:
raise HTTPException(400, f"Invalid action. Use one of: {', '.join(OME_POWER_ACTIONS)}")
if action in ("off", "force_off", "cycle") and not payload.confirm:
raise HTTPException(400, "confirm=true required for off/cycle actions")
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
if node.get("source") == "inventory" or (isinstance(device_id, int) and device_id < 0):
raise HTTPException(400, "Inventory-only hosts cannot be powered via OME")
if not (node.get("is_server") or node.get("is_idrac") or node.get("type") == 1000):
raise HTTPException(400, "Power control is only supported for servers / iDRAC endpoints")
power_state, label = OME_POWER_ACTIONS[action]
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
body = {
"Id": 0,
"JobName": f"Cockpit {label}",
"JobDescription": f"OME Cockpit power control · {node.get('name')} · {node.get('service_tag')}",
"State": "Enabled",
"Schedule": "startnow",
"JobType": {"Name": "DeviceAction_Task"},
"Targets": [
{
"Id": int(device_id),
"Data": "",
"TargetType": {"Id": 1000, "Name": "DEVICE"},
}
],
"Params": [
{"Key": "override", "Value": "true"},
{"Key": "powerState", "Value": str(power_state)},
{"Key": "operationName", "Value": "POWER_CONTROL"},
{"Key": "deviceTypes", "Value": "1000"},
],
}
r = await client.post(
f"{base}/api/JobService/Jobs",
headers={**headers, "Content-Type": "application/json"},
json=body,
)
if r.status_code >= 400:
raise HTTPException(
r.status_code,
f"OME power job rejected: {(r.text or '')[:300]}",
)
job = r.json() if r.content else {}
job_id = job.get("Id")
log.info(
"OME power %s for device %s (%s) → job %s",
action,
device_id,
node.get("service_tag"),
job_id,
)
return {
"ok": True,
"action": action,
"label": label,
"device_id": device_id,
"name": node.get("name"),
"service_tag": node.get("service_tag"),
"job_id": job_id,
"note": "OME accepted the job. Power state updates on the next fleet poll.",
}
finally:
await ome_session_delete(client, base, headers, sid)
@app.get("/api/devices/{device_id}/idrac-console")
async def device_idrac_console(device_id: int, request: Request):
"""Return iDRAC web / HTML5 console URLs — prefer same-origin proxy embed."""
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
ip = node.get("idrac_ip") or node.get("ip")
if not ip:
raise HTTPException(400, "No iDRAC / management IP for this device")
base = f"https://{ip}"
# Same-origin reverse proxy so X-Frame-Options / CSP cannot block the iframe
embed = f"/api/idrac-proxy/{device_id}/restgui/start.html?console"
urls = {
"web": base + "/",
"html5": base + "/console",
"restgui": base + "/restgui/start.html?console",
"viewer": base + "/virtualconsole",
"embed": str(request.base_url).rstrip("/") + embed,
"embed_path": embed,
}
return {
"device_id": device_id,
"name": node.get("name"),
"service_tag": node.get("service_tag"),
"ip": ip,
"powered_on": node.get("powered_on"),
"connected": node.get("connected"),
"urls": urls,
"primary": embed,
"note": (
"Console is proxied through Cockpit (same-origin) so it can run in this panel. "
"Log in with your iDRAC credentials. Popout still opens the iDRAC directly."
),
}
_IDRAC_HOP_HEADERS = {
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailers",
"transfer-encoding",
"upgrade",
"content-length",
"content-encoding",
"content-security-policy",
"content-security-policy-report-only",
"x-frame-options",
"x-xss-protection",
"strict-transport-security",
}
def _idrac_proxy_prefix(device_id: int) -> str:
return f"/api/idrac-proxy/{device_id}"
def _idrac_resolve(device_id: int) -> tuple[dict, str]:
node = _fleet_device(device_id)
if not node:
raise HTTPException(404, "Device not found in current fleet snapshot")
# Prefer explicit iDRAC management IP; never treat bare OS inventory as iDRAC
ip = node.get("idrac_ip")
if not ip and node.get("is_idrac"):
ip = node.get("ip")
if not ip:
st = (node.get("service_tag") or "").strip()
if st:
sibling = next(
(
d
for d in (STATE.get("devices") or [])
if d.get("is_idrac")
and (d.get("service_tag") or "").strip().upper() == st.upper()
and (d.get("idrac_ip") or d.get("ip"))
),
None,
)
if sibling:
node = sibling
ip = sibling.get("idrac_ip") or sibling.get("ip")
if not ip:
raise HTTPException(
400,
"No iDRAC management IP for this device — select an iDRAC endpoint (OOB), not the OS host",
)
try:
ipaddress.ip_address(ip)
except ValueError as e:
raise HTTPException(400, "Invalid management IP") from e
return node, ip
def _idrac_rewrite_location(value: str, device_id: int, idrac_ip: str) -> str:
prefix = _idrac_proxy_prefix(device_id)
if not value:
return value
if value.startswith("/"):
return prefix + value
try:
u = urllib.parse.urlparse(value)
except Exception:
return value
host = (u.hostname or "").lower()
if host == idrac_ip.lower() or host.endswith(".dell-atc.lan") or host.endswith(".internal."):
path = u.path or "/"
return prefix + path + (("?" + u.query) if u.query else "") + (("#" + u.fragment) if u.fragment else "")
return value
def _idrac_rewrite_cookie(value: str, device_id: int) -> str:
"""Point cookies at the proxy path; drop Domain/Secure so HTTP cockpit can store them."""
prefix = _idrac_proxy_prefix(device_id)
parts = []
for part in value.split(";"):
p = part.strip()
pl = p.lower()
if pl.startswith("domain="):
continue
if pl == "secure":
continue
if pl.startswith("path="):
parts.append(f"Path={prefix}/")
continue
parts.append(p)
if not any(p.lower().startswith("path=") for p in parts):
parts.append(f"Path={prefix}/")
return "; ".join(parts)
def _idrac_inject_bootstrap(html: bytes, device_id: int, idrac_ip: str) -> bytes:
"""Rewrite absolute paths + patch fetch/XHR/WebSocket so the SPA stays on the proxy."""
prefix = _idrac_proxy_prefix(device_id)
try:
text = html.decode("utf-8")
except UnicodeDecodeError:
text = html.decode("latin-1")
# Absolute root paths in markup
for attr in ("href", "src", "action"):
text = re.sub(
rf'({attr}\s*=\s*["\'])/(?!/)',
rf"\1{prefix}/",
text,
flags=re.I,
)
text = text.replace(f"https://{idrac_ip}/", f"{prefix}/")
text = text.replace(f"http://{idrac_ip}/", f"{prefix}/")
boot = f"""<script>(function(){{
var PREFIX={json.dumps(prefix)};
var IDRAC_IP={json.dumps(idrac_ip)};
function abs(u){{
if(!u || typeof u!=="string") return u;
try{{
if(u.indexOf("blob:")===0 || u.indexOf("data:")===0 || u.indexOf("mailto:")===0) return u;
var x=new URL(u, location.href);
var h=(x.hostname||"").toLowerCase();
if(h===IDRAC_IP.toLowerCase() || (h && h.indexOf("dell-atc.lan")!==-1)){{
return location.origin + PREFIX + (x.pathname||"/") + x.search + x.hash;
}}
if(x.origin===location.origin && x.pathname.indexOf(PREFIX)!==0){{
return location.origin + PREFIX + (x.pathname||"/") + x.search + x.hash;
}}
return u;
}}catch(e){{ return u; }}
}}
function wsAbs(u){{
var a=abs(u);
if(typeof a!=="string") return u;
if(location.protocol==="http:" && a.indexOf("wss://"+location.host)===0)
return "ws://"+location.host+a.slice(("wss://"+location.host).length);
if(location.protocol==="https:" && a.indexOf("ws://"+location.host)===0)
return "wss://"+location.host+a.slice(("ws://"+location.host).length);
return a;
}}
var _f=window.fetch;
window.fetch=function(input, init){{
if(typeof input==="string") input=abs(input);
else if(input && typeof Request!=="undefined" && input instanceof Request)
input=new Request(abs(input.url), input);
return _f.call(this, input, init);
}};
var _o=XMLHttpRequest.prototype.open;
XMLHttpRequest.prototype.open=function(method, url){{
var args=Array.prototype.slice.call(arguments, 2);
return _o.apply(this, [method, abs(url)].concat(args));
}};
var _WS=window.WebSocket;
window.WebSocket=function(url, protocols){{
var u=wsAbs(url);
return protocols===undefined ? new _WS(u) : new _WS(u, protocols);
}};
window.WebSocket.prototype=_WS.prototype;
window.WebSocket.CONNECTING=_WS.CONNECTING;
window.WebSocket.OPEN=_WS.OPEN;
window.WebSocket.CLOSING=_WS.CLOSING;
window.WebSocket.CLOSED=_WS.CLOSED;
}})();</script>"""
if re.search(r"<head[^>]*>", text, flags=re.I):
text = re.sub(r"(<head[^>]*>)", r"\1" + boot, text, count=1, flags=re.I)
else:
text = boot + text
return text.encode("utf-8")
def _idrac_rewrite_css(css: bytes, device_id: int, idrac_ip: str) -> bytes:
prefix = _idrac_proxy_prefix(device_id)
try:
text = css.decode("utf-8")
except UnicodeDecodeError:
text = css.decode("latin-1")
text = re.sub(r"url\((['\"]?)/", rf"url(\1{prefix}/", text)
text = text.replace(f"https://{idrac_ip}/", f"{prefix}/")
return text.encode("utf-8")
async def _idrac_proxy_http(device_id: int, path: str, request: Request):
import gzip
_, idrac_ip = _idrac_resolve(device_id)
prefix = _idrac_proxy_prefix(device_id)
rel = path or ""
qs = request.url.query
target = f"https://{idrac_ip}/{rel}" + (f"?{qs}" if qs else "")
# Forward headers; force gzip so iDRAC serves .js/.css (precompressed only)
fwd: dict[str, str] = {}
for k, v in request.headers.items():
lk = k.lower()
if lk in ("host", "content-length", "connection", "accept-encoding"):
continue
if lk == "referer" and v:
# Map our proxy referer back to iDRAC origin when possible
v = v.replace(str(request.base_url).rstrip("/") + prefix, f"https://{idrac_ip}")
v = v.replace(prefix, f"https://{idrac_ip}")
fwd[k] = v
fwd["Host"] = idrac_ip
fwd["Accept-Encoding"] = "gzip, deflate"
# Avoid long-lived upstream hangs
body = await request.body()
try:
async with httpx.AsyncClient(verify=False, timeout=120.0, follow_redirects=False) as client:
upstream = await client.request(
request.method,
target,
headers=fwd,
content=body if body else None,
)
except httpx.RequestError as e:
accept = (request.headers.get("accept") or "").lower()
dest = (request.headers.get("sec-fetch-dest") or "").lower()
if "text/html" in accept or dest == "iframe" or request.method == "GET":
err = (
str(e)
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
)
html = f"""<!DOCTYPE html><html><head><meta charset="utf-8"><title>iDRAC unreachable</title>
<style>
body{{margin:0;font:14px/1.45 system-ui,sans-serif;background:#0a0e14;color:#e8f0f8;
display:grid;place-items:center;min-height:100vh;padding:1.5rem}}
.card{{max-width:28rem;border:1px solid rgba(232,122,42,.45);border-radius:12px;padding:1.25rem 1.4rem;
background:rgba(20,12,8,.9)}}
h1{{margin:0 0 .4rem;font-size:1.05rem;color:#ffb06a}}
p{{margin:.35rem 0;color:#9eb4c8}}
code{{color:#fff;font-size:.85em}}
</style></head><body><div class="card">
<h1>iDRAC not reachable via proxy</h1>
<p>Target <code>{idrac_ip}</code> did not accept a connection from Cockpit.</p>
<p>Use an <strong>iDRAC / OOB</strong> endpoint (VLAN 40/41…), not the OS host IP.</p>
<p style="font-size:.78rem;opacity:.75">{err}</p>
</div></body></html>"""
return Response(content=html, status_code=502, media_type="text/html")
raise HTTPException(502, f"iDRAC proxy unreachable: {e}") from e
raw = upstream.content
enc = (upstream.headers.get("content-encoding") or "").lower()
if "gzip" in enc and raw:
try:
raw = gzip.decompress(raw)
except Exception:
pass
elif "deflate" in enc and raw:
try:
import zlib
raw = zlib.decompress(raw)
except Exception:
try:
import zlib
raw = zlib.decompress(raw, -zlib.MAX_WBITS)
except Exception:
pass
ctype = (upstream.headers.get("content-type") or "").lower()
if "text/html" in ctype and raw:
raw = _idrac_inject_bootstrap(raw, device_id, idrac_ip)
elif "text/css" in ctype and raw:
raw = _idrac_rewrite_css(raw, device_id, idrac_ip)
elif ("javascript" in ctype or "ecmascript" in ctype) and raw:
# Light touch: rewrite absolute iDRAC URLs embedded as strings
try:
t = raw.decode("utf-8")
except UnicodeDecodeError:
t = raw.decode("latin-1")
t = t.replace(f"https://{idrac_ip}", prefix)
t = t.replace(f"wss://{idrac_ip}", f"ws://{request.url.hostname}:{request.url.port or (443 if request.url.scheme == 'https' else 80)}{prefix}")
raw = t.encode("utf-8")
out_headers: list[tuple[str, str]] = []
for k, v in upstream.headers.multi_items():
lk = k.lower()
if lk in _IDRAC_HOP_HEADERS:
continue
if lk == "location":
out_headers.append((k, _idrac_rewrite_location(v, device_id, idrac_ip)))
continue
if lk == "set-cookie":
out_headers.append((k, _idrac_rewrite_cookie(v, device_id)))
continue
out_headers.append((k, v))
# Allow embedding in Cockpit UI
out_headers.append(("Content-Security-Policy", "frame-ancestors *"))
out_headers.append(("Cache-Control", "no-store"))
resp = Response(content=raw, status_code=upstream.status_code)
for k, v in out_headers:
resp.headers.append(k, v)
if not any(k.lower() == "content-type" for k, _ in out_headers):
ct = upstream.headers.get("content-type")
if ct:
resp.headers["content-type"] = ct
return resp
@app.api_route(
"/api/idrac-proxy/{device_id}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],
)
@app.api_route(
"/api/idrac-proxy/{device_id}/{path:path}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],
)
async def idrac_http_proxy(device_id: int, request: Request, path: str = ""):
return await _idrac_proxy_http(device_id, path, request)
@app.websocket("/api/idrac-proxy/{device_id}/{path:path}")
async def idrac_ws_proxy(ws: WebSocket, device_id: int, path: str):
"""Bridge browser WebSocket ↔ iDRAC (virtual console / live sessions)."""
import ssl
import websockets
_, idrac_ip = _idrac_resolve(device_id)
await ws.accept()
qs = ws.scope.get("query_string", b"").decode()
target = f"wss://{idrac_ip}/{path}" + (f"?{qs}" if qs else "")
sub = ws.scope.get("subprotocols") or []
ssl_ctx = ssl._create_unverified_context()
try:
async with websockets.connect(
target,
ssl=ssl_ctx,
subprotocols=list(sub) if sub else None,
open_timeout=30,
max_size=8 * 1024 * 1024,
) as upstream:
async def client_to_upstream():
while True:
msg = await ws.receive()
if msg.get("type") == "websocket.disconnect":
break
if "text" in msg and msg["text"] is not None:
await upstream.send(msg["text"])
elif "bytes" in msg and msg["bytes"] is not None:
await upstream.send(msg["bytes"])
async def upstream_to_client():
async for message in upstream:
if isinstance(message, (bytes, bytearray)):
await ws.send_bytes(message)
else:
await ws.send_text(message)
_done, pending = await asyncio.wait(
[
asyncio.create_task(client_to_upstream()),
asyncio.create_task(upstream_to_client()),
],
return_when=asyncio.FIRST_COMPLETED,
)
for t in pending:
t.cancel()
except HTTPException:
try:
await ws.close()
except Exception:
pass
raise
except Exception as e:
log.warning("iDRAC WS proxy error device=%s: %s", device_id, e)
try:
await ws.close(code=1011)
except Exception:
pass
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
@app.get("/api/devices/{device_id}/expansion")
async def device_expansion(device_id: int, force: bool = False):
if force and device_id in DETAIL_CACHE:
DETAIL_CACHE.pop(device_id, None)
detail = await device_detail(device_id)
return {
"device": detail.get("device"),
"expansion": detail.get("expansion")
or build_expansion_report((detail.get("device") or {}).get("model"), detail.get("inventory") or {}),
}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.websocket("/ws/fleet")
async def ws_fleet(ws: WebSocket):
await ws.accept()
CLIENTS.add(ws)
try:
await ws.send_json({"type": "snapshot", "data": {**STATE}})
while True:
await ws.receive_text()
except WebSocketDisconnect:
pass
finally:
CLIENTS.discard(ws)
class ChatIn(BaseModel):
message: str
history: list[dict] = Field(default_factory=list)
focus_device_id: int | None = None
model: str | None = None
class TicketIn(BaseModel):
title: str
body: str
created_by: str
assignee: str | None = None
priority: str = "normal"
class TicketMsgIn(BaseModel):
author: str
body: str
class TicketPatch(BaseModel):
status: str | None = None
assignee: str | None = None
priority: str | None = None
accepted_by: str | None = None
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def _device_by_id(device_id: int | None) -> dict | None:
if device_id is None:
return None
return next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
def _device_by_service_tag(st: str) -> dict | None:
needle = (st or "").strip().upper()
if not needle:
return None
for d in STATE.get("devices") or []:
if (d.get("service_tag") or "").strip().upper() == needle:
return d
return None
def _find_devices_fuzzy(query: str, limit: int = 8) -> list[dict]:
q = (query or "").strip().lower()
if not q:
return []
hits = []
for d in STATE.get("devices") or []:
blob = " ".join(
str(d.get(k) or "").lower()
for k in ("name", "service_tag", "ip", "idrac_ip", "rdp_host", "os_hostname", "model")
)
if q in blob:
hits.append(d)
hits.sort(key=lambda d: (0 if (d.get("service_tag") or "").lower() == q else 1, d.get("name") or ""))
return hits[:limit]
def _format_device_card(d: dict) -> str:
mem = d.get("memory_gb")
mem_s = f"{mem}GB/{d.get('dimm_count') or '?'}DIMM" if mem is not None else "unknown"
return (
"ST={st} | name={name} | model={model} | idrac={idrac} | "
"rdp={rdp} | os_host={osh} | ram={ram} | connected={conn} | status={status} | watts={watts}"
).format(
st=(d.get("service_tag") or "NONE"),
name=(d.get("name") or "")[:48],
model=(d.get("model") or "")[:36],
idrac=d.get("idrac_ip") or d.get("ip") or "—",
rdp=(",".join(d.get("rdp_ips") or []) or d.get("rdp_host") or "unresolved"),
osh=d.get("os_hostname") or "—",
ram=mem_s,
conn="yes" if d.get("connected") else "no",
status=d.get("status"),
watts=d.get("watts"),
)
async def ome_fetch_inventory_types(device_id: int, types: list[str]) -> dict[str, list]:
"""Fast parallel inventory fetch for chat tools (not full device landscape)."""
inventory: dict[str, list] = {}
async with httpx.AsyncClient(verify=False, timeout=45.0) as client:
base, headers, sid = await ome_session(client)
try:
async def one(inv_type: str):
try:
ir = await client.get(
f"{base}/api/DeviceService/Devices({device_id})/InventoryDetails('{inv_type}')",
headers=headers,
)
if ir.status_code == 200:
info = ir.json().get("InventoryInfo") or []
if info:
return inv_type, info
except Exception as e:
log.debug("chat inv %s failed: %s", inv_type, e)
return inv_type, []
results = await asyncio.gather(*[one(t) for t in types])
for inv_type, info in results:
if info:
inventory[inv_type] = info
finally:
await ome_session_delete(client, base, headers, sid)
return inventory
def _memory_from_dimms(mem_raw: list) -> dict:
"""Compact CURRENT memory summary from OME serverMemoryDevices."""
dimms = 0
total_gb = 0.0
for dimm in mem_raw or []:
gb = _mem_size_gb(dimm.get("Size"))
if gb is None:
continue
dimms += 1
total_gb += gb
return {
"memory_gb": round(total_gb, 1) if dimms else None,
"dimm_count": dimms or None,
}
async def fetch_fleet_memory(force: bool = False) -> dict:
"""Warm/cache CURRENT installed RAM for all managed servers (OME inventory)."""
cached = None if force else _cache_get("fleet_memory")
if cached is not None:
return cached
devices = [
d
for d in (STATE.get("devices") or [])
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
if (d.get("is_server") or d.get("is_idrac") or (d.get("type") == 1000))
and d.get("source") != "inventory"
and isinstance(d.get("id"), int)
and d.get("id") > 0
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
]
# Prefer unique device ids
seen: set[int] = set()
targets: list[dict] = []
for d in devices:
did = d.get("id")
if did is None or did in seen:
continue
seen.add(int(did))
targets.append(d)
rows: list[dict] = []
errors = 0
sem = asyncio.Semaphore(10)
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
async def one(node: dict):
nonlocal errors
did = int(node["id"])
async with sem:
try:
ir = await client.get(
f"{base}/api/DeviceService/Devices({did})/InventoryDetails('serverMemoryDevices')",
headers=headers,
)
mem_raw = []
if ir.status_code == 200:
mem_raw = ir.json().get("InventoryInfo") or []
stats = _memory_from_dimms(mem_raw)
return {
"id": did,
"service_tag": node.get("service_tag"),
"name": node.get("name"),
"model": node.get("model"),
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
"connected": bool(node.get("connected")),
"memory_gb": stats.get("memory_gb"),
"dimm_count": stats.get("dimm_count"),
}
except Exception as e:
errors += 1
log.debug("fleet memory %s: %s", did, e)
return {
"id": did,
"service_tag": node.get("service_tag"),
"name": node.get("name"),
"model": node.get("model"),
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
"connected": bool(node.get("connected")),
"memory_gb": None,
"dimm_count": None,
"error": str(e)[:80],
}
rows = list(await asyncio.gather(*[one(n) for n in targets]))
finally:
await ome_session_delete(client, base, headers, sid)
rows.sort(key=lambda r: ((r.get("name") or "").lower(), r.get("service_tag") or ""))
known = [r for r in rows if r.get("memory_gb") is not None]
payload = {
"source": "OME DeviceService InventoryDetails(serverMemoryDevices)",
"count": len(rows),
"with_memory": len(known),
"errors": errors,
"total_memory_gb": round(sum(r.get("memory_gb") or 0 for r in known), 1),
"servers": rows,
"updated_at": time.time(),
}
_cache_set("fleet_memory", payload)
# also stamp onto fleet nodes for UI/context
by_id = {r["id"]: r for r in rows if r.get("id") is not None}
for d in STATE.get("devices") or []:
hit = by_id.get(d.get("id"))
if hit:
d["memory_gb"] = hit.get("memory_gb")
d["dimm_count"] = hit.get("dimm_count")
log.info(
"Fleet memory cache: %s/%s servers with RAM (%.0f GB total)",
len(known),
len(rows),
payload["total_memory_gb"],
)
return payload
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
def build_fleet_context(focus_device_id: int | None = None, max_chars: int | None = None) -> str:
summary = STATE.get("summary") or {}
gpu = STATE.get("gpu") or {}
gsum = gpu.get("summary") or {}
devices = STATE.get("devices") or []
alerts = STATE.get("alerts") or []
events = STATE.get("events") or []
subnets = STATE.get("subnets") or []
ctx = STATE.get("context") or {}
limit = max_chars if max_chars is not None else settings.chat_system_chars
connected = [d for d in devices if d.get("connected")]
critical = [a for a in alerts if a.get("severity") == "Critical"][:8]
warnings = [a for a in alerts if a.get("severity") == "Warning"][:5]
hottest = (ctx.get("hottest") or [])[:5]
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
compliance = _cache_get("compliance") or REPORT_CACHE.get("compliance") or {}
comp_sum = (compliance or {}).get("summary") or {}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
st_lines = [
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"SERVICE TAG INDEX (canonical). idrac=BMC management IP. rdp=OS/Windows RDP candidates (DNS). Never call idrac the RDP IP.",
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
]
for d in sorted(devices, key=lambda x: (x.get("name") or "").lower()):
st = (d.get("service_tag") or "").strip() or "NONE"
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
rdp = ",".join((d.get("rdp_ips") or [])[:2]) or (d.get("rdp_host") or "—")
mem = d.get("memory_gb")
ram = f"{mem}GB" if mem is not None else "?"
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
st_lines.append(
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"- ST={st} | {name} | idrac={idrac} | rdp={rdp} | model={model} | ram={ram} | connected={conn}".format(
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
st=st,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
name=(d.get("name") or "")[:36],
idrac=d.get("idrac_ip") or d.get("ip") or "—",
rdp=rdp[:40],
model=(d.get("model") or "")[:26],
ram=ram,
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
conn="yes" if d.get("connected") else "no",
)
)
st_block = "\n".join(st_lines)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines = [
"You are OpenManage Cockpit Copilot for Dell ATC. Be concise and operational.",
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"ATC Datacenter Admins (escalate here when facts are missing):",
" - Jody van Dongen <jody.van.dongen@dell.com>",
" - Laurens Rammers <laurens.rammers@dell.com>",
"ACCURACY RULES (mandatory — never break these):",
"1) Use ONLY facts from this snapshot and any TOOL FACTS block. Never invent Service Tags, IPs, DIMM counts, firmware versions, RDP targets, port maps, VLAN members, or rack placements.",
"2) If a requested fact is not present in the snapshot/TOOL FACTS, or tools failed/returned empty: say exactly what is unknown, then tell the user to overleggen met Jody van Dongen and Laurens Rammers (emails above). Do not guess.",
"3) ALWAYS cite Service Tag (ST=...), model, idrac IP, and rdp IP (or 'rdp unresolved') when discussing a system.",
"4) Management/iDRAC IP is NOT the Windows RDP IP. Never recommend RDP to an iDRAC address.",
"5) Hardware CURRENT (OME inventory) is NOT Dell CATALOG MAX. Never state catalog maxima as installed capacity.",
"6) Prefer TOOL FACTS over the summary index when both are present. Quote TOOL FACTS numbers verbatim.",
"7) Never pretend MCP/tool output exists when no TOOL FACTS block was provided.",
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"",
"FLEET: total={t} connected={c} offline={o} watts={w} alerts_total={a}".format(
t=summary.get("total"),
c=summary.get("connected"),
o=summary.get("offline"),
w=summary.get("total_watts"),
a=summary.get("alerts_total"),
),
]
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
if comp_sum:
lines.append(
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"FIRMWARE COMPLIANCE summary (Dell baseline): outdated_devices={od} critical_components={cc} baseline={bn}".format(
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
od=comp_sum.get("outdated_devices"),
cc=comp_sum.get("critical_components"),
bn=(comp_sum.get("baseline_name") or "")[:40],
)
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
mem_cache = _cache_get("fleet_memory") or REPORT_CACHE.get("fleet_memory") or {}
mem_servers = mem_cache.get("servers") or []
if mem_servers:
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append(
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"MEMORY INDEX (CURRENT OME inventory, GB installed): known={k}/{n} total_gb={tg}".format(
k=mem_cache.get("with_memory"),
n=mem_cache.get("count"),
tg=mem_cache.get("total_memory_gb"),
)
)
ranked = sorted(
[r for r in mem_servers if r.get("memory_gb") is not None],
key=lambda r: (0 if r.get("connected") else 1, (r.get("name") or "").lower()),
)
for r in ranked[:55]:
lines.append(
"- ST={st} {name} model={model} ram={gb}GB dimms={dc} idrac={ip}".format(
st=r.get("service_tag") or "NONE",
name=(r.get("name") or "")[:28],
model=(r.get("model") or "")[:22],
gb=r.get("memory_gb"),
dc=r.get("dimm_count") or "?",
ip=r.get("idrac_ip") or "—",
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if len(ranked) > 55:
lines.append(f"- … +{len(ranked) - 55} more in TOOL FACTS list_fleet_memory")
else:
lines.append(
"MEMORY INDEX: not warmed yet — call list_fleet_memory tool / wait for background warm."
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
lines.append("CONNECTED (sample):")
for d in connected[:14]:
lines.append("- " + _format_device_card(d))
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append("SUBNETS:")
for s in subnets[:8]:
lines.append(
"- {cidr}: {conn}/{cnt} W={watts}".format(
cidr=s.get("cidr"),
conn=s.get("connected"),
cnt=s.get("count"),
watts=s.get("watts"),
)
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def _alert_line(a: dict, msg_len: int) -> str:
node = by_id.get(a.get("device_id")) or {}
st = node.get("service_tag") or a.get("service_tag") or "NONE"
return "- ST={st} {dev} idrac={ip}: {msg}".format(
st=st,
dev=(a.get("device") or node.get("name") or "")[:28],
ip=a.get("ip") or node.get("idrac_ip") or node.get("ip") or "—",
msg=(a.get("message") or "")[:msg_len],
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append("CRITICAL:")
if not critical:
lines.append("(none)")
for a in critical:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
lines.append(_alert_line(a, 100))
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append("WARNINGS:")
for a in warnings:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
lines.append(_alert_line(a, 80))
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append("DELTAS:")
if not events:
lines.append("(none)")
for e in events[:8]:
lines.append("- {title}: {text}".format(title=e.get("title"), text=(e.get("text") or "")[:100]))
lines.append("HOT POWER:")
for h in hottest:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
node = by_id.get(h.get("id")) or {}
lines.append(
"- ST={st} {name}: {watts}W".format(
st=(node.get("service_tag") or h.get("service_tag") or "NONE"),
name=(h.get("name") or node.get("name") or "")[:28],
watts=h.get("watts"),
)
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
lines.append(
"GPU atc-gpu-prod: avg_util={u}% power={p}W mem={mu}/{mt}MB".format(
u=gsum.get("avg_util"),
p=gsum.get("total_power_w"),
mu=gsum.get("total_mem_used_mb"),
mt=gsum.get("total_mem_mb"),
)
)
for g in (gpu.get("gpus") or [])[:8]:
lines.append(
"- GPU{i}: {util}% {temp}C {pw}W".format(
i=g.get("index"),
util=g.get("util_gpu"),
temp=g.get("temp_c"),
pw=g.get("power_w"),
)
)
if focus_device_id is not None:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
node = _device_by_id(focus_device_id)
lines.append("FOCUS DEVICE:")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
if node:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
lines.append(_format_device_card(node))
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
related = [a for a in alerts if a.get("device_id") == focus_device_id][:5]
for a in related:
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
lines.append(
"alert {sev}: {msg}".format(sev=a.get("severity"), msg=(a.get("message") or "")[:100])
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
else:
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
lines.append("device_id=%s missing from fleet snapshot" % focus_device_id)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
lines.append(
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"Reply with concrete Service Tags, idrac/rdp IPs, models, and next actions. "
"Quote TOOL FACTS verbatim for hardware numbers. "
"If anything is missing from facts: do not invent — escalate to Jody van Dongen and Laurens Rammers."
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
)
body = "\n".join(lines)
budget = max(800, limit - len(st_block) - 40)
if len(body) > budget:
body = body[: budget - 20] + "\n…[truncated]"
out = st_block + "\n\n" + body
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
if len(out) > limit:
out = out[: limit - 20] + "\n…[truncated]"
return out
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def plan_chat_tools(message: str, focus_device_id: int | None = None) -> list[tuple[str, dict]]:
"""Server-side tool planner — Llama3-GPTQ does not emit native tool_calls reliably."""
import re
msg = message or ""
low = msg.lower()
planned: list[tuple[str, dict]] = []
seen: set[str] = set()
def add(name: str, args: dict):
key = name + json.dumps(args, sort_keys=True, default=str)
if key in seen:
return
seen.add(key)
planned.append((name, args))
for st in re.findall(r"\b([A-Z0-9]{7})\b", msg.upper()):
if st in {"OMEPROD", "WINDOWS", "POWERED", "UNKNOWN", "SERVICE", "CONNECTED"}:
continue
if _device_by_service_tag(st):
add("lookup_device", {"query": st})
for m in re.finditer(r"\b(?:for|on|about|host|server|node)\s+([A-Za-z0-9._-]{4,40})", msg, re.I):
add("lookup_device", {"query": m.group(1)})
wants_hw = bool(
re.search(
r"\b(memory|ram|dimm|disk|drive|ssd|hdd|pcie|expansion|cpu|processor|core|slot|bay|installed|capacity)\b",
low,
)
)
wants_rdp = bool(re.search(r"\b(rdp|remote\s*desktop|windows|hyper-?v|os\s*ip|hostname)\b", low))
wants_fw = bool(re.search(r"\b(firmware|bios\s*version|outdated|compliance|firmware\s*catalog|update\s*catalog|baseline)\b", low))
wants_war = bool(re.search(r"\b(warrant|prosupport|support\s*end|days?\s*left)\b", low))
wants_alerts = bool(re.search(r"\b(alert|critical|warning|fault|health)\b", low))
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
wants_counts = bool(
re.search(
r"\b(hoeveel|how\s+many|count|aantal|total|fleet\s+size|connected|offline)\b",
low,
)
)
# Fleet summary questions still benefit from alerts + memory tools when relevant
if wants_counts and not planned:
add("list_alerts", {})
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
target_id = focus_device_id
target_st = None
if planned:
q0 = planned[0][1].get("query")
node = _device_by_service_tag(str(q0)) or ((_find_devices_fuzzy(str(q0), 1) or [None])[0])
if node:
target_id = node.get("id")
target_st = node.get("service_tag")
if wants_rdp and not any(n == "lookup_device" for n, _ in planned):
if focus_device_id:
node = _device_by_id(focus_device_id)
if node:
add("lookup_device", {"query": node.get("service_tag") or node.get("name") or str(focus_device_id)})
else:
n_lookups = 0
for d in STATE.get("devices") or []:
if d.get("is_windows") or d.get("rdp_host"):
add("lookup_device", {"query": d.get("service_tag") or d.get("name")})
n_lookups += 1
if n_lookups >= 4:
break
wants_fleet_mem = bool(
re.search(
r"\b(each|every|all|per\s+server|fleet|overview|elk|alle|iedere|elke)\b",
low,
)
) or bool(re.search(r"hoeveel\s+(memory|ram|geheugen)", low))
if wants_hw:
args: dict = {}
if target_id:
args["device_id"] = int(target_id)
if target_st:
args["service_tag"] = target_st
if args:
add("get_hardware", args)
if wants_fleet_mem:
add("list_fleet_memory", {})
elif wants_fleet_mem or not focus_device_id:
add("list_fleet_memory", {})
elif focus_device_id:
add("get_hardware", {"device_id": int(focus_device_id)})
if wants_fw:
args = {"outdated_only": True}
if target_id:
args["device_id"] = int(target_id)
if target_st:
args["service_tag"] = target_st
add("get_compliance", args)
if wants_war:
args = {}
if target_id:
args["device_id"] = int(target_id)
if target_st:
args["service_tag"] = target_st
if args:
add("get_warranty", args)
elif focus_device_id:
add("get_warranty", {"device_id": int(focus_device_id)})
if wants_alerts:
args = {}
if target_id:
args["device_id"] = int(target_id)
if target_st:
args["service_tag"] = target_st
add("list_alerts", args)
if focus_device_id and not planned:
node = _device_by_id(focus_device_id)
if node:
add("lookup_device", {"query": node.get("service_tag") or str(focus_device_id)})
return planned[:6]
async def run_chat_tool(name: str, args: dict) -> dict:
args = args or {}
try:
if name == "lookup_device":
q = str(args.get("query") or "").strip()
exact = _device_by_service_tag(q)
hits = [exact] if exact else _find_devices_fuzzy(q, 8)
return {
"tool": name,
"query": q,
"count": len(hits),
"devices": [
{
"id": d.get("id"),
"service_tag": d.get("service_tag"),
"name": d.get("name"),
"model": d.get("model"),
"idrac_ip": d.get("idrac_ip") or d.get("ip"),
"rdp_host": d.get("rdp_host"),
"rdp_ips": d.get("rdp_ips") or [],
"os_hostname": d.get("os_hostname"),
"is_windows": d.get("is_windows"),
"connected": d.get("connected"),
"status": d.get("status"),
"watts": d.get("watts"),
}
for d in hits
if d
],
}
if name == "get_hardware":
node = None
if args.get("device_id") is not None:
node = _device_by_id(int(args["device_id"]))
if not node and args.get("service_tag"):
node = _device_by_service_tag(str(args["service_tag"]))
if not node:
return {"tool": name, "error": "device not found in fleet snapshot"}
did = int(node["id"])
inv = await ome_fetch_inventory_types(
did,
[
"serverMemoryDevices",
"serverProcessors",
"serverArrayDisks",
"serverRaidControllers",
"serverDeviceCards",
"serverOperatingSystems",
],
)
report = build_expansion_report(node.get("model"), inv)
cur = report.get("current") or {}
exp = report.get("expansion") or {}
return {
"tool": name,
"device": {
"id": did,
"service_tag": node.get("service_tag"),
"name": node.get("name"),
"model": node.get("model"),
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
"rdp_ips": node.get("rdp_ips") or [],
"os_hostname": node.get("os_hostname"),
},
"current_ome_inventory": {
"memory_installed_gb": (cur.get("memory") or {}).get("installed_gb"),
"dimm_count": (cur.get("memory") or {}).get("dimm_count"),
"dimm_modules": [
{
"slot": m.get("slot"),
"size_gb": m.get("size_gb"),
"rated_mts": m.get("rated_mts"),
"operating_mts": m.get("operating_mts"),
}
for m in ((cur.get("memory") or {}).get("modules") or [])[:24]
],
"cpu_sockets_populated": (cur.get("cpu") or {}).get("sockets_populated"),
"cpu_cores_total": (cur.get("cpu") or {}).get("cores_total"),
"processors": (cur.get("cpu") or {}).get("processors") or [],
"disk_count": (cur.get("disks") or {}).get("count"),
"disk_capacity_gb": (cur.get("disks") or {}).get("capacity_gb"),
"disks": [
{
"bay": d.get("bay"),
"model": d.get("model"),
"size_gb": d.get("size_gb"),
"bus": d.get("bus"),
"media": d.get("media"),
}
for d in ((cur.get("disks") or {}).get("items") or [])[:20]
],
"os": cur.get("os") or [],
},
"catalog_max_not_measured": {
"source": exp.get("source"),
"catalog_known": exp.get("catalog_known"),
"catalog_model": exp.get("catalog_model"),
"memory_max_tb": (exp.get("memory") or {}).get("max_capacity_tb"),
"dimm_slots_max": (exp.get("memory") or {}).get("dimm_slots_max"),
"cpu_sockets_max": (exp.get("cpu") or {}).get("sockets_max"),
"drive_bays_max": (exp.get("disks") or {}).get("bays_max"),
"pcie_slots_max": (exp.get("pcie") or {}).get("slots_max"),
},
"findings": report.get("findings") or [],
}
if name == "list_fleet_memory":
mem = await fetch_fleet_memory(force=bool(args.get("force")))
servers = mem.get("servers") or []
# Compact rows for the LLM — include unknown so the model does not invent
slim = [
{
"service_tag": s.get("service_tag"),
"name": s.get("name"),
"model": s.get("model"),
"idrac_ip": s.get("idrac_ip"),
"connected": s.get("connected"),
"memory_gb": s.get("memory_gb"),
"dimm_count": s.get("dimm_count"),
}
for s in servers
]
return {
"tool": name,
"source": mem.get("source"),
"count": mem.get("count"),
"with_memory": mem.get("with_memory"),
"total_memory_gb": mem.get("total_memory_gb"),
"note": "memory_gb is CURRENT installed RAM from OME inventory (not catalog max).",
"servers": slim,
}
if name == "get_compliance":
compliance = await fetch_compliance(force=False)
comps = compliance.get("components") or []
node = None
if args.get("device_id") is not None:
node = _device_by_id(int(args["device_id"]))
if not node and args.get("service_tag"):
node = _device_by_service_tag(str(args["service_tag"]))
outdated_only = bool(args.get("outdated_only", True))
if node:
did = node.get("id")
st = (node.get("service_tag") or "").upper()
rows = [
c
for c in comps
if c.get("device_id") == did
or (st and (c.get("service_tag") or "").upper() == st)
]
else:
rows = list(comps)
if outdated_only:
rows = [
c
for c in rows
if str(c.get("compliance_status") or "").lower()
in ("critical", "warning", "downgrade", "non-compliant", "outdated")
or (
str(c.get("update_action") or "").lower()
not in ("", "equal", "compliant", "ok", "none")
and str(c.get("current_version") or "") != str(c.get("catalog_version") or "")
)
]
slim = [
{
"service_tag": c.get("service_tag"),
"device_name": c.get("device_name"),
"component": c.get("component"),
"current_version": c.get("current_version"),
"catalog_version": c.get("catalog_version"),
"compliance_status": c.get("compliance_status"),
"update_action": c.get("update_action"),
}
for c in rows[:40]
]
return {"tool": name, "summary": compliance.get("summary"), "count": len(slim), "components": slim}
if name == "get_warranty":
warranties = await fetch_warranties(force=False)
items = warranties.get("items") or warranties.get("warranties") or []
node = None
if args.get("device_id") is not None:
node = _device_by_id(int(args["device_id"]))
if not node and args.get("service_tag"):
node = _device_by_service_tag(str(args["service_tag"]))
if not node:
return {"tool": name, "error": "device not found"}
st = (node.get("service_tag") or "").upper()
rows = [
w
for w in items
if w.get("device_id") == node.get("id")
or (st and (w.get("service_tag") or "").upper() == st)
]
return {
"tool": name,
"service_tag": node.get("service_tag"),
"name": node.get("name"),
"warranties": rows[:8],
}
if name == "list_alerts":
alerts = STATE.get("alerts") or []
by_id = {d.get("id"): d for d in (STATE.get("devices") or [])}
node = None
if args.get("device_id") is not None:
node = _device_by_id(int(args["device_id"]))
if not node and args.get("service_tag"):
node = _device_by_service_tag(str(args["service_tag"]))
sev = (args.get("severity") or "").strip().lower()
out = []
for a in alerts:
if node and a.get("device_id") != node.get("id"):
continue
if sev and str(a.get("severity") or "").lower() != sev:
continue
d = by_id.get(a.get("device_id")) or {}
out.append(
{
"severity": a.get("severity"),
"service_tag": d.get("service_tag"),
"device": a.get("device") or d.get("name"),
"idrac_ip": a.get("ip") or d.get("idrac_ip") or d.get("ip"),
"message": (a.get("message") or "")[:180],
}
)
if len(out) >= 25:
break
return {"tool": name, "count": len(out), "alerts": out}
return {"tool": name, "error": f"unknown tool {name}"}
except Exception as e:
log.exception("chat tool %s failed", name)
return {"tool": name, "error": str(e)}
def format_tool_facts(results: list[dict]) -> str:
if not results:
return ""
parts = ["TOOL FACTS (live lookups — authoritative; do not contradict):"]
for r in results:
if r.get("tool") == "list_fleet_memory" and r.get("servers"):
lines = [
"list_fleet_memory CURRENT RAM:",
f"known={r.get('with_memory')}/{r.get('count')} total_gb={r.get('total_memory_gb')}",
"ST\tname\tmodel\tmemory_gb\tdimms\tidrac",
]
for s in r.get("servers") or []:
lines.append(
"{st}\t{name}\t{model}\t{gb}\t{dc}\t{ip}".format(
st=s.get("service_tag") or "NONE",
name=(s.get("name") or "")[:36],
model=(s.get("model") or "")[:24],
gb=s.get("memory_gb") if s.get("memory_gb") is not None else "unknown",
dc=s.get("dimm_count") if s.get("dimm_count") is not None else "unknown",
ip=s.get("idrac_ip") or "—",
)
)
parts.append("\n".join(lines)[:14000])
else:
parts.append(json.dumps(r, ensure_ascii=False, default=str)[:14000])
return "\n".join(parts)
@app.get("/api/fleet-memory")
async def api_fleet_memory(force: bool = False):
"""CURRENT installed RAM per server (OME inventory cache)."""
return await fetch_fleet_memory(force=force)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/api/gpu")
async def api_gpu():
if not STATE.get("gpu"):
STATE["gpu"] = await fetch_gpu()
return STATE.get("gpu") or {}
@app.get("/api/admins")
async def api_admins():
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
return {"admins": OPS_USERS, "users": OPS_USERS}
@app.get("/api/ops/users")
async def api_ops_users():
return {
"users": OPS_USERS,
"admins": [u for u in OPS_USERS if u.get("team") == "admin"],
"fde": [u for u in OPS_USERS if u.get("team") == "fde"],
"context": {
"cockpit": "OME Cockpit by Data Forward Deployed Engineers Mohamed El Kadi & Bart Sjerps",
"cluster": "Runs on the FDE cluster operated by Data Forward Deployed Engineers Mohamed El Kadi and Bart Sjerps",
"admins": "Jody van Dongen and Laurens Rammers — ATC datacenter administrators",
"fde": "Mo and Bart — both Data Forward Deployed Engineers deploying AI workloads",
},
}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
_OWUI_TOKEN: dict[str, Any] = {"token": None, "expires": 0.0}
async def _owui_token() -> str | None:
email = (settings.openwebui_email or "").strip()
password = settings.openwebui_password or ""
if not email or not password:
return None
now = time.time()
if _OWUI_TOKEN.get("token") and float(_OWUI_TOKEN.get("expires") or 0) > now:
return str(_OWUI_TOKEN["token"])
url = settings.openwebui_url.rstrip("/") + "/api/v1/auths/signin"
try:
async with httpx.AsyncClient(timeout=20.0) as client:
r = await client.post(url, json={"email": email, "password": password})
if r.status_code >= 400:
log.warning("Open WebUI signin failed: %s %s", r.status_code, r.text[:200])
return None
token = (r.json() or {}).get("token")
if token:
_OWUI_TOKEN["token"] = token
_OWUI_TOKEN["expires"] = now + 3500
return token
except Exception as e:
log.warning("Open WebUI signin error: %s", e)
return None
async def list_chat_models() -> list[dict]:
models: list[dict] = []
seen: set[str] = set()
token = await _owui_token()
if token:
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(
settings.openwebui_url.rstrip("/") + "/api/models",
headers={"Authorization": f"Bearer {token}"},
)
if r.status_code == 200:
for m in (r.json() or {}).get("data") or []:
mid = m.get("id")
if not mid or mid in seen:
continue
seen.add(mid)
models.append(
{
"id": mid,
"name": m.get("name") or mid,
"source": "openwebui",
"owned_by": m.get("owned_by"),
}
)
except Exception as e:
log.warning("Open WebUI models failed: %s", e)
try:
async with httpx.AsyncClient(timeout=10.0) as client:
r = await client.get(settings.vllm_url.rstrip("/") + "/models")
if r.status_code == 200:
for m in (r.json() or {}).get("data") or []:
mid = m.get("id")
if not mid or mid in seen:
continue
seen.add(mid)
models.append(
{
"id": mid,
"name": mid,
"source": "vllm",
"owned_by": m.get("owned_by") or "vllm",
}
)
except Exception as e:
log.warning("vLLM models failed: %s", e)
if not models:
models.append(
{
"id": settings.vllm_model,
"name": settings.vllm_model,
"source": "vllm",
"owned_by": "vllm",
}
)
return models
@app.get("/api/models")
async def api_models():
models = await list_chat_models()
return {"models": models, "default": settings.vllm_model}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
def format_fleet_memory_reply(result: dict) -> str:
"""Deterministic full table — LLMs truncate long fleet lists."""
servers = result.get("servers") or []
lines = [
"CURRENT installed RAM from OME inventory (not Dell catalog max).",
"known={k}/{n} · total_installed={tg} GB".format(
k=result.get("with_memory"),
n=result.get("count"),
tg=result.get("total_memory_gb"),
),
"",
"ST | name | model | memory_gb | dimms | idrac | connected",
]
for s in servers:
gb = s.get("memory_gb")
gb_s = "unknown" if gb is None else str(gb)
dc = s.get("dimm_count")
dc_s = "unknown" if dc is None else str(dc)
lines.append(
"{st} | {name} | {model} | {gb} | {dc} | {ip} | {conn}".format(
st=s.get("service_tag") or "NONE",
name=(s.get("name") or "")[:40],
model=(s.get("model") or "")[:28],
gb=gb_s,
dc=dc_s,
ip=s.get("idrac_ip") or "—",
conn="yes" if s.get("connected") else "no",
)
)
return "\n".join(lines)
def should_use_deterministic_fleet_memory(message: str, planned: list, tool_results: list[dict]) -> dict | None:
if not any(n == "list_fleet_memory" for n, _ in planned):
return None
# Prefer deterministic when the ask is clearly a full memory inventory
low = (message or "").lower()
inventoryish = bool(
re.search(r"\b(memory|ram|geheugen|dimm)\b", low)
and re.search(r"\b(elk|elke|iedere|alle|each|every|all|per\s+server|hoeveel|list|overview)\b", low)
)
# Or when list_fleet_memory is the only tool
only_mem = len(planned) == 1 and planned[0][0] == "list_fleet_memory"
if not (inventoryish or only_mem):
return None
for r in tool_results:
if r.get("tool") == "list_fleet_memory" and not r.get("error"):
return r
return None
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.post("/api/chat")
async def api_chat(payload: ChatIn):
gpu = STATE.get("gpu") or {}
gsum = gpu.get("summary") or {}
model = (payload.model or settings.vllm_model or "").strip() or settings.vllm_model
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
# 1) Plan + execute live tools BEFORE the LLM (accurate facts; Llama3-GPTQ lacks native tool_calls)
planned = plan_chat_tools(payload.message, payload.focus_device_id)
tool_results: list[dict] = []
if planned:
tool_results = list(
await asyncio.gather(*[run_chat_tool(name, args) for name, args in planned])
)
max_tokens = settings.vllm_max_tokens
if any(n == "list_fleet_memory" for n, _ in planned):
max_tokens = max(max_tokens, 1800)
# Full fleet memory lists: answer from TOOL FACTS directly (accurate, complete)
det = should_use_deterministic_fleet_memory(payload.message, planned, tool_results)
if det is not None:
gpu = STATE.get("gpu") or {}
gsum = gpu.get("summary") or {}
return {
"reply": format_fleet_memory_reply(det),
"model": model,
"backend": "ome-tools",
"gpu": gsum,
"context_bytes": 0,
"tools_used": [{"name": n, "args": a} for n, a in planned],
"tool_count": len(tool_results),
"deterministic": True,
}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
system = build_fleet_context(payload.focus_device_id, max_chars=settings.chat_system_chars)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
facts = format_tool_facts(tool_results)
if facts:
# Prefer tool facts; keep within budget (vLLM context is tight)
budget = max(1800, settings.chat_system_chars - 100)
# Accuracy rules + short fleet head, then authoritative TOOL FACTS
head = system
if len(system) > 2800:
head = system[:2800] + "\n…[fleet truncated for tools]"
combined = head + "\n\n" + facts
if len(combined) > budget:
# Keep TOOL FACTS intact; shrink head further
max_facts = min(len(facts), budget - 900)
facts_trim = facts[:max_facts]
head = system[: max(600, budget - len(facts_trim) - 40)]
combined = head + "\n…\n\n" + facts_trim
system = combined[:budget]
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
messages = [{"role": "system", "content": system}]
for h in (payload.history or [])[-4:]:
role = h.get("role")
content = h.get("content")
if role in ("user", "assistant") and content:
messages.append({"role": role, "content": str(content)[:1200]})
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
user_msg = payload.message[:2500]
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
escalate = (
"If any needed fact is absent, say it is unknown and instruct the user to overleggen met "
"Jody van Dongen (jody.van.dongen@dell.com) and Laurens Rammers (laurens.rammers@dell.com). "
"Never invent."
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
if tool_results:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
tool_errors = [r for r in tool_results if r.get("error")]
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
user_msg += (
"\n\n[System note: live TOOL FACTS were fetched for this question. "
"Treat TOOL FACTS as ground truth. Quote CURRENT vs CATALOG separately. "
"Do not hedge or invent. Say unknown only if a field is absent from TOOL FACTS. "
"For fleet lists: output EVERY row from TOOL FACTS in a compact table "
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"(ST | name | model | memory_gb | dimms | idrac) — do not summarize or omit. "
f"{escalate}]"
)
if tool_errors:
user_msg += (
"\n[System note: some tools failed — do not fill gaps from model knowledge. "
f"{escalate}]"
)
else:
user_msg += (
"\n\n[System note: no live TOOL FACTS were fetched for this turn. "
"Answer only from the fleet snapshot in the system message. "
f"{escalate}]"
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
)
messages.append({"role": "user", "content": user_msg})
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
owui_models = {"ome-copilot", "arena-model"}
use_owui = model in owui_models
try:
if use_owui:
token = await _owui_token()
if not token:
raise HTTPException(
502,
"Open WebUI auth not configured (set OPENWEBUI_EMAIL / OPENWEBUI_PASSWORD)",
)
url = settings.openwebui_url.rstrip("/") + "/api/chat/completions"
async with httpx.AsyncClient(timeout=180.0) as client:
r = await client.post(
url,
headers={"Authorization": f"Bearer {token}"},
json={
"model": model,
"messages": messages,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"temperature": 0.05,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"max_tokens": max_tokens,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"stream": False,
},
)
if r.status_code >= 400:
raise HTTPException(502, f"Open WebUI error {r.status_code}: {r.text[:500]}")
data = r.json()
backend = "openwebui"
else:
url = settings.vllm_url.rstrip("/") + "/chat/completions"
async with httpx.AsyncClient(timeout=180.0) as client:
r = await client.post(
url,
json={
"model": model,
"messages": messages,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"temperature": 0.05,
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"max_tokens": max_tokens,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
},
)
if r.status_code >= 400:
detail = r.text
if r.status_code == 400 and "context length" in detail.lower():
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
tight = build_fleet_context(payload.focus_device_id, max_chars=2800)
if facts:
tight = (tight + "\n\n" + facts)[:3800]
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
messages = [{"role": "system", "content": tight}, messages[-1]]
r = await client.post(
url,
json={
"model": model,
"messages": messages,
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"temperature": 0.05,
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
"max_tokens": 500,
},
)
detail = r.text
if r.status_code >= 400:
raise HTTPException(502, f"vLLM error {r.status_code}: {detail[:600]}")
data = r.json()
backend = "vllm"
text_out = (
((data.get("choices") or [{}])[0].get("message") or {}).get("content")
or ""
)
return {
"reply": text_out,
"model": model,
"backend": backend,
"gpu": gsum,
"context_bytes": len(system),
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"tools_used": [{"name": n, "args": a} for n, a in planned],
"tool_count": len(tool_results),
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
}
except HTTPException:
raise
except Exception as e:
raise HTTPException(502, f"Chat backend error: {e}") from e
def _admin_name(aid: str) -> str:
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
u = _ops_user(aid)
return u["name"] if u else aid
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/api/tickets")
async def list_tickets():
with _db() as conn:
rows = conn.execute(
"SELECT * FROM tickets ORDER BY updated_at DESC LIMIT 200"
).fetchall()
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
return {"tickets": [dict(r) for r in rows], "admins": OPS_USERS, "users": OPS_USERS}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.post("/api/tickets")
async def create_ticket(payload: TicketIn):
now = time.time()
assignee = payload.assignee
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
if not assignee or not _ops_user(assignee):
assignee = _default_assignee(payload.created_by)
if payload.created_by and not _ops_user(payload.created_by):
raise HTTPException(400, f"Unknown actor: {payload.created_by}")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
with _db() as conn:
cur = conn.execute(
"""
INSERT INTO tickets(title, body, status, priority, created_by, assignee, created_at, updated_at)
VALUES (?, ?, 'open', ?, ?, ?, ?, ?)
""",
(
payload.title.strip()[:200],
payload.body.strip()[:5000],
payload.priority,
payload.created_by,
assignee,
now,
now,
),
)
tid = cur.lastrowid
conn.execute(
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
(tid, payload.created_by, payload.body.strip()[:5000], now),
)
row = conn.execute("SELECT * FROM tickets WHERE id=?", (tid,)).fetchone()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("create")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
return dict(row)
@app.get("/api/tickets/{ticket_id}")
async def get_ticket(ticket_id: int):
with _db() as conn:
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
if not row:
raise HTTPException(404, "Ticket not found")
msgs = conn.execute(
"SELECT * FROM ticket_messages WHERE ticket_id=? ORDER BY created_at ASC",
(ticket_id,),
).fetchall()
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
return {"ticket": dict(row), "messages": [dict(m) for m in msgs], "admins": OPS_USERS, "users": OPS_USERS}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.post("/api/tickets/{ticket_id}/messages")
async def add_ticket_message(ticket_id: int, payload: TicketMsgIn):
now = time.time()
with _db() as conn:
row = conn.execute("SELECT id FROM tickets WHERE id=?", (ticket_id,)).fetchone()
if not row:
raise HTTPException(404, "Ticket not found")
conn.execute(
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
(ticket_id, payload.author, payload.body.strip()[:5000], now),
)
conn.execute("UPDATE tickets SET updated_at=? WHERE id=?", (now, ticket_id))
msgs = conn.execute(
"SELECT * FROM ticket_messages WHERE ticket_id=? ORDER BY created_at ASC",
(ticket_id,),
).fetchall()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("message")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
return {"messages": [dict(m) for m in msgs]}
@app.patch("/api/tickets/{ticket_id}")
async def patch_ticket(ticket_id: int, payload: TicketPatch):
now = time.time()
with _db() as conn:
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
if not row:
raise HTTPException(404, "Ticket not found")
status = payload.status or row["status"]
assignee = payload.assignee if payload.assignee is not None else row["assignee"]
priority = payload.priority or row["priority"]
# accepted_by: keep existing unless explicitly set; accepting sets status accepted
accepted_by = row["accepted_by"] if "accepted_by" in row.keys() else None
if payload.accepted_by is not None:
accepted_by = payload.accepted_by
if status == "open":
status = "accepted"
conn.execute(
"UPDATE tickets SET status=?, assignee=?, priority=?, accepted_by=?, updated_at=? WHERE id=?",
(status, assignee, priority, accepted_by, now, ticket_id),
)
if payload.accepted_by:
conn.execute(
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
(
ticket_id,
payload.accepted_by,
f"Ticket accepted by {_admin_name(payload.accepted_by)}",
now,
),
)
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("patch")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
return dict(row)
@app.delete("/api/tickets/{ticket_id}")
async def delete_ticket(ticket_id: int):
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("pre-delete")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
with _db() as conn:
row = conn.execute("SELECT id FROM tickets WHERE id=?", (ticket_id,)).fetchone()
if not row:
raise HTTPException(404, "Ticket not found")
conn.execute("DELETE FROM ticket_messages WHERE ticket_id=?", (ticket_id,))
conn.execute("DELETE FROM tickets WHERE id=?", (ticket_id,))
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
_backup_ops_db("delete")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
return {"ok": True, "deleted": ticket_id}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
class RackIn(BaseModel):
site_id: str
name: str
units: int = 42
sort_order: int | None = None
class RackPatch(BaseModel):
name: str | None = None
sort_order: int | None = None
class PlacementIn(BaseModel):
rack_id: int
u_start: int
u_height: int | None = None
@app.get("/api/network/fabric")
async def api_network_fabric():
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
ensure_network_inventory_in_state()
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
return build_network_fabric()
class VlanIn(BaseModel):
vlan_id: int | None = None
name: str
cidr: str
purpose: str = ""
site_id: str | None = None
color: str = "#00a8e8"
sort_order: int = 0
@app.get("/api/network/vlans")
async def api_network_vlans():
return build_vlan_map()
@app.post("/api/network/vlans")
async def api_create_vlan(payload: VlanIn):
now = time.time()
try:
ipaddress.ip_network(payload.cidr, strict=False)
except Exception as e:
raise HTTPException(400, f"Invalid cidr: {e}") from e
with _db() as conn:
cur = conn.execute(
"""
INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
payload.vlan_id,
payload.name.strip()[:80],
payload.cidr.strip(),
(payload.purpose or "")[:200],
payload.site_id,
payload.color or "#00a8e8",
payload.sort_order,
now,
now,
),
)
vid = cur.lastrowid
_backup_ops_db("vlan-create")
return {"id": vid, **build_vlan_map()}
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
class EndpointIn(BaseModel):
hostname: str | None = None
ip: str
role: str = "server"
kind: str = "host"
device_id: int | None = None
vlan_id: int | None = None
switch_id: int | None = None
switch_port: int | None = None
model: str | None = None
note: str | None = None
user_note: str | None = None
class EndpointPatch(BaseModel):
hostname: str | None = None
role: str | None = None
kind: str | None = None
device_id: int | None = None
vlan_id: int | None = None
switch_id: int | None = None
switch_port: int | None = None
model: str | None = None
note: str | None = None
user_note: str | None = None
clear_device_id: bool = False
@app.get("/api/network/endpoints")
async def api_list_endpoints():
ensure_network_inventory_in_state()
return {"endpoints": _load_network_endpoints()}
@app.post("/api/network/endpoints")
async def api_create_endpoint(payload: EndpointIn):
try:
ipaddress.ip_address(payload.ip.strip())
except Exception as e:
raise HTTPException(400, f"Invalid ip: {e}") from e
_upsert_network_endpoint(
hostname=(payload.hostname or "").strip() or None,
ip=payload.ip.strip(),
role=(payload.role or "server").strip()[:40],
kind=(payload.kind or "host").strip()[:40],
device_id=payload.device_id,
vlan_id=payload.vlan_id,
switch_id=payload.switch_id,
switch_port=payload.switch_port,
model=(payload.model or "").strip()[:80] or None,
note=(payload.note or "").strip()[:500] or None,
overwrite_identity=True,
)
if payload.user_note is not None:
with _db() as conn:
_ensure_network_endpoints_schema(conn)
conn.execute(
"UPDATE network_endpoints SET user_note=?, updated_at=? WHERE ip=?",
((payload.user_note or "")[:2000], time.time(), payload.ip.strip()),
)
ensure_network_inventory_in_state(force=True)
_backup_ops_db("endpoint-create")
row = next((e for e in _load_network_endpoints() if e.get("ip") == payload.ip.strip()), None)
return {"endpoint": row, "vlans": build_vlan_map()}
@app.patch("/api/network/endpoints/{endpoint_id}")
async def api_patch_endpoint(endpoint_id: int, payload: EndpointPatch):
with _db() as conn:
_ensure_network_endpoints_schema(conn)
row = conn.execute("SELECT * FROM network_endpoints WHERE id=?", (endpoint_id,)).fetchone()
if not row:
raise HTTPException(404, "Endpoint not found")
sets = ["updated_at=?"]
vals: list = [time.time()]
data = payload.model_dump(exclude_unset=True)
clear = bool(data.pop("clear_device_id", False))
mapping = {
"hostname": 80,
"role": 40,
"kind": 40,
"model": 80,
"note": 500,
"user_note": 2000,
}
for key, maxlen in mapping.items():
if key in data and data[key] is not None:
sets.append(f"{key}=?")
vals.append(str(data[key])[:maxlen])
for key in ("device_id", "vlan_id", "switch_id", "switch_port"):
if key in data and data[key] is not None:
sets.append(f"{key}=?")
vals.append(data[key])
if clear:
sets.append("device_id=NULL")
vals.append(endpoint_id)
conn.execute(f"UPDATE network_endpoints SET {', '.join(sets)} WHERE id=?", vals)
ensure_network_inventory_in_state(force=True)
_backup_ops_db("endpoint-patch")
row = next((e for e in _load_network_endpoints() if e.get("id") == endpoint_id), None)
return {"endpoint": row, "vlans": build_vlan_map()}
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
class PortLinkIn(BaseModel):
switch_port: int
device_id: int
device_port: str = ""
note: str = ""
@app.get("/api/network/switches/{switch_id}/ports")
async def api_switch_ports(switch_id: int):
return build_switch_portmap(switch_id)
@app.put("/api/network/switches/{switch_id}/ports/{port_num}")
async def api_put_port_link(switch_id: int, port_num: int, payload: PortLinkIn):
sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None)
if not sw:
raise HTTPException(404, "Switch not found")
nports = _switch_port_count(sw)
if port_num < 1 or port_num > nports:
raise HTTPException(400, f"Port must be 1..{nports}")
peer = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None)
if not peer:
raise HTTPException(404, "Target device not found in fleet")
now = time.time()
with _db() as conn:
conn.execute(
"""
INSERT INTO port_links(switch_id, switch_port, device_id, device_port, note, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(switch_id, switch_port) DO UPDATE SET
device_id=excluded.device_id,
device_port=excluded.device_port,
note=excluded.note,
updated_at=excluded.updated_at
""",
(
switch_id,
port_num,
payload.device_id,
(payload.device_port or "")[:80],
(payload.note or "")[:200],
now,
),
)
_backup_ops_db("port-link")
return build_switch_portmap(switch_id)
@app.delete("/api/network/switches/{switch_id}/ports/{port_num}")
async def api_delete_port_link(switch_id: int, port_num: int):
with _db() as conn:
conn.execute(
"DELETE FROM port_links WHERE switch_id=? AND switch_port=?",
(switch_id, port_num),
)
_backup_ops_db("port-unlink")
return build_switch_portmap(switch_id)
@app.get("/api/devices/{device_id}/nics")
async def api_device_nics(device_id: int):
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
"""OME serverNetworkInterfaces expanded to port/FQDD rows for wiring UI."""
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
node = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
if not node:
raise HTTPException(404, "Device not found")
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
nics: list[dict] = []
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
try:
inv = await ome_fetch_inventory_types(device_id, ["serverNetworkInterfaces"])
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
for card in inv.get("serverNetworkInterfaces") or []:
nic_id = str(card.get("NicId") or "").strip()
vendor = str(card.get("VendorName") or card.get("Manufacturer") or "").strip()
ports = card.get("Ports") or []
if not ports:
# Rare flat inventory shape
fqdd = card.get("Fqdd") or card.get("InstanceId") or nic_id
name = (
card.get("ProductName")
or card.get("DeviceDescription")
or fqdd
or "NIC"
)
nics.append(
{
"nic_id": nic_id or None,
"port_id": fqdd,
"fqdd": fqdd,
"name": name,
"mac": card.get("PermanentMACAddress") or card.get("CurrentMACAddress"),
"speed": card.get("LinkSpeed") or card.get("Speed"),
"link": card.get("LinkStatus"),
"vendor": vendor or None,
"label": str(fqdd or name),
}
)
continue
for port in ports:
port_id = str(port.get("PortId") or "").strip()
parts = port.get("Partitions") or []
part = parts[0] if parts else {}
fqdd = str(part.get("Fqdd") or port_id or "").strip()
mac = (
part.get("PermanentMacAddress")
or part.get("CurrentMacAddress")
or part.get("PermanentMACAddress")
or part.get("CurrentMACAddress")
or ""
)
product = str(port.get("ProductName") or "").strip()
# OME often appends " - AA:BB:..." — keep product without MAC
if " - " in product and mac and product.upper().endswith(str(mac).upper()):
product = product[: product.rfind(" - ")].strip()
elif " - " in product:
left, right = product.rsplit(" - ", 1)
if ":" in right and len(right.replace(":", "")) >= 12:
product = left.strip()
link = port.get("LinkStatus")
speed = port.get("LinkSpeed")
label_bits = [port_id or fqdd or nic_id or "NIC"]
if product:
label_bits.append(product)
if mac:
label_bits.append(str(mac))
if link:
label_bits.append(str(link))
if speed not in (None, "", 0, "0"):
label_bits.append(f"{speed} Mb/s")
nics.append(
{
"nic_id": nic_id or None,
"port_id": port_id or None,
"fqdd": fqdd or port_id or None,
"name": product or port_id or nic_id or "NIC",
"mac": mac or None,
"speed": speed,
"link": link,
"vendor": vendor or None,
"label": " · ".join(label_bits),
}
)
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
except Exception as e:
log.debug("nic fetch %s: %s", device_id, e)
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
# Prefer unique port_id/fqdd order as returned by OME
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
return {"device_id": device_id, "nics": nics, "count": len(nics)}
@app.get("/api/racks")
async def api_racks():
_seed_atc_racks()
return _racks_payload()
@app.post("/api/racks")
async def api_create_rack(payload: RackIn):
now = time.time()
site = (payload.site_id or "").upper()
if site not in ("ATC1", "ATC2"):
raise HTTPException(400, "site_id must be ATC1 or ATC2")
with _db() as conn:
row = conn.execute("SELECT id FROM sites WHERE id=?", (site,)).fetchone()
if not row:
conn.execute(
"INSERT INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
(site, site, 1 if site == "ATC1" else 2),
)
so = payload.sort_order
if so is None:
mx = conn.execute(
"SELECT COALESCE(MAX(sort_order), 0) AS m FROM racks WHERE site_id=?",
(site,),
).fetchone()["m"]
so = int(mx) + 1
cur = conn.execute(
"""
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
""",
(site, payload.name.strip()[:64], max(1, min(48, payload.units)), so, now, now),
)
rid = cur.lastrowid
_backup_ops_db("rack-create")
return {"id": rid, **_racks_payload()}
@app.patch("/api/racks/{rack_id}")
async def api_patch_rack(rack_id: int, payload: RackPatch):
now = time.time()
with _db() as conn:
row = conn.execute("SELECT * FROM racks WHERE id=?", (rack_id,)).fetchone()
if not row:
raise HTTPException(404, "Rack not found")
name = payload.name.strip()[:64] if payload.name else row["name"]
so = payload.sort_order if payload.sort_order is not None else row["sort_order"]
conn.execute(
"UPDATE racks SET name=?, sort_order=?, updated_at=? WHERE id=?",
(name, so, now, rack_id),
)
_backup_ops_db("rack-patch")
return _racks_payload()
class RackItemIn(BaseModel):
rack_id: int
u_start: int = 1
u_height: int | None = None
side: str = "front" # front|rear|left|right
device_id: int | None = None
catalog_sku: str | None = None
label: str | None = None
notes: str | None = None
class RackItemPatch(BaseModel):
rack_id: int | None = None
u_start: int | None = None
u_height: int | None = None
side: str | None = None
catalog_sku: str | None = None
label: str | None = None
notes: str | None = None
@app.get("/api/rack-catalog")
async def api_rack_catalog():
families: dict[str, list] = {}
for item in DELL_RACK_CATALOG:
families.setdefault(item["family"], []).append(item)
return {"items": DELL_RACK_CATALOG, "families": families, "count": len(DELL_RACK_CATALOG)}
@app.put("/api/racks/placements/{device_id}")
async def api_put_placement(device_id: int, payload: PlacementIn):
"""Place/move an OME device (compat). Writes rack_items."""
device = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
cat = match_rack_catalog((device or {}).get("model"), (device or {}).get("role"))
body = RackItemIn(
rack_id=payload.rack_id,
u_start=payload.u_start,
u_height=payload.u_height if payload.u_height is not None else _default_u_height(device),
side="front",
device_id=device_id,
catalog_sku=(cat or {}).get("sku"),
label=(device or {}).get("name"),
)
return await api_upsert_rack_item(body)
@app.delete("/api/racks/placements/{device_id}")
async def api_delete_placement(device_id: int):
with _db() as conn:
conn.execute("DELETE FROM rack_items WHERE device_id=?", (device_id,))
try:
conn.execute("DELETE FROM rack_placements WHERE device_id=?", (device_id,))
except Exception:
pass
_backup_ops_db("unplace")
return _racks_payload()
@app.post("/api/racks/items")
async def api_upsert_rack_item(payload: RackItemIn):
now = time.time()
side = (payload.side or "front").lower()
if side not in ("front", "rear", "left", "right"):
raise HTTPException(400, "side must be front|rear|left|right")
device = None
if payload.device_id is not None:
device = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None)
if not device:
raise HTTPException(404, "OME device not found")
sku = payload.catalog_sku
cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None) if sku else None
if not cat and device:
cat = match_rack_catalog(device.get("model"), device.get("role"))
sku = (cat or {}).get("sku")
if not device and not cat and not sku:
raise HTTPException(400, "Provide device_id or catalog_sku")
height = payload.u_height
# Prefer catalog U-height for known SKUs so 2U/3U/… always place correctly
if cat is not None:
cat_h = int(cat.get("u_height") or 0)
if cat_h > 0:
height = cat_h
elif height is None:
height = _default_u_height(device)
elif height is None:
height = _default_u_height(device)
if side in ("left", "right"):
u_start, height = 1, 42
else:
height = max(1, min(42, int(height)))
u_start = int(payload.u_start)
if u_start < 1 or u_start + height - 1 > 42:
raise HTTPException(400, f"Must fit U1–U42 (u_start={u_start} height={height})")
label = (payload.label or (device or {}).get("name") or (cat or {}).get("name") or "Device")[:120]
with _db() as conn:
rack = conn.execute("SELECT * FROM racks WHERE id=?", (payload.rack_id,)).fetchone()
if not rack:
raise HTTPException(404, "Rack not found")
if _rack_overlap(
conn,
payload.rack_id,
u_start if side not in ("left", "right") else 1,
height,
side=side,
exclude_device_id=payload.device_id,
):
raise HTTPException(409, "U range overlaps another item on this side")
existing = None
if payload.device_id is not None:
existing = conn.execute(
"SELECT id FROM rack_items WHERE device_id=?", (payload.device_id,)
).fetchone()
if existing:
conn.execute(
"""
UPDATE rack_items
SET rack_id=?, catalog_sku=?, label=?, u_start=?, u_height=?, side=?, notes=?, updated_at=?
WHERE id=?
""",
(
payload.rack_id,
sku,
label,
u_start if side not in ("left", "right") else 1,
height,
side,
(payload.notes or "")[:200],
now,
existing["id"],
),
)
item_id = existing["id"]
else:
cur = conn.execute(
"""
INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
payload.rack_id,
payload.device_id,
sku,
label,
u_start if side not in ("left", "right") else 1,
height,
side,
(payload.notes or "")[:200],
now,
),
)
item_id = cur.lastrowid
# keep legacy table in sync for OME devices
if payload.device_id is not None and side == "front":
try:
conn.execute(
"""
INSERT INTO rack_placements(device_id, rack_id, u_start, u_height, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(device_id) DO UPDATE SET
rack_id=excluded.rack_id, u_start=excluded.u_start,
u_height=excluded.u_height, updated_at=excluded.updated_at
""",
(payload.device_id, payload.rack_id, u_start, height, now),
)
except Exception:
pass
_backup_ops_db("rack-item")
payload_out = _racks_payload()
payload_out["item_id"] = item_id
return payload_out
@app.patch("/api/racks/items/{item_id}")
async def api_patch_rack_item(item_id: int, payload: RackItemPatch):
now = time.time()
with _db() as conn:
row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone()
if not row:
raise HTTPException(404, "Item not found")
rack_id = payload.rack_id if payload.rack_id is not None else row["rack_id"]
side = (payload.side or row["side"] or "front").lower()
u_start = payload.u_start if payload.u_start is not None else row["u_start"]
height = payload.u_height if payload.u_height is not None else row["u_height"]
if side in ("left", "right"):
u_start, height = 1, 42
else:
height = max(1, min(42, int(height)))
u_start = int(u_start)
if u_start < 1 or u_start + height - 1 > 42:
raise HTTPException(400, "Must fit U1–U42")
if _rack_overlap(conn, rack_id, u_start, height, side=side, exclude_item_id=item_id):
raise HTTPException(409, "U range overlaps another item on this side")
conn.execute(
"""
UPDATE rack_items
SET rack_id=?, catalog_sku=COALESCE(?, catalog_sku), label=COALESCE(?, label),
u_start=?, u_height=?, side=?, notes=COALESCE(?, notes), updated_at=?
WHERE id=?
""",
(
rack_id,
payload.catalog_sku,
payload.label,
u_start,
height,
side,
payload.notes,
now,
item_id,
),
)
_backup_ops_db("rack-item-patch")
return _racks_payload()
@app.delete("/api/racks/items/{item_id}")
async def api_delete_rack_item(item_id: int):
with _db() as conn:
row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone()
if not row:
raise HTTPException(404, "Item not found")
conn.execute("DELETE FROM rack_items WHERE id=?", (item_id,))
if row["device_id"] is not None:
try:
conn.execute("DELETE FROM rack_placements WHERE device_id=?", (row["device_id"],))
except Exception:
pass
_backup_ops_db("rack-item-delete")
return _racks_payload()
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
def _get_ssh_sem() -> asyncio.Semaphore:
global _ssh_sem
if _ssh_sem is None:
_ssh_sem = asyncio.Semaphore(SSH_MAX_SESSIONS)
return _ssh_sem
def _fleet_ssh_targets() -> set[str]:
"""Allow SSH only to IPs currently known in the OME fleet snapshot."""
allowed: set[str] = set()
for d in STATE.get("devices") or []:
ip = d.get("ip")
if ip and ip.count(".") == 3:
allowed.add(ip)
return allowed
@app.websocket("/ws/ssh")
async def ws_ssh(ws: WebSocket):
"""Browser terminal <-> SSH bridge. Isolated per connection; concurrency-limited."""
await ws.accept()
conn = None
process = None
reader_task = None
session_id = f"{id(ws)}-{time.time()}"
client_host = ""
acquired = False
async def send_json(payload: dict):
try:
await ws.send_text(json.dumps(payload))
except Exception:
pass
def _client_key() -> str:
try:
return ws.client.host if ws.client else "unknown"
except Exception:
return "unknown"
try:
# Limit concurrent sessions globally and per browser IP
ck = _client_key()
active_for_client = sum(1 for s in SSH_SESSIONS.values() if s.get("client") == ck)
if active_for_client >= SSH_MAX_PER_CLIENT:
await send_json({
"type": "error",
"message": f"Too many SSH sessions from this client (max {SSH_MAX_PER_CLIENT})",
})
await ws.close()
return
try:
await asyncio.wait_for(_get_ssh_sem().acquire(), timeout=8.0)
acquired = True
except asyncio.TimeoutError:
await send_json({
"type": "error",
"message": "SSH gateway busy — try again in a moment",
})
await ws.close()
return
SSH_SESSIONS[session_id] = {
"client": ck,
"started": time.time(),
"host": None,
"user": None,
}
raw = await asyncio.wait_for(ws.receive_text(), timeout=60.0)
try:
msg = json.loads(raw)
except Exception:
await send_json({"type": "error", "message": "Invalid auth payload"})
await ws.close()
return
if msg.get("type") != "auth":
await send_json({"type": "error", "message": "Expected auth message"})
await ws.close()
return
host = str(msg.get("host") or "").strip()
username = str(msg.get("username") or "").strip()
password = str(msg.get("password") or "")
try:
port = int(msg.get("port") or 22)
except Exception:
port = 22
if not host or not username:
await send_json({"type": "error", "message": "Host and username are required"})
await ws.close()
return
if port < 1 or port > 65535:
await send_json({"type": "error", "message": "Invalid port"})
await ws.close()
return
async with _lock:
allowed = set(_fleet_ssh_targets())
if host not in allowed:
await send_json({
"type": "error",
"message": f"Host {host} is not in the current OME fleet — SSH blocked",
})
await ws.close()
return
client_host = host
SSH_SESSIONS[session_id]["host"] = host
SSH_SESSIONS[session_id]["user"] = username
await send_json({"type": "status", "message": f"Connecting to {username}@{host}:{port}…"})
try:
conn = await asyncio.wait_for(
asyncssh.connect(
host,
port=port,
username=username,
password=password,
known_hosts=None,
client_keys=None,
preferred_auth=["password", "keyboard-interactive"],
keepalive_interval=30,
keepalive_count_max=3,
),
timeout=25.0,
)
except Exception as e:
await send_json({"type": "error", "message": f"SSH connect failed: {e}"})
await ws.close()
return
term = str(msg.get("term") or "xterm-256color")
cols = int(msg.get("cols") or 120)
rows = int(msg.get("rows") or 36)
process = await conn.create_process(
term_type=term,
term_size=(max(cols, 40), max(rows, 10)),
encoding="utf-8",
errors="replace",
)
await send_json({"type": "ready", "message": f"Connected as {username}@{host}"})
log.info("SSH session start id=%s client=%s target=%s@%s active=%s",
session_id, ck, username, host, len(SSH_SESSIONS))
async def pump_ssh_to_ws():
try:
while True:
data = await process.stdout.read(8192)
if not data:
try:
err = await asyncio.wait_for(process.stderr.read(1024), timeout=0.2)
except Exception:
err = ""
if err:
await ws.send_text(err if isinstance(err, str) else err.decode("utf-8", "replace"))
break
await ws.send_text(data if isinstance(data, str) else data.decode("utf-8", "replace"))
except Exception:
pass
try:
await send_json({"type": "status", "message": "SSH session ended"})
except Exception:
pass
reader_task = asyncio.create_task(pump_ssh_to_ws())
# Idle watchdog: close after 30 min without client input
last_input = time.time()
IDLE_LIMIT = 1800
while True:
try:
packet = await asyncio.wait_for(ws.receive(), timeout=30.0)
except asyncio.TimeoutError:
if time.time() - last_input > IDLE_LIMIT:
await send_json({"type": "status", "message": "SSH idle timeout"})
break
# keepalive ping to browser
await send_json({"type": "pong"})
continue
if packet.get("type") == "websocket.disconnect":
break
text = packet.get("text")
if text is None:
continue
last_input = time.time()
if text.startswith("{") and '"type"' in text[:48]:
try:
ctrl = json.loads(text)
except Exception:
process.stdin.write(text)
await process.stdin.drain()
continue
ctype = ctrl.get("type")
if ctype == "resize":
try:
process.change_terminal_size(int(ctrl.get("cols") or 80), int(ctrl.get("rows") or 24))
except Exception:
pass
elif ctype == "data":
process.stdin.write(str(ctrl.get("data") or ""))
await process.stdin.drain()
elif ctype == "ping":
await send_json({"type": "pong"})
continue
process.stdin.write(text)
await process.stdin.drain()
except WebSocketDisconnect:
pass
except Exception as e:
log.warning("SSH websocket error: %s", e)
try:
await send_json({"type": "error", "message": str(e)})
except Exception:
pass
finally:
SSH_SESSIONS.pop(session_id, None)
if reader_task:
reader_task.cancel()
try:
await reader_task
except Exception:
pass
try:
if process:
process.close()
await asyncio.wait_for(process.wait_closed(), timeout=3.0)
except Exception:
pass
try:
if conn:
conn.close()
await asyncio.wait_for(conn.wait_closed(), timeout=3.0)
except Exception:
pass
if acquired:
try:
_get_ssh_sem().release()
except Exception:
pass
try:
await ws.close()
except Exception:
pass
log.info("SSH session end id=%s target=%s active=%s", session_id, client_host, len(SSH_SESSIONS))
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
# ---------------------------------------------------------------------------
# Reports / compliance / warranty / export (read-only OME surfaces)
# ---------------------------------------------------------------------------
async def fetch_warranties(force: bool = False) -> dict:
if not force:
cached = _cache_get("warranties")
if cached is not None:
return cached
rows: list[dict] = []
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
skip = 0
while True:
r = await client.get(
f"{base}/api/WarrantyService/Warranties",
headers=headers,
params={"$top": 200, "$skip": skip},
)
r.raise_for_status()
chunk = (r.json() or {}).get("value") or []
if not chunk:
break
for w in chunk:
rows.append(
{
"id": w.get("Id"),
"device_id": w.get("DeviceId"),
"service_tag": w.get("DeviceIdentifier"),
"device_name": w.get("DeviceName"),
"model": w.get("DeviceModel"),
"service_level": w.get("ServiceLevelDescription") or w.get("ServiceLevelCode"),
"start_date": w.get("StartDate"),
"end_date": w.get("EndDate"),
"days_remaining": w.get("DaysRemaining"),
"ship_date": w.get("SystemShipDate"),
"state": w.get("State"),
"country": w.get("CountryLookupCode"),
}
)
skip += len(chunk)
if len(chunk) < 200:
break
finally:
await ome_session_delete(client, base, headers, sid)
# Prefer longest-remaining warranty per device
by_device: dict[int, dict] = {}
for w in rows:
did = w.get("device_id")
if did is None:
continue
prev = by_device.get(did)
if not prev or int(w.get("days_remaining") or 0) > int(prev.get("days_remaining") or 0):
by_device[did] = w
elif int(w.get("days_remaining") or 0) == int(prev.get("days_remaining") or 0):
# keep later end date
if str(w.get("end_date") or "") > str(prev.get("end_date") or ""):
by_device[did] = w
payload = {
"updated_at": time.time(),
"count": len(rows),
"items": rows,
"by_device": {str(k): v for k, v in by_device.items()},
}
return _cache_set("warranties", payload)
async def fetch_baselines(force: bool = False) -> dict:
if not force:
cached = _cache_get("baselines")
if cached is not None:
return cached
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(f"{base}/api/UpdateService/Baselines", headers=headers)
r.raise_for_status()
items = []
for b in (r.json() or {}).get("value") or []:
summary = b.get("ComplianceSummary") or {}
items.append(
{
"id": b.get("Id"),
"name": b.get("Name"),
"description": b.get("Description"),
"catalog_id": b.get("CatalogId"),
"catalog_type": b.get("CatalogType"),
"last_run": b.get("LastRun"),
"task_id": b.get("TaskId"),
"compliance_status": summary.get("ComplianceStatus"),
"critical": summary.get("NumberOfCritical"),
"warning": summary.get("NumberOfWarning"),
"normal": summary.get("NumberOfNormal"),
"downgrade": summary.get("NumberOfDowngrade"),
"unknown": summary.get("NumberOfUnknown"),
}
)
finally:
await ome_session_delete(client, base, headers, sid)
return _cache_set("baselines", {"updated_at": time.time(), "items": items})
async def fetch_catalogs(force: bool = False) -> dict:
if not force:
cached = _cache_get("catalogs")
if cached is not None:
return cached
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(f"{base}/api/UpdateService/Catalogs", headers=headers)
r.raise_for_status()
items = []
for c in (r.json() or {}).get("value") or []:
repo = c.get("Repository") or {}
items.append(
{
"id": c.get("Id"),
"filename": c.get("Filename") or c.get("SourcePath"),
"source": repo.get("Source") or c.get("Source"),
"repository_name": repo.get("Name"),
"repository_type": repo.get("RepositoryType") or c.get("RepositoryType"),
"owner": c.get("Owner"),
"status": (c.get("Status") or {}).get("Name") if isinstance(c.get("Status"), dict) else c.get("Status"),
"last_updated": c.get("LastUpdated") or c.get("BundlesUpdateTime"),
}
)
finally:
await ome_session_delete(client, base, headers, sid)
return _cache_set("catalogs", {"updated_at": time.time(), "items": items})
def _pick_primary_baseline(baselines: list[dict]) -> dict | None:
if not baselines:
return None
for b in baselines:
name = (b.get("name") or "").lower()
if "dell" in name and "online" in name:
return b
# most critical first
return sorted(baselines, key=lambda x: int(x.get("critical") or 0), reverse=True)[0]
async def fetch_compliance(force: bool = False, baseline_id: int | None = None) -> dict:
if not force and baseline_id is None:
cached = _cache_get("compliance")
if cached is not None:
return cached
bl = await fetch_baselines(force=force)
primary = None
if baseline_id is not None:
primary = next((b for b in bl["items"] if b.get("id") == baseline_id), None)
if primary is None:
primary = _pick_primary_baseline(bl["items"])
if not primary:
payload = {
"updated_at": time.time(),
"summary": {"outdated_devices": 0, "critical_components": 0, "baseline_name": None},
"devices": [],
"components": [],
"baselines": bl["items"],
}
return _cache_set("compliance", payload)
bid = primary["id"]
devices_out: list[dict] = []
components_out: list[dict] = []
fleet_by_id = {d.get("id"): d for d in STATE.get("devices") or []}
async with httpx.AsyncClient(verify=False, timeout=120.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(
f"{base}/api/UpdateService/Baselines({bid})/DeviceComplianceReports",
headers=headers,
)
r.raise_for_status()
for dcr in (r.json() or {}).get("value") or []:
did = dcr.get("DeviceId")
fleet = fleet_by_id.get(did) or {}
st = dcr.get("ServiceTag") or fleet.get("service_tag")
comps = dcr.get("ComponentComplianceReports") or []
device_row = {
"device_id": did,
"service_tag": st,
"name": fleet.get("name") or dcr.get("DeviceName") or st,
"model": dcr.get("DeviceModel") or fleet.get("model"),
"ip": fleet.get("ip"),
"firmware_status": dcr.get("FirmwareStatus"),
"compliance_status": dcr.get("ComplianceStatus"),
"reboot_required": dcr.get("RebootRequired"),
"component_count": len(comps),
"dell_uri": None,
}
devices_out.append(device_row)
for comp in comps:
uri = comp.get("Uri")
if uri and not device_row["dell_uri"]:
device_row["dell_uri"] = uri
components_out.append(
{
"device_id": did,
"service_tag": st,
"device_name": device_row["name"],
"model": device_row["model"],
"ip": device_row["ip"],
"component": comp.get("Name"),
"component_type": comp.get("ComponentType"),
"current_version": comp.get("CurrentVersion"),
"catalog_version": comp.get("Version"),
"update_action": comp.get("UpdateAction"),
"criticality": comp.get("Criticality"),
"compliance_status": comp.get("ComplianceStatus"),
"reboot_required": comp.get("RebootRequired"),
"dell_uri": uri,
"path": comp.get("Path"),
"baseline_id": bid,
"baseline_name": primary.get("name"),
"status_badge": (
"outdated"
if str(comp.get("UpdateAction") or "").upper() == "UPGRADE"
or str(comp.get("ComplianceStatus") or "").upper()
in ("CRITICAL", "WARNING", "NONCOMPLIANT", "NON-COMPLIANT")
else "current"
if str(comp.get("UpdateAction") or "").upper() in ("EQUAL", "NONE", "")
and str(comp.get("ComplianceStatus") or "").upper()
in ("", "OK", "COMPLIANT", "NORMAL", "DOWNGRADE")
else "unknown"
),
}
)
finally:
await ome_session_delete(client, base, headers, sid)
outdated = [
d
for d in devices_out
if str(d.get("compliance_status") or "").upper() in ("CRITICAL", "WARNING")
or str(d.get("firmware_status") or "").lower() in ("non-compliant", "noncompliant")
]
crit_comps = [
c
for c in components_out
if str(c.get("compliance_status") or "").upper() == "CRITICAL"
or str(c.get("update_action") or "").upper() == "UPGRADE"
]
payload = {
"updated_at": time.time(),
"baseline": primary,
"baselines": bl["items"],
"summary": {
"baseline_id": bid,
"baseline_name": primary.get("name"),
"outdated_devices": len(outdated),
"devices_in_report": len(devices_out),
"critical_components": len(crit_comps),
"components_total": len(components_out),
"compliance_status": primary.get("compliance_status"),
"last_run": primary.get("last_run"),
},
"devices": devices_out,
"components": components_out,
}
if baseline_id is None:
return _cache_set("compliance", payload)
return payload
async def fetch_report_defs(force: bool = False) -> dict:
if not force:
cached = _cache_get("report_defs")
if cached is not None:
return cached
items = []
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(f"{base}/api/ReportService/ReportDefs", headers=headers)
r.raise_for_status()
for d in (r.json() or {}).get("value") or []:
cols = [c.get("Name") for c in (d.get("ColumnNames") or []) if c.get("Name")]
items.append(
{
"id": d.get("Id"),
"name": d.get("Name"),
"description": d.get("Description"),
"category": d.get("Category") or d.get("FilterGroupName"),
"is_builtin": d.get("IsBuiltIn"),
"last_run": d.get("LastRunDate"),
"last_run_by": d.get("LastRunBy"),
"columns": cols,
}
)
finally:
await ome_session_delete(client, base, headers, sid)
items.sort(key=lambda x: ((x.get("category") or ""), (x.get("name") or "").lower()))
return _cache_set("report_defs", {"updated_at": time.time(), "items": items})
async def fetch_jobs(force: bool = False, top: int = 40) -> dict:
if not force:
cached = _cache_get("jobs")
if cached is not None:
return cached
items = []
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(
f"{base}/api/JobService/Jobs",
headers=headers,
params={"$top": top},
)
r.raise_for_status()
for j in (r.json() or {}).get("value") or []:
status = j.get("LastRunStatus") or {}
items.append(
{
"id": j.get("Id"),
"name": j.get("JobName") or j.get("Name"),
"status": status.get("Name") if isinstance(status, dict) else status,
"job_type": (j.get("JobType") or {}).get("Name")
if isinstance(j.get("JobType"), dict)
else j.get("JobType"),
"last_run": j.get("LastRunStatus") and (j.get("LastRunDate") or j.get("StartTime")),
"progress": j.get("Progress") or j.get("PercentComplete"),
"created_by": j.get("CreatedBy"),
}
)
finally:
await ome_session_delete(client, base, headers, sid)
return _cache_set("jobs", {"updated_at": time.time(), "items": items})
def build_fleet_report_rows(warranties: dict | None = None) -> list[dict]:
by_dev = (warranties or {}).get("by_device") or {}
rows = []
for d in STATE.get("devices") or []:
w = by_dev.get(str(d.get("id"))) or {}
rows.append(
{
"id": d.get("id"),
"name": d.get("name"),
"service_tag": d.get("service_tag"),
"model": d.get("model"),
"ip": d.get("ip"),
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
"idrac_ip": d.get("idrac_ip") or d.get("ip"),
"os_hostname": d.get("os_hostname"),
"rdp_host": d.get("rdp_host"),
"rdp_ips": d.get("rdp_ips") or [],
"os_ips": d.get("os_ips") or [],
"is_windows": d.get("is_windows"),
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
"subnet": d.get("subnet"),
"type": d.get("type"),
"sub_type": d.get("sub_type"),
"is_server": d.get("is_server"),
"is_idrac": d.get("is_idrac"),
"connected": d.get("connected"),
"powered_on": d.get("powered_on"),
"power_state": d.get("power_state"),
"status": d.get("status"),
"watts": d.get("watts"),
"avg_watts": d.get("avg_watts"),
"peak_watts": d.get("peak_watts"),
"last_status_time": d.get("last_status_time"),
"last_inventory_time": d.get("last_inventory_time"),
"warranty_end": w.get("end_date"),
"warranty_days_remaining": w.get("days_remaining"),
"warranty_service_level": w.get("service_level"),
}
)
rows.sort(key=lambda r: (r.get("name") or "").lower())
return rows
@app.get("/api/warranties")
async def api_warranties(force: bool = False):
return await fetch_warranties(force=force)
@app.get("/api/compliance")
async def api_compliance(force: bool = False, baseline_id: int | None = None):
return await fetch_compliance(force=force, baseline_id=baseline_id)
@app.get("/api/baselines")
async def api_baselines(force: bool = False):
return await fetch_baselines(force=force)
@app.get("/api/catalogs")
async def api_catalogs(force: bool = False):
return await fetch_catalogs(force=force)
@app.get("/api/ome/report-defs")
async def api_report_defs(force: bool = False):
return await fetch_report_defs(force=force)
@app.get("/api/ome/jobs")
async def api_ome_jobs(force: bool = False):
return await fetch_jobs(force=force)
class ReportRunIn(BaseModel):
report_def_id: int
@app.post("/api/ome/reports/run")
async def api_ome_report_run(payload: ReportRunIn):
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.post(
f"{base}/api/ReportService/Actions/ReportService.RunReport",
headers=headers,
json={"ReportDefId": payload.report_def_id},
)
if r.status_code >= 400:
raise HTTPException(r.status_code, r.text[:500])
job_id = r.json() if isinstance(r.json(), (int, str)) else (r.json() or {}).get("Id") or r.text
return {
"job_id": job_id,
"report_def_id": payload.report_def_id,
"message": "Report job started in OME. Poll results shortly.",
"results_url": f"/api/ome/reports/{payload.report_def_id}/results",
}
finally:
await ome_session_delete(client, base, headers, sid)
@app.get("/api/ome/reports/{report_def_id}/results")
async def api_ome_report_results(report_def_id: int):
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
base, headers, sid = await ome_session(client)
try:
r = await client.get(
f"{base}/api/ReportService/ReportDefs({report_def_id})/ReportResults",
headers=headers,
)
if r.status_code >= 400:
raise HTTPException(
r.status_code,
(r.json().get("error", {}) or {}).get("message")
if r.headers.get("content-type", "").startswith("application/json")
else r.text[:500],
)
return r.json()
finally:
await ome_session_delete(client, base, headers, sid)
@app.get("/api/reports/fleet")
async def api_reports_fleet(force: bool = False):
warranties = await fetch_warranties(force=force)
rows = build_fleet_report_rows(warranties)
return {
"updated_at": STATE.get("updated_at"),
"summary": STATE.get("summary"),
"ome": STATE.get("ome"),
"count": len(rows),
"rows": rows,
}
@app.get("/api/reports/firmware")
async def api_reports_firmware(force: bool = False, baseline_id: int | None = None):
return await fetch_compliance(force=force, baseline_id=baseline_id)
@app.get("/api/reports/brief")
async def api_reports_brief(force: bool = False):
warranties = await fetch_warranties(force=force)
compliance = await fetch_compliance(force=force)
fleet_rows = build_fleet_report_rows(warranties)
critical = [a for a in (STATE.get("alerts") or []) if a.get("severity") == "Critical"][:15]
expiring = sorted(
[
w
for w in (warranties.get("items") or [])
if w.get("days_remaining") is not None and int(w.get("days_remaining") or 0) <= 90
],
key=lambda x: int(x.get("days_remaining") or 0),
)[:25]
outdated = [
d
for d in (compliance.get("devices") or [])
if str(d.get("compliance_status") or "").upper() in ("CRITICAL", "WARNING")
or str(d.get("firmware_status") or "").lower() in ("non-compliant", "noncompliant")
]
return {
"generated_at": time.time(),
"ome": STATE.get("ome"),
"summary": STATE.get("summary"),
"compliance_summary": compliance.get("summary"),
"critical_alerts": critical,
"outdated_devices": outdated[:40],
"warranty_expiring": expiring,
"fleet": fleet_rows,
"baseline": compliance.get("baseline"),
}
@app.get("/api/export/{kind}")
async def api_export(kind: str, fmt: str = "csv", force: bool = False, baseline_id: int | None = None):
kind = kind.lower().strip()
fmt = fmt.lower().strip()
if fmt not in ("csv", "json"):
raise HTTPException(400, "fmt must be csv or json")
if kind in ("fleet", "inventory"):
data = await api_reports_fleet(force=force)
rows = data["rows"]
filename = f"ome-fleet-{int(time.time())}"
elif kind in ("firmware", "compliance"):
data = await fetch_compliance(force=force, baseline_id=baseline_id)
rows = data.get("components") or []
filename = f"ome-firmware-compliance-{int(time.time())}"
elif kind == "warranty":
data = await fetch_warranties(force=force)
rows = data.get("items") or []
filename = f"ome-warranties-{int(time.time())}"
elif kind == "brief":
data = await api_reports_brief(force=force)
if fmt == "json":
return Response(
content=json.dumps(data, indent=2, default=str),
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="ome-customer-brief-{int(time.time())}.json"'},
)
# flatten brief as fleet CSV appendix
rows = data.get("fleet") or []
filename = f"ome-customer-brief-fleet-{int(time.time())}"
else:
raise HTTPException(404, "Unknown export kind. Use fleet|firmware|warranty|brief")
if fmt == "json":
body = json.dumps({"kind": kind, "count": len(rows), "rows": rows}, indent=2, default=str)
return Response(
content=body,
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="{filename}.json"'},
)
csv_body = rows_to_csv(rows)
return Response(
content=csv_body,
media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="{filename}.csv"'},
)
@app.get("/api/devices/{device_id}/warranty")
async def device_warranty(device_id: int, force: bool = False):
warranties = await fetch_warranties(force=force)
items = [w for w in warranties.get("items") or [] if w.get("device_id") == device_id]
primary = (warranties.get("by_device") or {}).get(str(device_id))
return {"device_id": device_id, "primary": primary, "items": items}
@app.get("/api/devices/{device_id}/compliance")
async def device_compliance(device_id: int, force: bool = False):
compliance = await fetch_compliance(force=force)
device = next((d for d in compliance.get("devices") or [] if d.get("device_id") == device_id), None)
comps = [c for c in compliance.get("components") or [] if c.get("device_id") == device_id]
return {
"device_id": device_id,
"baseline": compliance.get("baseline"),
"device": device,
"components": comps,
}
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/api/ssh/sessions")
async def api_ssh_sessions():
"""Ops visibility: active SSH bridges (no secrets)."""
now = time.time()
sessions = [
{
"id": sid,
"client": meta.get("client"),
"host": meta.get("host"),
"user": meta.get("user"),
"age_sec": int(now - float(meta.get("started") or now)),
}
for sid, meta in list(SSH_SESSIONS.items())
]
return {"active": len(sessions), "max": SSH_MAX_SESSIONS, "sessions": sessions}
STATIC_DIR = Path("/ui")
@app.get("/")
async def index():
resp = FileResponse(STATIC_DIR / "index.html")
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
resp.headers["Pragma"] = "no-cache"
return resp
@app.get("/styles.css")
async def styles():
return FileResponse(STATIC_DIR / "styles.css", media_type="text/css")
@app.get("/app.js")
async def app_js():
return FileResponse(STATIC_DIR / "app.js", media_type="application/javascript")
@app.get("/ops.js")
async def ops_js():
return FileResponse(STATIC_DIR / "ops.js", media_type="application/javascript")
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
@app.get("/network.js")
async def network_js():
return FileResponse(STATIC_DIR / "network.js", media_type="application/javascript")
Add iDRAC Console wall, power control, and proxy embeds.
2026-07-19 00:16:19 +02:00
@app.get("/console.js")
async def console_js():
return FileResponse(STATIC_DIR / "console.js", media_type="application/javascript")
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/ssh.js")
async def ssh_js():
return FileResponse(STATIC_DIR / "ssh.js", media_type="application/javascript")
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
@app.get("/rdp.js")
async def rdp_js():
return FileResponse(STATIC_DIR / "rdp.js", media_type="application/javascript")
@app.get("/rdp-popout.html")
async def rdp_popout_html():
resp = FileResponse(STATIC_DIR / "rdp-popout.html")
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
return resp
@app.get("/api/rdp.rdp")
async def api_rdp_file(host: str, port: int = 3389, username: str = ""):
"""Download a standard .rdp file for the local Remote Desktop client."""
host = (host or "").strip()
if not host or len(host) > 253 or any(c in host for c in "\r\n\x00"):
raise HTTPException(400, "Invalid host")
if port < 1 or port > 65535:
raise HTTPException(400, "Invalid port")
username = (username or "").strip()
if len(username) > 256 or any(c in username for c in "\r\n\x00"):
raise HTTPException(400, "Invalid username")
full = f"{host}:{port}" if port != 3389 else host
lines = [
"screen mode id:i:2",
"use multimon:i:0",
"desktopwidth:i:1920",
"desktopheight:i:1080",
"session bpp:i:32",
"compression:i:1",
"keyboardhook:i:2",
"audiocapturemode:i:0",
"videoplaybackmode:i:1",
"connection type:i:7",
"networkautodetect:i:1",
"bandwidthautodetect:i:1",
"displayconnectionbar:i:1",
"bitmapcachepersistenable:i:1",
f"full address:s:{full}",
"audiomode:i:0",
"redirectclipboard:i:1",
"autoreconnection enabled:i:1",
"authentication level:i:2",
"prompt for credentials:i:1",
"negotiate security layer:i:1",
"gatewayusagemethod:i:4",
"gatewaycredentialssource:i:4",
"gatewayprofileusagemethod:i:0",
]
if username:
lines.insert(15, f"username:s:{username}")
body = "\r\n".join(lines) + "\r\n"
safe = "".join(c if c.isalnum() or c in ".-_" else "_" for c in host)[:80] or "session"
return Response(
content=body,
media_type="application/x-rdp",
headers={
"Content-Disposition": f'attachment; filename="{safe}.rdp"',
"Cache-Control": "no-store",
},
)
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/vendor/xterm/xterm.css")
async def vendor_xterm_css():
return FileResponse(STATIC_DIR / "vendor/xterm/xterm.css", media_type="text/css")
@app.get("/vendor/xterm/xterm.min.js")
async def vendor_xterm_js():
return FileResponse(
STATIC_DIR / "vendor/xterm/xterm.min.js",
media_type="application/javascript",
)
@app.get("/vendor/xterm/xterm-addon-fit.min.js")
async def vendor_xterm_fit_js():
return FileResponse(
STATIC_DIR / "vendor/xterm/xterm-addon-fit.min.js",
media_type="application/javascript",
)
Add Reports drawer, Dell catalog compliance, and Service Tag chat context.
2026-07-17 15:15:57 +02:00
@app.get("/reports.js")
async def reports_js():
return FileResponse(STATIC_DIR / "reports.js", media_type="application/javascript")
Add VLAN inventory, Present decks, and network fabric mapping.
2026-07-18 21:56:30 +02:00
@app.get("/present.js")
async def present_js():
return FileResponse(STATIC_DIR / "present.js", media_type="application/javascript")
@app.get("/logos/{filename}")
async def logo_asset(filename: str):
"""SVG / image logos for Present architecture slides."""
safe = Path(filename).name
if not safe or safe != filename or ".." in filename:
raise HTTPException(400, "invalid filename")
path = STATIC_DIR / "logos" / safe
if not path.is_file():
raise HTTPException(404, "logo not found")
media = "image/svg+xml" if safe.lower().endswith(".svg") else "image/png"
resp = FileResponse(path, media_type=media)
resp.headers["Cache-Control"] = "public, max-age=86400"
return resp
Initial import of Dell OpenManage Cockpit for ATC.
2026-07-17 04:00:38 +02:00
@app.get("/dell.png")
async def dell_png():
return FileResponse(STATIC_DIR / "dell.png", media_type="image/png")
@app.get("/dell-mark.png")
async def dell_mark():
return FileResponse(STATIC_DIR / "dell-mark.png", media_type="image/png")
@app.get("/dell.svg")
async def dell_svg():
return FileResponse(STATIC_DIR / "dell.png", media_type="image/png")
Add Network Design Studio with multi-U Dell rack designer.
2026-07-18 01:18:14 +02:00
@app.get("/assets/dell/{filename}")
async def dell_asset(filename: str):
"""Serve Dell product face photos for rack Design Studio."""
safe = Path(filename).name
if not safe or safe != filename or ".." in filename:
raise HTTPException(400, "invalid filename")
path = STATIC_DIR / "assets" / "dell" / safe
if not path.is_file():
raise HTTPException(404, "asset not found")
media = "image/jpeg" if safe.lower().endswith((".jpg", ".jpeg")) else "image/png"
resp = FileResponse(path, media_type=media)
resp.headers["Cache-Control"] = "public, max-age=86400"
return resp
Reference in New Issue Copy Permalink