root
1e87f22006
Seed all ATC/FDE VLANs with live host inventory and editable notes, fix cluster membership on the map, and ship Present/network UI polish. Co-authored-by: Cursor <cursoragent@cursor.com>
6095 lines
242 KiB
Python
6095 lines
242 KiB
Python
|
||
import asyncio
|
||
import json
|
||
import re
|
||
import os
|
||
import socket
|
||
import sqlite3
|
||
import ipaddress
|
||
import logging
|
||
import time
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import asyncssh
|
||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from fastapi.responses import FileResponse, Response
|
||
from pydantic import BaseModel, Field
|
||
from pydantic_settings import BaseSettings
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
log = logging.getLogger("ome-cockpit")
|
||
|
||
|
||
class Settings(BaseSettings):
|
||
ome_url: str = "https://cov-omeprod01.dell-atc.lan"
|
||
ome_user: str = "admin"
|
||
ome_password: str = ""
|
||
poll_interval: float = 15.0
|
||
openwebui_url: str = "http://atc-portal01.dell-atc.lan:3080"
|
||
cors_origins: str = "*"
|
||
power_fetch_limit: int = 40
|
||
gpu_metrics_url: str = "http://10.0.10.106:9110"
|
||
vllm_url: str = "http://10.0.10.106:8000/v1"
|
||
vllm_model: str = "llama3-70b-gptq"
|
||
vllm_max_model_len: int = 4096
|
||
vllm_max_tokens: int = 1100
|
||
chat_system_chars: int = 12000
|
||
openwebui_email: str = ""
|
||
openwebui_password: str = ""
|
||
cockpit_data: str = "/data"
|
||
reports_cache_ttl: float = 600.0
|
||
|
||
class Config:
|
||
env_file = ".env"
|
||
|
||
|
||
settings = Settings()
|
||
app = FastAPI(title="OME Cockpit API", version="1.2.0")
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=[o.strip() for o in settings.cors_origins.split(",")],
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
STATE: dict[str, Any] = {
|
||
"updated_at": 0,
|
||
"ome": {},
|
||
"summary": {},
|
||
"subnets": [],
|
||
"devices": [],
|
||
"groups": [],
|
||
"models": [],
|
||
"alerts": [],
|
||
"events": [],
|
||
"context": {},
|
||
"gpu": {},
|
||
"pulse": 0,
|
||
"openwebui_url": settings.openwebui_url,
|
||
}
|
||
CLIENTS: set[WebSocket] = set()
|
||
SSH_SESSIONS: dict[str, dict] = {}
|
||
SSH_MAX_SESSIONS = 25
|
||
SSH_MAX_PER_CLIENT = 3
|
||
_ssh_sem: asyncio.Semaphore | None = None
|
||
_lock = asyncio.Lock()
|
||
DETAIL_CACHE: dict[int, dict] = {}
|
||
PREV_DEVICES: dict[int, dict] = {}
|
||
EVENT_FEED: list[dict] = []
|
||
REPORT_CACHE: dict[str, Any] = {
|
||
"warranties": None,
|
||
"warranties_ts": 0.0,
|
||
"compliance": None,
|
||
"compliance_ts": 0.0,
|
||
"baselines": None,
|
||
"baselines_ts": 0.0,
|
||
"catalogs": None,
|
||
"catalogs_ts": 0.0,
|
||
"report_defs": None,
|
||
"report_defs_ts": 0.0,
|
||
"jobs": None,
|
||
"jobs_ts": 0.0,
|
||
}
|
||
|
||
|
||
async def ome_session(client: httpx.AsyncClient) -> tuple[str, dict, str]:
|
||
"""Create an OME API session; returns (base, headers, session_id)."""
|
||
base = settings.ome_url.rstrip("/")
|
||
r = await client.post(
|
||
f"{base}/api/SessionService/Sessions",
|
||
json={
|
||
"UserName": settings.ome_user,
|
||
"Password": settings.ome_password,
|
||
"SessionType": "API",
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
token = r.headers.get("X-Auth-Token")
|
||
sid = str((r.json() or {}).get("Id") or "")
|
||
headers = {"X-Auth-Token": token, "Accept": "application/json"}
|
||
return base, headers, sid
|
||
|
||
|
||
async def ome_session_delete(client: httpx.AsyncClient, base: str, headers: dict, sid: str) -> None:
|
||
if not sid:
|
||
return
|
||
try:
|
||
await client.delete(f"{base}/api/SessionService/Sessions('{sid}')", headers=headers)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _cache_get(key: str) -> Any | None:
|
||
ts = float(REPORT_CACHE.get(f"{key}_ts") or 0)
|
||
if REPORT_CACHE.get(key) is not None and time.time() - ts < settings.reports_cache_ttl:
|
||
return REPORT_CACHE.get(key)
|
||
return None
|
||
|
||
|
||
def _cache_set(key: str, value: Any) -> Any:
|
||
REPORT_CACHE[key] = value
|
||
REPORT_CACHE[f"{key}_ts"] = time.time()
|
||
return value
|
||
|
||
|
||
def csv_escape(val: Any) -> str:
|
||
s = "" if val is None else str(val)
|
||
if any(c in s for c in (",", '"', "\n", "\r")):
|
||
return '"' + s.replace('"', '""') + '"'
|
||
return s
|
||
|
||
|
||
def rows_to_csv(rows: list[dict], columns: list[str] | None = None) -> str:
|
||
if not rows and not columns:
|
||
return ""
|
||
cols = columns or list(rows[0].keys())
|
||
lines = [",".join(cols)]
|
||
for row in rows:
|
||
lines.append(",".join(csv_escape(row.get(c)) for c in cols))
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Dell PowerEdge expansion / memory population catalog (from Dell Tech Guides
|
||
# + Installation & Service Manuals / KB 000141539). Used for empty-slot maps,
|
||
# max capacity, and best-practice guidance — not a live scrape.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_DELL_LINKS = {
|
||
# Verified live PDFs / product docs (checked 2026-07-17: %PDF or HTTP 200 HTML).
|
||
"spec_r740xd": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740xd-spec-sheet.pdf",
|
||
"spec_r740": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740-spec-sheet.pdf",
|
||
"tech_r740": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R740_R740xd_Technical_Guide.pdf",
|
||
"own_r740xd": "https://dl.dell.com/topicspdf/poweredge-r740xd_owners-manual_en-us.pdf",
|
||
"docs_r740xd": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740xd/docs",
|
||
"docs_r740": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740/docs",
|
||
"spec_r640": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r640-spec-sheet.pdf",
|
||
"own_r640": "https://dl.dell.com/topicspdf/poweredge-r640_owners-manual_en-us.pdf",
|
||
"docs_r640": "https://www.dell.com/support/product-details/en-us/product/poweredge-r640/docs",
|
||
"spec_r440": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r440-spec-sheet.pdf",
|
||
"docs_r440": "https://www.dell.com/support/product-details/en-us/product/poweredge-r440/docs",
|
||
"spec_r6415": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R6415_Spec_Sheet_EN.pdf",
|
||
"ref_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_reference-guide_en-us.pdf",
|
||
"own_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_owners-manual_en-us.pdf",
|
||
"docs_r6415": "https://www.dell.com/support/product-details/en-us/product/poweredge-r6415/docs",
|
||
"spec_r7425": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge-R7425-Spec-Sheet.pdf",
|
||
"own_r7425": "https://dl.dell.com/topicspdf/poweredge-r7425_owners-manual_en-us.pdf",
|
||
"docs_r7425": "https://www.dell.com/support/product-details/en-us/product/poweredge-r7425/docs",
|
||
"spec_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-spec-sheet.pdf",
|
||
"tech_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-technical-guide.pdf",
|
||
"docs_r660": "https://www.dell.com/support/product-details/en-us/product/poweredge-r660/docs",
|
||
"spec_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-spec-sheet.pdf",
|
||
"tech_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-technical-guide.pdf",
|
||
"docs_xr5610": "https://www.dell.com/support/product-details/en-us/product/poweredge-xr5610/docs",
|
||
"support_home": "https://www.dell.com/support/home",
|
||
}
|
||
|
||
|
||
# 14G dual-socket channel map (R640/R740/R740xd family)
|
||
_CH14 = {
|
||
"A": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5},
|
||
"B": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5},
|
||
}
|
||
|
||
|
||
# Dell datacenter rack catalog — visual U heights + face styles for rack designer
|
||
DELL_RACK_CATALOG: list[dict[str, Any]] = [
|
||
# --- PowerEdge 1U ---
|
||
{"sku": "PE-R440", "name": "PowerEdge R440", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r440"]},
|
||
{"sku": "PE-R640", "name": "PowerEdge R640", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r640"]},
|
||
{"sku": "PE-R650", "name": "PowerEdge R650", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650"]},
|
||
{"sku": "PE-R650xs", "name": "PowerEdge R650xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650xs"]},
|
||
{"sku": "PE-R660", "name": "PowerEdge R660", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660"]},
|
||
{"sku": "PE-R660xs", "name": "PowerEdge R660xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660xs"]},
|
||
{"sku": "PE-R6615", "name": "PowerEdge R6615", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6615"]},
|
||
{"sku": "PE-R6415", "name": "PowerEdge R6415", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6415"]},
|
||
{"sku": "PE-R6515", "name": "PowerEdge R6515", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6515"]},
|
||
{"sku": "PE-R6525", "name": "PowerEdge R6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6525"]},
|
||
{"sku": "PE-R6715", "name": "PowerEdge R6715", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6715"]},
|
||
{"sku": "PE-R670", "name": "PowerEdge R670", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r670"]},
|
||
{"sku": "PE-XR5610", "name": "PowerEdge XR5610", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["xr5610"]},
|
||
{"sku": "PE-C6520", "name": "PowerEdge C6520", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["c6520"]},
|
||
{"sku": "PE-C6525", "name": "PowerEdge C6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c6525"]},
|
||
{"sku": "PE-C4140", "name": "PowerEdge C4140", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c4140"]},
|
||
# --- PowerEdge 2U ---
|
||
{"sku": "PE-R740", "name": "PowerEdge R740", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740"]},
|
||
{"sku": "PE-R740xd", "name": "PowerEdge R740xd", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd"]},
|
||
{"sku": "PE-R740xd2", "name": "PowerEdge R740xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd2"]},
|
||
{"sku": "PE-R7425", "name": "PowerEdge R7425", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7425"]},
|
||
{"sku": "PE-R750", "name": "PowerEdge R750", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750"]},
|
||
{"sku": "PE-R750xa", "name": "PowerEdge R750xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750xa"]},
|
||
{"sku": "PE-R7515", "name": "PowerEdge R7515", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7515"]},
|
||
{"sku": "PE-R7525", "name": "PowerEdge R7525", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7525"]},
|
||
{"sku": "PE-R760", "name": "PowerEdge R760", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760"]},
|
||
{"sku": "PE-R760xd2", "name": "PowerEdge R760xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xd2"]},
|
||
{"sku": "PE-R760xa", "name": "PowerEdge R760xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xa"]},
|
||
{"sku": "PE-R770", "name": "PowerEdge R770", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r770"]},
|
||
{"sku": "PE-R840", "name": "PowerEdge R840", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r840"]},
|
||
{"sku": "PE-R940", "name": "PowerEdge R940", "family": "PowerEdge", "category": "server", "u_height": 3, "face": "server_4u", "match": ["r940"]},
|
||
{"sku": "PE-R940xa", "name": "PowerEdge R940xa", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["r940xa"]},
|
||
{"sku": "PE-R340", "name": "PowerEdge R340", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r340"]},
|
||
{"sku": "PE-R450", "name": "PowerEdge R450", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r450"]},
|
||
{"sku": "PE-R550", "name": "PowerEdge R550", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r550"]},
|
||
{"sku": "PE-T640", "name": "PowerEdge T640", "family": "PowerEdge", "category": "server", "u_height": 5, "face": "server_4u", "match": ["t640"]},
|
||
{"sku": "PE-DSS8440", "name": "DSS8440", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["dss8440"]},
|
||
{"sku": "PE-XE9680", "name": "PowerEdge XE9680", "family": "PowerEdge", "category": "server", "u_height": 6, "face": "server_4u", "match": ["xe9680"]},
|
||
{"sku": "PE-XE8640", "name": "PowerEdge XE8640", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["xe8640"]},
|
||
# --- MX / modular ---
|
||
{"sku": "PE-MX7000", "name": "PowerEdge MX7000", "family": "PowerEdge MX", "category": "chassis", "u_height": 7, "face": "chassis_mx", "match": ["mx7000"]},
|
||
{"sku": "PE-MX740c", "name": "PowerEdge MX740c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx740c"]},
|
||
{"sku": "PE-MX750c", "name": "PowerEdge MX750c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx750c"]},
|
||
{"sku": "PE-M640", "name": "PowerEdge M640", "family": "PowerEdge", "category": "blade", "u_height": 0, "face": "blade", "match": ["m640"]},
|
||
# --- Networking / PowerSwitch / Force10 ---
|
||
{"sku": "PS-S4048", "name": "PowerSwitch S4048-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4048", "force10"]},
|
||
{"sku": "PS-S4148", "name": "PowerSwitch S4148-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4148"]},
|
||
{"sku": "PS-S5248", "name": "PowerSwitch S5248F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5248"]},
|
||
{"sku": "PS-S5296", "name": "PowerSwitch S5296F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5296"]},
|
||
{"sku": "PS-N3248", "name": "PowerSwitch N3248TE-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3248"]},
|
||
{"sku": "PS-Z9264", "name": "PowerSwitch Z9264F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 2, "face": "switch_2u", "match": ["z9264"]},
|
||
{"sku": "PS-Z9432", "name": "PowerSwitch Z9432F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9432"]},
|
||
{"sku": "PC-5548", "name": "PowerConnect 5548", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 5548", "pc5548"]},
|
||
{"sku": "PC-6248", "name": "PowerConnect 6248", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 6248", "pc6248"]},
|
||
{"sku": "PC-7048", "name": "PowerConnect 7048", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7048", "pc7048"]},
|
||
{"sku": "PC-8024", "name": "PowerConnect 8024F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8024", "pc8024"]},
|
||
{"sku": "PC-8132", "name": "PowerConnect 8132F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8132", "pc8132"]},
|
||
{"sku": "MX-9116n", "name": "MX9116n Fabric Switching Engine", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx9116"]},
|
||
{"sku": "MX-5108n", "name": "MX5108n Ethernet Switch", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx5108"]},
|
||
# --- Storage ---
|
||
{"sku": "ME-4012", "name": "PowerVault ME4012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4012"]},
|
||
{"sku": "ME-4024", "name": "PowerVault ME4024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4024"]},
|
||
{"sku": "ME-5012", "name": "PowerVault ME5012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5012"]},
|
||
{"sku": "ME-5024", "name": "PowerVault ME5024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5024"]},
|
||
{"sku": "SC-AllFlash", "name": "SC Series Array", "family": "PowerStore/SC", "category": "storage", "u_height": 3, "face": "storage_3u", "match": ["sc "]},
|
||
{"sku": "PST-5000T", "name": "PowerStore 5000T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 5000"]},
|
||
{"sku": "PST-1200T", "name": "PowerStore 1200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 1200"]},
|
||
{"sku": "PST-3200T", "name": "PowerStore 3200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 3200"]},
|
||
{"sku": "DD-6900", "name": "PowerProtect DD6900", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd6900"]},
|
||
{"sku": "DD-9400", "name": "PowerProtect DD9400", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd9400"]},
|
||
{"sku": "UNITY-380", "name": "Unity XT 380", "family": "Unity", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["unity"]},
|
||
# --- VxRail appliances ---
|
||
{"sku": "VX-E460F", "name": "VxRail E460F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["e460f", "vxrail e"]},
|
||
{"sku": "VX-V470F", "name": "VxRail V470F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["v470f"]},
|
||
{"sku": "VX-P570F", "name": "VxRail P570F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["p570f"]},
|
||
{"sku": "VX-V670F", "name": "VxRail V670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["v670f"]},
|
||
{"sku": "VX-P670F", "name": "VxRail P670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["p670f"]},
|
||
{"sku": "VX-S570", "name": "VxRail S570", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["s570"]},
|
||
# --- Networking extras ---
|
||
{"sku": "PS-S5232", "name": "PowerSwitch S5232F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5232"]},
|
||
{"sku": "PS-S5212", "name": "PowerSwitch S5212F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5212"]},
|
||
{"sku": "PS-N3224", "name": "PowerSwitch N3224T-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3224"]},
|
||
{"sku": "PS-Z9100", "name": "PowerSwitch Z9100-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9100"]},
|
||
{"sku": "PC-7024", "name": "PowerConnect 7024", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7024"]},
|
||
{"sku": "PC-8100", "name": "PowerConnect 8100", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8100"]},
|
||
# --- PDU / misc ---
|
||
{"sku": "PDU-Geist-0U", "name": "Geist Rack PDU (0U vertical)", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["geist", "mg03", "mgs3"]},
|
||
{"sku": "PDU-1U", "name": "Rack PDU 1U horizontal", "family": "PDU", "category": "pdu", "u_height": 1, "face": "pdu_1u", "match": []},
|
||
{"sku": "PDU-APC-0U", "name": "APC Rack PDU 0U", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["apc"]},
|
||
{"sku": "PANEL-1U", "name": "Blanking panel 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []},
|
||
{"sku": "PANEL-2U", "name": "Blanking panel 2U", "family": "Accessory", "category": "blank", "u_height": 2, "face": "blank_2u", "match": []},
|
||
{"sku": "CABLE-MGMT-1U", "name": "Cable management arm 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []},
|
||
{"sku": "KVM-1U", "name": "Rack KVM 1U", "family": "Accessory", "category": "custom", "u_height": 1, "face": "custom", "match": ["kvm"]},
|
||
{"sku": "CUSTOM", "name": "Custom device", "family": "Custom", "category": "custom", "u_height": 1, "face": "custom", "match": []},
|
||
]
|
||
|
||
|
||
FACE_IMAGE_MAP: dict[str, str] = {
|
||
"server_1u": "/assets/dell/face-server-1u.jpg?v=bezel2",
|
||
"server_2u": "/assets/dell/face-server-2u.jpg?v=bezel2",
|
||
"server_4u": "/assets/dell/face-server-4u.jpg?v=bezel2",
|
||
"switch_1u": "/assets/dell/face-switch-1u.jpg?v=bezel2",
|
||
"switch_2u": "/assets/dell/face-switch-1u.jpg?v=bezel2",
|
||
"storage_2u": "/assets/dell/face-storage-2u.jpg?v=bezel2",
|
||
"storage_3u": "/assets/dell/face-storage-2u.jpg?v=bezel2",
|
||
"chassis_mx": "/assets/dell/face-chassis-mx.jpg?v=bezel2",
|
||
"pdu_1u": "/assets/dell/face-pdu-1u.jpg?v=bezel2",
|
||
"pdu_0u": "/assets/dell/face-pdu-1u.jpg?v=bezel2",
|
||
"blank_1u": "/assets/dell/face-blank-1u.jpg?v=bezel2",
|
||
"blank_2u": "/assets/dell/face-blank-1u.jpg?v=bezel2",
|
||
"blade": "/assets/dell/face-chassis-mx.jpg?v=bezel2",
|
||
"fabric": "/assets/dell/face-switch-1u.jpg?v=bezel2",
|
||
"custom": "/assets/dell/face-server-1u.jpg?v=bezel2",
|
||
}
|
||
|
||
# Official Dell product shots by SKU (processed to dark background to match faces)
|
||
SKU_IMAGE_MAP: dict[str, str] = {
|
||
# Prefer shared dark face art for visual consistency across all PowerEdge 1U/2U
|
||
}
|
||
|
||
|
||
def catalog_image_for(item: dict) -> str:
|
||
sku = item.get("sku") or ""
|
||
if sku in SKU_IMAGE_MAP:
|
||
return SKU_IMAGE_MAP[sku]
|
||
face = item.get("face") or "custom"
|
||
return FACE_IMAGE_MAP.get(face, FACE_IMAGE_MAP["custom"])
|
||
|
||
|
||
def enrich_catalog_images() -> None:
|
||
for item in DELL_RACK_CATALOG:
|
||
item["image"] = catalog_image_for(item)
|
||
|
||
|
||
enrich_catalog_images()
|
||
|
||
|
||
def match_rack_catalog(model: str | None, role: str | None = None) -> dict | None:
|
||
m = (model or "").lower()
|
||
best = None
|
||
for item in DELL_RACK_CATALOG:
|
||
for token in item.get("match") or []:
|
||
if token and token in m:
|
||
best = item
|
||
break
|
||
if best:
|
||
break
|
||
if best:
|
||
return best
|
||
if role == "switch":
|
||
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PS-S4048"), None)
|
||
if role == "storage":
|
||
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "ME-4024"), None)
|
||
if role == "chassis":
|
||
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-MX7000"), None)
|
||
if role == "pdu":
|
||
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PDU-Geist-0U"), None)
|
||
if role == "server":
|
||
return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-R740"), None)
|
||
return None
|
||
|
||
MODEL_EXPANSION: dict[str, dict[str, Any]] = {
|
||
"PowerEdge R740xd": {
|
||
"generation": "14G",
|
||
"dimm_slots": 24,
|
||
"dimms_per_cpu": 12,
|
||
"channels_per_cpu": 6,
|
||
"max_memory_tb": 3.0,
|
||
"max_memory_note": "Up to 3 TB with LRDIMM (+ DCPMM configs); typical RDIMM max lower per CPU SKU",
|
||
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 8,
|
||
"pcie_note": "Up to 8× PCIe 3.0 via riser options; slot count depends on riser kit + CPU count",
|
||
"drive_bays_max": 24,
|
||
"drive_note": "Chassis-dependent: up to 24× 2.5\" (or mixed 3.5\"/NVMe mid-tray options on xd)",
|
||
"cpu_sockets": 2,
|
||
"best_practices": [
|
||
"Populate identical DIMMs across all six channels per CPU (6 or 12 DIMMs/CPU) for best bandwidth.",
|
||
"White release-tab sockets first, then black (channel rank 1 then rank 2).",
|
||
"Do not mix RDIMM and LRDIMM in the same system.",
|
||
"Unbalanced channel population reduces memory bandwidth / can lower effective speed.",
|
||
"Performance Optimized (4 DIMMs/CPU): slots 1,2,4,5 — (8 DIMMs/CPU): 1,2,4,5,7,8,10,11 (Dell KB 000141539).",
|
||
],
|
||
"performance_notes": [
|
||
"Peak memory bandwidth needs one matched DIMM in every channel (6 per CPU) before adding the second rank (slots 7–12).",
|
||
"Mixing capacities: place largest DIMMs in the first populated sockets of each channel.",
|
||
"DCPMM: up to 6 PMem + 6 DRAM per CPU; full 12-slot population recommended for App Direct / Memory Mode.",
|
||
],
|
||
"docs": [
|
||
{"title": "R740xd Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740xd"], "kind": "spec"},
|
||
{"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"},
|
||
{"title": "R740xd Owner's Manual (PDF)", "url": _DELL_LINKS["own_r740xd"], "kind": "manual"},
|
||
{"title": "Dell Support · R740xd docs", "url": _DELL_LINKS["docs_r740xd"], "kind": "support"},
|
||
],
|
||
"channel_map": _CH14,
|
||
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
|
||
},
|
||
"PowerEdge R740": {
|
||
"generation": "14G",
|
||
"dimm_slots": 24,
|
||
"dimms_per_cpu": 12,
|
||
"channels_per_cpu": 6,
|
||
"max_memory_tb": 3.0,
|
||
"max_memory_note": "Up to 3 TB with LRDIMM/DCPMM depending on CPU",
|
||
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 8,
|
||
"pcie_note": "Up to 8× PCIe 3.0 (riser dependent)",
|
||
"drive_bays_max": 16,
|
||
"drive_note": "Up to 16× 2.5\" or 8× 3.5\" depending on chassis kit",
|
||
"cpu_sockets": 2,
|
||
"best_practices": [
|
||
"Same 14G rules as R740xd: balance channels; white tabs before black.",
|
||
"Performance Optimized 4/8 DIMM/CPU populations per Dell KB 000141539.",
|
||
],
|
||
"performance_notes": [
|
||
"Populate 6 identical DIMMs per CPU before doubling up channels for max throughput.",
|
||
],
|
||
"docs": [
|
||
{"title": "R740 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740"], "kind": "spec"},
|
||
{"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"},
|
||
{"title": "Dell Support · R740 docs", "url": _DELL_LINKS["docs_r740"], "kind": "support"},
|
||
],
|
||
"channel_map": _CH14,
|
||
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
|
||
},
|
||
"PowerEdge R640": {
|
||
"generation": "14G",
|
||
"dimm_slots": 24,
|
||
"dimms_per_cpu": 12,
|
||
"channels_per_cpu": 6,
|
||
"max_memory_tb": 3.0,
|
||
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "DCPMM/PMem"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 3,
|
||
"pcie_note": "1U: up to 3× PCIe 3.0 (+ rNDC)",
|
||
"drive_bays_max": 10,
|
||
"drive_note": "Up to 10× 2.5\" (NVMe options vary)",
|
||
"cpu_sockets": 2,
|
||
"best_practices": [
|
||
"14G dual-socket population: identical DIMMs across 6 channels per CPU.",
|
||
"White-tab sockets first; avoid RDIMM+LRDIMM mix.",
|
||
],
|
||
"performance_notes": [
|
||
"1U thermal/airflow limits denser GPU/NVMe kits — verify riser/backplane BOM.",
|
||
],
|
||
"docs": [
|
||
{"title": "R640 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r640"], "kind": "spec"},
|
||
{"title": "R640 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r640"], "kind": "manual"},
|
||
{"title": "Dell Support · R640 docs", "url": _DELL_LINKS["docs_r640"], "kind": "support"},
|
||
],
|
||
"channel_map": _CH14,
|
||
"slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)],
|
||
},
|
||
"PowerEdge R440": {
|
||
"generation": "14G",
|
||
"dimm_slots": 16,
|
||
"dimms_per_cpu": 8,
|
||
"channels_per_cpu": 6,
|
||
"max_memory_tb": 0.512,
|
||
"memory_types": ["DDR4 RDIMM"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 3,
|
||
"drive_bays_max": 10,
|
||
"cpu_sockets": 2,
|
||
"best_practices": ["Populate channels evenly; match DIMM size/speed/rank per Dell 14G guidance."],
|
||
"performance_notes": ["Entry 14G — keep DIMM speeds matched to CPU QPI/UPI memory controller limits."],
|
||
"docs": [
|
||
{"title": "R440 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r440"], "kind": "spec"},
|
||
{"title": "Dell Support · R440 docs", "url": _DELL_LINKS["docs_r440"], "kind": "support"},
|
||
],
|
||
"slot_labels": [f"A{i}" for i in range(1, 9)] + [f"B{i}" for i in range(1, 9)],
|
||
},
|
||
"PowerEdge R6415": {
|
||
"generation": "14G-AMD",
|
||
"dimm_slots": 16,
|
||
"dimms_per_cpu": 16,
|
||
"channels_per_cpu": 8,
|
||
"max_memory_tb": 1.0,
|
||
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 3,
|
||
"drive_bays_max": 10,
|
||
"cpu_sockets": 1,
|
||
"best_practices": [
|
||
"AMD EPYC Naples/Rome 14G: for 4 DIMMs use slots 1,3,5,7 (Dell KB 000141539).",
|
||
"Populate evenly across memory channels for Infinity Fabric bandwidth.",
|
||
],
|
||
"performance_notes": [
|
||
"EPYC memory performance scales strongly with channels populated — prefer 8 identical DIMMs when possible.",
|
||
],
|
||
"docs": [
|
||
{"title": "R6415 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r6415"], "kind": "spec"},
|
||
{"title": "R6415 Tech Specs / Reference (PDF)", "url": _DELL_LINKS["ref_r6415"], "kind": "tech"},
|
||
{"title": "R6415 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r6415"], "kind": "manual"},
|
||
{"title": "Dell Support · R6415 docs", "url": _DELL_LINKS["docs_r6415"], "kind": "support"},
|
||
],
|
||
"slot_labels": [f"A{i}" for i in range(1, 17)],
|
||
},
|
||
"PowerEdge R7425": {
|
||
"generation": "14G-AMD",
|
||
"dimm_slots": 32,
|
||
"dimms_per_cpu": 16,
|
||
"channels_per_cpu": 8,
|
||
"max_memory_tb": 2.0,
|
||
"memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"],
|
||
"max_dimm_speed_mts": 2666,
|
||
"pcie_slots_max": 8,
|
||
"drive_bays_max": 24,
|
||
"cpu_sockets": 2,
|
||
"best_practices": [
|
||
"Dual EPYC: mirror CPU1 and CPU2 DIMM populations.",
|
||
"4 DIMMs/CPU → slots 1,3,5,7 per Dell AMD 14G guidance.",
|
||
],
|
||
"performance_notes": ["Balance both sockets identically to avoid NUMA imbalance."],
|
||
"docs": [
|
||
{"title": "R7425 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r7425"], "kind": "spec"},
|
||
{"title": "R7425 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r7425"], "kind": "manual"},
|
||
{"title": "Dell Support · R7425 docs", "url": _DELL_LINKS["docs_r7425"], "kind": "support"},
|
||
],
|
||
"slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)],
|
||
},
|
||
"PowerEdge R660": {
|
||
"generation": "16G",
|
||
"dimm_slots": 32,
|
||
"dimms_per_cpu": 16,
|
||
"channels_per_cpu": 8,
|
||
"max_memory_tb": 8.0,
|
||
"memory_types": ["DDR5 RDIMM"],
|
||
"max_dimm_speed_mts": 4800,
|
||
"pcie_slots_max": 3,
|
||
"drive_bays_max": 10,
|
||
"cpu_sockets": 2,
|
||
"best_practices": [
|
||
"16G DDR5: populate all memory channels evenly; follow white/black tab order in the ISM for the SKU.",
|
||
"Use matched DDR5 RDIMM kits; mixing ranks/speeds can downclock the entire domain.",
|
||
],
|
||
"performance_notes": [
|
||
"Sapphire Rapids / Emerald Rapids memory bandwidth benefits strongly from full channel population.",
|
||
],
|
||
"docs": [
|
||
{"title": "R660 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r660"], "kind": "spec"},
|
||
{"title": "R660 Technical Guide (PDF)", "url": _DELL_LINKS["tech_r660"], "kind": "tech"},
|
||
{"title": "Dell Support · R660 docs", "url": _DELL_LINKS["docs_r660"], "kind": "support"},
|
||
],
|
||
"slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)],
|
||
},
|
||
"PowerEdge XR5610": {
|
||
"generation": "16G-XR",
|
||
"dimm_slots": 8,
|
||
"dimms_per_cpu": 8,
|
||
"channels_per_cpu": 8,
|
||
"max_memory_tb": 1.0,
|
||
"max_memory_note": "Dell Spec Sheet: 8× DDR5 RDIMM, up to 1 TB, up to 5600 MT/s (CPU/SKU dependent)",
|
||
"memory_types": ["DDR5 RDIMM"],
|
||
"max_dimm_speed_mts": 5600,
|
||
"pcie_slots_max": 2,
|
||
"pcie_note": "Spec Sheet: up to 2× PCIe Gen5 x16 (+ OCP 3.0) — riser/GPU option dependent",
|
||
"drive_bays_max": 4,
|
||
"drive_note": "Spec Sheet: up to 4× 2.5\" SAS/SATA/NVMe front; BOSS-N1 M.2 is separate boot — not front-bay count",
|
||
"cpu_sockets": 1,
|
||
"best_practices": [
|
||
"Populate DDR5 channels evenly with matched RDIMMs.",
|
||
"Confirm exact DIMM map in the XR5610 Technical Guide / ISM for this BOM.",
|
||
],
|
||
"performance_notes": [
|
||
"Single-socket edge platform: memory bandwidth scales with channels populated.",
|
||
],
|
||
"docs": [
|
||
{"title": "XR5610 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_xr5610"], "kind": "spec"},
|
||
{"title": "XR5610 Technical Guide (PDF)", "url": _DELL_LINKS["tech_xr5610"], "kind": "tech"},
|
||
{"title": "Dell Support · XR5610 docs", "url": _DELL_LINKS["docs_xr5610"], "kind": "support"},
|
||
],
|
||
"slot_labels": [f"A{i}" for i in range(1, 9)],
|
||
},
|
||
}
|
||
|
||
|
||
def _normalize_model_key(model: str | None) -> str:
|
||
m = (model or "").strip()
|
||
if not m:
|
||
return ""
|
||
if m in MODEL_EXPANSION:
|
||
return m
|
||
# fuzzy family match
|
||
for key in MODEL_EXPANSION:
|
||
if key.lower() in m.lower() or m.lower() in key.lower():
|
||
return key
|
||
# strip suffixes like "vSAN Ready Node"
|
||
for key in MODEL_EXPANSION:
|
||
short = key.replace("PowerEdge ", "")
|
||
if short.lower() in m.lower():
|
||
return key
|
||
return m
|
||
|
||
|
||
def _parse_dimm_socket(name: str | None) -> tuple[str | None, int | None]:
|
||
import re
|
||
|
||
s = name or ""
|
||
m = re.search(r"(?:DIMM\.Socket\.|DIMM\s*)([A-D])(\d{1,2})", s, re.I)
|
||
if not m:
|
||
m = re.search(r"\b([A-D])(\d{1,2})\b", s)
|
||
if not m:
|
||
return None, None
|
||
return m.group(1).upper(), int(m.group(2))
|
||
|
||
|
||
def _disk_size_gb(raw: Any) -> float | None:
|
||
"""OME serverArrayDisks.Size is already capacity in GB (e.g. '446.62')."""
|
||
if raw is None or raw == "":
|
||
return None
|
||
try:
|
||
return round(float(str(raw).strip()), 2)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _mem_size_gb(raw: Any) -> float | None:
|
||
"""OME serverMemoryDevices.Size is capacity in MB."""
|
||
if raw is None or raw == "":
|
||
return None
|
||
try:
|
||
return round(float(raw) / 1024.0, 1)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def build_expansion_report(model: str | None, inventory: dict) -> dict:
|
||
"""Split live OME facts (current) from Dell catalog max expansion (recommendations)."""
|
||
key = _normalize_model_key(model)
|
||
catalog_known = bool(MODEL_EXPANSION.get(key))
|
||
spec = dict(MODEL_EXPANSION.get(key) or {})
|
||
if not catalog_known:
|
||
spec = {
|
||
"generation": None,
|
||
"dimm_slots": None,
|
||
"max_memory_tb": None,
|
||
"pcie_slots_max": None,
|
||
"drive_bays_max": None,
|
||
"cpu_sockets": None,
|
||
"best_practices": [
|
||
"No Cockpit catalog entry for this model — verify DIMM/CPU/drive maxima in the Dell Tech Specs / ISM for the exact Service Tag BOM.",
|
||
],
|
||
"performance_notes": [],
|
||
"docs": [{"title": "Dell Support home", "url": _DELL_LINKS["support_home"]}],
|
||
"slot_labels": [],
|
||
"channel_map": {},
|
||
"memory_types": [],
|
||
"max_memory_note": "Maximum not asserted — catalog unknown for this model.",
|
||
"drive_note": "Bay maximum not asserted — catalog unknown for this model.",
|
||
"pcie_note": "PCIe maximum not asserted — catalog unknown for this model.",
|
||
}
|
||
|
||
mem_raw = inventory.get("serverMemoryDevices") or []
|
||
cpu_raw = inventory.get("serverProcessors") or []
|
||
disk_raw = inventory.get("serverArrayDisks") or []
|
||
cards_raw = inventory.get("serverDeviceCards") or []
|
||
raid_raw = inventory.get("serverRaidControllers") or []
|
||
os_raw = inventory.get("serverOperatingSystems") or []
|
||
|
||
dimm_modules: list[dict] = []
|
||
for dimm in mem_raw:
|
||
bank, num = _parse_dimm_socket(
|
||
dimm.get("Name") or dimm.get("InstanceId") or dimm.get("DeviceDescription")
|
||
)
|
||
if not bank or not num:
|
||
continue
|
||
label = f"{bank}{num}"
|
||
size_gb = _mem_size_gb(dimm.get("Size"))
|
||
dimm_modules.append(
|
||
{
|
||
"slot": label,
|
||
"cpu_bank": bank,
|
||
"slot_num": num,
|
||
"size_gb": size_gb,
|
||
"size_mb": dimm.get("Size"),
|
||
"rated_mts": dimm.get("Speed"),
|
||
"operating_mts": dimm.get("CurrentOperatingSpeed"),
|
||
"rank": dimm.get("Rank"),
|
||
"type": dimm.get("TypeDetails") or dimm.get("MemoryType") or "DDR",
|
||
"manufacturer": dimm.get("Manufacturer"),
|
||
"part_number": dimm.get("PartNumber"),
|
||
"serial": dimm.get("SerialNumber"),
|
||
"status": dimm.get("Status"),
|
||
}
|
||
)
|
||
dimm_modules.sort(key=lambda s: (s.get("cpu_bank") or "", s.get("slot_num") or 0))
|
||
mem_total_gb = round(sum(s.get("size_gb") or 0 for s in dimm_modules), 1)
|
||
rated_speeds = sorted({s["rated_mts"] for s in dimm_modules if s.get("rated_mts") is not None})
|
||
op_speeds = sorted({s["operating_mts"] for s in dimm_modules if s.get("operating_mts") is not None})
|
||
|
||
cpus: list[dict] = []
|
||
for p in cpu_raw:
|
||
cpus.append(
|
||
{
|
||
"slot": p.get("SlotNumber") or p.get("InstanceId"),
|
||
"model": p.get("ModelName") or p.get("BrandName"),
|
||
"cores": p.get("NumberOfEnabledCores") or p.get("NumberOfCores"),
|
||
"threads": p.get("NumberOfEnabledThreads") or p.get("NumberOfThreads"),
|
||
"current_mhz": p.get("CurrentSpeed"),
|
||
"max_mhz": p.get("MaxSpeed"),
|
||
"status": p.get("Status"),
|
||
}
|
||
)
|
||
cpu_cores_total = sum(int(c.get("cores") or 0) for c in cpus)
|
||
|
||
disks: list[dict] = []
|
||
for d in disk_raw:
|
||
disks.append(
|
||
{
|
||
"bay": d.get("SlotNumber"),
|
||
"model": d.get("ModelNumber"),
|
||
"vendor": d.get("VendorName"),
|
||
"media": d.get("MediaType"),
|
||
"size_gb": _disk_size_gb(d.get("Size")),
|
||
"bus": d.get("BusType"),
|
||
"status": d.get("StatusString") or d.get("Status"),
|
||
"serial": d.get("SerialNumber"),
|
||
"enclosure": d.get("EnclosureId"),
|
||
"form_factor": d.get("FormFactor"),
|
||
}
|
||
)
|
||
disks.sort(key=lambda x: (str(x.get("bus") or ""), x.get("bay") is None, x.get("bay") or 0))
|
||
disk_total_gb = round(sum(d.get("size_gb") or 0 for d in disks), 2)
|
||
|
||
adapters: list[dict] = []
|
||
for c in cards_raw:
|
||
adapters.append(
|
||
{
|
||
"slot": str(c.get("SlotNumber") or ""),
|
||
"description": c.get("Description"),
|
||
"manufacturer": c.get("Manufacturer"),
|
||
"slot_type": c.get("SlotType"),
|
||
"width": c.get("DatabusWidth"),
|
||
}
|
||
)
|
||
for r in raid_raw:
|
||
adapters.append(
|
||
{
|
||
"slot": str(r.get("PciSlot") or r.get("Fqdd") or "RAID"),
|
||
"description": r.get("Name") or r.get("DeviceDescription"),
|
||
"manufacturer": "Dell PERC",
|
||
"slot_type": "RAID",
|
||
"firmware": r.get("FirmwareVersion"),
|
||
"cache_mb": r.get("CacheSizeInMb"),
|
||
}
|
||
)
|
||
|
||
os_info = []
|
||
for o in os_raw:
|
||
os_info.append(
|
||
{
|
||
"os_name": (o.get("OsName") or "").strip() or None,
|
||
"os_version": (o.get("OsVersion") or "").strip() or None,
|
||
"hostname": (o.get("Hostname") or "").strip() or None,
|
||
}
|
||
)
|
||
|
||
current = {
|
||
"source": "OME DeviceService inventory (live)",
|
||
"memory": {
|
||
"installed_gb": mem_total_gb,
|
||
"dimm_count": len(dimm_modules),
|
||
"modules": dimm_modules,
|
||
"rated_mts": rated_speeds,
|
||
"operating_mts": op_speeds,
|
||
"banks_seen": sorted({m["cpu_bank"] for m in dimm_modules if m.get("cpu_bank")}),
|
||
},
|
||
"cpu": {
|
||
"sockets_populated": len(cpus),
|
||
"cores_total": cpu_cores_total,
|
||
"processors": cpus,
|
||
},
|
||
"disks": {
|
||
"count": len(disks),
|
||
"capacity_gb": disk_total_gb,
|
||
"items": disks,
|
||
},
|
||
"adapters": adapters,
|
||
"os": os_info,
|
||
}
|
||
|
||
populated_map = {m["slot"]: m for m in dimm_modules}
|
||
labels = list(spec.get("slot_labels") or [])
|
||
slot_map: list[dict] = []
|
||
for label in labels:
|
||
bank = label[0]
|
||
num = int(label[1:] or 0)
|
||
ch_map = (spec.get("channel_map") or {}).get(bank) or {}
|
||
if label in populated_map:
|
||
row = dict(populated_map[label])
|
||
row["populated"] = True
|
||
row["channel"] = ch_map.get(num)
|
||
row["rank_color"] = "white" if num <= 6 else "black"
|
||
row["speed_mts"] = row.get("operating_mts") or row.get("rated_mts")
|
||
slot_map.append(row)
|
||
else:
|
||
slot_map.append(
|
||
{
|
||
"slot": label,
|
||
"cpu_bank": bank,
|
||
"slot_num": num,
|
||
"channel": ch_map.get(num),
|
||
"rank_color": "white" if num <= 6 else "black",
|
||
"populated": False,
|
||
}
|
||
)
|
||
for label, row in populated_map.items():
|
||
if label not in labels:
|
||
extra = dict(row)
|
||
extra["populated"] = True
|
||
extra["speed_mts"] = extra.get("operating_mts") or extra.get("rated_mts")
|
||
slot_map.append(extra)
|
||
|
||
empty_n = sum(1 for s in slot_map if not s.get("populated")) if labels else None
|
||
dimm_slots_catalog = spec.get("dimm_slots")
|
||
max_tb = spec.get("max_memory_tb")
|
||
headroom_gb = None
|
||
if max_tb is not None:
|
||
try:
|
||
headroom_gb = round(float(max_tb) * 1024 - mem_total_gb, 1)
|
||
except Exception:
|
||
headroom_gb = None
|
||
|
||
drive_max = spec.get("drive_bays_max")
|
||
free_bays = (int(drive_max) - len(disks)) if catalog_known and isinstance(drive_max, int) else None
|
||
cpu_max = spec.get("cpu_sockets")
|
||
free_cpu = (int(cpu_max) - len(cpus)) if catalog_known and isinstance(cpu_max, int) else None
|
||
|
||
findings: list[dict] = []
|
||
if catalog_known and labels:
|
||
by_bank: dict[str, list] = {}
|
||
for s in slot_map:
|
||
by_bank.setdefault(s.get("cpu_bank") or "?", []).append(s)
|
||
for bank, bank_slots in by_bank.items():
|
||
ch_counts: dict[int, int] = {}
|
||
for s in bank_slots:
|
||
if s.get("populated") and s.get("channel") is not None:
|
||
ch_counts[int(s["channel"])] = ch_counts.get(int(s["channel"]), 0) + 1
|
||
if ch_counts and spec.get("channels_per_cpu"):
|
||
vals = list(ch_counts.values())
|
||
if max(vals) - min(vals) > 0 or len(ch_counts) < int(spec["channels_per_cpu"]):
|
||
findings.append(
|
||
{
|
||
"severity": "warning",
|
||
"code": "unbalanced_channels",
|
||
"text": f"Bank {bank}: channel population {dict(sorted(ch_counts.items()))} — Dell guidance prefers even channel fill for bandwidth.",
|
||
}
|
||
)
|
||
if len(op_speeds) > 1:
|
||
findings.append(
|
||
{
|
||
"severity": "warning",
|
||
"code": "mixed_operating_speeds",
|
||
"text": f"OME reports mixed operating speeds {op_speeds} MT/s — controller usually clocks all DIMMs to the lowest.",
|
||
}
|
||
)
|
||
if rated_speeds and op_speeds and max(op_speeds) < max(rated_speeds):
|
||
findings.append(
|
||
{
|
||
"severity": "info",
|
||
"code": "downclocked",
|
||
"text": f"DIMMs rated {rated_speeds} MT/s, operating at {op_speeds} MT/s (CPU/population limited).",
|
||
}
|
||
)
|
||
|
||
expansion = {
|
||
"source": "Dell model catalog in Cockpit (not live measurement)" if catalog_known else "Catalog unknown",
|
||
"catalog_known": catalog_known,
|
||
"catalog_model": key if catalog_known else None,
|
||
"generation": spec.get("generation"),
|
||
"memory": {
|
||
"max_capacity_tb": max_tb,
|
||
"max_capacity_gb": round(float(max_tb) * 1024, 1) if max_tb is not None else None,
|
||
"dimm_slots_max": dimm_slots_catalog,
|
||
"dimms_installed": len(dimm_modules),
|
||
"empty_slots_estimate": empty_n if catalog_known else None,
|
||
"headroom_gb": headroom_gb if catalog_known else None,
|
||
"supported_types": spec.get("memory_types") or [],
|
||
"max_rated_mts": spec.get("max_dimm_speed_mts"),
|
||
"note": spec.get("max_memory_note")
|
||
or (
|
||
f"Catalog max {max_tb} TB / {dimm_slots_catalog} DIMM slots"
|
||
if catalog_known
|
||
else "No catalog max — do not treat empty map as measured free slots."
|
||
),
|
||
"slot_map": slot_map if catalog_known else [],
|
||
},
|
||
"cpu": {
|
||
"sockets_max": cpu_max,
|
||
"sockets_populated": len(cpus),
|
||
"sockets_free_estimate": free_cpu,
|
||
"note": (
|
||
f"Chassis supports up to {cpu_max} CPU socket(s); OME sees {len(cpus)} populated."
|
||
if catalog_known and cpu_max is not None
|
||
else "CPU socket maximum not in Cockpit catalog for this model."
|
||
),
|
||
},
|
||
"disks": {
|
||
"bays_max": drive_max if catalog_known else None,
|
||
"drives_installed": len(disks),
|
||
"bays_free_estimate": free_bays,
|
||
"note": spec.get("drive_note")
|
||
or (
|
||
"Bay maximum from Dell chassis options — verify BOM; M.2/BOSS may not count as front bays."
|
||
if catalog_known
|
||
else "Drive bay maximum unknown for this model."
|
||
),
|
||
},
|
||
"pcie": {
|
||
"slots_max": spec.get("pcie_slots_max") if catalog_known else None,
|
||
"note": spec.get("pcie_note")
|
||
or ("PCIe max depends on riser kit." if catalog_known else "PCIe maximum unknown for this model."),
|
||
},
|
||
"best_practices": spec.get("best_practices") or [],
|
||
"performance_notes": spec.get("performance_notes") or [],
|
||
"docs": spec.get("docs") or [],
|
||
}
|
||
|
||
return {
|
||
"model": model,
|
||
"catalog_key": key if catalog_known else None,
|
||
"generic": not catalog_known,
|
||
"current": current,
|
||
"expansion": expansion,
|
||
"findings": findings,
|
||
"spec": {
|
||
"generation": expansion.get("generation"),
|
||
"dimm_slots": expansion["memory"].get("dimm_slots_max"),
|
||
"max_memory_tb": expansion["memory"].get("max_capacity_tb"),
|
||
"max_memory_note": expansion["memory"].get("note"),
|
||
"memory_types": expansion["memory"].get("supported_types"),
|
||
"max_dimm_speed_mts": expansion["memory"].get("max_rated_mts"),
|
||
"pcie_slots_max": expansion["pcie"].get("slots_max"),
|
||
"pcie_note": expansion["pcie"].get("note"),
|
||
"drive_bays_max": expansion["disks"].get("bays_max"),
|
||
"drive_note": expansion["disks"].get("note"),
|
||
"cpu_sockets": expansion["cpu"].get("sockets_max"),
|
||
"best_practices": expansion.get("best_practices") or [],
|
||
"performance_notes": expansion.get("performance_notes") or [],
|
||
"docs": expansion.get("docs") or [],
|
||
},
|
||
"memory": {
|
||
"total_gb": mem_total_gb,
|
||
"populated": len(dimm_modules),
|
||
"empty": empty_n,
|
||
"headroom_gb": headroom_gb,
|
||
"slots": slot_map if slot_map else [{**m, "populated": True, "speed_mts": m.get("operating_mts") or m.get("rated_mts")} for m in dimm_modules],
|
||
"banks": current["memory"]["banks_seen"],
|
||
},
|
||
"processors": cpus,
|
||
"disks": {
|
||
"installed": len(disks),
|
||
"max_bays": drive_max if catalog_known else None,
|
||
"free_bays_estimate": free_bays,
|
||
"items": [{**d, "slot": d.get("bay"), "name": d.get("model")} for d in disks],
|
||
},
|
||
"pcie": {
|
||
"max_slots": expansion["pcie"].get("slots_max"),
|
||
"items": adapters,
|
||
"note": expansion["pcie"].get("note"),
|
||
},
|
||
}
|
||
|
||
|
||
|
||
def subnet_of(ip: str | None) -> str:
|
||
if not ip or ip.count(".") != 3:
|
||
return "unknown"
|
||
try:
|
||
return str(ipaddress.ip_network(f"{ip}/24", strict=False))
|
||
except Exception:
|
||
parts = ip.split(".")
|
||
return f"{parts[0]}.{parts[1]}.{parts[2]}.0/24"
|
||
|
||
|
||
def pick_ip(device: dict) -> str | None:
|
||
for m in device.get("DeviceManagement") or []:
|
||
addr = m.get("NetworkAddress")
|
||
if addr and addr.count(".") == 3:
|
||
return addr
|
||
return None
|
||
|
||
|
||
def pick_management(device: dict) -> dict:
|
||
"""Extract iDRAC/management IP plus OS hostname hints from OME DeviceManagement."""
|
||
mgmt = (device.get("DeviceManagement") or [{}])[0] or {}
|
||
idrac_ip = None
|
||
for m in device.get("DeviceManagement") or []:
|
||
addr = m.get("NetworkAddress")
|
||
if addr and str(addr).count(".") == 3:
|
||
idrac_ip = addr
|
||
mgmt = m
|
||
break
|
||
os_hostname = (mgmt.get("InstrumentationName") or "").strip() or None
|
||
dns_name = (mgmt.get("DnsName") or "").strip() or None
|
||
return {
|
||
"idrac_ip": idrac_ip,
|
||
"os_hostname": os_hostname,
|
||
"mgmt_dns_name": dns_name,
|
||
"mac": mgmt.get("MacAddress"),
|
||
}
|
||
|
||
|
||
async def resolve_host_ips(hostname: str | None) -> list[str]:
|
||
if not hostname or hostname.count(".") == 3:
|
||
return []
|
||
loop = asyncio.get_running_loop()
|
||
try:
|
||
infos = await loop.getaddrinfo(hostname, None, family=socket.AF_INET)
|
||
return sorted({i[4][0] for i in infos})
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
async def enrich_rdp_targets(devices: list[dict]) -> None:
|
||
"""DNS-resolve OS hostnames from OME InstrumentationName → RDP candidate IPs."""
|
||
hostnames: set[str] = set()
|
||
for n in devices:
|
||
h = (n.get("os_hostname") or "").strip()
|
||
if h and h.count(".") != 3:
|
||
hostnames.add(h)
|
||
# also try short name
|
||
if "." in h:
|
||
hostnames.add(h.split(".", 1)[0])
|
||
|
||
resolved: dict[str, list[str]] = {}
|
||
sem = asyncio.Semaphore(20)
|
||
|
||
async def one(h: str):
|
||
async with sem:
|
||
resolved[h] = await resolve_host_ips(h)
|
||
|
||
if hostnames:
|
||
await asyncio.gather(*[one(h) for h in hostnames])
|
||
|
||
for n in devices:
|
||
idrac = n.get("idrac_ip") or n.get("ip")
|
||
h = (n.get("os_hostname") or "").strip()
|
||
ips: list[str] = []
|
||
seen: set[str] = set()
|
||
for candidate in (h, h.split(".", 1)[0] if h and "." in h else None):
|
||
if not candidate:
|
||
continue
|
||
for ip in resolved.get(candidate) or []:
|
||
if ip not in seen:
|
||
seen.add(ip)
|
||
ips.append(ip)
|
||
# Prefer OS IPs that differ from iDRAC
|
||
os_ips = [ip for ip in ips if ip != idrac]
|
||
rdp_ips = os_ips or ips
|
||
n["rdp_ips"] = rdp_ips
|
||
n["rdp_host"] = rdp_ips[0] if rdp_ips else None
|
||
n["os_ips"] = os_ips
|
||
# Windows hint from name / later OS inventory
|
||
blob = f"{n.get('name') or ''} {n.get('os_hostname') or ''} {n.get('os_name') or ''}".lower()
|
||
n["is_windows"] = any(k in blob for k in ("windows", "hyperv", "hyper-v", "-hv", "hv0")) or bool(
|
||
n.get("os_name") and "windows" in str(n.get("os_name")).lower()
|
||
)
|
||
|
||
|
||
async def fetch_power(client: httpx.AsyncClient, base: str, headers: dict, device_id: int) -> dict:
|
||
try:
|
||
r = await client.get(f"{base}/api/DeviceService/Devices({device_id})/Power", headers=headers)
|
||
if r.status_code != 200:
|
||
return {}
|
||
p = r.json()
|
||
def num(key):
|
||
v = p.get(key)
|
||
try:
|
||
return float(v) if v not in (None, "") else None
|
||
except Exception:
|
||
return None
|
||
return {
|
||
"watts": num("power"),
|
||
"avg_watts": num("avgPower"),
|
||
"peak_watts": num("peakPower"),
|
||
"min_watts": num("minimumPower"),
|
||
"energy_kwh": num("systemEnergyConsumption"),
|
||
"power_unit": p.get("powerUnit") or "watt",
|
||
}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
async def ome_fetch() -> dict:
|
||
base = settings.ome_url.rstrip("/")
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
r = await client.post(
|
||
f"{base}/api/SessionService/Sessions",
|
||
json={
|
||
"UserName": settings.ome_user,
|
||
"Password": settings.ome_password,
|
||
"SessionType": "API",
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
token = r.headers.get("X-Auth-Token")
|
||
sid = r.json().get("Id")
|
||
headers = {"X-Auth-Token": token, "Accept": "application/json"}
|
||
try:
|
||
info = (await client.get(f"{base}/api/ApplicationService/Info", headers=headers)).json()
|
||
devices_resp = (
|
||
await client.get(f"{base}/api/DeviceService/Devices?$top=5000", headers=headers)
|
||
).json()
|
||
groups_resp = (
|
||
await client.get(f"{base}/api/GroupService/Groups?$top=200", headers=headers)
|
||
).json()
|
||
alerts_raw = []
|
||
alerts_total = None
|
||
try:
|
||
alerts_resp = (
|
||
await client.get(
|
||
f"{base}/api/AlertService/Alerts?$top=200",
|
||
headers=headers,
|
||
)
|
||
).json()
|
||
alerts_raw = alerts_resp.get("value") or []
|
||
alerts_total = alerts_resp.get("@odata.count")
|
||
except Exception as e:
|
||
log.warning("OME alerts fetch failed: %s", e)
|
||
|
||
devices_raw = devices_resp.get("value") or []
|
||
# power for connected servers first
|
||
power_targets = [
|
||
d.get("Id")
|
||
for d in devices_raw
|
||
if d.get("Type") == 1000 and d.get("ConnectionState") and d.get("Id")
|
||
][: settings.power_fetch_limit]
|
||
# also a few offline servers for comparison
|
||
offline_ids = [
|
||
d.get("Id")
|
||
for d in devices_raw
|
||
if d.get("Type") == 1000 and not d.get("ConnectionState") and d.get("Id")
|
||
][:5]
|
||
power_targets = list(dict.fromkeys(power_targets + offline_ids))
|
||
|
||
sem = asyncio.Semaphore(8)
|
||
|
||
async def limited(did):
|
||
async with sem:
|
||
return did, await fetch_power(client, base, headers, did)
|
||
|
||
power_map = {}
|
||
if power_targets:
|
||
results = await asyncio.gather(*[limited(i) for i in power_targets])
|
||
power_map = {i: p for i, p in results if p}
|
||
finally:
|
||
if sid:
|
||
try:
|
||
await client.delete(
|
||
f"{base}/api/SessionService/Sessions('{sid}')",
|
||
headers=headers,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
devices = []
|
||
model_counts: dict[str, int] = defaultdict(int)
|
||
idrac_count = server_count = connected = powered = 0
|
||
switch_count = chassis_count = pdu_count = storage_count = 0
|
||
total_watts = 0.0
|
||
watts_samples = 0
|
||
status_counts: dict[str, int] = defaultdict(int)
|
||
|
||
for d in devices_raw:
|
||
mgmt = pick_management(d)
|
||
ip = mgmt.get("idrac_ip") or pick_ip(d)
|
||
subnet = subnet_of(ip)
|
||
dtype = d.get("Type")
|
||
sub = d.get("SubDeviceType") or ""
|
||
model = d.get("Model") or "Unknown"
|
||
is_server = dtype == 1000
|
||
is_idrac = sub == "iDRAC"
|
||
is_switch = dtype == 7000 or "switch" in (sub or "").lower() or "switch" in model.lower()
|
||
is_chassis = dtype == 2000 or "chassis" in (sub or "").lower()
|
||
is_pdu = dtype == 100
|
||
is_storage = dtype == 5000 or "storage" in (sub or "").lower()
|
||
if is_switch:
|
||
role = "switch"
|
||
elif is_chassis:
|
||
role = "chassis"
|
||
elif is_pdu:
|
||
role = "pdu"
|
||
elif is_storage:
|
||
role = "storage"
|
||
elif is_server:
|
||
role = "server"
|
||
else:
|
||
role = "other"
|
||
conn = bool(d.get("ConnectionState"))
|
||
power_state = d.get("PowerState")
|
||
status = str(d.get("Status") or "")
|
||
status_counts[status] += 1
|
||
if is_server:
|
||
model_counts[model] += 1
|
||
if is_idrac:
|
||
idrac_count += 1
|
||
if is_switch:
|
||
switch_count += 1
|
||
if is_chassis:
|
||
chassis_count += 1
|
||
if is_pdu:
|
||
pdu_count += 1
|
||
if is_storage:
|
||
storage_count += 1
|
||
if is_server:
|
||
server_count += 1
|
||
if conn:
|
||
connected += 1
|
||
if power_state == 17:
|
||
powered += 1
|
||
|
||
pwr = power_map.get(d.get("Id")) or {}
|
||
watts = pwr.get("watts")
|
||
if watts is not None:
|
||
total_watts += watts
|
||
watts_samples += 1
|
||
|
||
node = {
|
||
"id": d.get("Id"),
|
||
"name": d.get("DeviceName") or f"device-{d.get('Id')}",
|
||
"model": model,
|
||
"service_tag": d.get("DeviceServiceTag")
|
||
or d.get("Identifier")
|
||
or d.get("ChassisServiceTag"),
|
||
"type": dtype,
|
||
"sub_type": sub,
|
||
"connected": conn,
|
||
"power_state": power_state,
|
||
"powered_on": power_state == 17,
|
||
"status": status,
|
||
"ip": ip,
|
||
"idrac_ip": mgmt.get("idrac_ip") or ip,
|
||
"os_hostname": mgmt.get("os_hostname"),
|
||
"mgmt_dns_name": mgmt.get("mgmt_dns_name"),
|
||
"rdp_host": None,
|
||
"rdp_ips": [],
|
||
"os_ips": [],
|
||
"is_windows": False,
|
||
"subnet": subnet,
|
||
"role": role,
|
||
"is_server": is_server,
|
||
"is_idrac": is_idrac,
|
||
"is_switch": is_switch,
|
||
"is_chassis": is_chassis,
|
||
"is_pdu": is_pdu,
|
||
"is_storage": is_storage,
|
||
"chassis_service_tag": d.get("ChassisServiceTag"),
|
||
"idrac_url": f"https://{ip}" if ip else None,
|
||
"watts": watts,
|
||
"avg_watts": pwr.get("avg_watts"),
|
||
"peak_watts": pwr.get("peak_watts"),
|
||
"energy_kwh": pwr.get("energy_kwh"),
|
||
"last_status_time": d.get("LastStatusTime"),
|
||
"last_inventory_time": d.get("LastInventoryTime"),
|
||
}
|
||
node["map_cidrs"] = [subnet] if subnet and subnet != "unknown" else []
|
||
node["extra_ips"] = []
|
||
devices.append(node)
|
||
|
||
await enrich_rdp_targets(devices)
|
||
sync_endpoints_from_fleet(devices)
|
||
apply_network_endpoints(devices)
|
||
subnets = build_subnet_summaries(devices)
|
||
|
||
groups = [
|
||
{"id": g.get("Id"), "name": g.get("Name")}
|
||
for g in (groups_resp.get("value") or [])
|
||
if g.get("Name")
|
||
][:60]
|
||
|
||
models = [
|
||
{"name": m, "count": c}
|
||
for m, c in sorted(model_counts.items(), key=lambda x: -x[1])
|
||
]
|
||
|
||
# --- realtime context: fleet deltas + alerts ---
|
||
global PREV_DEVICES, EVENT_FEED
|
||
now = time.time()
|
||
new_events: list[dict] = []
|
||
first_poll = not PREV_DEVICES
|
||
current_ids: set[int] = set()
|
||
|
||
def _role(n: dict) -> str:
|
||
if n.get("is_switch"):
|
||
return "switch"
|
||
if n.get("is_chassis"):
|
||
return "chassis"
|
||
if n.get("is_idrac"):
|
||
return "iDRAC"
|
||
if n.get("is_server"):
|
||
return "server"
|
||
if n.get("is_storage"):
|
||
return "storage"
|
||
if n.get("is_pdu"):
|
||
return "PDU"
|
||
return n.get("role") or "device"
|
||
|
||
for n in devices:
|
||
did = n.get("id")
|
||
if did is None:
|
||
continue
|
||
current_ids.add(did)
|
||
prev = PREV_DEVICES.get(did)
|
||
if not prev:
|
||
# First poll only seeds baseline; later polls raise new-node alerts
|
||
if not first_poll and (n.get("is_server") or n.get("is_idrac")):
|
||
role = _role(n)
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "device_new",
|
||
"severity": "critical",
|
||
"device_id": did,
|
||
"title": f"New {role} discovered",
|
||
"text": (
|
||
f"{n.get('name')} · {n.get('ip') or 'no IP'} · "
|
||
f"{n.get('model') or 'unknown model'} · {n.get('subnet') or 'unknown subnet'}"
|
||
),
|
||
"notify": True,
|
||
"role": role,
|
||
"ip": n.get("ip"),
|
||
"model": n.get("model"),
|
||
"subnet": n.get("subnet"),
|
||
"name": n.get("name"),
|
||
})
|
||
continue
|
||
if bool(prev.get("connected")) != bool(n.get("connected")):
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "connection",
|
||
"severity": "info" if n.get("connected") else "warning",
|
||
"device_id": did,
|
||
"title": n.get("name"),
|
||
"text": "Connected" if n.get("connected") else "Went offline",
|
||
})
|
||
if bool(prev.get("powered_on")) != bool(n.get("powered_on")):
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "power_state",
|
||
"severity": "info" if n.get("powered_on") else "warning",
|
||
"device_id": did,
|
||
"title": n.get("name"),
|
||
"text": "Powered on" if n.get("powered_on") else "Powered off",
|
||
})
|
||
pw, pp = n.get("watts"), prev.get("watts")
|
||
if pw is not None and pp is not None and abs(pw - pp) >= 40:
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "power_watt",
|
||
"severity": "info",
|
||
"device_id": did,
|
||
"title": n.get("name"),
|
||
"text": f"Power {round(pp)}W → {round(pw)}W",
|
||
})
|
||
if str(prev.get("status")) != str(n.get("status")):
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "status",
|
||
"severity": "warning",
|
||
"device_id": did,
|
||
"title": n.get("name"),
|
||
"text": f"Status {prev.get('status')} → {n.get('status')}",
|
||
})
|
||
|
||
if not first_poll:
|
||
for did, prev in PREV_DEVICES.items():
|
||
if did in current_ids:
|
||
continue
|
||
if not (prev.get("is_server") or prev.get("is_idrac")):
|
||
continue
|
||
role = _role(prev)
|
||
new_events.append({
|
||
"ts": now,
|
||
"kind": "device_removed",
|
||
"severity": "warning",
|
||
"device_id": did,
|
||
"title": f"{role} left inventory",
|
||
"text": (
|
||
f"{prev.get('name')} · {prev.get('ip') or 'no IP'} · "
|
||
f"{prev.get('model') or 'unknown model'}"
|
||
),
|
||
"notify": True,
|
||
"role": role,
|
||
"name": prev.get("name"),
|
||
"ip": prev.get("ip"),
|
||
"model": prev.get("model"),
|
||
})
|
||
|
||
PREV_DEVICES = {n["id"]: n for n in devices if n.get("id") is not None}
|
||
if new_events:
|
||
EVENT_FEED = (new_events + EVENT_FEED)[:120]
|
||
|
||
sev_map = {"Critical": 0, "Warning": 0, "Normal": 0, "Info": 0, "Unknown": 0}
|
||
alerts = []
|
||
for a in alerts_raw:
|
||
sev = a.get("SeverityName") or "Unknown"
|
||
if sev not in sev_map:
|
||
sev_map[sev] = 0
|
||
sev_map[sev] = sev_map.get(sev, 0) + 1
|
||
alerts.append({
|
||
"id": a.get("Id"),
|
||
"severity": sev,
|
||
"device_id": a.get("AlertDeviceId") or a.get("AlertEntityId"),
|
||
"device": a.get("AlertDeviceName") or a.get("AlertEntityName"),
|
||
"ip": a.get("AlertDeviceIpAddress"),
|
||
"category": a.get("CategoryName"),
|
||
"subcategory": a.get("SubCategoryName"),
|
||
"message": a.get("Message"),
|
||
"message_id": a.get("AlertMessageId"),
|
||
"status": a.get("StatusName"),
|
||
"time": a.get("TimeStamp"),
|
||
"action": a.get("RecommendedAction"),
|
||
})
|
||
|
||
hottest = sorted(
|
||
[n for n in devices if n.get("watts") is not None],
|
||
key=lambda x: x.get("watts") or 0,
|
||
reverse=True,
|
||
)[:5]
|
||
recent_status = sorted(
|
||
[n for n in devices if n.get("last_status_time")],
|
||
key=lambda x: str(x.get("last_status_time")),
|
||
reverse=True,
|
||
)[:5]
|
||
|
||
context = {
|
||
"refreshed_at": now,
|
||
"poll_seconds": settings.poll_interval,
|
||
"alerts_total": alerts_total,
|
||
"alert_severity": sev_map,
|
||
"events_new": len(new_events),
|
||
"notifications": [e for e in new_events if e.get("notify")],
|
||
"hottest": [
|
||
{"id": n["id"], "name": n["name"], "watts": n["watts"], "subnet": n["subnet"]}
|
||
for n in hottest
|
||
],
|
||
"recent_status": [
|
||
{
|
||
"id": n["id"],
|
||
"name": n["name"],
|
||
"time": n.get("last_status_time"),
|
||
"connected": n.get("connected"),
|
||
"status": n.get("status"),
|
||
}
|
||
for n in recent_status
|
||
],
|
||
"focus_hint": (
|
||
f"{connected} connected · {round(total_watts)} W sampled · "
|
||
f"{sev_map.get('Critical', 0)} critical / {sev_map.get('Warning', 0)} warning alerts in feed"
|
||
),
|
||
}
|
||
|
||
return {
|
||
"ome": {
|
||
"name": info.get("Name") or "OpenManage Enterprise",
|
||
"version": info.get("Version"),
|
||
"build": info.get("BuildNumber"),
|
||
"url": base,
|
||
"fqdn": "cov-omeprod01.dell-atc.lan",
|
||
"console_url": f"{base}/management-console/ome/",
|
||
},
|
||
"summary": {
|
||
"total": len(devices),
|
||
"idracs": idrac_count,
|
||
"servers": server_count,
|
||
"switches": switch_count,
|
||
"chassis": chassis_count,
|
||
"pdus": pdu_count,
|
||
"storage": storage_count,
|
||
"connected": connected,
|
||
"powered_on": powered,
|
||
"offline": max(0, server_count - connected),
|
||
"status_counts": dict(status_counts),
|
||
"total_watts": round(total_watts, 1),
|
||
"avg_node_watts": round(total_watts / watts_samples, 1) if watts_samples else None,
|
||
"power_samples": watts_samples,
|
||
"alerts_critical": sev_map.get("Critical", 0),
|
||
"alerts_warning": sev_map.get("Warning", 0),
|
||
"alerts_total": alerts_total,
|
||
},
|
||
"subnets": subnets,
|
||
"devices": devices,
|
||
"groups": groups,
|
||
"models": models,
|
||
"alerts": alerts,
|
||
"events": EVENT_FEED[:40],
|
||
"context": context,
|
||
"openwebui_url": settings.openwebui_url,
|
||
"updated_at": now,
|
||
}
|
||
|
||
|
||
async def broadcast(payload: dict):
|
||
dead = []
|
||
for ws in list(CLIENTS):
|
||
try:
|
||
await ws.send_json(payload)
|
||
except Exception:
|
||
dead.append(ws)
|
||
for ws in dead:
|
||
CLIENTS.discard(ws)
|
||
|
||
|
||
|
||
def _seed_atc_racks() -> None:
|
||
"""Idempotent seed: ATC1 (8×42U) + ATC2 (6×42U)."""
|
||
now = time.time()
|
||
with _db() as conn:
|
||
n = conn.execute("SELECT COUNT(*) AS c FROM racks").fetchone()["c"]
|
||
if n > 0:
|
||
return
|
||
conn.execute(
|
||
"INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
|
||
("ATC1", "ATC1", 1),
|
||
)
|
||
conn.execute(
|
||
"INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
|
||
("ATC2", "ATC2", 2),
|
||
)
|
||
for i in range(1, 9):
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
|
||
VALUES (?, ?, 42, ?, ?, ?)
|
||
""",
|
||
("ATC1", f"ATC1-R{i:02d}", i, now, now),
|
||
)
|
||
for i in range(1, 7):
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
|
||
VALUES (?, ?, 42, ?, ?, ?)
|
||
""",
|
||
("ATC2", f"ATC2-R{i:02d}", i, now, now),
|
||
)
|
||
log.info("Seeded ATC racks: ATC1×8 + ATC2×6 (42U)")
|
||
|
||
|
||
def build_network_fabric(devices: list[dict] | None = None) -> dict:
|
||
"""Inferred switch↔server / chassis↔blade edges (no LLDP)."""
|
||
devices = list(devices if devices is not None else (STATE.get("devices") or []))
|
||
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
|
||
edges: list[dict] = []
|
||
seen: set[tuple] = set()
|
||
|
||
def add_edge(a: int, b: int, kind: str):
|
||
if a is None or b is None or a == b:
|
||
return
|
||
key = (min(a, b), max(a, b), kind)
|
||
if key in seen:
|
||
return
|
||
seen.add(key)
|
||
da, db = by_id.get(a) or {}, by_id.get(b) or {}
|
||
edges.append(
|
||
{
|
||
"from_id": a,
|
||
"to_id": b,
|
||
"kind": kind,
|
||
"confidence": "inferred",
|
||
"note": (
|
||
"management subnet"
|
||
if kind == "subnet"
|
||
else "MX chassis / blade"
|
||
),
|
||
"live": bool(da.get("connected") and db.get("connected")),
|
||
}
|
||
)
|
||
|
||
switches = [d for d in devices if d.get("is_switch")]
|
||
for sw in switches:
|
||
cidr = sw.get("subnet") or "unknown"
|
||
if cidr == "unknown":
|
||
continue
|
||
for d in devices:
|
||
if d.get("id") == sw.get("id"):
|
||
continue
|
||
if d.get("subnet") != cidr:
|
||
continue
|
||
if d.get("is_switch") or d.get("is_pdu"):
|
||
continue
|
||
# Prefer linking servers/storage/chassis endpoints
|
||
if d.get("is_server") or d.get("is_storage") or d.get("is_chassis") or d.get("is_idrac"):
|
||
add_edge(sw["id"], d["id"], "subnet")
|
||
|
||
chassis_list = [d for d in devices if d.get("is_chassis")]
|
||
for ch in chassis_list:
|
||
cst = (ch.get("service_tag") or "").upper()
|
||
for d in devices:
|
||
if not d.get("is_server"):
|
||
continue
|
||
model = (d.get("model") or "").upper()
|
||
blade_cst = (d.get("chassis_service_tag") or "").upper()
|
||
if cst and blade_cst and blade_cst == cst:
|
||
add_edge(ch["id"], d["id"], "chassis")
|
||
elif "MX" in model and "MX7000" not in model:
|
||
# fallback: all MX blades to MX chassis in same environment
|
||
add_edge(ch["id"], d["id"], "chassis")
|
||
|
||
return {
|
||
"edges": edges,
|
||
"note": "Edges inferred from management subnet / MX chassis — not LLDP.",
|
||
"switches": [
|
||
{
|
||
"id": s.get("id"),
|
||
"name": s.get("name"),
|
||
"service_tag": s.get("service_tag"),
|
||
"ip": s.get("ip") or s.get("idrac_ip"),
|
||
"subnet": s.get("subnet"),
|
||
"connected": s.get("connected"),
|
||
"model": s.get("model"),
|
||
"connected_device_ids": [
|
||
e["to_id"] if e["from_id"] == s.get("id") else e["from_id"]
|
||
for e in edges
|
||
if e["kind"] == "subnet"
|
||
and (e["from_id"] == s.get("id") or e["to_id"] == s.get("id"))
|
||
],
|
||
}
|
||
for s in switches
|
||
],
|
||
"summary": {
|
||
"devices": len(devices),
|
||
"edges": len(edges),
|
||
"switches": len(switches),
|
||
"chassis": sum(1 for d in devices if d.get("is_chassis")),
|
||
},
|
||
}
|
||
|
||
|
||
def _default_u_height(device: dict | None) -> int:
|
||
if not device:
|
||
return 1
|
||
cat = match_rack_catalog(device.get("model"), device.get("role"))
|
||
if cat is not None:
|
||
uh = int(cat.get("u_height") or 0)
|
||
return uh if uh > 0 else 1 # 0U vertical PDUs place as 1U marker unless side=left/right
|
||
if device.get("is_chassis"):
|
||
return 7
|
||
if device.get("is_storage"):
|
||
return 2
|
||
if device.get("is_switch"):
|
||
return 1
|
||
return 1
|
||
|
||
|
||
def _rack_overlap(
|
||
conn: sqlite3.Connection,
|
||
rack_id: int,
|
||
u_start: int,
|
||
u_height: int,
|
||
side: str = "front",
|
||
exclude_item_id: int | None = None,
|
||
exclude_device_id: int | None = None,
|
||
) -> bool:
|
||
if side in ("left", "right"):
|
||
return False # side PDUs do not consume front/rear U map
|
||
u_end = u_start + max(u_height, 1) - 1
|
||
rows = conn.execute(
|
||
"SELECT id, device_id, u_start, u_height, side FROM rack_items WHERE rack_id=?",
|
||
(rack_id,),
|
||
).fetchall()
|
||
for r in rows:
|
||
if exclude_item_id is not None and r["id"] == exclude_item_id:
|
||
continue
|
||
if exclude_device_id is not None and r["device_id"] == exclude_device_id:
|
||
continue
|
||
rside = (r["side"] or "front")
|
||
if rside in ("left", "right"):
|
||
continue
|
||
if rside != side:
|
||
continue
|
||
a, b = r["u_start"], r["u_start"] + r["u_height"] - 1
|
||
if not (u_end < a or u_start > b):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _enrich_rack_item(item: dict, devices: dict) -> dict:
|
||
out = dict(item)
|
||
dev = devices.get(item.get("device_id")) if item.get("device_id") is not None else None
|
||
sku = item.get("catalog_sku")
|
||
cat = None
|
||
if sku:
|
||
cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None)
|
||
if not cat and dev:
|
||
cat = match_rack_catalog(dev.get("model"), dev.get("role"))
|
||
if cat:
|
||
out["catalog"] = {k: cat[k] for k in ("sku", "name", "family", "category", "u_height", "face", "image") if k in cat}
|
||
out["face"] = cat.get("face")
|
||
out["image"] = cat.get("image") or catalog_image_for(cat)
|
||
out["category"] = cat.get("category")
|
||
else:
|
||
out["catalog"] = None
|
||
out["face"] = "custom"
|
||
out["image"] = FACE_IMAGE_MAP["custom"]
|
||
out["category"] = (dev or {}).get("role") or "custom"
|
||
if dev:
|
||
out["device"] = {
|
||
"id": dev.get("id"),
|
||
"name": dev.get("name"),
|
||
"model": dev.get("model"),
|
||
"service_tag": dev.get("service_tag"),
|
||
"role": dev.get("role"),
|
||
"ip": dev.get("ip") or dev.get("idrac_ip"),
|
||
"connected": dev.get("connected"),
|
||
"is_switch": dev.get("is_switch"),
|
||
"is_server": dev.get("is_server"),
|
||
"is_storage": dev.get("is_storage"),
|
||
"is_chassis": dev.get("is_chassis"),
|
||
"is_pdu": dev.get("is_pdu"),
|
||
}
|
||
out["label"] = item.get("label") or dev.get("name")
|
||
out["model"] = dev.get("model")
|
||
else:
|
||
out["device"] = None
|
||
out["label"] = item.get("label") or ((cat or {}).get("name") if cat else "Custom")
|
||
out["model"] = (cat or {}).get("name")
|
||
return out
|
||
|
||
|
||
def _racks_payload() -> dict:
|
||
_migrate_rack_items()
|
||
devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None}
|
||
with _db() as conn:
|
||
sites = [dict(r) for r in conn.execute("SELECT * FROM sites ORDER BY sort_order").fetchall()]
|
||
racks = [dict(r) for r in conn.execute("SELECT * FROM racks ORDER BY site_id, sort_order").fetchall()]
|
||
items = [dict(r) for r in conn.execute("SELECT * FROM rack_items ORDER BY u_start DESC").fetchall()]
|
||
placed_ids = {i["device_id"] for i in items if i.get("device_id") is not None}
|
||
by_site: dict[str, list] = {}
|
||
for rk in racks:
|
||
card = dict(rk)
|
||
card["items"] = []
|
||
card["placements"] = [] # backward compat alias
|
||
for it in items:
|
||
if it["rack_id"] != rk["id"]:
|
||
continue
|
||
enriched = _enrich_rack_item(it, devices)
|
||
card["items"].append(enriched)
|
||
# compat shape for older UI bits
|
||
card["placements"].append(
|
||
{
|
||
"device_id": it.get("device_id"),
|
||
"rack_id": it["rack_id"],
|
||
"u_start": it["u_start"],
|
||
"u_height": it["u_height"],
|
||
"item_id": it["id"],
|
||
"side": it.get("side") or "front",
|
||
"face": enriched.get("face"),
|
||
"catalog": enriched.get("catalog"),
|
||
"label": enriched.get("label"),
|
||
"device": enriched.get("device") or {
|
||
"id": it.get("device_id"),
|
||
"name": enriched.get("label"),
|
||
"model": enriched.get("model"),
|
||
"role": enriched.get("category"),
|
||
"service_tag": None,
|
||
},
|
||
}
|
||
)
|
||
by_site.setdefault(rk["site_id"], []).append(card)
|
||
site_out = [{**s, "racks": by_site.get(s["id"], [])} for s in sites]
|
||
unplaced = []
|
||
for d in STATE.get("devices") or []:
|
||
if d.get("id") in placed_ids:
|
||
continue
|
||
if not (d.get("is_server") or d.get("is_switch") or d.get("is_chassis") or d.get("is_storage") or d.get("is_pdu")):
|
||
continue
|
||
cat = match_rack_catalog(d.get("model"), d.get("role"))
|
||
unplaced.append(
|
||
{
|
||
"id": d.get("id"),
|
||
"name": d.get("name"),
|
||
"model": d.get("model"),
|
||
"service_tag": d.get("service_tag"),
|
||
"role": d.get("role"),
|
||
"ip": d.get("ip") or d.get("idrac_ip"),
|
||
"connected": d.get("connected"),
|
||
"default_u_height": _default_u_height(d),
|
||
"catalog_sku": (cat or {}).get("sku"),
|
||
"face": (cat or {}).get("face") or "custom",
|
||
"image": (cat or {}).get("image") or catalog_image_for(cat or {"face": "custom"}),
|
||
"is_switch": d.get("is_switch"),
|
||
"is_server": d.get("is_server"),
|
||
"is_storage": d.get("is_storage"),
|
||
"is_chassis": d.get("is_chassis"),
|
||
"is_pdu": d.get("is_pdu"),
|
||
}
|
||
)
|
||
unplaced.sort(key=lambda x: ((x.get("role") or ""), (x.get("name") or "").lower()))
|
||
return {
|
||
"sites": site_out,
|
||
"items": items,
|
||
"placements": items,
|
||
"unplaced": unplaced,
|
||
"catalog": DELL_RACK_CATALOG,
|
||
}
|
||
|
||
|
||
def _switch_port_count(device: dict | None) -> int:
|
||
"""Default front-panel port count from model (OME has no LLDP ports here)."""
|
||
model = ((device or {}).get("model") or "").lower()
|
||
if "s4048" in model:
|
||
return 48
|
||
if "s5248" in model or "s5296" in model:
|
||
return 48
|
||
if "s4148" in model:
|
||
return 48
|
||
if "mx9116" in model:
|
||
return 16
|
||
if "mx5108" in model:
|
||
return 8
|
||
return 48
|
||
|
||
|
||
def _port_label(port_num: int, total: int = 48) -> str:
|
||
return f"Eth1/{port_num}"
|
||
|
||
|
||
def build_switch_portmap(switch_id: int) -> dict:
|
||
devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None}
|
||
sw = devices.get(switch_id)
|
||
if not sw or not sw.get("is_switch"):
|
||
# still allow if type 7000 missing flag somehow
|
||
sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None)
|
||
if not sw:
|
||
raise HTTPException(404, "Switch not found in fleet")
|
||
nports = _switch_port_count(sw)
|
||
with _db() as conn:
|
||
rows = conn.execute(
|
||
"SELECT * FROM port_links WHERE switch_id=? ORDER BY switch_port",
|
||
(switch_id,),
|
||
).fetchall()
|
||
links = [dict(r) for r in rows]
|
||
by_port = {int(l["switch_port"]): l for l in links}
|
||
ports = []
|
||
for i in range(1, nports + 1):
|
||
link = by_port.get(i)
|
||
peer = devices.get(link["device_id"]) if link else None
|
||
ports.append(
|
||
{
|
||
"port": i,
|
||
"label": _port_label(i, nports),
|
||
"wired": bool(link),
|
||
"link": (
|
||
{
|
||
"id": link["id"],
|
||
"device_id": link["device_id"],
|
||
"device_port": link.get("device_port") or "",
|
||
"note": link.get("note") or "",
|
||
"device": {
|
||
"id": peer.get("id"),
|
||
"name": peer.get("name"),
|
||
"service_tag": peer.get("service_tag"),
|
||
"ip": peer.get("ip") or peer.get("idrac_ip"),
|
||
"model": peer.get("model"),
|
||
"role": peer.get("role"),
|
||
"connected": peer.get("connected"),
|
||
}
|
||
if peer
|
||
else None,
|
||
}
|
||
if link
|
||
else None
|
||
),
|
||
}
|
||
)
|
||
return {
|
||
"switch": {
|
||
"id": sw.get("id"),
|
||
"name": sw.get("name"),
|
||
"model": sw.get("model"),
|
||
"service_tag": sw.get("service_tag"),
|
||
"ip": sw.get("ip") or sw.get("idrac_ip"),
|
||
"subnet": sw.get("subnet"),
|
||
"connected": sw.get("connected"),
|
||
"port_count": nports,
|
||
},
|
||
"ports": ports,
|
||
"wired_count": sum(1 for p in ports if p["wired"]),
|
||
"note": "Port wiring is manual (OME has no LLDP port map for this switch). Map switch port → server NIC/port.",
|
||
}
|
||
|
||
|
||
def _seed_atc_vlans() -> None:
|
||
"""Seed known ATC lab VLANs/subnets (editable). Ensures new catalog entries are added."""
|
||
now = time.time()
|
||
defaults = [
|
||
(40, "iDRAC / OOB A", "10.0.40.0/24", "Out-of-band management (has S4048)", "ATC1", "#00a8e8", 1),
|
||
(41, "iDRAC / OOB B", "10.0.41.0/24", "Out-of-band management", "ATC1", "#3dffe0", 2),
|
||
(42, "iDRAC / OOB C", "10.0.42.0/24", "Out-of-band management", "ATC2", "#7c5cff", 3),
|
||
(10, "Production / OS", "10.0.10.0/24", "Hypervisor & Windows OS / RDP", "ATC1", "#ff9a3c", 4),
|
||
(11, "Lab / OS", "10.0.11.0/24", "Lab OS network", "ATC2", "#ff5c5c", 5),
|
||
(20, "Shared services", "10.0.20.0/24", "Mgmt / demo / shared ATC services", "ATC1", "#a3e635", 6),
|
||
(120, "Secondary OS", "10.0.120.0/24", "Secondary OS / RDP path", "ATC1", "#f0d060", 7),
|
||
(90, "FDE cluster / AI", "10.0.90.0/24", "FDE platform · DNS · HAProxy · OPNsense · Proxmox (dell-fde.lan)", "FDE", "#38bdf8", 8),
|
||
]
|
||
with _db() as conn:
|
||
existing = {
|
||
int(r["vlan_id"])
|
||
for r in conn.execute("SELECT vlan_id FROM vlans WHERE vlan_id IS NOT NULL").fetchall()
|
||
if r["vlan_id"] is not None
|
||
}
|
||
added = 0
|
||
for vlan_id, name, cidr, purpose, site, color, so in defaults:
|
||
if vlan_id in existing:
|
||
continue
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(vlan_id, name, cidr, purpose, site, color, so, now, now),
|
||
)
|
||
added += 1
|
||
if added:
|
||
log.info("Seeded ATC VLAN catalog (+%s new, %s known defaults)", added, len(defaults))
|
||
|
||
|
||
|
||
|
||
def _synthetic_device_id(ip: str) -> int:
|
||
"""Stable negative id for inventory-only hosts (safe for JS Number)."""
|
||
try:
|
||
a, b, c, d = (int(x) for x in ip.split("."))
|
||
return -(a * 1_000_000 + b * 10_000 + c * 100 + d)
|
||
except Exception:
|
||
return -abs(hash(ip)) % 1_000_000_000
|
||
|
||
|
||
def _vlan_id_for_ip(ip: str) -> int | None:
|
||
try:
|
||
return int(ip.split(".")[2])
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _ensure_network_endpoints_schema(conn) -> None:
|
||
conn.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS network_endpoints (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
hostname TEXT,
|
||
ip TEXT NOT NULL UNIQUE,
|
||
role TEXT NOT NULL DEFAULT 'server',
|
||
kind TEXT NOT NULL DEFAULT 'physical',
|
||
device_id INTEGER,
|
||
vlan_id INTEGER,
|
||
switch_id INTEGER,
|
||
switch_port INTEGER,
|
||
model TEXT,
|
||
note TEXT,
|
||
user_note TEXT,
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
)
|
||
"""
|
||
)
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(network_endpoints)").fetchall()}
|
||
if "user_note" not in cols:
|
||
conn.execute("ALTER TABLE network_endpoints ADD COLUMN user_note TEXT")
|
||
|
||
|
||
def _upsert_network_endpoint(
|
||
*,
|
||
hostname: str | None,
|
||
ip: str,
|
||
role: str = "server",
|
||
kind: str = "host",
|
||
device_id: int | None = None,
|
||
vlan_id: int | None = None,
|
||
switch_id: int | None = None,
|
||
switch_port: int | None = None,
|
||
model: str | None = None,
|
||
note: str | None = None,
|
||
overwrite_identity: bool = False,
|
||
clear_device_id: bool = False,
|
||
) -> None:
|
||
"""Insert/update inventory row. Never clobber user_note. Soft-update identity fields."""
|
||
ip = (ip or "").strip()
|
||
if not ip or ip.count(".") != 3:
|
||
return
|
||
now = time.time()
|
||
vlan_id = vlan_id if vlan_id is not None else _vlan_id_for_ip(ip)
|
||
with _db() as conn:
|
||
_ensure_network_endpoints_schema(conn)
|
||
row = conn.execute("SELECT * FROM network_endpoints WHERE ip=?", (ip,)).fetchone()
|
||
if not row:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO network_endpoints(
|
||
hostname, ip, role, kind, device_id, vlan_id, switch_id, switch_port,
|
||
model, note, user_note, created_at, updated_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
||
""",
|
||
(
|
||
hostname,
|
||
ip,
|
||
role or "server",
|
||
kind or "host",
|
||
device_id,
|
||
vlan_id,
|
||
switch_id,
|
||
switch_port,
|
||
model,
|
||
note,
|
||
now,
|
||
now,
|
||
),
|
||
)
|
||
return
|
||
# Update carefully
|
||
sets = ["updated_at=?"]
|
||
vals: list = [now]
|
||
if overwrite_identity or not row["hostname"]:
|
||
if hostname:
|
||
sets.append("hostname=?")
|
||
vals.append(hostname)
|
||
if overwrite_identity or not row["model"]:
|
||
if model:
|
||
sets.append("model=?")
|
||
vals.append(model)
|
||
if overwrite_identity or not row["note"]:
|
||
if note:
|
||
sets.append("note=?")
|
||
vals.append(note)
|
||
if overwrite_identity and note is not None:
|
||
sets.append("note=?")
|
||
vals.append(note)
|
||
if overwrite_identity and model is not None:
|
||
if "model=?" not in sets:
|
||
sets.append("model=?")
|
||
vals.append(model)
|
||
if overwrite_identity and hostname is not None:
|
||
if "hostname=?" not in sets:
|
||
sets.append("hostname=?")
|
||
vals.append(hostname)
|
||
if role:
|
||
sets.append("role=?")
|
||
vals.append(role)
|
||
if kind:
|
||
sets.append("kind=?")
|
||
vals.append(kind)
|
||
if vlan_id is not None:
|
||
sets.append("vlan_id=?")
|
||
vals.append(vlan_id)
|
||
if clear_device_id:
|
||
sets.append("device_id=NULL")
|
||
elif device_id is not None and (overwrite_identity or row["device_id"] is None):
|
||
sets.append("device_id=?")
|
||
vals.append(device_id)
|
||
if switch_id is not None and (overwrite_identity or row["switch_id"] is None):
|
||
sets.append("switch_id=?")
|
||
vals.append(switch_id)
|
||
if switch_port is not None and (overwrite_identity or row["switch_port"] is None):
|
||
sets.append("switch_port=?")
|
||
vals.append(switch_port)
|
||
vals.append(ip)
|
||
conn.execute(f"UPDATE network_endpoints SET {', '.join(sets)} WHERE ip=?", vals)
|
||
|
||
|
||
def _known_host_inventory() -> list[dict]:
|
||
"""Static + discovered lab hosts (OS / FDE / shared services)."""
|
||
return [
|
||
# VLAN 90 — FDE (physical Proxmox nodes are R640 per ops; OME idrac-proxmox R740xd is a different box)
|
||
{"hostname": "router.dell-fde.lan", "ip": "10.0.90.1", "role": "gateway", "kind": "appliance", "vlan_id": 90, "model": None, "note": "FDE gateway"},
|
||
{"hostname": "ns.dell-fde.lan", "ip": "10.0.90.2", "role": "dns", "kind": "vm", "vlan_id": 90, "note": "FDE DNS"},
|
||
{"hostname": "haproxy.dell-fde.lan", "ip": "10.0.90.3", "role": "lb", "kind": "vm", "vlan_id": 90, "note": "FDE HAProxy"},
|
||
{"hostname": "caddy.dell-fde.lan", "ip": "10.0.90.4", "role": "proxy", "kind": "vm", "vlan_id": 90, "note": "FDE Caddy"},
|
||
{"hostname": "opnsense01.dell-fde.lan", "ip": "10.0.90.5", "role": "firewall", "kind": "vm", "vlan_id": 90, "note": "FDE OPNsense 1"},
|
||
{"hostname": "opnsense02.dell-fde.lan", "ip": "10.0.90.6", "role": "firewall", "kind": "vm", "vlan_id": 90, "note": "FDE OPNsense 2"},
|
||
{
|
||
"hostname": "proxmox01.dell-fde.lan",
|
||
"ip": "10.0.90.21",
|
||
"role": "server",
|
||
"kind": "physical",
|
||
"vlan_id": 90,
|
||
"model": "PowerEdge R640",
|
||
"device_id": None,
|
||
"switch_id": 51950,
|
||
"note": "FDE Proxmox node 1 · R640 (ops). Candidate iDRAC: C90XK53/B90XK53 on VLAN41 — not the R740xd idrac-proxmox.",
|
||
"overwrite_identity": True,
|
||
"clear_device_id": True,
|
||
},
|
||
{
|
||
"hostname": "proxmox02.dell-fde.lan",
|
||
"ip": "10.0.90.22",
|
||
"role": "server",
|
||
"kind": "physical",
|
||
"vlan_id": 90,
|
||
"model": "PowerEdge R640",
|
||
"device_id": None,
|
||
"switch_id": 51950,
|
||
"note": "FDE Proxmox node 2 · R640 (ops). Candidate iDRAC: C90XK53/B90XK53 on VLAN41.",
|
||
"overwrite_identity": True,
|
||
"clear_device_id": True,
|
||
},
|
||
{"hostname": "fde-dns.fde-dns.lan", "ip": "10.0.90.41", "role": "dns", "kind": "vm", "vlan_id": 90},
|
||
{"hostname": "dns-server.fde-dns.lan", "ip": "10.0.90.42", "role": "dns", "kind": "vm", "vlan_id": 90},
|
||
{"hostname": "db01.dell-fde.lan", "ip": "10.0.90.181", "role": "database", "kind": "vm", "vlan_id": 90},
|
||
# VLAN 10 — Production OS
|
||
{"hostname": "cop-hv01.dell-atc.lan", "ip": "10.0.10.13", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
|
||
{"hostname": "cop-dc01.dell-atc.lan", "ip": "10.0.10.15", "role": "dc", "kind": "vm", "vlan_id": 10},
|
||
{"hostname": "cop-dc02.dell-atc.lan", "ip": "10.0.10.16", "role": "dc", "kind": "vm", "vlan_id": 10},
|
||
{"hostname": "cop-vsan-vdi01.dell-atc.lan", "ip": "10.0.10.41", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
|
||
{"hostname": "cop-vsan-vdi02.dell-atc.lan", "ip": "10.0.10.42", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
|
||
{"hostname": "cop-vsan-vdi03.dell-atc.lan", "ip": "10.0.10.43", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
|
||
{"hostname": "cop-vsan-vdi04.dell-atc.lan", "ip": "10.0.10.44", "role": "hypervisor", "kind": "physical", "vlan_id": 10},
|
||
{"hostname": "pve01.dell-atc.dell.nl", "ip": "10.0.10.65", "role": "server", "kind": "physical", "vlan_id": 10, "note": "Proxmox VE (ATC)"},
|
||
{"hostname": "atc-gpu-prod.dell-atc.lan", "ip": "10.0.10.106", "role": "gpu", "kind": "physical", "vlan_id": 10},
|
||
# VLAN 20 — Shared services
|
||
{"hostname": "cov-dc03.dell-atc.lan", "ip": "10.0.20.15", "role": "dc", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-omedemo01.dell-atc.lan", "ip": "10.0.20.22", "role": "ome", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-cloudiq01.dell-atc.lan", "ip": "10.0.20.23", "role": "cloudiq", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-scg01.dell-atc.lan", "ip": "10.0.20.31", "role": "scg", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-vcsa-vsan01.dell-atc.lan", "ip": "10.0.20.40", "role": "vcenter", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "photon-machine.dell-atc.lan", "ip": "10.0.20.102", "role": "host", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-grafana.dell-atc.lan", "ip": "10.0.20.103", "role": "monitoring", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-mgt01.dell-atc.lan", "ip": "10.0.20.104", "role": "mgmt", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-dataflow01.dell-atc.lan", "ip": "10.0.20.105", "role": "dataflow", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-gpu-dev.dell-atc.lan", "ip": "10.0.20.106", "role": "gpu", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-spearmint.dell-atc.lan", "ip": "10.0.20.108", "role": "app", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-file01.dell-atc.lan", "ip": "10.0.20.109", "role": "files", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-nas.dell-atc.lan", "ip": "10.0.20.110", "role": "nas", "kind": "appliance", "vlan_id": 20},
|
||
{"hostname": "atc-objectscale.dell-atc.lan", "ip": "10.0.20.111", "role": "object", "kind": "appliance", "vlan_id": 20},
|
||
{"hostname": "atc-db01.dell-atc.lan", "ip": "10.0.20.112", "role": "database", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-backup.dell-atc.lan", "ip": "10.0.20.113", "role": "backup", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "opnsense03.dell-atc.lan", "ip": "10.0.20.114", "role": "firewall", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "atc-git.dell-atc.lan", "ip": "10.0.20.118", "role": "git", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cvd-jody01.dell-atc.lan", "ip": "10.0.20.119", "role": "desktop", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "emj-wilcor01.dell-atc.lan", "ip": "10.0.20.133", "role": "desktop", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "cov-wac01.dell-atc.lan", "ip": "10.0.20.147", "role": "wac", "kind": "vm", "vlan_id": 20},
|
||
{"hostname": "CVP-MGMT01.dell-atc.lan", "ip": "10.0.20.164", "role": "mgmt", "kind": "vm", "vlan_id": 20},
|
||
]
|
||
|
||
|
||
def _seed_network_endpoints() -> None:
|
||
"""Ensure schema + known host inventory + correct FDE Proxmox identity."""
|
||
with _db() as conn:
|
||
_ensure_network_endpoints_schema(conn)
|
||
for h in _known_host_inventory():
|
||
_upsert_network_endpoint(
|
||
hostname=h.get("hostname"),
|
||
ip=h["ip"],
|
||
role=h.get("role") or "server",
|
||
kind=h.get("kind") or "host",
|
||
device_id=h.get("device_id"),
|
||
vlan_id=h.get("vlan_id"),
|
||
switch_id=h.get("switch_id"),
|
||
switch_port=h.get("switch_port"),
|
||
model=h.get("model"),
|
||
note=h.get("note"),
|
||
overwrite_identity=bool(h.get("overwrite_identity")),
|
||
clear_device_id=bool(h.get("clear_device_id")),
|
||
)
|
||
|
||
|
||
def sync_endpoints_from_fleet(devices: list[dict] | None = None) -> int:
|
||
"""Upsert every OME fleet mgmt IP into network_endpoints (enables notes on all systems)."""
|
||
_seed_network_endpoints()
|
||
devices = devices if devices is not None else (STATE.get("devices") or [])
|
||
n = 0
|
||
for d in devices:
|
||
if d.get("source") == "inventory":
|
||
continue
|
||
did = d.get("id")
|
||
if not isinstance(did, int) or did <= 0:
|
||
continue
|
||
ip = d.get("idrac_ip") or d.get("ip")
|
||
if not ip:
|
||
continue
|
||
role = d.get("role") or "server"
|
||
kind = "switch" if d.get("is_switch") else ("pdu" if d.get("is_pdu") else ("chassis" if d.get("is_chassis") else "physical"))
|
||
_upsert_network_endpoint(
|
||
hostname=d.get("name"),
|
||
ip=ip,
|
||
role=role,
|
||
kind=kind,
|
||
device_id=did,
|
||
vlan_id=_vlan_id_for_ip(ip),
|
||
model=d.get("model"),
|
||
note=f"OME {d.get('service_tag') or did}",
|
||
)
|
||
n += 1
|
||
# Also register secondary OS IPs from rdp/os/extra
|
||
for sip in (d.get("rdp_ips") or []) + (d.get("os_ips") or []) + (d.get("extra_ips") or []):
|
||
if not sip or sip == ip:
|
||
continue
|
||
_upsert_network_endpoint(
|
||
hostname=d.get("cluster_hostname") or d.get("os_hostname") or d.get("name"),
|
||
ip=sip,
|
||
role=role,
|
||
kind="os",
|
||
device_id=did,
|
||
vlan_id=_vlan_id_for_ip(sip),
|
||
model=d.get("model"),
|
||
note=f"OS/secondary IP for OME {d.get('service_tag') or did}",
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def _load_network_endpoints() -> list[dict]:
|
||
_seed_network_endpoints()
|
||
with _db() as conn:
|
||
_ensure_network_endpoints_schema(conn)
|
||
return [dict(r) for r in conn.execute("SELECT * FROM network_endpoints ORDER BY ip").fetchall()]
|
||
|
||
|
||
def _load_vlan_catalog_by_cidr() -> dict[str, dict]:
|
||
_seed_atc_vlans()
|
||
with _db() as conn:
|
||
rows = [dict(r) for r in conn.execute("SELECT * FROM vlans").fetchall()]
|
||
out = {}
|
||
for r in rows:
|
||
cidr = (r.get("cidr") or "").strip()
|
||
if cidr:
|
||
out[cidr] = r
|
||
return out
|
||
|
||
|
||
def _device_ips(d: dict) -> list[str]:
|
||
out: list[str] = []
|
||
for k in ("ip", "idrac_ip"):
|
||
v = d.get(k)
|
||
if v and v not in out:
|
||
out.append(v)
|
||
for key in ("rdp_ips", "os_ips", "extra_ips"):
|
||
for v in d.get(key) or []:
|
||
if v and v not in out:
|
||
out.append(v)
|
||
return out
|
||
|
||
|
||
def _refresh_map_cidrs(d: dict) -> None:
|
||
cidrs: list[str] = []
|
||
seen: set[str] = set()
|
||
primary = d.get("subnet")
|
||
if primary and primary != "unknown" and primary not in seen:
|
||
seen.add(primary)
|
||
cidrs.append(primary)
|
||
for ip in _device_ips(d):
|
||
c = subnet_of(ip)
|
||
if c != "unknown" and c not in seen:
|
||
seen.add(c)
|
||
cidrs.append(c)
|
||
d["map_cidrs"] = cidrs
|
||
|
||
|
||
def apply_network_endpoints(devices: list[dict]) -> None:
|
||
"""Attach inventory IPs to OME devices and inject synthetic hosts for unlinked endpoints."""
|
||
endpoints = _load_network_endpoints()
|
||
if not endpoints:
|
||
for d in devices:
|
||
_refresh_map_cidrs(d)
|
||
return
|
||
|
||
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
|
||
# Index existing IPs on devices to avoid duplicate synthetics for OME mgmt IPs
|
||
device_ips: set[str] = set()
|
||
for d in devices:
|
||
for ip in _device_ips(d):
|
||
device_ips.add(ip)
|
||
|
||
existing_ids = set(by_id)
|
||
used_synth: set[int] = set()
|
||
|
||
for ep in endpoints:
|
||
ip = (ep.get("ip") or "").strip()
|
||
if not ip or ip.count(".") != 3:
|
||
continue
|
||
hostname = (ep.get("hostname") or ip).strip()
|
||
did = ep.get("device_id")
|
||
user_note = ep.get("user_note") or ""
|
||
sys_note = ep.get("note") or ""
|
||
|
||
if did is not None and did in by_id:
|
||
d = by_id[did]
|
||
extras = list(d.get("extra_ips") or [])
|
||
# Only treat as extra if not already primary mgmt IP
|
||
if ip not in extras and ip != d.get("ip") and ip != d.get("idrac_ip"):
|
||
extras.append(ip)
|
||
d["extra_ips"] = extras
|
||
if ip != d.get("ip") and ip != d.get("idrac_ip"):
|
||
os_ips = list(d.get("os_ips") or [])
|
||
if ip not in os_ips:
|
||
os_ips.append(ip)
|
||
d["os_ips"] = os_ips
|
||
rdp = list(d.get("rdp_ips") or [])
|
||
if ip not in rdp:
|
||
rdp.append(ip)
|
||
d["rdp_ips"] = rdp
|
||
d["cluster_hostname"] = hostname
|
||
d["endpoint_id"] = ep.get("id")
|
||
d["endpoint_note"] = sys_note
|
||
d["user_note"] = user_note
|
||
if ep.get("model") and (d.get("source") == "inventory" or not d.get("model")):
|
||
pass # keep OME model authoritative for linked devices
|
||
if ep.get("switch_id") is not None:
|
||
d["endpoint_switch_id"] = ep.get("switch_id")
|
||
d["endpoint_switch_port"] = ep.get("switch_port")
|
||
_refresh_map_cidrs(d)
|
||
continue
|
||
|
||
# Skip synthetic if this IP already belongs to an OME device in the fleet
|
||
if ip in device_ips:
|
||
# Still attach notes onto the matching device if we can find it
|
||
for d in devices:
|
||
if ip in _device_ips(d) or d.get("ip") == ip or d.get("idrac_ip") == ip:
|
||
d["endpoint_id"] = ep.get("id")
|
||
d["endpoint_note"] = sys_note
|
||
d["user_note"] = user_note
|
||
break
|
||
continue
|
||
|
||
sid = _synthetic_device_id(ip)
|
||
if sid in existing_ids or sid in used_synth:
|
||
continue
|
||
used_synth.add(sid)
|
||
cidr = subnet_of(ip)
|
||
role = ep.get("role") or "server"
|
||
devices.append(
|
||
{
|
||
"id": sid,
|
||
"name": hostname,
|
||
"model": ep.get("model") or "Inventory host",
|
||
"service_tag": None,
|
||
"type": 1000,
|
||
"sub_type": "inventory",
|
||
"connected": True,
|
||
"power_state": None,
|
||
"powered_on": None,
|
||
"status": "inventory",
|
||
"ip": ip,
|
||
"idrac_ip": None,
|
||
"os_hostname": hostname,
|
||
"mgmt_dns_name": hostname,
|
||
"rdp_host": ip,
|
||
"rdp_ips": [ip],
|
||
"os_ips": [ip],
|
||
"extra_ips": [ip],
|
||
"is_windows": False,
|
||
"subnet": cidr,
|
||
"map_cidrs": [cidr],
|
||
"role": role,
|
||
"is_server": True,
|
||
"is_idrac": False,
|
||
"is_switch": role == "switch",
|
||
"is_chassis": False,
|
||
"is_pdu": role == "pdu",
|
||
"is_storage": role == "storage",
|
||
"chassis_service_tag": None,
|
||
"idrac_url": None,
|
||
"watts": None,
|
||
"source": "inventory",
|
||
"endpoint_id": ep.get("id"),
|
||
"endpoint_note": sys_note,
|
||
"user_note": user_note,
|
||
"endpoint_switch_id": ep.get("switch_id"),
|
||
"endpoint_switch_port": ep.get("switch_port"),
|
||
"cluster_hostname": hostname,
|
||
}
|
||
)
|
||
|
||
for d in devices:
|
||
_refresh_map_cidrs(d)
|
||
|
||
|
||
def build_subnet_summaries(devices: list[dict]) -> list[dict]:
|
||
"""Aggregate devices by map_cidrs (mgmt + secondary/inventory IPs) and attach VLAN catalog labels."""
|
||
catalog = _load_vlan_catalog_by_cidr()
|
||
buckets: dict[str, list[dict]] = defaultdict(list)
|
||
for d in devices:
|
||
cidrs = d.get("map_cidrs") or ([d.get("subnet")] if d.get("subnet") else [])
|
||
for c in cidrs:
|
||
if not c or c == "unknown":
|
||
continue
|
||
buckets[c].append(d)
|
||
|
||
for cidr in catalog:
|
||
buckets.setdefault(cidr, [])
|
||
|
||
subnets = []
|
||
for cidr, members in sorted(buckets.items(), key=lambda x: (-len(x[1]), x[0])):
|
||
seen: set = set()
|
||
uniq = []
|
||
for m in members:
|
||
mid = m.get("id")
|
||
if mid in seen:
|
||
continue
|
||
seen.add(mid)
|
||
uniq.append(m)
|
||
ids = [m["id"] for m in uniq if m.get("id") is not None]
|
||
sub_watts = sum(n["watts"] or 0 for n in uniq if n.get("watts") is not None)
|
||
switches = [n for n in uniq if n.get("is_switch")]
|
||
meta = catalog.get(cidr) or {}
|
||
subnets.append(
|
||
{
|
||
"cidr": cidr,
|
||
"count": len(uniq),
|
||
"connected": sum(1 for n in uniq if n.get("connected")),
|
||
"watts": round(sub_watts, 1) if sub_watts else None,
|
||
"device_ids": ids,
|
||
"has_switch": bool(switches),
|
||
"switches": [
|
||
{
|
||
"id": s["id"],
|
||
"name": s["name"],
|
||
"service_tag": s.get("service_tag"),
|
||
"ip": s.get("ip"),
|
||
}
|
||
for s in switches
|
||
],
|
||
"vlan_id": meta.get("vlan_id"),
|
||
"vlan_name": meta.get("name"),
|
||
"vlan_color": meta.get("color"),
|
||
"site_id": meta.get("site_id"),
|
||
}
|
||
)
|
||
return subnets
|
||
|
||
|
||
def _all_port_links() -> list[dict]:
|
||
with _db() as conn:
|
||
conn.execute(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS port_links (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
switch_id INTEGER NOT NULL,
|
||
switch_port INTEGER NOT NULL,
|
||
device_id INTEGER NOT NULL,
|
||
device_port TEXT,
|
||
note TEXT,
|
||
updated_at REAL NOT NULL,
|
||
UNIQUE(switch_id, switch_port)
|
||
)
|
||
"""
|
||
)
|
||
return [dict(r) for r in conn.execute("SELECT * FROM port_links").fetchall()]
|
||
|
||
|
||
_NETWORK_INV_TS = 0.0
|
||
|
||
|
||
def ensure_network_inventory_in_state(force: bool = False) -> None:
|
||
"""Merge inventory endpoints into live fleet STATE (safe between OME polls)."""
|
||
global _NETWORK_INV_TS
|
||
now = time.time()
|
||
if not force and STATE.get("devices") and (now - _NETWORK_INV_TS) < 8:
|
||
return
|
||
devices = list(STATE.get("devices") or [])
|
||
sync_endpoints_from_fleet(devices)
|
||
devices = [d for d in devices if d.get("source") != "inventory"]
|
||
apply_network_endpoints(devices)
|
||
STATE["devices"] = devices
|
||
STATE["subnets"] = build_subnet_summaries(devices)
|
||
_NETWORK_INV_TS = now
|
||
|
||
|
||
def build_vlan_map() -> dict:
|
||
"""VLANs from catalog + membership from fleet IPs + inventory endpoints; switch attachment matrix."""
|
||
_seed_atc_vlans()
|
||
_seed_network_endpoints()
|
||
_migrate_rack_items()
|
||
ensure_network_inventory_in_state()
|
||
devices = list(STATE.get("devices") or [])
|
||
|
||
with _db() as conn:
|
||
rows = [dict(r) for r in conn.execute("SELECT * FROM vlans ORDER BY sort_order, vlan_id").fetchall()]
|
||
|
||
def in_cidr(ip: str, cidr: str) -> bool:
|
||
try:
|
||
return ipaddress.ip_address(ip) in ipaddress.ip_network(cidr, strict=False)
|
||
except Exception:
|
||
return False
|
||
|
||
switches_by_id = {d.get("id"): d for d in devices if d.get("is_switch")}
|
||
links = _all_port_links()
|
||
links_by_device: dict[int, list[dict]] = defaultdict(list)
|
||
for link in links:
|
||
did = link.get("device_id")
|
||
if did is None:
|
||
continue
|
||
sw = switches_by_id.get(link.get("switch_id")) or {}
|
||
links_by_device[int(did)].append(
|
||
{
|
||
"switch_id": link.get("switch_id"),
|
||
"switch_name": sw.get("name") or f"switch-{link.get('switch_id')}",
|
||
"switch_ip": sw.get("ip") or sw.get("idrac_ip"),
|
||
"switch_port": link.get("switch_port"),
|
||
"device_port": link.get("device_port") or "",
|
||
"note": link.get("note") or "",
|
||
}
|
||
)
|
||
|
||
for d in devices:
|
||
sid = d.get("endpoint_switch_id")
|
||
if sid is None:
|
||
continue
|
||
did = d.get("id")
|
||
if did is None:
|
||
continue
|
||
already = {(x["switch_id"], x.get("switch_port")) for x in links_by_device.get(did, [])}
|
||
port = d.get("endpoint_switch_port")
|
||
key = (sid, port)
|
||
if key in already:
|
||
continue
|
||
sw = switches_by_id.get(sid) or {}
|
||
links_by_device[int(did)].append(
|
||
{
|
||
"switch_id": sid,
|
||
"switch_name": sw.get("name") or f"switch-{sid}",
|
||
"switch_ip": sw.get("ip") or sw.get("idrac_ip"),
|
||
"switch_port": port,
|
||
"device_port": "",
|
||
"note": "from inventory endpoint",
|
||
"inferred": True,
|
||
}
|
||
)
|
||
|
||
eps_by_ip = {e.get("ip"): e for e in _load_network_endpoints() if e.get("ip")}
|
||
|
||
vlans = []
|
||
for v in rows:
|
||
members = []
|
||
for d in devices:
|
||
matched = [ip for ip in _device_ips(d) if in_cidr(ip, v["cidr"])]
|
||
if not matched:
|
||
continue
|
||
mid = d.get("id")
|
||
display = d.get("cluster_hostname") or d.get("name")
|
||
ep = None
|
||
for ip in matched:
|
||
if ip in eps_by_ip:
|
||
ep = eps_by_ip[ip]
|
||
break
|
||
model = d.get("model")
|
||
if d.get("source") == "inventory" and ep and ep.get("model"):
|
||
model = ep.get("model")
|
||
members.append(
|
||
{
|
||
"id": mid,
|
||
"name": display,
|
||
"ome_name": d.get("name"),
|
||
"service_tag": d.get("service_tag"),
|
||
"role": d.get("role"),
|
||
"connected": d.get("connected"),
|
||
"ips": matched,
|
||
"model": model,
|
||
"source": d.get("source") or "ome",
|
||
"switch_ports": links_by_device.get(mid, []),
|
||
"endpoint_id": d.get("endpoint_id") or (ep or {}).get("id"),
|
||
"endpoint_note": d.get("endpoint_note") or (ep or {}).get("note") or "",
|
||
"user_note": d.get("user_note") or (ep or {}).get("user_note") or "",
|
||
}
|
||
)
|
||
members.sort(key=lambda m: (m.get("name") or "").lower())
|
||
vlans.append(
|
||
{
|
||
**v,
|
||
"member_count": len(members),
|
||
"connected_count": sum(1 for m in members if m.get("connected")),
|
||
"members": members[:250],
|
||
}
|
||
)
|
||
|
||
matrix = []
|
||
for d in devices:
|
||
if not (d.get("is_server") or d.get("source") == "inventory"):
|
||
continue
|
||
if d.get("is_switch") or d.get("is_pdu"):
|
||
continue
|
||
ips = _device_ips(d)
|
||
member_vlans = []
|
||
for v in rows:
|
||
matched = [ip for ip in ips if in_cidr(ip, v["cidr"])]
|
||
if matched:
|
||
member_vlans.append(
|
||
{
|
||
"vlan_id": v.get("vlan_id"),
|
||
"name": v.get("name"),
|
||
"cidr": v.get("cidr"),
|
||
"color": v.get("color"),
|
||
"ips": matched,
|
||
}
|
||
)
|
||
if not member_vlans and not links_by_device.get(d.get("id")):
|
||
continue
|
||
matrix.append(
|
||
{
|
||
"id": d.get("id"),
|
||
"name": d.get("cluster_hostname") or d.get("name"),
|
||
"ome_name": d.get("name"),
|
||
"service_tag": d.get("service_tag"),
|
||
"model": d.get("model"),
|
||
"role": d.get("role"),
|
||
"connected": d.get("connected"),
|
||
"source": d.get("source") or "ome",
|
||
"ips": ips,
|
||
"vlans": member_vlans,
|
||
"switch_ports": links_by_device.get(d.get("id"), []),
|
||
"endpoint_id": d.get("endpoint_id"),
|
||
"endpoint_note": d.get("endpoint_note") or "",
|
||
"user_note": d.get("user_note") or "",
|
||
}
|
||
)
|
||
matrix.sort(key=lambda m: (m.get("name") or "").lower())
|
||
|
||
seen = {v["cidr"] for v in rows}
|
||
discovered = []
|
||
buckets: dict[str, int] = {}
|
||
for d in devices:
|
||
for ip in _device_ips(d):
|
||
if ip.count(".") != 3:
|
||
continue
|
||
cidr = ".".join(ip.split(".")[:3]) + ".0/24"
|
||
if cidr.startswith("127."):
|
||
continue
|
||
buckets[cidr] = buckets.get(cidr, 0) + 1
|
||
for cidr, cnt in sorted(buckets.items(), key=lambda x: -x[1]):
|
||
if cidr not in seen:
|
||
discovered.append({"cidr": cidr, "device_ips": cnt, "in_catalog": False})
|
||
|
||
return {
|
||
"vlans": vlans,
|
||
"attachment_matrix": matrix,
|
||
"port_links": [
|
||
{
|
||
"switch_id": l.get("switch_id"),
|
||
"switch_port": l.get("switch_port"),
|
||
"device_id": l.get("device_id"),
|
||
"device_port": l.get("device_port"),
|
||
"note": l.get("note"),
|
||
}
|
||
for l in links
|
||
],
|
||
"discovered_subnets": discovered,
|
||
"note": "VLAN catalog + OME IPs + inventory endpoints. Switch ports from Fabric wiring and endpoint hints.",
|
||
}
|
||
|
||
|
||
|
||
def _migrate_rack_items() -> None:
|
||
"""Copy legacy rack_placements into flexible rack_items once."""
|
||
with _db() as conn:
|
||
n = conn.execute("SELECT COUNT(*) AS c FROM rack_items").fetchone()["c"]
|
||
if n > 0:
|
||
return
|
||
try:
|
||
rows = conn.execute("SELECT * FROM rack_placements").fetchall()
|
||
except Exception:
|
||
return
|
||
now = time.time()
|
||
for r in rows:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at)
|
||
VALUES (?, ?, NULL, NULL, ?, ?, 'front', NULL, ?)
|
||
""",
|
||
(r["rack_id"], r["device_id"], r["u_start"], r["u_height"], now),
|
||
)
|
||
if rows:
|
||
log.info("Migrated %s rack_placements → rack_items", len(rows))
|
||
|
||
OPS_USERS = [
|
||
{
|
||
"id": "jody",
|
||
"name": "Jody van Dongen",
|
||
"role": "ATC Datacenter Admin",
|
||
"team": "admin",
|
||
"email": "jody.van.dongen@dell.com",
|
||
"focus": "OME fleet · racks · warranty / compliance",
|
||
},
|
||
{
|
||
"id": "laurens",
|
||
"name": "Laurens Rammers",
|
||
"role": "ATC Datacenter Admin",
|
||
"team": "admin",
|
||
"email": "laurens.rammers@dell.com",
|
||
"focus": "Datacenter ops · handoffs · escalation",
|
||
},
|
||
{
|
||
"id": "mo",
|
||
"name": "Mohamed El Kadi",
|
||
"role": "Data Forward Deployed Engineer",
|
||
"team": "fde",
|
||
"email": "mohamed.el.kadi@dell.com",
|
||
"focus": "OME Cockpit · OpenManage AI · AI workloads on FDE cluster",
|
||
},
|
||
{
|
||
"id": "bart",
|
||
"name": "Bart Sjerps",
|
||
"role": "Data Forward Deployed Engineer",
|
||
"team": "fde",
|
||
"email": "bart.sjerps@dell.com",
|
||
"focus": "FDE cluster · AI workload deployment with Mo",
|
||
},
|
||
]
|
||
# Back-compat alias used by older ticket endpoints / UI
|
||
ADMINS = OPS_USERS
|
||
|
||
|
||
def _ops_user(uid: str) -> dict | None:
|
||
for u in OPS_USERS:
|
||
if u["id"] == uid:
|
||
return u
|
||
return None
|
||
|
||
|
||
def _default_assignee(created_by: str) -> str:
|
||
"""Admins hand off to each other; FDE hand off to an admin by default."""
|
||
if created_by == "jody":
|
||
return "laurens"
|
||
if created_by == "laurens":
|
||
return "jody"
|
||
if created_by in ("mo", "bart"):
|
||
return "jody"
|
||
return "jody"
|
||
|
||
DATA_DIR = Path(settings.cockpit_data)
|
||
DB_PATH = DATA_DIR / "ops.db"
|
||
|
||
|
||
def _db():
|
||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
conn = sqlite3.connect(DB_PATH)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.execute("PRAGMA foreign_keys=ON")
|
||
return conn
|
||
|
||
|
||
def _backup_ops_db(reason: str = "manual") -> Path | None:
|
||
"""Snapshot ops.db so rebuilds / wipes do not silently lose ticket history."""
|
||
try:
|
||
if not DB_PATH.exists() or DB_PATH.stat().st_size < 100:
|
||
return None
|
||
bak_dir = DATA_DIR / "backups"
|
||
bak_dir.mkdir(parents=True, exist_ok=True)
|
||
stamp = time.strftime("%Y%m%d-%H%M%S")
|
||
dest = bak_dir / f"ops-{stamp}-{reason}-{int(time.time()*1000)%100000}.db"
|
||
import shutil
|
||
shutil.copy2(DB_PATH, dest)
|
||
# keep last 20 backups
|
||
olds = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True)
|
||
for stale in olds[20:]:
|
||
try:
|
||
stale.unlink()
|
||
except Exception:
|
||
pass
|
||
return dest
|
||
except Exception as e:
|
||
log.warning("ops.db backup failed: %s", e)
|
||
return None
|
||
|
||
|
||
def _restore_ops_db_if_empty() -> bool:
|
||
"""If tickets table is empty but a backup exists, restore the newest backup with tickets."""
|
||
try:
|
||
with _db() as conn:
|
||
n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
|
||
if n > 0:
|
||
return False
|
||
bak_dir = DATA_DIR / "backups"
|
||
if not bak_dir.exists():
|
||
return False
|
||
candidates = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True)
|
||
for cand in candidates:
|
||
try:
|
||
c2 = sqlite3.connect(cand)
|
||
c2.row_factory = sqlite3.Row
|
||
cnt = c2.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
|
||
c2.close()
|
||
if cnt <= 0:
|
||
continue
|
||
import shutil
|
||
# preserve empty current as .empty
|
||
if DB_PATH.exists():
|
||
shutil.copy2(DB_PATH, bak_dir / f"ops-empty-before-restore-{int(time.time())}.db")
|
||
shutil.copy2(cand, DB_PATH)
|
||
log.warning("Restored ops.db from backup %s (%s tickets)", cand.name, cnt)
|
||
return True
|
||
except Exception as e:
|
||
log.debug("skip backup %s: %s", cand, e)
|
||
return False
|
||
except Exception as e:
|
||
log.warning("ops.db restore check failed: %s", e)
|
||
return False
|
||
|
||
|
||
def init_db():
|
||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
with _db() as conn:
|
||
conn.executescript(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS tickets (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
title TEXT NOT NULL,
|
||
body TEXT NOT NULL,
|
||
status TEXT NOT NULL DEFAULT 'open',
|
||
priority TEXT NOT NULL DEFAULT 'normal',
|
||
created_by TEXT NOT NULL,
|
||
assignee TEXT,
|
||
accepted_by TEXT,
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS ticket_messages (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
ticket_id INTEGER NOT NULL REFERENCES tickets(id) ON DELETE CASCADE,
|
||
author TEXT NOT NULL,
|
||
body TEXT NOT NULL,
|
||
created_at REAL NOT NULL
|
||
);
|
||
"""
|
||
)
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(tickets)").fetchall()}
|
||
if "accepted_by" not in cols:
|
||
conn.execute("ALTER TABLE tickets ADD COLUMN accepted_by TEXT")
|
||
conn.executescript(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS sites (
|
||
id TEXT PRIMARY KEY,
|
||
name TEXT NOT NULL,
|
||
sort_order INTEGER NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS racks (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
site_id TEXT NOT NULL REFERENCES sites(id),
|
||
name TEXT NOT NULL,
|
||
units INTEGER NOT NULL DEFAULT 42,
|
||
sort_order INTEGER NOT NULL,
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS rack_placements (
|
||
device_id INTEGER PRIMARY KEY,
|
||
rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE,
|
||
u_start INTEGER NOT NULL,
|
||
u_height INTEGER NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS rack_items (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE,
|
||
device_id INTEGER,
|
||
catalog_sku TEXT,
|
||
label TEXT,
|
||
u_start INTEGER NOT NULL,
|
||
u_height INTEGER NOT NULL,
|
||
side TEXT NOT NULL DEFAULT 'front',
|
||
notes TEXT,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS port_links (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
switch_id INTEGER NOT NULL,
|
||
switch_port INTEGER NOT NULL,
|
||
device_id INTEGER NOT NULL,
|
||
device_port TEXT,
|
||
note TEXT,
|
||
updated_at REAL NOT NULL,
|
||
UNIQUE(switch_id, switch_port)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS vlans (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
vlan_id INTEGER,
|
||
name TEXT NOT NULL,
|
||
cidr TEXT NOT NULL,
|
||
purpose TEXT,
|
||
site_id TEXT,
|
||
color TEXT,
|
||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS network_endpoints (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
hostname TEXT,
|
||
ip TEXT NOT NULL UNIQUE,
|
||
role TEXT NOT NULL DEFAULT 'server',
|
||
kind TEXT NOT NULL DEFAULT 'physical',
|
||
device_id INTEGER,
|
||
vlan_id INTEGER,
|
||
switch_id INTEGER,
|
||
switch_port INTEGER,
|
||
model TEXT,
|
||
note TEXT,
|
||
created_at REAL NOT NULL,
|
||
updated_at REAL NOT NULL
|
||
);
|
||
"""
|
||
)
|
||
_restore_ops_db_if_empty()
|
||
_seed_atc_racks()
|
||
_seed_atc_vlans()
|
||
_seed_network_endpoints()
|
||
n = 0
|
||
try:
|
||
with _db() as conn:
|
||
n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"]
|
||
except Exception:
|
||
pass
|
||
log.info("Ops tickets DB ready at %s (%s tickets)", DB_PATH, n)
|
||
|
||
|
||
async def fetch_gpu() -> dict:
|
||
url = settings.gpu_metrics_url.rstrip("/") + "/api/gpu"
|
||
try:
|
||
async with httpx.AsyncClient(timeout=4.0) as client:
|
||
r = await client.get(url)
|
||
if r.status_code == 200:
|
||
return r.json()
|
||
except Exception as e:
|
||
log.warning("GPU metrics fetch failed: %s", e)
|
||
return {
|
||
"host": "atc-gpu-prod",
|
||
"gpus": [],
|
||
"error": "unreachable",
|
||
"updated_at": time.time(),
|
||
"model": settings.vllm_model,
|
||
}
|
||
|
||
|
||
async def poll_loop():
|
||
while True:
|
||
try:
|
||
data = await ome_fetch()
|
||
gpu = await fetch_gpu()
|
||
data["gpu"] = gpu
|
||
async with _lock:
|
||
STATE.update(data)
|
||
STATE["pulse"] = int(STATE.get("pulse") or 0) + 1
|
||
await broadcast({"type": "snapshot", "data": {**STATE}})
|
||
log.info(
|
||
"OME refresh: %s devices, %s connected, %s W sampled, GPU util avg %s",
|
||
STATE["summary"].get("total"),
|
||
STATE["summary"].get("connected"),
|
||
STATE["summary"].get("total_watts"),
|
||
(gpu.get("summary") or {}).get("avg_util"),
|
||
)
|
||
except Exception as e:
|
||
log.exception("OME poll failed: %s", e)
|
||
await broadcast({"type": "error", "message": str(e)})
|
||
await asyncio.sleep(settings.poll_interval)
|
||
|
||
|
||
async def gpu_loop():
|
||
"""Faster GPU pulse so matrix feels live during inference."""
|
||
while True:
|
||
try:
|
||
gpu = await fetch_gpu()
|
||
async with _lock:
|
||
STATE["gpu"] = gpu
|
||
await broadcast({"type": "gpu", "data": gpu})
|
||
except Exception as e:
|
||
log.warning("gpu loop: %s", e)
|
||
await asyncio.sleep(2.0)
|
||
|
||
|
||
@app.on_event("startup")
|
||
async def startup():
|
||
init_db()
|
||
_backup_ops_db("startup")
|
||
asyncio.create_task(poll_loop())
|
||
asyncio.create_task(gpu_loop())
|
||
asyncio.create_task(warm_reports_cache())
|
||
|
||
|
||
async def warm_reports_cache():
|
||
"""Background warm of warranty/compliance/memory for chat + inspector."""
|
||
await asyncio.sleep(8)
|
||
try:
|
||
await fetch_warranties()
|
||
await fetch_compliance()
|
||
log.info(
|
||
"Reports cache warmed: warranties=%s compliance_outdated=%s",
|
||
(REPORT_CACHE.get("warranties") or {}).get("count"),
|
||
((REPORT_CACHE.get("compliance") or {}).get("summary") or {}).get("outdated_devices"),
|
||
)
|
||
except Exception as e:
|
||
log.warning("Reports cache warm failed: %s", e)
|
||
# Wait until fleet snapshot exists, then warm memory inventory for chat tools
|
||
for _ in range(40):
|
||
if STATE.get("devices"):
|
||
break
|
||
await asyncio.sleep(2)
|
||
try:
|
||
mem = await fetch_fleet_memory(force=False)
|
||
log.info(
|
||
"Memory cache warmed: %s/%s servers",
|
||
mem.get("with_memory"),
|
||
mem.get("count"),
|
||
)
|
||
except Exception as e:
|
||
log.warning("Memory cache warm failed: %s", e)
|
||
|
||
|
||
@app.get("/api/health")
|
||
async def health():
|
||
return {
|
||
"status": "ok",
|
||
"updated_at": STATE.get("updated_at"),
|
||
"pulse": STATE.get("pulse"),
|
||
}
|
||
|
||
|
||
@app.get("/api/fleet")
|
||
async def fleet():
|
||
ensure_network_inventory_in_state()
|
||
return {**STATE}
|
||
|
||
|
||
@app.get("/api/devices/{device_id}")
|
||
async def device_detail(device_id: int):
|
||
# lightweight detail: return fleet node + optional inventory slices
|
||
node = next((d for d in STATE.get("devices") or [] if d.get("id") == device_id), None)
|
||
if not node:
|
||
raise HTTPException(404, "Device not found in current fleet snapshot")
|
||
|
||
if device_id in DETAIL_CACHE and time.time() - DETAIL_CACHE[device_id].get("_ts", 0) < 300:
|
||
return DETAIL_CACHE[device_id]
|
||
|
||
base = settings.ome_url.rstrip("/")
|
||
detail = {"device": node, "inventory": {}, "power": {}}
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
r = await client.post(
|
||
f"{base}/api/SessionService/Sessions",
|
||
json={
|
||
"UserName": settings.ome_user,
|
||
"Password": settings.ome_password,
|
||
"SessionType": "API",
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
token = r.headers.get("X-Auth-Token")
|
||
sid = r.json().get("Id")
|
||
headers = {"X-Auth-Token": token, "Accept": "application/json"}
|
||
try:
|
||
detail["power"] = await fetch_power(client, base, headers, device_id)
|
||
|
||
# Discover all inventory types OME has for this device (app/firmware landscape)
|
||
inv_types: list[str] = []
|
||
try:
|
||
tr = await client.get(
|
||
f"{base}/api/DeviceService/Devices({device_id})/InventoryTypes",
|
||
headers=headers,
|
||
)
|
||
if tr.status_code == 200:
|
||
inv_types = list((tr.json() or {}).get("InventoryTypes") or [])
|
||
except Exception:
|
||
inv_types = []
|
||
# Ensure core + software landscape types are always attempted
|
||
for required in (
|
||
"serverProcessors",
|
||
"serverMemoryDevices",
|
||
"serverArrayDisks",
|
||
"serverPowerSupplies",
|
||
"serverNetworkInterfaces",
|
||
"deviceLocation",
|
||
"deviceManagement",
|
||
"subsystemRollupStatus",
|
||
"serverOperatingSystems",
|
||
"deviceSoftware",
|
||
"deviceLicense",
|
||
"deviceFru",
|
||
"deviceCapabilities",
|
||
"serverRaidControllers",
|
||
"serverDeviceCards",
|
||
"deviceBaseboards",
|
||
"serverDellVideos",
|
||
"serverFcCards",
|
||
"serverVirtualFlashes",
|
||
"serverStorageEnclosures",
|
||
"serverSupportedPowerStates",
|
||
"serverBiosSystemProfileSettings",
|
||
"deviceInventory",
|
||
):
|
||
if required not in inv_types:
|
||
inv_types.append(required)
|
||
|
||
detail["inventory_types"] = inv_types
|
||
for inv_type in inv_types:
|
||
try:
|
||
ir = await client.get(
|
||
f"{base}/api/DeviceService/Devices({device_id})/InventoryDetails('{inv_type}')",
|
||
headers=headers,
|
||
)
|
||
if ir.status_code == 200:
|
||
info = ir.json().get("InventoryInfo") or []
|
||
if info:
|
||
detail["inventory"][inv_type] = info
|
||
except Exception:
|
||
pass
|
||
|
||
# Normalized application / firmware landscape view
|
||
software = detail["inventory"].get("deviceSoftware") or []
|
||
os_info = detail["inventory"].get("serverOperatingSystems") or []
|
||
mgmt = detail["inventory"].get("deviceManagement") or []
|
||
licenses = detail["inventory"].get("deviceLicense") or []
|
||
def _stype(s: dict) -> str:
|
||
return str(s.get("SoftwareType") or "").strip().upper()
|
||
|
||
fw_types = {"BIOS", "FRMW", "FIRMWARE", "IDRAC", "USC", "BMC", "CPLD", "DCSM"}
|
||
drv_types = {"DRVR", "DRIVER", "DRV"}
|
||
firmware = [s for s in software if _stype(s) in fw_types or "FW" in _stype(s)]
|
||
drivers = [s for s in software if _stype(s) in drv_types or "DRIVER" in _stype(s)]
|
||
fw_ids = {id(s) for s in firmware}
|
||
drv_ids = {id(s) for s in drivers}
|
||
applications = [s for s in software if id(s) not in fw_ids and id(s) not in drv_ids]
|
||
detail["landscape"] = {
|
||
"os": os_info,
|
||
"software": software,
|
||
"management": mgmt,
|
||
"licenses": licenses,
|
||
"firmware": firmware,
|
||
"drivers": drivers,
|
||
"applications": applications,
|
||
}
|
||
detail["expansion"] = build_expansion_report(node.get("model"), detail.get("inventory") or {})
|
||
finally:
|
||
if sid:
|
||
try:
|
||
await client.delete(
|
||
f"{base}/api/SessionService/Sessions('{sid}')",
|
||
headers=headers,
|
||
)
|
||
except Exception:
|
||
pass
|
||
detail["_ts"] = time.time()
|
||
DETAIL_CACHE[device_id] = detail
|
||
return detail
|
||
|
||
|
||
@app.get("/api/devices/{device_id}/expansion")
|
||
async def device_expansion(device_id: int, force: bool = False):
|
||
if force and device_id in DETAIL_CACHE:
|
||
DETAIL_CACHE.pop(device_id, None)
|
||
detail = await device_detail(device_id)
|
||
return {
|
||
"device": detail.get("device"),
|
||
"expansion": detail.get("expansion")
|
||
or build_expansion_report((detail.get("device") or {}).get("model"), detail.get("inventory") or {}),
|
||
}
|
||
|
||
|
||
@app.websocket("/ws/fleet")
|
||
async def ws_fleet(ws: WebSocket):
|
||
await ws.accept()
|
||
CLIENTS.add(ws)
|
||
try:
|
||
await ws.send_json({"type": "snapshot", "data": {**STATE}})
|
||
while True:
|
||
await ws.receive_text()
|
||
except WebSocketDisconnect:
|
||
pass
|
||
finally:
|
||
CLIENTS.discard(ws)
|
||
|
||
|
||
class ChatIn(BaseModel):
|
||
message: str
|
||
history: list[dict] = Field(default_factory=list)
|
||
focus_device_id: int | None = None
|
||
model: str | None = None
|
||
|
||
|
||
class TicketIn(BaseModel):
|
||
title: str
|
||
body: str
|
||
created_by: str
|
||
assignee: str | None = None
|
||
priority: str = "normal"
|
||
|
||
|
||
class TicketMsgIn(BaseModel):
|
||
author: str
|
||
body: str
|
||
|
||
|
||
class TicketPatch(BaseModel):
|
||
status: str | None = None
|
||
assignee: str | None = None
|
||
priority: str | None = None
|
||
accepted_by: str | None = None
|
||
|
||
|
||
def _device_by_id(device_id: int | None) -> dict | None:
|
||
if device_id is None:
|
||
return None
|
||
return next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
|
||
|
||
|
||
def _device_by_service_tag(st: str) -> dict | None:
|
||
needle = (st or "").strip().upper()
|
||
if not needle:
|
||
return None
|
||
for d in STATE.get("devices") or []:
|
||
if (d.get("service_tag") or "").strip().upper() == needle:
|
||
return d
|
||
return None
|
||
|
||
|
||
def _find_devices_fuzzy(query: str, limit: int = 8) -> list[dict]:
|
||
q = (query or "").strip().lower()
|
||
if not q:
|
||
return []
|
||
hits = []
|
||
for d in STATE.get("devices") or []:
|
||
blob = " ".join(
|
||
str(d.get(k) or "").lower()
|
||
for k in ("name", "service_tag", "ip", "idrac_ip", "rdp_host", "os_hostname", "model")
|
||
)
|
||
if q in blob:
|
||
hits.append(d)
|
||
hits.sort(key=lambda d: (0 if (d.get("service_tag") or "").lower() == q else 1, d.get("name") or ""))
|
||
return hits[:limit]
|
||
|
||
|
||
def _format_device_card(d: dict) -> str:
|
||
mem = d.get("memory_gb")
|
||
mem_s = f"{mem}GB/{d.get('dimm_count') or '?'}DIMM" if mem is not None else "unknown"
|
||
return (
|
||
"ST={st} | name={name} | model={model} | idrac={idrac} | "
|
||
"rdp={rdp} | os_host={osh} | ram={ram} | connected={conn} | status={status} | watts={watts}"
|
||
).format(
|
||
st=(d.get("service_tag") or "NONE"),
|
||
name=(d.get("name") or "")[:48],
|
||
model=(d.get("model") or "")[:36],
|
||
idrac=d.get("idrac_ip") or d.get("ip") or "—",
|
||
rdp=(",".join(d.get("rdp_ips") or []) or d.get("rdp_host") or "unresolved"),
|
||
osh=d.get("os_hostname") or "—",
|
||
ram=mem_s,
|
||
conn="yes" if d.get("connected") else "no",
|
||
status=d.get("status"),
|
||
watts=d.get("watts"),
|
||
)
|
||
|
||
|
||
async def ome_fetch_inventory_types(device_id: int, types: list[str]) -> dict[str, list]:
|
||
"""Fast parallel inventory fetch for chat tools (not full device landscape)."""
|
||
inventory: dict[str, list] = {}
|
||
async with httpx.AsyncClient(verify=False, timeout=45.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
|
||
async def one(inv_type: str):
|
||
try:
|
||
ir = await client.get(
|
||
f"{base}/api/DeviceService/Devices({device_id})/InventoryDetails('{inv_type}')",
|
||
headers=headers,
|
||
)
|
||
if ir.status_code == 200:
|
||
info = ir.json().get("InventoryInfo") or []
|
||
if info:
|
||
return inv_type, info
|
||
except Exception as e:
|
||
log.debug("chat inv %s failed: %s", inv_type, e)
|
||
return inv_type, []
|
||
|
||
results = await asyncio.gather(*[one(t) for t in types])
|
||
for inv_type, info in results:
|
||
if info:
|
||
inventory[inv_type] = info
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
return inventory
|
||
|
||
|
||
|
||
def _memory_from_dimms(mem_raw: list) -> dict:
|
||
"""Compact CURRENT memory summary from OME serverMemoryDevices."""
|
||
dimms = 0
|
||
total_gb = 0.0
|
||
for dimm in mem_raw or []:
|
||
gb = _mem_size_gb(dimm.get("Size"))
|
||
if gb is None:
|
||
continue
|
||
dimms += 1
|
||
total_gb += gb
|
||
return {
|
||
"memory_gb": round(total_gb, 1) if dimms else None,
|
||
"dimm_count": dimms or None,
|
||
}
|
||
|
||
|
||
async def fetch_fleet_memory(force: bool = False) -> dict:
|
||
"""Warm/cache CURRENT installed RAM for all managed servers (OME inventory)."""
|
||
cached = None if force else _cache_get("fleet_memory")
|
||
if cached is not None:
|
||
return cached
|
||
|
||
devices = [
|
||
d
|
||
for d in (STATE.get("devices") or [])
|
||
if (d.get("is_server") or d.get("is_idrac") or (d.get("type") == 1000))
|
||
and d.get("source") != "inventory"
|
||
and isinstance(d.get("id"), int)
|
||
and d.get("id") > 0
|
||
]
|
||
# Prefer unique device ids
|
||
seen: set[int] = set()
|
||
targets: list[dict] = []
|
||
for d in devices:
|
||
did = d.get("id")
|
||
if did is None or did in seen:
|
||
continue
|
||
seen.add(int(did))
|
||
targets.append(d)
|
||
|
||
rows: list[dict] = []
|
||
errors = 0
|
||
sem = asyncio.Semaphore(10)
|
||
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
|
||
async def one(node: dict):
|
||
nonlocal errors
|
||
did = int(node["id"])
|
||
async with sem:
|
||
try:
|
||
ir = await client.get(
|
||
f"{base}/api/DeviceService/Devices({did})/InventoryDetails('serverMemoryDevices')",
|
||
headers=headers,
|
||
)
|
||
mem_raw = []
|
||
if ir.status_code == 200:
|
||
mem_raw = ir.json().get("InventoryInfo") or []
|
||
stats = _memory_from_dimms(mem_raw)
|
||
return {
|
||
"id": did,
|
||
"service_tag": node.get("service_tag"),
|
||
"name": node.get("name"),
|
||
"model": node.get("model"),
|
||
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
|
||
"connected": bool(node.get("connected")),
|
||
"memory_gb": stats.get("memory_gb"),
|
||
"dimm_count": stats.get("dimm_count"),
|
||
}
|
||
except Exception as e:
|
||
errors += 1
|
||
log.debug("fleet memory %s: %s", did, e)
|
||
return {
|
||
"id": did,
|
||
"service_tag": node.get("service_tag"),
|
||
"name": node.get("name"),
|
||
"model": node.get("model"),
|
||
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
|
||
"connected": bool(node.get("connected")),
|
||
"memory_gb": None,
|
||
"dimm_count": None,
|
||
"error": str(e)[:80],
|
||
}
|
||
|
||
rows = list(await asyncio.gather(*[one(n) for n in targets]))
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
|
||
rows.sort(key=lambda r: ((r.get("name") or "").lower(), r.get("service_tag") or ""))
|
||
known = [r for r in rows if r.get("memory_gb") is not None]
|
||
payload = {
|
||
"source": "OME DeviceService InventoryDetails(serverMemoryDevices)",
|
||
"count": len(rows),
|
||
"with_memory": len(known),
|
||
"errors": errors,
|
||
"total_memory_gb": round(sum(r.get("memory_gb") or 0 for r in known), 1),
|
||
"servers": rows,
|
||
"updated_at": time.time(),
|
||
}
|
||
_cache_set("fleet_memory", payload)
|
||
# also stamp onto fleet nodes for UI/context
|
||
by_id = {r["id"]: r for r in rows if r.get("id") is not None}
|
||
for d in STATE.get("devices") or []:
|
||
hit = by_id.get(d.get("id"))
|
||
if hit:
|
||
d["memory_gb"] = hit.get("memory_gb")
|
||
d["dimm_count"] = hit.get("dimm_count")
|
||
log.info(
|
||
"Fleet memory cache: %s/%s servers with RAM (%.0f GB total)",
|
||
len(known),
|
||
len(rows),
|
||
payload["total_memory_gb"],
|
||
)
|
||
return payload
|
||
|
||
|
||
|
||
def build_fleet_context(focus_device_id: int | None = None, max_chars: int | None = None) -> str:
|
||
summary = STATE.get("summary") or {}
|
||
gpu = STATE.get("gpu") or {}
|
||
gsum = gpu.get("summary") or {}
|
||
devices = STATE.get("devices") or []
|
||
alerts = STATE.get("alerts") or []
|
||
events = STATE.get("events") or []
|
||
subnets = STATE.get("subnets") or []
|
||
ctx = STATE.get("context") or {}
|
||
limit = max_chars if max_chars is not None else settings.chat_system_chars
|
||
|
||
connected = [d for d in devices if d.get("connected")]
|
||
critical = [a for a in alerts if a.get("severity") == "Critical"][:8]
|
||
warnings = [a for a in alerts if a.get("severity") == "Warning"][:5]
|
||
hottest = (ctx.get("hottest") or [])[:5]
|
||
compliance = _cache_get("compliance") or REPORT_CACHE.get("compliance") or {}
|
||
comp_sum = (compliance or {}).get("summary") or {}
|
||
by_id = {d.get("id"): d for d in devices if d.get("id") is not None}
|
||
|
||
st_lines = [
|
||
"SERVICE TAG INDEX (canonical). idrac=BMC management IP. rdp=OS/Windows RDP candidates (DNS). Never call idrac the RDP IP.",
|
||
]
|
||
for d in sorted(devices, key=lambda x: (x.get("name") or "").lower()):
|
||
st = (d.get("service_tag") or "").strip() or "NONE"
|
||
rdp = ",".join((d.get("rdp_ips") or [])[:2]) or (d.get("rdp_host") or "—")
|
||
mem = d.get("memory_gb")
|
||
ram = f"{mem}GB" if mem is not None else "?"
|
||
st_lines.append(
|
||
"- ST={st} | {name} | idrac={idrac} | rdp={rdp} | model={model} | ram={ram} | connected={conn}".format(
|
||
st=st,
|
||
name=(d.get("name") or "")[:36],
|
||
idrac=d.get("idrac_ip") or d.get("ip") or "—",
|
||
rdp=rdp[:40],
|
||
model=(d.get("model") or "")[:26],
|
||
ram=ram,
|
||
conn="yes" if d.get("connected") else "no",
|
||
)
|
||
)
|
||
st_block = "\n".join(st_lines)
|
||
|
||
lines = [
|
||
"You are OpenManage Cockpit Copilot for Dell ATC. Be concise and operational.",
|
||
"ATC Datacenter Admins (escalate here when facts are missing):",
|
||
" - Jody van Dongen <jody.van.dongen@dell.com>",
|
||
" - Laurens Rammers <laurens.rammers@dell.com>",
|
||
"ACCURACY RULES (mandatory — never break these):",
|
||
"1) Use ONLY facts from this snapshot and any TOOL FACTS block. Never invent Service Tags, IPs, DIMM counts, firmware versions, RDP targets, port maps, VLAN members, or rack placements.",
|
||
"2) If a requested fact is not present in the snapshot/TOOL FACTS, or tools failed/returned empty: say exactly what is unknown, then tell the user to overleggen met Jody van Dongen and Laurens Rammers (emails above). Do not guess.",
|
||
"3) ALWAYS cite Service Tag (ST=...), model, idrac IP, and rdp IP (or 'rdp unresolved') when discussing a system.",
|
||
"4) Management/iDRAC IP is NOT the Windows RDP IP. Never recommend RDP to an iDRAC address.",
|
||
"5) Hardware CURRENT (OME inventory) is NOT Dell CATALOG MAX. Never state catalog maxima as installed capacity.",
|
||
"6) Prefer TOOL FACTS over the summary index when both are present. Quote TOOL FACTS numbers verbatim.",
|
||
"7) Never pretend MCP/tool output exists when no TOOL FACTS block was provided.",
|
||
"",
|
||
"FLEET: total={t} connected={c} offline={o} watts={w} alerts_total={a}".format(
|
||
t=summary.get("total"),
|
||
c=summary.get("connected"),
|
||
o=summary.get("offline"),
|
||
w=summary.get("total_watts"),
|
||
a=summary.get("alerts_total"),
|
||
),
|
||
]
|
||
if comp_sum:
|
||
lines.append(
|
||
"FIRMWARE COMPLIANCE summary (Dell baseline): outdated_devices={od} critical_components={cc} baseline={bn}".format(
|
||
od=comp_sum.get("outdated_devices"),
|
||
cc=comp_sum.get("critical_components"),
|
||
bn=(comp_sum.get("baseline_name") or "")[:40],
|
||
)
|
||
)
|
||
|
||
mem_cache = _cache_get("fleet_memory") or REPORT_CACHE.get("fleet_memory") or {}
|
||
mem_servers = mem_cache.get("servers") or []
|
||
if mem_servers:
|
||
lines.append(
|
||
"MEMORY INDEX (CURRENT OME inventory, GB installed): known={k}/{n} total_gb={tg}".format(
|
||
k=mem_cache.get("with_memory"),
|
||
n=mem_cache.get("count"),
|
||
tg=mem_cache.get("total_memory_gb"),
|
||
)
|
||
)
|
||
ranked = sorted(
|
||
[r for r in mem_servers if r.get("memory_gb") is not None],
|
||
key=lambda r: (0 if r.get("connected") else 1, (r.get("name") or "").lower()),
|
||
)
|
||
for r in ranked[:55]:
|
||
lines.append(
|
||
"- ST={st} {name} model={model} ram={gb}GB dimms={dc} idrac={ip}".format(
|
||
st=r.get("service_tag") or "NONE",
|
||
name=(r.get("name") or "")[:28],
|
||
model=(r.get("model") or "")[:22],
|
||
gb=r.get("memory_gb"),
|
||
dc=r.get("dimm_count") or "?",
|
||
ip=r.get("idrac_ip") or "—",
|
||
)
|
||
)
|
||
if len(ranked) > 55:
|
||
lines.append(f"- … +{len(ranked) - 55} more in TOOL FACTS list_fleet_memory")
|
||
else:
|
||
lines.append(
|
||
"MEMORY INDEX: not warmed yet — call list_fleet_memory tool / wait for background warm."
|
||
)
|
||
|
||
lines.append("CONNECTED (sample):")
|
||
for d in connected[:14]:
|
||
lines.append("- " + _format_device_card(d))
|
||
|
||
lines.append("SUBNETS:")
|
||
for s in subnets[:8]:
|
||
lines.append(
|
||
"- {cidr}: {conn}/{cnt} W={watts}".format(
|
||
cidr=s.get("cidr"),
|
||
conn=s.get("connected"),
|
||
cnt=s.get("count"),
|
||
watts=s.get("watts"),
|
||
)
|
||
)
|
||
|
||
def _alert_line(a: dict, msg_len: int) -> str:
|
||
node = by_id.get(a.get("device_id")) or {}
|
||
st = node.get("service_tag") or a.get("service_tag") or "NONE"
|
||
return "- ST={st} {dev} idrac={ip}: {msg}".format(
|
||
st=st,
|
||
dev=(a.get("device") or node.get("name") or "")[:28],
|
||
ip=a.get("ip") or node.get("idrac_ip") or node.get("ip") or "—",
|
||
msg=(a.get("message") or "")[:msg_len],
|
||
)
|
||
|
||
lines.append("CRITICAL:")
|
||
if not critical:
|
||
lines.append("(none)")
|
||
for a in critical:
|
||
lines.append(_alert_line(a, 100))
|
||
|
||
lines.append("WARNINGS:")
|
||
for a in warnings:
|
||
lines.append(_alert_line(a, 80))
|
||
|
||
lines.append("DELTAS:")
|
||
if not events:
|
||
lines.append("(none)")
|
||
for e in events[:8]:
|
||
lines.append("- {title}: {text}".format(title=e.get("title"), text=(e.get("text") or "")[:100]))
|
||
|
||
lines.append("HOT POWER:")
|
||
for h in hottest:
|
||
node = by_id.get(h.get("id")) or {}
|
||
lines.append(
|
||
"- ST={st} {name}: {watts}W".format(
|
||
st=(node.get("service_tag") or h.get("service_tag") or "NONE"),
|
||
name=(h.get("name") or node.get("name") or "")[:28],
|
||
watts=h.get("watts"),
|
||
)
|
||
)
|
||
|
||
lines.append(
|
||
"GPU atc-gpu-prod: avg_util={u}% power={p}W mem={mu}/{mt}MB".format(
|
||
u=gsum.get("avg_util"),
|
||
p=gsum.get("total_power_w"),
|
||
mu=gsum.get("total_mem_used_mb"),
|
||
mt=gsum.get("total_mem_mb"),
|
||
)
|
||
)
|
||
for g in (gpu.get("gpus") or [])[:8]:
|
||
lines.append(
|
||
"- GPU{i}: {util}% {temp}C {pw}W".format(
|
||
i=g.get("index"),
|
||
util=g.get("util_gpu"),
|
||
temp=g.get("temp_c"),
|
||
pw=g.get("power_w"),
|
||
)
|
||
)
|
||
|
||
if focus_device_id is not None:
|
||
node = _device_by_id(focus_device_id)
|
||
lines.append("FOCUS DEVICE:")
|
||
if node:
|
||
lines.append(_format_device_card(node))
|
||
related = [a for a in alerts if a.get("device_id") == focus_device_id][:5]
|
||
for a in related:
|
||
lines.append(
|
||
"alert {sev}: {msg}".format(sev=a.get("severity"), msg=(a.get("message") or "")[:100])
|
||
)
|
||
else:
|
||
lines.append("device_id=%s missing from fleet snapshot" % focus_device_id)
|
||
|
||
lines.append(
|
||
"Reply with concrete Service Tags, idrac/rdp IPs, models, and next actions. "
|
||
"Quote TOOL FACTS verbatim for hardware numbers. "
|
||
"If anything is missing from facts: do not invent — escalate to Jody van Dongen and Laurens Rammers."
|
||
)
|
||
body = "\n".join(lines)
|
||
budget = max(800, limit - len(st_block) - 40)
|
||
if len(body) > budget:
|
||
body = body[: budget - 20] + "\n…[truncated]"
|
||
out = st_block + "\n\n" + body
|
||
if len(out) > limit:
|
||
out = out[: limit - 20] + "\n…[truncated]"
|
||
return out
|
||
|
||
|
||
def plan_chat_tools(message: str, focus_device_id: int | None = None) -> list[tuple[str, dict]]:
|
||
"""Server-side tool planner — Llama3-GPTQ does not emit native tool_calls reliably."""
|
||
import re
|
||
|
||
msg = message or ""
|
||
low = msg.lower()
|
||
planned: list[tuple[str, dict]] = []
|
||
seen: set[str] = set()
|
||
|
||
def add(name: str, args: dict):
|
||
key = name + json.dumps(args, sort_keys=True, default=str)
|
||
if key in seen:
|
||
return
|
||
seen.add(key)
|
||
planned.append((name, args))
|
||
|
||
for st in re.findall(r"\b([A-Z0-9]{7})\b", msg.upper()):
|
||
if st in {"OMEPROD", "WINDOWS", "POWERED", "UNKNOWN", "SERVICE", "CONNECTED"}:
|
||
continue
|
||
if _device_by_service_tag(st):
|
||
add("lookup_device", {"query": st})
|
||
|
||
for m in re.finditer(r"\b(?:for|on|about|host|server|node)\s+([A-Za-z0-9._-]{4,40})", msg, re.I):
|
||
add("lookup_device", {"query": m.group(1)})
|
||
|
||
wants_hw = bool(
|
||
re.search(
|
||
r"\b(memory|ram|dimm|disk|drive|ssd|hdd|pcie|expansion|cpu|processor|core|slot|bay|installed|capacity)\b",
|
||
low,
|
||
)
|
||
)
|
||
wants_rdp = bool(re.search(r"\b(rdp|remote\s*desktop|windows|hyper-?v|os\s*ip|hostname)\b", low))
|
||
wants_fw = bool(re.search(r"\b(firmware|bios\s*version|outdated|compliance|firmware\s*catalog|update\s*catalog|baseline)\b", low))
|
||
wants_war = bool(re.search(r"\b(warrant|prosupport|support\s*end|days?\s*left)\b", low))
|
||
wants_alerts = bool(re.search(r"\b(alert|critical|warning|fault|health)\b", low))
|
||
wants_counts = bool(
|
||
re.search(
|
||
r"\b(hoeveel|how\s+many|count|aantal|total|fleet\s+size|connected|offline)\b",
|
||
low,
|
||
)
|
||
)
|
||
# Fleet summary questions still benefit from alerts + memory tools when relevant
|
||
if wants_counts and not planned:
|
||
add("list_alerts", {})
|
||
|
||
target_id = focus_device_id
|
||
target_st = None
|
||
if planned:
|
||
q0 = planned[0][1].get("query")
|
||
node = _device_by_service_tag(str(q0)) or ((_find_devices_fuzzy(str(q0), 1) or [None])[0])
|
||
if node:
|
||
target_id = node.get("id")
|
||
target_st = node.get("service_tag")
|
||
|
||
if wants_rdp and not any(n == "lookup_device" for n, _ in planned):
|
||
if focus_device_id:
|
||
node = _device_by_id(focus_device_id)
|
||
if node:
|
||
add("lookup_device", {"query": node.get("service_tag") or node.get("name") or str(focus_device_id)})
|
||
else:
|
||
n_lookups = 0
|
||
for d in STATE.get("devices") or []:
|
||
if d.get("is_windows") or d.get("rdp_host"):
|
||
add("lookup_device", {"query": d.get("service_tag") or d.get("name")})
|
||
n_lookups += 1
|
||
if n_lookups >= 4:
|
||
break
|
||
|
||
wants_fleet_mem = bool(
|
||
re.search(
|
||
r"\b(each|every|all|per\s+server|fleet|overview|elk|alle|iedere|elke)\b",
|
||
low,
|
||
)
|
||
) or bool(re.search(r"hoeveel\s+(memory|ram|geheugen)", low))
|
||
|
||
if wants_hw:
|
||
args: dict = {}
|
||
if target_id:
|
||
args["device_id"] = int(target_id)
|
||
if target_st:
|
||
args["service_tag"] = target_st
|
||
if args:
|
||
add("get_hardware", args)
|
||
if wants_fleet_mem:
|
||
add("list_fleet_memory", {})
|
||
elif wants_fleet_mem or not focus_device_id:
|
||
add("list_fleet_memory", {})
|
||
elif focus_device_id:
|
||
add("get_hardware", {"device_id": int(focus_device_id)})
|
||
|
||
if wants_fw:
|
||
args = {"outdated_only": True}
|
||
if target_id:
|
||
args["device_id"] = int(target_id)
|
||
if target_st:
|
||
args["service_tag"] = target_st
|
||
add("get_compliance", args)
|
||
|
||
if wants_war:
|
||
args = {}
|
||
if target_id:
|
||
args["device_id"] = int(target_id)
|
||
if target_st:
|
||
args["service_tag"] = target_st
|
||
if args:
|
||
add("get_warranty", args)
|
||
elif focus_device_id:
|
||
add("get_warranty", {"device_id": int(focus_device_id)})
|
||
|
||
if wants_alerts:
|
||
args = {}
|
||
if target_id:
|
||
args["device_id"] = int(target_id)
|
||
if target_st:
|
||
args["service_tag"] = target_st
|
||
add("list_alerts", args)
|
||
|
||
if focus_device_id and not planned:
|
||
node = _device_by_id(focus_device_id)
|
||
if node:
|
||
add("lookup_device", {"query": node.get("service_tag") or str(focus_device_id)})
|
||
|
||
return planned[:6]
|
||
|
||
|
||
async def run_chat_tool(name: str, args: dict) -> dict:
|
||
args = args or {}
|
||
try:
|
||
if name == "lookup_device":
|
||
q = str(args.get("query") or "").strip()
|
||
exact = _device_by_service_tag(q)
|
||
hits = [exact] if exact else _find_devices_fuzzy(q, 8)
|
||
return {
|
||
"tool": name,
|
||
"query": q,
|
||
"count": len(hits),
|
||
"devices": [
|
||
{
|
||
"id": d.get("id"),
|
||
"service_tag": d.get("service_tag"),
|
||
"name": d.get("name"),
|
||
"model": d.get("model"),
|
||
"idrac_ip": d.get("idrac_ip") or d.get("ip"),
|
||
"rdp_host": d.get("rdp_host"),
|
||
"rdp_ips": d.get("rdp_ips") or [],
|
||
"os_hostname": d.get("os_hostname"),
|
||
"is_windows": d.get("is_windows"),
|
||
"connected": d.get("connected"),
|
||
"status": d.get("status"),
|
||
"watts": d.get("watts"),
|
||
}
|
||
for d in hits
|
||
if d
|
||
],
|
||
}
|
||
|
||
if name == "get_hardware":
|
||
node = None
|
||
if args.get("device_id") is not None:
|
||
node = _device_by_id(int(args["device_id"]))
|
||
if not node and args.get("service_tag"):
|
||
node = _device_by_service_tag(str(args["service_tag"]))
|
||
if not node:
|
||
return {"tool": name, "error": "device not found in fleet snapshot"}
|
||
did = int(node["id"])
|
||
inv = await ome_fetch_inventory_types(
|
||
did,
|
||
[
|
||
"serverMemoryDevices",
|
||
"serverProcessors",
|
||
"serverArrayDisks",
|
||
"serverRaidControllers",
|
||
"serverDeviceCards",
|
||
"serverOperatingSystems",
|
||
],
|
||
)
|
||
report = build_expansion_report(node.get("model"), inv)
|
||
cur = report.get("current") or {}
|
||
exp = report.get("expansion") or {}
|
||
return {
|
||
"tool": name,
|
||
"device": {
|
||
"id": did,
|
||
"service_tag": node.get("service_tag"),
|
||
"name": node.get("name"),
|
||
"model": node.get("model"),
|
||
"idrac_ip": node.get("idrac_ip") or node.get("ip"),
|
||
"rdp_ips": node.get("rdp_ips") or [],
|
||
"os_hostname": node.get("os_hostname"),
|
||
},
|
||
"current_ome_inventory": {
|
||
"memory_installed_gb": (cur.get("memory") or {}).get("installed_gb"),
|
||
"dimm_count": (cur.get("memory") or {}).get("dimm_count"),
|
||
"dimm_modules": [
|
||
{
|
||
"slot": m.get("slot"),
|
||
"size_gb": m.get("size_gb"),
|
||
"rated_mts": m.get("rated_mts"),
|
||
"operating_mts": m.get("operating_mts"),
|
||
}
|
||
for m in ((cur.get("memory") or {}).get("modules") or [])[:24]
|
||
],
|
||
"cpu_sockets_populated": (cur.get("cpu") or {}).get("sockets_populated"),
|
||
"cpu_cores_total": (cur.get("cpu") or {}).get("cores_total"),
|
||
"processors": (cur.get("cpu") or {}).get("processors") or [],
|
||
"disk_count": (cur.get("disks") or {}).get("count"),
|
||
"disk_capacity_gb": (cur.get("disks") or {}).get("capacity_gb"),
|
||
"disks": [
|
||
{
|
||
"bay": d.get("bay"),
|
||
"model": d.get("model"),
|
||
"size_gb": d.get("size_gb"),
|
||
"bus": d.get("bus"),
|
||
"media": d.get("media"),
|
||
}
|
||
for d in ((cur.get("disks") or {}).get("items") or [])[:20]
|
||
],
|
||
"os": cur.get("os") or [],
|
||
},
|
||
"catalog_max_not_measured": {
|
||
"source": exp.get("source"),
|
||
"catalog_known": exp.get("catalog_known"),
|
||
"catalog_model": exp.get("catalog_model"),
|
||
"memory_max_tb": (exp.get("memory") or {}).get("max_capacity_tb"),
|
||
"dimm_slots_max": (exp.get("memory") or {}).get("dimm_slots_max"),
|
||
"cpu_sockets_max": (exp.get("cpu") or {}).get("sockets_max"),
|
||
"drive_bays_max": (exp.get("disks") or {}).get("bays_max"),
|
||
"pcie_slots_max": (exp.get("pcie") or {}).get("slots_max"),
|
||
},
|
||
"findings": report.get("findings") or [],
|
||
}
|
||
|
||
if name == "list_fleet_memory":
|
||
mem = await fetch_fleet_memory(force=bool(args.get("force")))
|
||
servers = mem.get("servers") or []
|
||
# Compact rows for the LLM — include unknown so the model does not invent
|
||
slim = [
|
||
{
|
||
"service_tag": s.get("service_tag"),
|
||
"name": s.get("name"),
|
||
"model": s.get("model"),
|
||
"idrac_ip": s.get("idrac_ip"),
|
||
"connected": s.get("connected"),
|
||
"memory_gb": s.get("memory_gb"),
|
||
"dimm_count": s.get("dimm_count"),
|
||
}
|
||
for s in servers
|
||
]
|
||
return {
|
||
"tool": name,
|
||
"source": mem.get("source"),
|
||
"count": mem.get("count"),
|
||
"with_memory": mem.get("with_memory"),
|
||
"total_memory_gb": mem.get("total_memory_gb"),
|
||
"note": "memory_gb is CURRENT installed RAM from OME inventory (not catalog max).",
|
||
"servers": slim,
|
||
}
|
||
|
||
if name == "get_compliance":
|
||
compliance = await fetch_compliance(force=False)
|
||
comps = compliance.get("components") or []
|
||
node = None
|
||
if args.get("device_id") is not None:
|
||
node = _device_by_id(int(args["device_id"]))
|
||
if not node and args.get("service_tag"):
|
||
node = _device_by_service_tag(str(args["service_tag"]))
|
||
outdated_only = bool(args.get("outdated_only", True))
|
||
if node:
|
||
did = node.get("id")
|
||
st = (node.get("service_tag") or "").upper()
|
||
rows = [
|
||
c
|
||
for c in comps
|
||
if c.get("device_id") == did
|
||
or (st and (c.get("service_tag") or "").upper() == st)
|
||
]
|
||
else:
|
||
rows = list(comps)
|
||
if outdated_only:
|
||
rows = [
|
||
c
|
||
for c in rows
|
||
if str(c.get("compliance_status") or "").lower()
|
||
in ("critical", "warning", "downgrade", "non-compliant", "outdated")
|
||
or (
|
||
str(c.get("update_action") or "").lower()
|
||
not in ("", "equal", "compliant", "ok", "none")
|
||
and str(c.get("current_version") or "") != str(c.get("catalog_version") or "")
|
||
)
|
||
]
|
||
slim = [
|
||
{
|
||
"service_tag": c.get("service_tag"),
|
||
"device_name": c.get("device_name"),
|
||
"component": c.get("component"),
|
||
"current_version": c.get("current_version"),
|
||
"catalog_version": c.get("catalog_version"),
|
||
"compliance_status": c.get("compliance_status"),
|
||
"update_action": c.get("update_action"),
|
||
}
|
||
for c in rows[:40]
|
||
]
|
||
return {"tool": name, "summary": compliance.get("summary"), "count": len(slim), "components": slim}
|
||
|
||
if name == "get_warranty":
|
||
warranties = await fetch_warranties(force=False)
|
||
items = warranties.get("items") or warranties.get("warranties") or []
|
||
node = None
|
||
if args.get("device_id") is not None:
|
||
node = _device_by_id(int(args["device_id"]))
|
||
if not node and args.get("service_tag"):
|
||
node = _device_by_service_tag(str(args["service_tag"]))
|
||
if not node:
|
||
return {"tool": name, "error": "device not found"}
|
||
st = (node.get("service_tag") or "").upper()
|
||
rows = [
|
||
w
|
||
for w in items
|
||
if w.get("device_id") == node.get("id")
|
||
or (st and (w.get("service_tag") or "").upper() == st)
|
||
]
|
||
return {
|
||
"tool": name,
|
||
"service_tag": node.get("service_tag"),
|
||
"name": node.get("name"),
|
||
"warranties": rows[:8],
|
||
}
|
||
|
||
if name == "list_alerts":
|
||
alerts = STATE.get("alerts") or []
|
||
by_id = {d.get("id"): d for d in (STATE.get("devices") or [])}
|
||
node = None
|
||
if args.get("device_id") is not None:
|
||
node = _device_by_id(int(args["device_id"]))
|
||
if not node and args.get("service_tag"):
|
||
node = _device_by_service_tag(str(args["service_tag"]))
|
||
sev = (args.get("severity") or "").strip().lower()
|
||
out = []
|
||
for a in alerts:
|
||
if node and a.get("device_id") != node.get("id"):
|
||
continue
|
||
if sev and str(a.get("severity") or "").lower() != sev:
|
||
continue
|
||
d = by_id.get(a.get("device_id")) or {}
|
||
out.append(
|
||
{
|
||
"severity": a.get("severity"),
|
||
"service_tag": d.get("service_tag"),
|
||
"device": a.get("device") or d.get("name"),
|
||
"idrac_ip": a.get("ip") or d.get("idrac_ip") or d.get("ip"),
|
||
"message": (a.get("message") or "")[:180],
|
||
}
|
||
)
|
||
if len(out) >= 25:
|
||
break
|
||
return {"tool": name, "count": len(out), "alerts": out}
|
||
|
||
return {"tool": name, "error": f"unknown tool {name}"}
|
||
except Exception as e:
|
||
log.exception("chat tool %s failed", name)
|
||
return {"tool": name, "error": str(e)}
|
||
|
||
|
||
def format_tool_facts(results: list[dict]) -> str:
|
||
if not results:
|
||
return ""
|
||
parts = ["TOOL FACTS (live lookups — authoritative; do not contradict):"]
|
||
for r in results:
|
||
if r.get("tool") == "list_fleet_memory" and r.get("servers"):
|
||
lines = [
|
||
"list_fleet_memory CURRENT RAM:",
|
||
f"known={r.get('with_memory')}/{r.get('count')} total_gb={r.get('total_memory_gb')}",
|
||
"ST\tname\tmodel\tmemory_gb\tdimms\tidrac",
|
||
]
|
||
for s in r.get("servers") or []:
|
||
lines.append(
|
||
"{st}\t{name}\t{model}\t{gb}\t{dc}\t{ip}".format(
|
||
st=s.get("service_tag") or "NONE",
|
||
name=(s.get("name") or "")[:36],
|
||
model=(s.get("model") or "")[:24],
|
||
gb=s.get("memory_gb") if s.get("memory_gb") is not None else "unknown",
|
||
dc=s.get("dimm_count") if s.get("dimm_count") is not None else "unknown",
|
||
ip=s.get("idrac_ip") or "—",
|
||
)
|
||
)
|
||
parts.append("\n".join(lines)[:14000])
|
||
else:
|
||
parts.append(json.dumps(r, ensure_ascii=False, default=str)[:14000])
|
||
return "\n".join(parts)
|
||
|
||
|
||
|
||
|
||
|
||
@app.get("/api/fleet-memory")
|
||
async def api_fleet_memory(force: bool = False):
|
||
"""CURRENT installed RAM per server (OME inventory cache)."""
|
||
return await fetch_fleet_memory(force=force)
|
||
|
||
|
||
@app.get("/api/gpu")
|
||
async def api_gpu():
|
||
if not STATE.get("gpu"):
|
||
STATE["gpu"] = await fetch_gpu()
|
||
return STATE.get("gpu") or {}
|
||
|
||
|
||
@app.get("/api/admins")
|
||
async def api_admins():
|
||
return {"admins": OPS_USERS, "users": OPS_USERS}
|
||
|
||
|
||
@app.get("/api/ops/users")
|
||
async def api_ops_users():
|
||
return {
|
||
"users": OPS_USERS,
|
||
"admins": [u for u in OPS_USERS if u.get("team") == "admin"],
|
||
"fde": [u for u in OPS_USERS if u.get("team") == "fde"],
|
||
"context": {
|
||
"cockpit": "OME Cockpit by Data Forward Deployed Engineers Mohamed El Kadi & Bart Sjerps",
|
||
"cluster": "Runs on the FDE cluster operated by Data Forward Deployed Engineers Mohamed El Kadi and Bart Sjerps",
|
||
"admins": "Jody van Dongen and Laurens Rammers — ATC datacenter administrators",
|
||
"fde": "Mo and Bart — both Data Forward Deployed Engineers deploying AI workloads",
|
||
},
|
||
}
|
||
|
||
|
||
|
||
_OWUI_TOKEN: dict[str, Any] = {"token": None, "expires": 0.0}
|
||
|
||
|
||
async def _owui_token() -> str | None:
|
||
email = (settings.openwebui_email or "").strip()
|
||
password = settings.openwebui_password or ""
|
||
if not email or not password:
|
||
return None
|
||
now = time.time()
|
||
if _OWUI_TOKEN.get("token") and float(_OWUI_TOKEN.get("expires") or 0) > now:
|
||
return str(_OWUI_TOKEN["token"])
|
||
url = settings.openwebui_url.rstrip("/") + "/api/v1/auths/signin"
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||
r = await client.post(url, json={"email": email, "password": password})
|
||
if r.status_code >= 400:
|
||
log.warning("Open WebUI signin failed: %s %s", r.status_code, r.text[:200])
|
||
return None
|
||
token = (r.json() or {}).get("token")
|
||
if token:
|
||
_OWUI_TOKEN["token"] = token
|
||
_OWUI_TOKEN["expires"] = now + 3500
|
||
return token
|
||
except Exception as e:
|
||
log.warning("Open WebUI signin error: %s", e)
|
||
return None
|
||
|
||
|
||
async def list_chat_models() -> list[dict]:
|
||
models: list[dict] = []
|
||
seen: set[str] = set()
|
||
|
||
token = await _owui_token()
|
||
if token:
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
r = await client.get(
|
||
settings.openwebui_url.rstrip("/") + "/api/models",
|
||
headers={"Authorization": f"Bearer {token}"},
|
||
)
|
||
if r.status_code == 200:
|
||
for m in (r.json() or {}).get("data") or []:
|
||
mid = m.get("id")
|
||
if not mid or mid in seen:
|
||
continue
|
||
seen.add(mid)
|
||
models.append(
|
||
{
|
||
"id": mid,
|
||
"name": m.get("name") or mid,
|
||
"source": "openwebui",
|
||
"owned_by": m.get("owned_by"),
|
||
}
|
||
)
|
||
except Exception as e:
|
||
log.warning("Open WebUI models failed: %s", e)
|
||
|
||
try:
|
||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||
r = await client.get(settings.vllm_url.rstrip("/") + "/models")
|
||
if r.status_code == 200:
|
||
for m in (r.json() or {}).get("data") or []:
|
||
mid = m.get("id")
|
||
if not mid or mid in seen:
|
||
continue
|
||
seen.add(mid)
|
||
models.append(
|
||
{
|
||
"id": mid,
|
||
"name": mid,
|
||
"source": "vllm",
|
||
"owned_by": m.get("owned_by") or "vllm",
|
||
}
|
||
)
|
||
except Exception as e:
|
||
log.warning("vLLM models failed: %s", e)
|
||
|
||
if not models:
|
||
models.append(
|
||
{
|
||
"id": settings.vllm_model,
|
||
"name": settings.vllm_model,
|
||
"source": "vllm",
|
||
"owned_by": "vllm",
|
||
}
|
||
)
|
||
return models
|
||
|
||
|
||
@app.get("/api/models")
|
||
async def api_models():
|
||
models = await list_chat_models()
|
||
return {"models": models, "default": settings.vllm_model}
|
||
|
||
|
||
|
||
def format_fleet_memory_reply(result: dict) -> str:
|
||
"""Deterministic full table — LLMs truncate long fleet lists."""
|
||
servers = result.get("servers") or []
|
||
lines = [
|
||
"CURRENT installed RAM from OME inventory (not Dell catalog max).",
|
||
"known={k}/{n} · total_installed={tg} GB".format(
|
||
k=result.get("with_memory"),
|
||
n=result.get("count"),
|
||
tg=result.get("total_memory_gb"),
|
||
),
|
||
"",
|
||
"ST | name | model | memory_gb | dimms | idrac | connected",
|
||
]
|
||
for s in servers:
|
||
gb = s.get("memory_gb")
|
||
gb_s = "unknown" if gb is None else str(gb)
|
||
dc = s.get("dimm_count")
|
||
dc_s = "unknown" if dc is None else str(dc)
|
||
lines.append(
|
||
"{st} | {name} | {model} | {gb} | {dc} | {ip} | {conn}".format(
|
||
st=s.get("service_tag") or "NONE",
|
||
name=(s.get("name") or "")[:40],
|
||
model=(s.get("model") or "")[:28],
|
||
gb=gb_s,
|
||
dc=dc_s,
|
||
ip=s.get("idrac_ip") or "—",
|
||
conn="yes" if s.get("connected") else "no",
|
||
)
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def should_use_deterministic_fleet_memory(message: str, planned: list, tool_results: list[dict]) -> dict | None:
|
||
if not any(n == "list_fleet_memory" for n, _ in planned):
|
||
return None
|
||
# Prefer deterministic when the ask is clearly a full memory inventory
|
||
low = (message or "").lower()
|
||
inventoryish = bool(
|
||
re.search(r"\b(memory|ram|geheugen|dimm)\b", low)
|
||
and re.search(r"\b(elk|elke|iedere|alle|each|every|all|per\s+server|hoeveel|list|overview)\b", low)
|
||
)
|
||
# Or when list_fleet_memory is the only tool
|
||
only_mem = len(planned) == 1 and planned[0][0] == "list_fleet_memory"
|
||
if not (inventoryish or only_mem):
|
||
return None
|
||
for r in tool_results:
|
||
if r.get("tool") == "list_fleet_memory" and not r.get("error"):
|
||
return r
|
||
return None
|
||
|
||
|
||
@app.post("/api/chat")
|
||
async def api_chat(payload: ChatIn):
|
||
gpu = STATE.get("gpu") or {}
|
||
gsum = gpu.get("summary") or {}
|
||
model = (payload.model or settings.vllm_model or "").strip() or settings.vllm_model
|
||
|
||
# 1) Plan + execute live tools BEFORE the LLM (accurate facts; Llama3-GPTQ lacks native tool_calls)
|
||
planned = plan_chat_tools(payload.message, payload.focus_device_id)
|
||
tool_results: list[dict] = []
|
||
if planned:
|
||
tool_results = list(
|
||
await asyncio.gather(*[run_chat_tool(name, args) for name, args in planned])
|
||
)
|
||
|
||
max_tokens = settings.vllm_max_tokens
|
||
if any(n == "list_fleet_memory" for n, _ in planned):
|
||
max_tokens = max(max_tokens, 1800)
|
||
|
||
# Full fleet memory lists: answer from TOOL FACTS directly (accurate, complete)
|
||
det = should_use_deterministic_fleet_memory(payload.message, planned, tool_results)
|
||
if det is not None:
|
||
gpu = STATE.get("gpu") or {}
|
||
gsum = gpu.get("summary") or {}
|
||
return {
|
||
"reply": format_fleet_memory_reply(det),
|
||
"model": model,
|
||
"backend": "ome-tools",
|
||
"gpu": gsum,
|
||
"context_bytes": 0,
|
||
"tools_used": [{"name": n, "args": a} for n, a in planned],
|
||
"tool_count": len(tool_results),
|
||
"deterministic": True,
|
||
}
|
||
|
||
system = build_fleet_context(payload.focus_device_id, max_chars=settings.chat_system_chars)
|
||
facts = format_tool_facts(tool_results)
|
||
if facts:
|
||
# Prefer tool facts; keep within budget (vLLM context is tight)
|
||
budget = max(1800, settings.chat_system_chars - 100)
|
||
# Accuracy rules + short fleet head, then authoritative TOOL FACTS
|
||
head = system
|
||
if len(system) > 2800:
|
||
head = system[:2800] + "\n…[fleet truncated for tools]"
|
||
combined = head + "\n\n" + facts
|
||
if len(combined) > budget:
|
||
# Keep TOOL FACTS intact; shrink head further
|
||
max_facts = min(len(facts), budget - 900)
|
||
facts_trim = facts[:max_facts]
|
||
head = system[: max(600, budget - len(facts_trim) - 40)]
|
||
combined = head + "\n…\n\n" + facts_trim
|
||
system = combined[:budget]
|
||
|
||
messages = [{"role": "system", "content": system}]
|
||
for h in (payload.history or [])[-4:]:
|
||
role = h.get("role")
|
||
content = h.get("content")
|
||
if role in ("user", "assistant") and content:
|
||
messages.append({"role": role, "content": str(content)[:1200]})
|
||
user_msg = payload.message[:2500]
|
||
escalate = (
|
||
"If any needed fact is absent, say it is unknown and instruct the user to overleggen met "
|
||
"Jody van Dongen (jody.van.dongen@dell.com) and Laurens Rammers (laurens.rammers@dell.com). "
|
||
"Never invent."
|
||
)
|
||
if tool_results:
|
||
tool_errors = [r for r in tool_results if r.get("error")]
|
||
user_msg += (
|
||
"\n\n[System note: live TOOL FACTS were fetched for this question. "
|
||
"Treat TOOL FACTS as ground truth. Quote CURRENT vs CATALOG separately. "
|
||
"Do not hedge or invent. Say unknown only if a field is absent from TOOL FACTS. "
|
||
"For fleet lists: output EVERY row from TOOL FACTS in a compact table "
|
||
"(ST | name | model | memory_gb | dimms | idrac) — do not summarize or omit. "
|
||
f"{escalate}]"
|
||
)
|
||
if tool_errors:
|
||
user_msg += (
|
||
"\n[System note: some tools failed — do not fill gaps from model knowledge. "
|
||
f"{escalate}]"
|
||
)
|
||
else:
|
||
user_msg += (
|
||
"\n\n[System note: no live TOOL FACTS were fetched for this turn. "
|
||
"Answer only from the fleet snapshot in the system message. "
|
||
f"{escalate}]"
|
||
)
|
||
messages.append({"role": "user", "content": user_msg})
|
||
|
||
owui_models = {"ome-copilot", "arena-model"}
|
||
use_owui = model in owui_models
|
||
|
||
try:
|
||
if use_owui:
|
||
token = await _owui_token()
|
||
if not token:
|
||
raise HTTPException(
|
||
502,
|
||
"Open WebUI auth not configured (set OPENWEBUI_EMAIL / OPENWEBUI_PASSWORD)",
|
||
)
|
||
url = settings.openwebui_url.rstrip("/") + "/api/chat/completions"
|
||
async with httpx.AsyncClient(timeout=180.0) as client:
|
||
r = await client.post(
|
||
url,
|
||
headers={"Authorization": f"Bearer {token}"},
|
||
json={
|
||
"model": model,
|
||
"messages": messages,
|
||
"temperature": 0.05,
|
||
"max_tokens": max_tokens,
|
||
"stream": False,
|
||
},
|
||
)
|
||
if r.status_code >= 400:
|
||
raise HTTPException(502, f"Open WebUI error {r.status_code}: {r.text[:500]}")
|
||
data = r.json()
|
||
backend = "openwebui"
|
||
else:
|
||
url = settings.vllm_url.rstrip("/") + "/chat/completions"
|
||
async with httpx.AsyncClient(timeout=180.0) as client:
|
||
r = await client.post(
|
||
url,
|
||
json={
|
||
"model": model,
|
||
"messages": messages,
|
||
"temperature": 0.05,
|
||
"max_tokens": max_tokens,
|
||
},
|
||
)
|
||
if r.status_code >= 400:
|
||
detail = r.text
|
||
if r.status_code == 400 and "context length" in detail.lower():
|
||
tight = build_fleet_context(payload.focus_device_id, max_chars=2800)
|
||
if facts:
|
||
tight = (tight + "\n\n" + facts)[:3800]
|
||
messages = [{"role": "system", "content": tight}, messages[-1]]
|
||
r = await client.post(
|
||
url,
|
||
json={
|
||
"model": model,
|
||
"messages": messages,
|
||
"temperature": 0.05,
|
||
"max_tokens": 500,
|
||
},
|
||
)
|
||
detail = r.text
|
||
if r.status_code >= 400:
|
||
raise HTTPException(502, f"vLLM error {r.status_code}: {detail[:600]}")
|
||
data = r.json()
|
||
backend = "vllm"
|
||
|
||
text_out = (
|
||
((data.get("choices") or [{}])[0].get("message") or {}).get("content")
|
||
or ""
|
||
)
|
||
return {
|
||
"reply": text_out,
|
||
"model": model,
|
||
"backend": backend,
|
||
"gpu": gsum,
|
||
"context_bytes": len(system),
|
||
"tools_used": [{"name": n, "args": a} for n, a in planned],
|
||
"tool_count": len(tool_results),
|
||
}
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Chat backend error: {e}") from e
|
||
|
||
|
||
|
||
|
||
def _admin_name(aid: str) -> str:
|
||
u = _ops_user(aid)
|
||
return u["name"] if u else aid
|
||
|
||
|
||
@app.get("/api/tickets")
|
||
async def list_tickets():
|
||
with _db() as conn:
|
||
rows = conn.execute(
|
||
"SELECT * FROM tickets ORDER BY updated_at DESC LIMIT 200"
|
||
).fetchall()
|
||
return {"tickets": [dict(r) for r in rows], "admins": OPS_USERS, "users": OPS_USERS}
|
||
|
||
|
||
@app.post("/api/tickets")
|
||
async def create_ticket(payload: TicketIn):
|
||
now = time.time()
|
||
assignee = payload.assignee
|
||
if not assignee or not _ops_user(assignee):
|
||
assignee = _default_assignee(payload.created_by)
|
||
if payload.created_by and not _ops_user(payload.created_by):
|
||
raise HTTPException(400, f"Unknown actor: {payload.created_by}")
|
||
with _db() as conn:
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO tickets(title, body, status, priority, created_by, assignee, created_at, updated_at)
|
||
VALUES (?, ?, 'open', ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
payload.title.strip()[:200],
|
||
payload.body.strip()[:5000],
|
||
payload.priority,
|
||
payload.created_by,
|
||
assignee,
|
||
now,
|
||
now,
|
||
),
|
||
)
|
||
tid = cur.lastrowid
|
||
conn.execute(
|
||
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
|
||
(tid, payload.created_by, payload.body.strip()[:5000], now),
|
||
)
|
||
row = conn.execute("SELECT * FROM tickets WHERE id=?", (tid,)).fetchone()
|
||
_backup_ops_db("create")
|
||
return dict(row)
|
||
|
||
|
||
@app.get("/api/tickets/{ticket_id}")
|
||
async def get_ticket(ticket_id: int):
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Ticket not found")
|
||
msgs = conn.execute(
|
||
"SELECT * FROM ticket_messages WHERE ticket_id=? ORDER BY created_at ASC",
|
||
(ticket_id,),
|
||
).fetchall()
|
||
return {"ticket": dict(row), "messages": [dict(m) for m in msgs], "admins": OPS_USERS, "users": OPS_USERS}
|
||
|
||
|
||
@app.post("/api/tickets/{ticket_id}/messages")
|
||
async def add_ticket_message(ticket_id: int, payload: TicketMsgIn):
|
||
now = time.time()
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT id FROM tickets WHERE id=?", (ticket_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Ticket not found")
|
||
conn.execute(
|
||
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
|
||
(ticket_id, payload.author, payload.body.strip()[:5000], now),
|
||
)
|
||
conn.execute("UPDATE tickets SET updated_at=? WHERE id=?", (now, ticket_id))
|
||
msgs = conn.execute(
|
||
"SELECT * FROM ticket_messages WHERE ticket_id=? ORDER BY created_at ASC",
|
||
(ticket_id,),
|
||
).fetchall()
|
||
_backup_ops_db("message")
|
||
return {"messages": [dict(m) for m in msgs]}
|
||
|
||
|
||
@app.patch("/api/tickets/{ticket_id}")
|
||
async def patch_ticket(ticket_id: int, payload: TicketPatch):
|
||
now = time.time()
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Ticket not found")
|
||
status = payload.status or row["status"]
|
||
assignee = payload.assignee if payload.assignee is not None else row["assignee"]
|
||
priority = payload.priority or row["priority"]
|
||
# accepted_by: keep existing unless explicitly set; accepting sets status accepted
|
||
accepted_by = row["accepted_by"] if "accepted_by" in row.keys() else None
|
||
if payload.accepted_by is not None:
|
||
accepted_by = payload.accepted_by
|
||
if status == "open":
|
||
status = "accepted"
|
||
conn.execute(
|
||
"UPDATE tickets SET status=?, assignee=?, priority=?, accepted_by=?, updated_at=? WHERE id=?",
|
||
(status, assignee, priority, accepted_by, now, ticket_id),
|
||
)
|
||
if payload.accepted_by:
|
||
conn.execute(
|
||
"INSERT INTO ticket_messages(ticket_id, author, body, created_at) VALUES (?, ?, ?, ?)",
|
||
(
|
||
ticket_id,
|
||
payload.accepted_by,
|
||
f"Ticket accepted by {_admin_name(payload.accepted_by)}",
|
||
now,
|
||
),
|
||
)
|
||
row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone()
|
||
_backup_ops_db("patch")
|
||
return dict(row)
|
||
|
||
|
||
@app.delete("/api/tickets/{ticket_id}")
|
||
async def delete_ticket(ticket_id: int):
|
||
_backup_ops_db("pre-delete")
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT id FROM tickets WHERE id=?", (ticket_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Ticket not found")
|
||
conn.execute("DELETE FROM ticket_messages WHERE ticket_id=?", (ticket_id,))
|
||
conn.execute("DELETE FROM tickets WHERE id=?", (ticket_id,))
|
||
_backup_ops_db("delete")
|
||
return {"ok": True, "deleted": ticket_id}
|
||
|
||
|
||
class RackIn(BaseModel):
|
||
site_id: str
|
||
name: str
|
||
units: int = 42
|
||
sort_order: int | None = None
|
||
|
||
|
||
class RackPatch(BaseModel):
|
||
name: str | None = None
|
||
sort_order: int | None = None
|
||
|
||
|
||
class PlacementIn(BaseModel):
|
||
rack_id: int
|
||
u_start: int
|
||
u_height: int | None = None
|
||
|
||
|
||
@app.get("/api/network/fabric")
|
||
async def api_network_fabric():
|
||
ensure_network_inventory_in_state()
|
||
return build_network_fabric()
|
||
|
||
|
||
class VlanIn(BaseModel):
|
||
vlan_id: int | None = None
|
||
name: str
|
||
cidr: str
|
||
purpose: str = ""
|
||
site_id: str | None = None
|
||
color: str = "#00a8e8"
|
||
sort_order: int = 0
|
||
|
||
|
||
@app.get("/api/network/vlans")
|
||
async def api_network_vlans():
|
||
return build_vlan_map()
|
||
|
||
|
||
@app.post("/api/network/vlans")
|
||
async def api_create_vlan(payload: VlanIn):
|
||
now = time.time()
|
||
try:
|
||
ipaddress.ip_network(payload.cidr, strict=False)
|
||
except Exception as e:
|
||
raise HTTPException(400, f"Invalid cidr: {e}") from e
|
||
with _db() as conn:
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
payload.vlan_id,
|
||
payload.name.strip()[:80],
|
||
payload.cidr.strip(),
|
||
(payload.purpose or "")[:200],
|
||
payload.site_id,
|
||
payload.color or "#00a8e8",
|
||
payload.sort_order,
|
||
now,
|
||
now,
|
||
),
|
||
)
|
||
vid = cur.lastrowid
|
||
_backup_ops_db("vlan-create")
|
||
return {"id": vid, **build_vlan_map()}
|
||
|
||
|
||
class EndpointIn(BaseModel):
|
||
hostname: str | None = None
|
||
ip: str
|
||
role: str = "server"
|
||
kind: str = "host"
|
||
device_id: int | None = None
|
||
vlan_id: int | None = None
|
||
switch_id: int | None = None
|
||
switch_port: int | None = None
|
||
model: str | None = None
|
||
note: str | None = None
|
||
user_note: str | None = None
|
||
|
||
|
||
class EndpointPatch(BaseModel):
|
||
hostname: str | None = None
|
||
role: str | None = None
|
||
kind: str | None = None
|
||
device_id: int | None = None
|
||
vlan_id: int | None = None
|
||
switch_id: int | None = None
|
||
switch_port: int | None = None
|
||
model: str | None = None
|
||
note: str | None = None
|
||
user_note: str | None = None
|
||
clear_device_id: bool = False
|
||
|
||
|
||
@app.get("/api/network/endpoints")
|
||
async def api_list_endpoints():
|
||
ensure_network_inventory_in_state()
|
||
return {"endpoints": _load_network_endpoints()}
|
||
|
||
|
||
@app.post("/api/network/endpoints")
|
||
async def api_create_endpoint(payload: EndpointIn):
|
||
try:
|
||
ipaddress.ip_address(payload.ip.strip())
|
||
except Exception as e:
|
||
raise HTTPException(400, f"Invalid ip: {e}") from e
|
||
_upsert_network_endpoint(
|
||
hostname=(payload.hostname or "").strip() or None,
|
||
ip=payload.ip.strip(),
|
||
role=(payload.role or "server").strip()[:40],
|
||
kind=(payload.kind or "host").strip()[:40],
|
||
device_id=payload.device_id,
|
||
vlan_id=payload.vlan_id,
|
||
switch_id=payload.switch_id,
|
||
switch_port=payload.switch_port,
|
||
model=(payload.model or "").strip()[:80] or None,
|
||
note=(payload.note or "").strip()[:500] or None,
|
||
overwrite_identity=True,
|
||
)
|
||
if payload.user_note is not None:
|
||
with _db() as conn:
|
||
_ensure_network_endpoints_schema(conn)
|
||
conn.execute(
|
||
"UPDATE network_endpoints SET user_note=?, updated_at=? WHERE ip=?",
|
||
((payload.user_note or "")[:2000], time.time(), payload.ip.strip()),
|
||
)
|
||
ensure_network_inventory_in_state(force=True)
|
||
_backup_ops_db("endpoint-create")
|
||
row = next((e for e in _load_network_endpoints() if e.get("ip") == payload.ip.strip()), None)
|
||
return {"endpoint": row, "vlans": build_vlan_map()}
|
||
|
||
|
||
@app.patch("/api/network/endpoints/{endpoint_id}")
|
||
async def api_patch_endpoint(endpoint_id: int, payload: EndpointPatch):
|
||
with _db() as conn:
|
||
_ensure_network_endpoints_schema(conn)
|
||
row = conn.execute("SELECT * FROM network_endpoints WHERE id=?", (endpoint_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Endpoint not found")
|
||
sets = ["updated_at=?"]
|
||
vals: list = [time.time()]
|
||
data = payload.model_dump(exclude_unset=True)
|
||
clear = bool(data.pop("clear_device_id", False))
|
||
mapping = {
|
||
"hostname": 80,
|
||
"role": 40,
|
||
"kind": 40,
|
||
"model": 80,
|
||
"note": 500,
|
||
"user_note": 2000,
|
||
}
|
||
for key, maxlen in mapping.items():
|
||
if key in data and data[key] is not None:
|
||
sets.append(f"{key}=?")
|
||
vals.append(str(data[key])[:maxlen])
|
||
for key in ("device_id", "vlan_id", "switch_id", "switch_port"):
|
||
if key in data and data[key] is not None:
|
||
sets.append(f"{key}=?")
|
||
vals.append(data[key])
|
||
if clear:
|
||
sets.append("device_id=NULL")
|
||
vals.append(endpoint_id)
|
||
conn.execute(f"UPDATE network_endpoints SET {', '.join(sets)} WHERE id=?", vals)
|
||
ensure_network_inventory_in_state(force=True)
|
||
_backup_ops_db("endpoint-patch")
|
||
row = next((e for e in _load_network_endpoints() if e.get("id") == endpoint_id), None)
|
||
return {"endpoint": row, "vlans": build_vlan_map()}
|
||
|
||
|
||
class PortLinkIn(BaseModel):
|
||
switch_port: int
|
||
device_id: int
|
||
device_port: str = ""
|
||
note: str = ""
|
||
|
||
|
||
@app.get("/api/network/switches/{switch_id}/ports")
|
||
async def api_switch_ports(switch_id: int):
|
||
return build_switch_portmap(switch_id)
|
||
|
||
|
||
@app.put("/api/network/switches/{switch_id}/ports/{port_num}")
|
||
async def api_put_port_link(switch_id: int, port_num: int, payload: PortLinkIn):
|
||
sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None)
|
||
if not sw:
|
||
raise HTTPException(404, "Switch not found")
|
||
nports = _switch_port_count(sw)
|
||
if port_num < 1 or port_num > nports:
|
||
raise HTTPException(400, f"Port must be 1..{nports}")
|
||
peer = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None)
|
||
if not peer:
|
||
raise HTTPException(404, "Target device not found in fleet")
|
||
now = time.time()
|
||
with _db() as conn:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO port_links(switch_id, switch_port, device_id, device_port, note, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(switch_id, switch_port) DO UPDATE SET
|
||
device_id=excluded.device_id,
|
||
device_port=excluded.device_port,
|
||
note=excluded.note,
|
||
updated_at=excluded.updated_at
|
||
""",
|
||
(
|
||
switch_id,
|
||
port_num,
|
||
payload.device_id,
|
||
(payload.device_port or "")[:80],
|
||
(payload.note or "")[:200],
|
||
now,
|
||
),
|
||
)
|
||
_backup_ops_db("port-link")
|
||
return build_switch_portmap(switch_id)
|
||
|
||
|
||
@app.delete("/api/network/switches/{switch_id}/ports/{port_num}")
|
||
async def api_delete_port_link(switch_id: int, port_num: int):
|
||
with _db() as conn:
|
||
conn.execute(
|
||
"DELETE FROM port_links WHERE switch_id=? AND switch_port=?",
|
||
(switch_id, port_num),
|
||
)
|
||
_backup_ops_db("port-unlink")
|
||
return build_switch_portmap(switch_id)
|
||
|
||
|
||
@app.get("/api/devices/{device_id}/nics")
|
||
async def api_device_nics(device_id: int):
|
||
"""OME serverNetworkInterfaces expanded to port/FQDD rows for wiring UI."""
|
||
node = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
|
||
if not node:
|
||
raise HTTPException(404, "Device not found")
|
||
nics: list[dict] = []
|
||
try:
|
||
inv = await ome_fetch_inventory_types(device_id, ["serverNetworkInterfaces"])
|
||
for card in inv.get("serverNetworkInterfaces") or []:
|
||
nic_id = str(card.get("NicId") or "").strip()
|
||
vendor = str(card.get("VendorName") or card.get("Manufacturer") or "").strip()
|
||
ports = card.get("Ports") or []
|
||
if not ports:
|
||
# Rare flat inventory shape
|
||
fqdd = card.get("Fqdd") or card.get("InstanceId") or nic_id
|
||
name = (
|
||
card.get("ProductName")
|
||
or card.get("DeviceDescription")
|
||
or fqdd
|
||
or "NIC"
|
||
)
|
||
nics.append(
|
||
{
|
||
"nic_id": nic_id or None,
|
||
"port_id": fqdd,
|
||
"fqdd": fqdd,
|
||
"name": name,
|
||
"mac": card.get("PermanentMACAddress") or card.get("CurrentMACAddress"),
|
||
"speed": card.get("LinkSpeed") or card.get("Speed"),
|
||
"link": card.get("LinkStatus"),
|
||
"vendor": vendor or None,
|
||
"label": str(fqdd or name),
|
||
}
|
||
)
|
||
continue
|
||
for port in ports:
|
||
port_id = str(port.get("PortId") or "").strip()
|
||
parts = port.get("Partitions") or []
|
||
part = parts[0] if parts else {}
|
||
fqdd = str(part.get("Fqdd") or port_id or "").strip()
|
||
mac = (
|
||
part.get("PermanentMacAddress")
|
||
or part.get("CurrentMacAddress")
|
||
or part.get("PermanentMACAddress")
|
||
or part.get("CurrentMACAddress")
|
||
or ""
|
||
)
|
||
product = str(port.get("ProductName") or "").strip()
|
||
# OME often appends " - AA:BB:..." — keep product without MAC
|
||
if " - " in product and mac and product.upper().endswith(str(mac).upper()):
|
||
product = product[: product.rfind(" - ")].strip()
|
||
elif " - " in product:
|
||
left, right = product.rsplit(" - ", 1)
|
||
if ":" in right and len(right.replace(":", "")) >= 12:
|
||
product = left.strip()
|
||
link = port.get("LinkStatus")
|
||
speed = port.get("LinkSpeed")
|
||
label_bits = [port_id or fqdd or nic_id or "NIC"]
|
||
if product:
|
||
label_bits.append(product)
|
||
if mac:
|
||
label_bits.append(str(mac))
|
||
if link:
|
||
label_bits.append(str(link))
|
||
if speed not in (None, "", 0, "0"):
|
||
label_bits.append(f"{speed} Mb/s")
|
||
nics.append(
|
||
{
|
||
"nic_id": nic_id or None,
|
||
"port_id": port_id or None,
|
||
"fqdd": fqdd or port_id or None,
|
||
"name": product or port_id or nic_id or "NIC",
|
||
"mac": mac or None,
|
||
"speed": speed,
|
||
"link": link,
|
||
"vendor": vendor or None,
|
||
"label": " · ".join(label_bits),
|
||
}
|
||
)
|
||
except Exception as e:
|
||
log.debug("nic fetch %s: %s", device_id, e)
|
||
# Prefer unique port_id/fqdd order as returned by OME
|
||
return {"device_id": device_id, "nics": nics, "count": len(nics)}
|
||
|
||
|
||
|
||
@app.get("/api/racks")
|
||
async def api_racks():
|
||
_seed_atc_racks()
|
||
return _racks_payload()
|
||
|
||
|
||
@app.post("/api/racks")
|
||
async def api_create_rack(payload: RackIn):
|
||
now = time.time()
|
||
site = (payload.site_id or "").upper()
|
||
if site not in ("ATC1", "ATC2"):
|
||
raise HTTPException(400, "site_id must be ATC1 or ATC2")
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT id FROM sites WHERE id=?", (site,)).fetchone()
|
||
if not row:
|
||
conn.execute(
|
||
"INSERT INTO sites(id, name, sort_order) VALUES (?, ?, ?)",
|
||
(site, site, 1 if site == "ATC1" else 2),
|
||
)
|
||
so = payload.sort_order
|
||
if so is None:
|
||
mx = conn.execute(
|
||
"SELECT COALESCE(MAX(sort_order), 0) AS m FROM racks WHERE site_id=?",
|
||
(site,),
|
||
).fetchone()["m"]
|
||
so = int(mx) + 1
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(site, payload.name.strip()[:64], max(1, min(48, payload.units)), so, now, now),
|
||
)
|
||
rid = cur.lastrowid
|
||
_backup_ops_db("rack-create")
|
||
return {"id": rid, **_racks_payload()}
|
||
|
||
|
||
@app.patch("/api/racks/{rack_id}")
|
||
async def api_patch_rack(rack_id: int, payload: RackPatch):
|
||
now = time.time()
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT * FROM racks WHERE id=?", (rack_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Rack not found")
|
||
name = payload.name.strip()[:64] if payload.name else row["name"]
|
||
so = payload.sort_order if payload.sort_order is not None else row["sort_order"]
|
||
conn.execute(
|
||
"UPDATE racks SET name=?, sort_order=?, updated_at=? WHERE id=?",
|
||
(name, so, now, rack_id),
|
||
)
|
||
_backup_ops_db("rack-patch")
|
||
return _racks_payload()
|
||
|
||
|
||
|
||
class RackItemIn(BaseModel):
|
||
rack_id: int
|
||
u_start: int = 1
|
||
u_height: int | None = None
|
||
side: str = "front" # front|rear|left|right
|
||
device_id: int | None = None
|
||
catalog_sku: str | None = None
|
||
label: str | None = None
|
||
notes: str | None = None
|
||
|
||
|
||
class RackItemPatch(BaseModel):
|
||
rack_id: int | None = None
|
||
u_start: int | None = None
|
||
u_height: int | None = None
|
||
side: str | None = None
|
||
catalog_sku: str | None = None
|
||
label: str | None = None
|
||
notes: str | None = None
|
||
|
||
|
||
@app.get("/api/rack-catalog")
|
||
async def api_rack_catalog():
|
||
families: dict[str, list] = {}
|
||
for item in DELL_RACK_CATALOG:
|
||
families.setdefault(item["family"], []).append(item)
|
||
return {"items": DELL_RACK_CATALOG, "families": families, "count": len(DELL_RACK_CATALOG)}
|
||
|
||
|
||
@app.put("/api/racks/placements/{device_id}")
|
||
async def api_put_placement(device_id: int, payload: PlacementIn):
|
||
"""Place/move an OME device (compat). Writes rack_items."""
|
||
device = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None)
|
||
cat = match_rack_catalog((device or {}).get("model"), (device or {}).get("role"))
|
||
body = RackItemIn(
|
||
rack_id=payload.rack_id,
|
||
u_start=payload.u_start,
|
||
u_height=payload.u_height if payload.u_height is not None else _default_u_height(device),
|
||
side="front",
|
||
device_id=device_id,
|
||
catalog_sku=(cat or {}).get("sku"),
|
||
label=(device or {}).get("name"),
|
||
)
|
||
return await api_upsert_rack_item(body)
|
||
|
||
|
||
@app.delete("/api/racks/placements/{device_id}")
|
||
async def api_delete_placement(device_id: int):
|
||
with _db() as conn:
|
||
conn.execute("DELETE FROM rack_items WHERE device_id=?", (device_id,))
|
||
try:
|
||
conn.execute("DELETE FROM rack_placements WHERE device_id=?", (device_id,))
|
||
except Exception:
|
||
pass
|
||
_backup_ops_db("unplace")
|
||
return _racks_payload()
|
||
|
||
|
||
@app.post("/api/racks/items")
|
||
async def api_upsert_rack_item(payload: RackItemIn):
|
||
now = time.time()
|
||
side = (payload.side or "front").lower()
|
||
if side not in ("front", "rear", "left", "right"):
|
||
raise HTTPException(400, "side must be front|rear|left|right")
|
||
device = None
|
||
if payload.device_id is not None:
|
||
device = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None)
|
||
if not device:
|
||
raise HTTPException(404, "OME device not found")
|
||
sku = payload.catalog_sku
|
||
cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None) if sku else None
|
||
if not cat and device:
|
||
cat = match_rack_catalog(device.get("model"), device.get("role"))
|
||
sku = (cat or {}).get("sku")
|
||
if not device and not cat and not sku:
|
||
raise HTTPException(400, "Provide device_id or catalog_sku")
|
||
height = payload.u_height
|
||
# Prefer catalog U-height for known SKUs so 2U/3U/… always place correctly
|
||
if cat is not None:
|
||
cat_h = int(cat.get("u_height") or 0)
|
||
if cat_h > 0:
|
||
height = cat_h
|
||
elif height is None:
|
||
height = _default_u_height(device)
|
||
elif height is None:
|
||
height = _default_u_height(device)
|
||
if side in ("left", "right"):
|
||
u_start, height = 1, 42
|
||
else:
|
||
height = max(1, min(42, int(height)))
|
||
u_start = int(payload.u_start)
|
||
if u_start < 1 or u_start + height - 1 > 42:
|
||
raise HTTPException(400, f"Must fit U1–U42 (u_start={u_start} height={height})")
|
||
label = (payload.label or (device or {}).get("name") or (cat or {}).get("name") or "Device")[:120]
|
||
with _db() as conn:
|
||
rack = conn.execute("SELECT * FROM racks WHERE id=?", (payload.rack_id,)).fetchone()
|
||
if not rack:
|
||
raise HTTPException(404, "Rack not found")
|
||
if _rack_overlap(
|
||
conn,
|
||
payload.rack_id,
|
||
u_start if side not in ("left", "right") else 1,
|
||
height,
|
||
side=side,
|
||
exclude_device_id=payload.device_id,
|
||
):
|
||
raise HTTPException(409, "U range overlaps another item on this side")
|
||
existing = None
|
||
if payload.device_id is not None:
|
||
existing = conn.execute(
|
||
"SELECT id FROM rack_items WHERE device_id=?", (payload.device_id,)
|
||
).fetchone()
|
||
if existing:
|
||
conn.execute(
|
||
"""
|
||
UPDATE rack_items
|
||
SET rack_id=?, catalog_sku=?, label=?, u_start=?, u_height=?, side=?, notes=?, updated_at=?
|
||
WHERE id=?
|
||
""",
|
||
(
|
||
payload.rack_id,
|
||
sku,
|
||
label,
|
||
u_start if side not in ("left", "right") else 1,
|
||
height,
|
||
side,
|
||
(payload.notes or "")[:200],
|
||
now,
|
||
existing["id"],
|
||
),
|
||
)
|
||
item_id = existing["id"]
|
||
else:
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
payload.rack_id,
|
||
payload.device_id,
|
||
sku,
|
||
label,
|
||
u_start if side not in ("left", "right") else 1,
|
||
height,
|
||
side,
|
||
(payload.notes or "")[:200],
|
||
now,
|
||
),
|
||
)
|
||
item_id = cur.lastrowid
|
||
# keep legacy table in sync for OME devices
|
||
if payload.device_id is not None and side == "front":
|
||
try:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO rack_placements(device_id, rack_id, u_start, u_height, updated_at)
|
||
VALUES (?, ?, ?, ?, ?)
|
||
ON CONFLICT(device_id) DO UPDATE SET
|
||
rack_id=excluded.rack_id, u_start=excluded.u_start,
|
||
u_height=excluded.u_height, updated_at=excluded.updated_at
|
||
""",
|
||
(payload.device_id, payload.rack_id, u_start, height, now),
|
||
)
|
||
except Exception:
|
||
pass
|
||
_backup_ops_db("rack-item")
|
||
payload_out = _racks_payload()
|
||
payload_out["item_id"] = item_id
|
||
return payload_out
|
||
|
||
|
||
@app.patch("/api/racks/items/{item_id}")
|
||
async def api_patch_rack_item(item_id: int, payload: RackItemPatch):
|
||
now = time.time()
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Item not found")
|
||
rack_id = payload.rack_id if payload.rack_id is not None else row["rack_id"]
|
||
side = (payload.side or row["side"] or "front").lower()
|
||
u_start = payload.u_start if payload.u_start is not None else row["u_start"]
|
||
height = payload.u_height if payload.u_height is not None else row["u_height"]
|
||
if side in ("left", "right"):
|
||
u_start, height = 1, 42
|
||
else:
|
||
height = max(1, min(42, int(height)))
|
||
u_start = int(u_start)
|
||
if u_start < 1 or u_start + height - 1 > 42:
|
||
raise HTTPException(400, "Must fit U1–U42")
|
||
if _rack_overlap(conn, rack_id, u_start, height, side=side, exclude_item_id=item_id):
|
||
raise HTTPException(409, "U range overlaps another item on this side")
|
||
conn.execute(
|
||
"""
|
||
UPDATE rack_items
|
||
SET rack_id=?, catalog_sku=COALESCE(?, catalog_sku), label=COALESCE(?, label),
|
||
u_start=?, u_height=?, side=?, notes=COALESCE(?, notes), updated_at=?
|
||
WHERE id=?
|
||
""",
|
||
(
|
||
rack_id,
|
||
payload.catalog_sku,
|
||
payload.label,
|
||
u_start,
|
||
height,
|
||
side,
|
||
payload.notes,
|
||
now,
|
||
item_id,
|
||
),
|
||
)
|
||
_backup_ops_db("rack-item-patch")
|
||
return _racks_payload()
|
||
|
||
|
||
@app.delete("/api/racks/items/{item_id}")
|
||
async def api_delete_rack_item(item_id: int):
|
||
with _db() as conn:
|
||
row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone()
|
||
if not row:
|
||
raise HTTPException(404, "Item not found")
|
||
conn.execute("DELETE FROM rack_items WHERE id=?", (item_id,))
|
||
if row["device_id"] is not None:
|
||
try:
|
||
conn.execute("DELETE FROM rack_placements WHERE device_id=?", (row["device_id"],))
|
||
except Exception:
|
||
pass
|
||
_backup_ops_db("rack-item-delete")
|
||
return _racks_payload()
|
||
|
||
|
||
|
||
|
||
|
||
def _get_ssh_sem() -> asyncio.Semaphore:
|
||
global _ssh_sem
|
||
if _ssh_sem is None:
|
||
_ssh_sem = asyncio.Semaphore(SSH_MAX_SESSIONS)
|
||
return _ssh_sem
|
||
|
||
|
||
def _fleet_ssh_targets() -> set[str]:
|
||
"""Allow SSH only to IPs currently known in the OME fleet snapshot."""
|
||
allowed: set[str] = set()
|
||
for d in STATE.get("devices") or []:
|
||
ip = d.get("ip")
|
||
if ip and ip.count(".") == 3:
|
||
allowed.add(ip)
|
||
return allowed
|
||
|
||
|
||
@app.websocket("/ws/ssh")
|
||
async def ws_ssh(ws: WebSocket):
|
||
"""Browser terminal <-> SSH bridge. Isolated per connection; concurrency-limited."""
|
||
await ws.accept()
|
||
conn = None
|
||
process = None
|
||
reader_task = None
|
||
session_id = f"{id(ws)}-{time.time()}"
|
||
client_host = ""
|
||
acquired = False
|
||
|
||
async def send_json(payload: dict):
|
||
try:
|
||
await ws.send_text(json.dumps(payload))
|
||
except Exception:
|
||
pass
|
||
|
||
def _client_key() -> str:
|
||
try:
|
||
return ws.client.host if ws.client else "unknown"
|
||
except Exception:
|
||
return "unknown"
|
||
|
||
try:
|
||
# Limit concurrent sessions globally and per browser IP
|
||
ck = _client_key()
|
||
active_for_client = sum(1 for s in SSH_SESSIONS.values() if s.get("client") == ck)
|
||
if active_for_client >= SSH_MAX_PER_CLIENT:
|
||
await send_json({
|
||
"type": "error",
|
||
"message": f"Too many SSH sessions from this client (max {SSH_MAX_PER_CLIENT})",
|
||
})
|
||
await ws.close()
|
||
return
|
||
|
||
try:
|
||
await asyncio.wait_for(_get_ssh_sem().acquire(), timeout=8.0)
|
||
acquired = True
|
||
except asyncio.TimeoutError:
|
||
await send_json({
|
||
"type": "error",
|
||
"message": "SSH gateway busy — try again in a moment",
|
||
})
|
||
await ws.close()
|
||
return
|
||
|
||
SSH_SESSIONS[session_id] = {
|
||
"client": ck,
|
||
"started": time.time(),
|
||
"host": None,
|
||
"user": None,
|
||
}
|
||
|
||
raw = await asyncio.wait_for(ws.receive_text(), timeout=60.0)
|
||
try:
|
||
msg = json.loads(raw)
|
||
except Exception:
|
||
await send_json({"type": "error", "message": "Invalid auth payload"})
|
||
await ws.close()
|
||
return
|
||
|
||
if msg.get("type") != "auth":
|
||
await send_json({"type": "error", "message": "Expected auth message"})
|
||
await ws.close()
|
||
return
|
||
|
||
host = str(msg.get("host") or "").strip()
|
||
username = str(msg.get("username") or "").strip()
|
||
password = str(msg.get("password") or "")
|
||
try:
|
||
port = int(msg.get("port") or 22)
|
||
except Exception:
|
||
port = 22
|
||
|
||
if not host or not username:
|
||
await send_json({"type": "error", "message": "Host and username are required"})
|
||
await ws.close()
|
||
return
|
||
if port < 1 or port > 65535:
|
||
await send_json({"type": "error", "message": "Invalid port"})
|
||
await ws.close()
|
||
return
|
||
|
||
async with _lock:
|
||
allowed = set(_fleet_ssh_targets())
|
||
if host not in allowed:
|
||
await send_json({
|
||
"type": "error",
|
||
"message": f"Host {host} is not in the current OME fleet — SSH blocked",
|
||
})
|
||
await ws.close()
|
||
return
|
||
|
||
client_host = host
|
||
SSH_SESSIONS[session_id]["host"] = host
|
||
SSH_SESSIONS[session_id]["user"] = username
|
||
await send_json({"type": "status", "message": f"Connecting to {username}@{host}:{port}…"})
|
||
|
||
try:
|
||
conn = await asyncio.wait_for(
|
||
asyncssh.connect(
|
||
host,
|
||
port=port,
|
||
username=username,
|
||
password=password,
|
||
known_hosts=None,
|
||
client_keys=None,
|
||
preferred_auth=["password", "keyboard-interactive"],
|
||
keepalive_interval=30,
|
||
keepalive_count_max=3,
|
||
),
|
||
timeout=25.0,
|
||
)
|
||
except Exception as e:
|
||
await send_json({"type": "error", "message": f"SSH connect failed: {e}"})
|
||
await ws.close()
|
||
return
|
||
|
||
term = str(msg.get("term") or "xterm-256color")
|
||
cols = int(msg.get("cols") or 120)
|
||
rows = int(msg.get("rows") or 36)
|
||
process = await conn.create_process(
|
||
term_type=term,
|
||
term_size=(max(cols, 40), max(rows, 10)),
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
await send_json({"type": "ready", "message": f"Connected as {username}@{host}"})
|
||
log.info("SSH session start id=%s client=%s target=%s@%s active=%s",
|
||
session_id, ck, username, host, len(SSH_SESSIONS))
|
||
|
||
async def pump_ssh_to_ws():
|
||
try:
|
||
while True:
|
||
data = await process.stdout.read(8192)
|
||
if not data:
|
||
try:
|
||
err = await asyncio.wait_for(process.stderr.read(1024), timeout=0.2)
|
||
except Exception:
|
||
err = ""
|
||
if err:
|
||
await ws.send_text(err if isinstance(err, str) else err.decode("utf-8", "replace"))
|
||
break
|
||
await ws.send_text(data if isinstance(data, str) else data.decode("utf-8", "replace"))
|
||
except Exception:
|
||
pass
|
||
try:
|
||
await send_json({"type": "status", "message": "SSH session ended"})
|
||
except Exception:
|
||
pass
|
||
|
||
reader_task = asyncio.create_task(pump_ssh_to_ws())
|
||
|
||
# Idle watchdog: close after 30 min without client input
|
||
last_input = time.time()
|
||
IDLE_LIMIT = 1800
|
||
|
||
while True:
|
||
try:
|
||
packet = await asyncio.wait_for(ws.receive(), timeout=30.0)
|
||
except asyncio.TimeoutError:
|
||
if time.time() - last_input > IDLE_LIMIT:
|
||
await send_json({"type": "status", "message": "SSH idle timeout"})
|
||
break
|
||
# keepalive ping to browser
|
||
await send_json({"type": "pong"})
|
||
continue
|
||
|
||
if packet.get("type") == "websocket.disconnect":
|
||
break
|
||
text = packet.get("text")
|
||
if text is None:
|
||
continue
|
||
last_input = time.time()
|
||
if text.startswith("{") and '"type"' in text[:48]:
|
||
try:
|
||
ctrl = json.loads(text)
|
||
except Exception:
|
||
process.stdin.write(text)
|
||
await process.stdin.drain()
|
||
continue
|
||
ctype = ctrl.get("type")
|
||
if ctype == "resize":
|
||
try:
|
||
process.change_terminal_size(int(ctrl.get("cols") or 80), int(ctrl.get("rows") or 24))
|
||
except Exception:
|
||
pass
|
||
elif ctype == "data":
|
||
process.stdin.write(str(ctrl.get("data") or ""))
|
||
await process.stdin.drain()
|
||
elif ctype == "ping":
|
||
await send_json({"type": "pong"})
|
||
continue
|
||
process.stdin.write(text)
|
||
await process.stdin.drain()
|
||
|
||
except WebSocketDisconnect:
|
||
pass
|
||
except Exception as e:
|
||
log.warning("SSH websocket error: %s", e)
|
||
try:
|
||
await send_json({"type": "error", "message": str(e)})
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
SSH_SESSIONS.pop(session_id, None)
|
||
if reader_task:
|
||
reader_task.cancel()
|
||
try:
|
||
await reader_task
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if process:
|
||
process.close()
|
||
await asyncio.wait_for(process.wait_closed(), timeout=3.0)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if conn:
|
||
conn.close()
|
||
await asyncio.wait_for(conn.wait_closed(), timeout=3.0)
|
||
except Exception:
|
||
pass
|
||
if acquired:
|
||
try:
|
||
_get_ssh_sem().release()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
await ws.close()
|
||
except Exception:
|
||
pass
|
||
log.info("SSH session end id=%s target=%s active=%s", session_id, client_host, len(SSH_SESSIONS))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Reports / compliance / warranty / export (read-only OME surfaces)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def fetch_warranties(force: bool = False) -> dict:
|
||
if not force:
|
||
cached = _cache_get("warranties")
|
||
if cached is not None:
|
||
return cached
|
||
rows: list[dict] = []
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
skip = 0
|
||
while True:
|
||
r = await client.get(
|
||
f"{base}/api/WarrantyService/Warranties",
|
||
headers=headers,
|
||
params={"$top": 200, "$skip": skip},
|
||
)
|
||
r.raise_for_status()
|
||
chunk = (r.json() or {}).get("value") or []
|
||
if not chunk:
|
||
break
|
||
for w in chunk:
|
||
rows.append(
|
||
{
|
||
"id": w.get("Id"),
|
||
"device_id": w.get("DeviceId"),
|
||
"service_tag": w.get("DeviceIdentifier"),
|
||
"device_name": w.get("DeviceName"),
|
||
"model": w.get("DeviceModel"),
|
||
"service_level": w.get("ServiceLevelDescription") or w.get("ServiceLevelCode"),
|
||
"start_date": w.get("StartDate"),
|
||
"end_date": w.get("EndDate"),
|
||
"days_remaining": w.get("DaysRemaining"),
|
||
"ship_date": w.get("SystemShipDate"),
|
||
"state": w.get("State"),
|
||
"country": w.get("CountryLookupCode"),
|
||
}
|
||
)
|
||
skip += len(chunk)
|
||
if len(chunk) < 200:
|
||
break
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
# Prefer longest-remaining warranty per device
|
||
by_device: dict[int, dict] = {}
|
||
for w in rows:
|
||
did = w.get("device_id")
|
||
if did is None:
|
||
continue
|
||
prev = by_device.get(did)
|
||
if not prev or int(w.get("days_remaining") or 0) > int(prev.get("days_remaining") or 0):
|
||
by_device[did] = w
|
||
elif int(w.get("days_remaining") or 0) == int(prev.get("days_remaining") or 0):
|
||
# keep later end date
|
||
if str(w.get("end_date") or "") > str(prev.get("end_date") or ""):
|
||
by_device[did] = w
|
||
payload = {
|
||
"updated_at": time.time(),
|
||
"count": len(rows),
|
||
"items": rows,
|
||
"by_device": {str(k): v for k, v in by_device.items()},
|
||
}
|
||
return _cache_set("warranties", payload)
|
||
|
||
|
||
async def fetch_baselines(force: bool = False) -> dict:
|
||
if not force:
|
||
cached = _cache_get("baselines")
|
||
if cached is not None:
|
||
return cached
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(f"{base}/api/UpdateService/Baselines", headers=headers)
|
||
r.raise_for_status()
|
||
items = []
|
||
for b in (r.json() or {}).get("value") or []:
|
||
summary = b.get("ComplianceSummary") or {}
|
||
items.append(
|
||
{
|
||
"id": b.get("Id"),
|
||
"name": b.get("Name"),
|
||
"description": b.get("Description"),
|
||
"catalog_id": b.get("CatalogId"),
|
||
"catalog_type": b.get("CatalogType"),
|
||
"last_run": b.get("LastRun"),
|
||
"task_id": b.get("TaskId"),
|
||
"compliance_status": summary.get("ComplianceStatus"),
|
||
"critical": summary.get("NumberOfCritical"),
|
||
"warning": summary.get("NumberOfWarning"),
|
||
"normal": summary.get("NumberOfNormal"),
|
||
"downgrade": summary.get("NumberOfDowngrade"),
|
||
"unknown": summary.get("NumberOfUnknown"),
|
||
}
|
||
)
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
return _cache_set("baselines", {"updated_at": time.time(), "items": items})
|
||
|
||
|
||
async def fetch_catalogs(force: bool = False) -> dict:
|
||
if not force:
|
||
cached = _cache_get("catalogs")
|
||
if cached is not None:
|
||
return cached
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(f"{base}/api/UpdateService/Catalogs", headers=headers)
|
||
r.raise_for_status()
|
||
items = []
|
||
for c in (r.json() or {}).get("value") or []:
|
||
repo = c.get("Repository") or {}
|
||
items.append(
|
||
{
|
||
"id": c.get("Id"),
|
||
"filename": c.get("Filename") or c.get("SourcePath"),
|
||
"source": repo.get("Source") or c.get("Source"),
|
||
"repository_name": repo.get("Name"),
|
||
"repository_type": repo.get("RepositoryType") or c.get("RepositoryType"),
|
||
"owner": c.get("Owner"),
|
||
"status": (c.get("Status") or {}).get("Name") if isinstance(c.get("Status"), dict) else c.get("Status"),
|
||
"last_updated": c.get("LastUpdated") or c.get("BundlesUpdateTime"),
|
||
}
|
||
)
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
return _cache_set("catalogs", {"updated_at": time.time(), "items": items})
|
||
|
||
|
||
def _pick_primary_baseline(baselines: list[dict]) -> dict | None:
|
||
if not baselines:
|
||
return None
|
||
for b in baselines:
|
||
name = (b.get("name") or "").lower()
|
||
if "dell" in name and "online" in name:
|
||
return b
|
||
# most critical first
|
||
return sorted(baselines, key=lambda x: int(x.get("critical") or 0), reverse=True)[0]
|
||
|
||
|
||
async def fetch_compliance(force: bool = False, baseline_id: int | None = None) -> dict:
|
||
if not force and baseline_id is None:
|
||
cached = _cache_get("compliance")
|
||
if cached is not None:
|
||
return cached
|
||
bl = await fetch_baselines(force=force)
|
||
primary = None
|
||
if baseline_id is not None:
|
||
primary = next((b for b in bl["items"] if b.get("id") == baseline_id), None)
|
||
if primary is None:
|
||
primary = _pick_primary_baseline(bl["items"])
|
||
if not primary:
|
||
payload = {
|
||
"updated_at": time.time(),
|
||
"summary": {"outdated_devices": 0, "critical_components": 0, "baseline_name": None},
|
||
"devices": [],
|
||
"components": [],
|
||
"baselines": bl["items"],
|
||
}
|
||
return _cache_set("compliance", payload)
|
||
|
||
bid = primary["id"]
|
||
devices_out: list[dict] = []
|
||
components_out: list[dict] = []
|
||
fleet_by_id = {d.get("id"): d for d in STATE.get("devices") or []}
|
||
async with httpx.AsyncClient(verify=False, timeout=120.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(
|
||
f"{base}/api/UpdateService/Baselines({bid})/DeviceComplianceReports",
|
||
headers=headers,
|
||
)
|
||
r.raise_for_status()
|
||
for dcr in (r.json() or {}).get("value") or []:
|
||
did = dcr.get("DeviceId")
|
||
fleet = fleet_by_id.get(did) or {}
|
||
st = dcr.get("ServiceTag") or fleet.get("service_tag")
|
||
comps = dcr.get("ComponentComplianceReports") or []
|
||
device_row = {
|
||
"device_id": did,
|
||
"service_tag": st,
|
||
"name": fleet.get("name") or dcr.get("DeviceName") or st,
|
||
"model": dcr.get("DeviceModel") or fleet.get("model"),
|
||
"ip": fleet.get("ip"),
|
||
"firmware_status": dcr.get("FirmwareStatus"),
|
||
"compliance_status": dcr.get("ComplianceStatus"),
|
||
"reboot_required": dcr.get("RebootRequired"),
|
||
"component_count": len(comps),
|
||
"dell_uri": None,
|
||
}
|
||
devices_out.append(device_row)
|
||
for comp in comps:
|
||
uri = comp.get("Uri")
|
||
if uri and not device_row["dell_uri"]:
|
||
device_row["dell_uri"] = uri
|
||
components_out.append(
|
||
{
|
||
"device_id": did,
|
||
"service_tag": st,
|
||
"device_name": device_row["name"],
|
||
"model": device_row["model"],
|
||
"ip": device_row["ip"],
|
||
"component": comp.get("Name"),
|
||
"component_type": comp.get("ComponentType"),
|
||
"current_version": comp.get("CurrentVersion"),
|
||
"catalog_version": comp.get("Version"),
|
||
"update_action": comp.get("UpdateAction"),
|
||
"criticality": comp.get("Criticality"),
|
||
"compliance_status": comp.get("ComplianceStatus"),
|
||
"reboot_required": comp.get("RebootRequired"),
|
||
"dell_uri": uri,
|
||
"path": comp.get("Path"),
|
||
"baseline_id": bid,
|
||
"baseline_name": primary.get("name"),
|
||
"status_badge": (
|
||
"outdated"
|
||
if str(comp.get("UpdateAction") or "").upper() == "UPGRADE"
|
||
or str(comp.get("ComplianceStatus") or "").upper()
|
||
in ("CRITICAL", "WARNING", "NONCOMPLIANT", "NON-COMPLIANT")
|
||
else "current"
|
||
if str(comp.get("UpdateAction") or "").upper() in ("EQUAL", "NONE", "")
|
||
and str(comp.get("ComplianceStatus") or "").upper()
|
||
in ("", "OK", "COMPLIANT", "NORMAL", "DOWNGRADE")
|
||
else "unknown"
|
||
),
|
||
}
|
||
)
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
|
||
outdated = [
|
||
d
|
||
for d in devices_out
|
||
if str(d.get("compliance_status") or "").upper() in ("CRITICAL", "WARNING")
|
||
or str(d.get("firmware_status") or "").lower() in ("non-compliant", "noncompliant")
|
||
]
|
||
crit_comps = [
|
||
c
|
||
for c in components_out
|
||
if str(c.get("compliance_status") or "").upper() == "CRITICAL"
|
||
or str(c.get("update_action") or "").upper() == "UPGRADE"
|
||
]
|
||
payload = {
|
||
"updated_at": time.time(),
|
||
"baseline": primary,
|
||
"baselines": bl["items"],
|
||
"summary": {
|
||
"baseline_id": bid,
|
||
"baseline_name": primary.get("name"),
|
||
"outdated_devices": len(outdated),
|
||
"devices_in_report": len(devices_out),
|
||
"critical_components": len(crit_comps),
|
||
"components_total": len(components_out),
|
||
"compliance_status": primary.get("compliance_status"),
|
||
"last_run": primary.get("last_run"),
|
||
},
|
||
"devices": devices_out,
|
||
"components": components_out,
|
||
}
|
||
if baseline_id is None:
|
||
return _cache_set("compliance", payload)
|
||
return payload
|
||
|
||
|
||
async def fetch_report_defs(force: bool = False) -> dict:
|
||
if not force:
|
||
cached = _cache_get("report_defs")
|
||
if cached is not None:
|
||
return cached
|
||
items = []
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(f"{base}/api/ReportService/ReportDefs", headers=headers)
|
||
r.raise_for_status()
|
||
for d in (r.json() or {}).get("value") or []:
|
||
cols = [c.get("Name") for c in (d.get("ColumnNames") or []) if c.get("Name")]
|
||
items.append(
|
||
{
|
||
"id": d.get("Id"),
|
||
"name": d.get("Name"),
|
||
"description": d.get("Description"),
|
||
"category": d.get("Category") or d.get("FilterGroupName"),
|
||
"is_builtin": d.get("IsBuiltIn"),
|
||
"last_run": d.get("LastRunDate"),
|
||
"last_run_by": d.get("LastRunBy"),
|
||
"columns": cols,
|
||
}
|
||
)
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
items.sort(key=lambda x: ((x.get("category") or ""), (x.get("name") or "").lower()))
|
||
return _cache_set("report_defs", {"updated_at": time.time(), "items": items})
|
||
|
||
|
||
async def fetch_jobs(force: bool = False, top: int = 40) -> dict:
|
||
if not force:
|
||
cached = _cache_get("jobs")
|
||
if cached is not None:
|
||
return cached
|
||
items = []
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(
|
||
f"{base}/api/JobService/Jobs",
|
||
headers=headers,
|
||
params={"$top": top},
|
||
)
|
||
r.raise_for_status()
|
||
for j in (r.json() or {}).get("value") or []:
|
||
status = j.get("LastRunStatus") or {}
|
||
items.append(
|
||
{
|
||
"id": j.get("Id"),
|
||
"name": j.get("JobName") or j.get("Name"),
|
||
"status": status.get("Name") if isinstance(status, dict) else status,
|
||
"job_type": (j.get("JobType") or {}).get("Name")
|
||
if isinstance(j.get("JobType"), dict)
|
||
else j.get("JobType"),
|
||
"last_run": j.get("LastRunStatus") and (j.get("LastRunDate") or j.get("StartTime")),
|
||
"progress": j.get("Progress") or j.get("PercentComplete"),
|
||
"created_by": j.get("CreatedBy"),
|
||
}
|
||
)
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
return _cache_set("jobs", {"updated_at": time.time(), "items": items})
|
||
|
||
|
||
def build_fleet_report_rows(warranties: dict | None = None) -> list[dict]:
|
||
by_dev = (warranties or {}).get("by_device") or {}
|
||
rows = []
|
||
for d in STATE.get("devices") or []:
|
||
w = by_dev.get(str(d.get("id"))) or {}
|
||
rows.append(
|
||
{
|
||
"id": d.get("id"),
|
||
"name": d.get("name"),
|
||
"service_tag": d.get("service_tag"),
|
||
"model": d.get("model"),
|
||
"ip": d.get("ip"),
|
||
"idrac_ip": d.get("idrac_ip") or d.get("ip"),
|
||
"os_hostname": d.get("os_hostname"),
|
||
"rdp_host": d.get("rdp_host"),
|
||
"rdp_ips": d.get("rdp_ips") or [],
|
||
"os_ips": d.get("os_ips") or [],
|
||
"is_windows": d.get("is_windows"),
|
||
"subnet": d.get("subnet"),
|
||
"type": d.get("type"),
|
||
"sub_type": d.get("sub_type"),
|
||
"is_server": d.get("is_server"),
|
||
"is_idrac": d.get("is_idrac"),
|
||
"connected": d.get("connected"),
|
||
"powered_on": d.get("powered_on"),
|
||
"power_state": d.get("power_state"),
|
||
"status": d.get("status"),
|
||
"watts": d.get("watts"),
|
||
"avg_watts": d.get("avg_watts"),
|
||
"peak_watts": d.get("peak_watts"),
|
||
"last_status_time": d.get("last_status_time"),
|
||
"last_inventory_time": d.get("last_inventory_time"),
|
||
"warranty_end": w.get("end_date"),
|
||
"warranty_days_remaining": w.get("days_remaining"),
|
||
"warranty_service_level": w.get("service_level"),
|
||
}
|
||
)
|
||
rows.sort(key=lambda r: (r.get("name") or "").lower())
|
||
return rows
|
||
|
||
|
||
@app.get("/api/warranties")
|
||
async def api_warranties(force: bool = False):
|
||
return await fetch_warranties(force=force)
|
||
|
||
|
||
@app.get("/api/compliance")
|
||
async def api_compliance(force: bool = False, baseline_id: int | None = None):
|
||
return await fetch_compliance(force=force, baseline_id=baseline_id)
|
||
|
||
|
||
@app.get("/api/baselines")
|
||
async def api_baselines(force: bool = False):
|
||
return await fetch_baselines(force=force)
|
||
|
||
|
||
@app.get("/api/catalogs")
|
||
async def api_catalogs(force: bool = False):
|
||
return await fetch_catalogs(force=force)
|
||
|
||
|
||
@app.get("/api/ome/report-defs")
|
||
async def api_report_defs(force: bool = False):
|
||
return await fetch_report_defs(force=force)
|
||
|
||
|
||
@app.get("/api/ome/jobs")
|
||
async def api_ome_jobs(force: bool = False):
|
||
return await fetch_jobs(force=force)
|
||
|
||
|
||
class ReportRunIn(BaseModel):
|
||
report_def_id: int
|
||
|
||
|
||
@app.post("/api/ome/reports/run")
|
||
async def api_ome_report_run(payload: ReportRunIn):
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.post(
|
||
f"{base}/api/ReportService/Actions/ReportService.RunReport",
|
||
headers=headers,
|
||
json={"ReportDefId": payload.report_def_id},
|
||
)
|
||
if r.status_code >= 400:
|
||
raise HTTPException(r.status_code, r.text[:500])
|
||
job_id = r.json() if isinstance(r.json(), (int, str)) else (r.json() or {}).get("Id") or r.text
|
||
return {
|
||
"job_id": job_id,
|
||
"report_def_id": payload.report_def_id,
|
||
"message": "Report job started in OME. Poll results shortly.",
|
||
"results_url": f"/api/ome/reports/{payload.report_def_id}/results",
|
||
}
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
|
||
|
||
@app.get("/api/ome/reports/{report_def_id}/results")
|
||
async def api_ome_report_results(report_def_id: int):
|
||
async with httpx.AsyncClient(verify=False, timeout=60.0) as client:
|
||
base, headers, sid = await ome_session(client)
|
||
try:
|
||
r = await client.get(
|
||
f"{base}/api/ReportService/ReportDefs({report_def_id})/ReportResults",
|
||
headers=headers,
|
||
)
|
||
if r.status_code >= 400:
|
||
raise HTTPException(
|
||
r.status_code,
|
||
(r.json().get("error", {}) or {}).get("message")
|
||
if r.headers.get("content-type", "").startswith("application/json")
|
||
else r.text[:500],
|
||
)
|
||
return r.json()
|
||
finally:
|
||
await ome_session_delete(client, base, headers, sid)
|
||
|
||
|
||
@app.get("/api/reports/fleet")
|
||
async def api_reports_fleet(force: bool = False):
|
||
warranties = await fetch_warranties(force=force)
|
||
rows = build_fleet_report_rows(warranties)
|
||
return {
|
||
"updated_at": STATE.get("updated_at"),
|
||
"summary": STATE.get("summary"),
|
||
"ome": STATE.get("ome"),
|
||
"count": len(rows),
|
||
"rows": rows,
|
||
}
|
||
|
||
|
||
@app.get("/api/reports/firmware")
|
||
async def api_reports_firmware(force: bool = False, baseline_id: int | None = None):
|
||
return await fetch_compliance(force=force, baseline_id=baseline_id)
|
||
|
||
|
||
@app.get("/api/reports/brief")
|
||
async def api_reports_brief(force: bool = False):
|
||
warranties = await fetch_warranties(force=force)
|
||
compliance = await fetch_compliance(force=force)
|
||
fleet_rows = build_fleet_report_rows(warranties)
|
||
critical = [a for a in (STATE.get("alerts") or []) if a.get("severity") == "Critical"][:15]
|
||
expiring = sorted(
|
||
[
|
||
w
|
||
for w in (warranties.get("items") or [])
|
||
if w.get("days_remaining") is not None and int(w.get("days_remaining") or 0) <= 90
|
||
],
|
||
key=lambda x: int(x.get("days_remaining") or 0),
|
||
)[:25]
|
||
outdated = [
|
||
d
|
||
for d in (compliance.get("devices") or [])
|
||
if str(d.get("compliance_status") or "").upper() in ("CRITICAL", "WARNING")
|
||
or str(d.get("firmware_status") or "").lower() in ("non-compliant", "noncompliant")
|
||
]
|
||
return {
|
||
"generated_at": time.time(),
|
||
"ome": STATE.get("ome"),
|
||
"summary": STATE.get("summary"),
|
||
"compliance_summary": compliance.get("summary"),
|
||
"critical_alerts": critical,
|
||
"outdated_devices": outdated[:40],
|
||
"warranty_expiring": expiring,
|
||
"fleet": fleet_rows,
|
||
"baseline": compliance.get("baseline"),
|
||
}
|
||
|
||
|
||
@app.get("/api/export/{kind}")
|
||
async def api_export(kind: str, fmt: str = "csv", force: bool = False, baseline_id: int | None = None):
|
||
kind = kind.lower().strip()
|
||
fmt = fmt.lower().strip()
|
||
if fmt not in ("csv", "json"):
|
||
raise HTTPException(400, "fmt must be csv or json")
|
||
|
||
if kind in ("fleet", "inventory"):
|
||
data = await api_reports_fleet(force=force)
|
||
rows = data["rows"]
|
||
filename = f"ome-fleet-{int(time.time())}"
|
||
elif kind in ("firmware", "compliance"):
|
||
data = await fetch_compliance(force=force, baseline_id=baseline_id)
|
||
rows = data.get("components") or []
|
||
filename = f"ome-firmware-compliance-{int(time.time())}"
|
||
elif kind == "warranty":
|
||
data = await fetch_warranties(force=force)
|
||
rows = data.get("items") or []
|
||
filename = f"ome-warranties-{int(time.time())}"
|
||
elif kind == "brief":
|
||
data = await api_reports_brief(force=force)
|
||
if fmt == "json":
|
||
return Response(
|
||
content=json.dumps(data, indent=2, default=str),
|
||
media_type="application/json",
|
||
headers={"Content-Disposition": f'attachment; filename="ome-customer-brief-{int(time.time())}.json"'},
|
||
)
|
||
# flatten brief as fleet CSV appendix
|
||
rows = data.get("fleet") or []
|
||
filename = f"ome-customer-brief-fleet-{int(time.time())}"
|
||
else:
|
||
raise HTTPException(404, "Unknown export kind. Use fleet|firmware|warranty|brief")
|
||
|
||
if fmt == "json":
|
||
body = json.dumps({"kind": kind, "count": len(rows), "rows": rows}, indent=2, default=str)
|
||
return Response(
|
||
content=body,
|
||
media_type="application/json",
|
||
headers={"Content-Disposition": f'attachment; filename="{filename}.json"'},
|
||
)
|
||
csv_body = rows_to_csv(rows)
|
||
return Response(
|
||
content=csv_body,
|
||
media_type="text/csv",
|
||
headers={"Content-Disposition": f'attachment; filename="{filename}.csv"'},
|
||
)
|
||
|
||
|
||
@app.get("/api/devices/{device_id}/warranty")
|
||
async def device_warranty(device_id: int, force: bool = False):
|
||
warranties = await fetch_warranties(force=force)
|
||
items = [w for w in warranties.get("items") or [] if w.get("device_id") == device_id]
|
||
primary = (warranties.get("by_device") or {}).get(str(device_id))
|
||
return {"device_id": device_id, "primary": primary, "items": items}
|
||
|
||
|
||
@app.get("/api/devices/{device_id}/compliance")
|
||
async def device_compliance(device_id: int, force: bool = False):
|
||
compliance = await fetch_compliance(force=force)
|
||
device = next((d for d in compliance.get("devices") or [] if d.get("device_id") == device_id), None)
|
||
comps = [c for c in compliance.get("components") or [] if c.get("device_id") == device_id]
|
||
return {
|
||
"device_id": device_id,
|
||
"baseline": compliance.get("baseline"),
|
||
"device": device,
|
||
"components": comps,
|
||
}
|
||
|
||
|
||
@app.get("/api/ssh/sessions")
|
||
async def api_ssh_sessions():
|
||
"""Ops visibility: active SSH bridges (no secrets)."""
|
||
now = time.time()
|
||
sessions = [
|
||
{
|
||
"id": sid,
|
||
"client": meta.get("client"),
|
||
"host": meta.get("host"),
|
||
"user": meta.get("user"),
|
||
"age_sec": int(now - float(meta.get("started") or now)),
|
||
}
|
||
for sid, meta in list(SSH_SESSIONS.items())
|
||
]
|
||
return {"active": len(sessions), "max": SSH_MAX_SESSIONS, "sessions": sessions}
|
||
|
||
|
||
|
||
|
||
|
||
STATIC_DIR = Path("/ui")
|
||
|
||
|
||
@app.get("/")
|
||
async def index():
|
||
resp = FileResponse(STATIC_DIR / "index.html")
|
||
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||
resp.headers["Pragma"] = "no-cache"
|
||
return resp
|
||
|
||
|
||
@app.get("/styles.css")
|
||
async def styles():
|
||
return FileResponse(STATIC_DIR / "styles.css", media_type="text/css")
|
||
|
||
|
||
@app.get("/app.js")
|
||
async def app_js():
|
||
return FileResponse(STATIC_DIR / "app.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/ops.js")
|
||
async def ops_js():
|
||
return FileResponse(STATIC_DIR / "ops.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/network.js")
|
||
async def network_js():
|
||
return FileResponse(STATIC_DIR / "network.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/ssh.js")
|
||
async def ssh_js():
|
||
return FileResponse(STATIC_DIR / "ssh.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/rdp.js")
|
||
async def rdp_js():
|
||
return FileResponse(STATIC_DIR / "rdp.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/rdp-popout.html")
|
||
async def rdp_popout_html():
|
||
resp = FileResponse(STATIC_DIR / "rdp-popout.html")
|
||
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||
return resp
|
||
|
||
|
||
@app.get("/api/rdp.rdp")
|
||
async def api_rdp_file(host: str, port: int = 3389, username: str = ""):
|
||
"""Download a standard .rdp file for the local Remote Desktop client."""
|
||
host = (host or "").strip()
|
||
if not host or len(host) > 253 or any(c in host for c in "\r\n\x00"):
|
||
raise HTTPException(400, "Invalid host")
|
||
if port < 1 or port > 65535:
|
||
raise HTTPException(400, "Invalid port")
|
||
username = (username or "").strip()
|
||
if len(username) > 256 or any(c in username for c in "\r\n\x00"):
|
||
raise HTTPException(400, "Invalid username")
|
||
full = f"{host}:{port}" if port != 3389 else host
|
||
lines = [
|
||
"screen mode id:i:2",
|
||
"use multimon:i:0",
|
||
"desktopwidth:i:1920",
|
||
"desktopheight:i:1080",
|
||
"session bpp:i:32",
|
||
"compression:i:1",
|
||
"keyboardhook:i:2",
|
||
"audiocapturemode:i:0",
|
||
"videoplaybackmode:i:1",
|
||
"connection type:i:7",
|
||
"networkautodetect:i:1",
|
||
"bandwidthautodetect:i:1",
|
||
"displayconnectionbar:i:1",
|
||
"bitmapcachepersistenable:i:1",
|
||
f"full address:s:{full}",
|
||
"audiomode:i:0",
|
||
"redirectclipboard:i:1",
|
||
"autoreconnection enabled:i:1",
|
||
"authentication level:i:2",
|
||
"prompt for credentials:i:1",
|
||
"negotiate security layer:i:1",
|
||
"gatewayusagemethod:i:4",
|
||
"gatewaycredentialssource:i:4",
|
||
"gatewayprofileusagemethod:i:0",
|
||
]
|
||
if username:
|
||
lines.insert(15, f"username:s:{username}")
|
||
body = "\r\n".join(lines) + "\r\n"
|
||
safe = "".join(c if c.isalnum() or c in ".-_" else "_" for c in host)[:80] or "session"
|
||
return Response(
|
||
content=body,
|
||
media_type="application/x-rdp",
|
||
headers={
|
||
"Content-Disposition": f'attachment; filename="{safe}.rdp"',
|
||
"Cache-Control": "no-store",
|
||
},
|
||
)
|
||
|
||
|
||
@app.get("/vendor/xterm/xterm.css")
|
||
async def vendor_xterm_css():
|
||
return FileResponse(STATIC_DIR / "vendor/xterm/xterm.css", media_type="text/css")
|
||
|
||
|
||
@app.get("/vendor/xterm/xterm.min.js")
|
||
async def vendor_xterm_js():
|
||
return FileResponse(
|
||
STATIC_DIR / "vendor/xterm/xterm.min.js",
|
||
media_type="application/javascript",
|
||
)
|
||
|
||
|
||
@app.get("/vendor/xterm/xterm-addon-fit.min.js")
|
||
async def vendor_xterm_fit_js():
|
||
return FileResponse(
|
||
STATIC_DIR / "vendor/xterm/xterm-addon-fit.min.js",
|
||
media_type="application/javascript",
|
||
)
|
||
|
||
|
||
@app.get("/reports.js")
|
||
async def reports_js():
|
||
return FileResponse(STATIC_DIR / "reports.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/present.js")
|
||
async def present_js():
|
||
return FileResponse(STATIC_DIR / "present.js", media_type="application/javascript")
|
||
|
||
|
||
@app.get("/logos/{filename}")
|
||
async def logo_asset(filename: str):
|
||
"""SVG / image logos for Present architecture slides."""
|
||
safe = Path(filename).name
|
||
if not safe or safe != filename or ".." in filename:
|
||
raise HTTPException(400, "invalid filename")
|
||
path = STATIC_DIR / "logos" / safe
|
||
if not path.is_file():
|
||
raise HTTPException(404, "logo not found")
|
||
media = "image/svg+xml" if safe.lower().endswith(".svg") else "image/png"
|
||
resp = FileResponse(path, media_type=media)
|
||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||
return resp
|
||
|
||
|
||
@app.get("/dell.png")
|
||
async def dell_png():
|
||
return FileResponse(STATIC_DIR / "dell.png", media_type="image/png")
|
||
|
||
|
||
@app.get("/dell-mark.png")
|
||
async def dell_mark():
|
||
return FileResponse(STATIC_DIR / "dell-mark.png", media_type="image/png")
|
||
|
||
|
||
@app.get("/dell.svg")
|
||
async def dell_svg():
|
||
return FileResponse(STATIC_DIR / "dell.png", media_type="image/png")
|
||
|
||
|
||
@app.get("/assets/dell/{filename}")
|
||
async def dell_asset(filename: str):
|
||
"""Serve Dell product face photos for rack Design Studio."""
|
||
safe = Path(filename).name
|
||
if not safe or safe != filename or ".." in filename:
|
||
raise HTTPException(400, "invalid filename")
|
||
path = STATIC_DIR / "assets" / "dell" / safe
|
||
if not path.is_file():
|
||
raise HTTPException(404, "asset not found")
|
||
media = "image/jpeg" if safe.lower().endswith((".jpg", ".jpeg")) else "image/png"
|
||
resp = FileResponse(path, media_type=media)
|
||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||
return resp
|