diff --git a/api/main.py b/api/main.py index b5f0040..f1ac8d0 100644 --- a/api/main.py +++ b/api/main.py @@ -1,7 +1,9 @@ import asyncio import json +import re import os +import socket import sqlite3 import ipaddress import logging @@ -34,8 +36,8 @@ class Settings(BaseSettings): vllm_url: str = "http://10.0.10.106:8000/v1" vllm_model: str = "llama3-70b-gptq" vllm_max_model_len: int = 4096 - vllm_max_tokens: int = 700 - chat_system_chars: int = 9000 + vllm_max_tokens: int = 1100 + chat_system_chars: int = 12000 openwebui_email: str = "" openwebui_password: str = "" cockpit_data: str = "/data" @@ -152,6 +154,819 @@ def rows_to_csv(rows: list[dict], columns: list[str] | None = None) -> str: return "\n".join(lines) + "\n" +# --------------------------------------------------------------------------- +# Dell PowerEdge expansion / memory population catalog (from Dell Tech Guides +# + Installation & Service Manuals / KB 000141539). Used for empty-slot maps, +# max capacity, and best-practice guidance — not a live scrape. +# --------------------------------------------------------------------------- + +_DELL_LINKS = { + # Verified live PDFs / product docs (checked 2026-07-17: %PDF or HTTP 200 HTML). + "spec_r740xd": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740xd-spec-sheet.pdf", + "spec_r740": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r740-spec-sheet.pdf", + "tech_r740": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R740_R740xd_Technical_Guide.pdf", + "own_r740xd": "https://dl.dell.com/topicspdf/poweredge-r740xd_owners-manual_en-us.pdf", + "docs_r740xd": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740xd/docs", + "docs_r740": "https://www.dell.com/support/product-details/en-us/product/poweredge-r740/docs", + "spec_r640": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r640-spec-sheet.pdf", + "own_r640": "https://dl.dell.com/topicspdf/poweredge-r640_owners-manual_en-us.pdf", + "docs_r640": "https://www.dell.com/support/product-details/en-us/product/poweredge-r640/docs", + "spec_r440": "https://i.dell.com/sites/csdocuments/Product_Docs/en/poweredge-r440-spec-sheet.pdf", + "docs_r440": "https://www.dell.com/support/product-details/en-us/product/poweredge-r440/docs", + "spec_r6415": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge_R6415_Spec_Sheet_EN.pdf", + "ref_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_reference-guide_en-us.pdf", + "own_r6415": "https://dl.dell.com/topicspdf/poweredge-r6415_owners-manual_en-us.pdf", + "docs_r6415": "https://www.dell.com/support/product-details/en-us/product/poweredge-r6415/docs", + "spec_r7425": "https://i.dell.com/sites/csdocuments/Shared-Content_data-Sheets_Documents/en/aa/PowerEdge-R7425-Spec-Sheet.pdf", + "own_r7425": "https://dl.dell.com/topicspdf/poweredge-r7425_owners-manual_en-us.pdf", + "docs_r7425": "https://www.dell.com/support/product-details/en-us/product/poweredge-r7425/docs", + "spec_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-spec-sheet.pdf", + "tech_r660": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-r660-technical-guide.pdf", + "docs_r660": "https://www.dell.com/support/product-details/en-us/product/poweredge-r660/docs", + "spec_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-spec-sheet.pdf", + "tech_xr5610": "https://www.delltechnologies.com/asset/en-us/products/servers/technical-support/poweredge-xr5610-technical-guide.pdf", + "docs_xr5610": "https://www.dell.com/support/product-details/en-us/product/poweredge-xr5610/docs", + "support_home": "https://www.dell.com/support/home", +} + + +# 14G dual-socket channel map (R640/R740/R740xd family) +_CH14 = { + "A": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5}, + "B": {1: 0, 7: 0, 2: 1, 8: 1, 3: 2, 9: 2, 4: 3, 10: 3, 5: 4, 11: 4, 6: 5, 12: 5}, +} + + +# Dell datacenter rack catalog — visual U heights + face styles for rack designer +DELL_RACK_CATALOG: list[dict[str, Any]] = [ + # --- PowerEdge 1U --- + {"sku": "PE-R440", "name": "PowerEdge R440", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r440"]}, + {"sku": "PE-R640", "name": "PowerEdge R640", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r640"]}, + {"sku": "PE-R650", "name": "PowerEdge R650", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650"]}, + {"sku": "PE-R650xs", "name": "PowerEdge R650xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r650xs"]}, + {"sku": "PE-R660", "name": "PowerEdge R660", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660"]}, + {"sku": "PE-R660xs", "name": "PowerEdge R660xs", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r660xs"]}, + {"sku": "PE-R6615", "name": "PowerEdge R6615", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6615"]}, + {"sku": "PE-R6415", "name": "PowerEdge R6415", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6415"]}, + {"sku": "PE-R6515", "name": "PowerEdge R6515", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6515"]}, + {"sku": "PE-R6525", "name": "PowerEdge R6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6525"]}, + {"sku": "PE-R6715", "name": "PowerEdge R6715", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r6715"]}, + {"sku": "PE-R670", "name": "PowerEdge R670", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r670"]}, + {"sku": "PE-XR5610", "name": "PowerEdge XR5610", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["xr5610"]}, + {"sku": "PE-C6520", "name": "PowerEdge C6520", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["c6520"]}, + {"sku": "PE-C6525", "name": "PowerEdge C6525", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c6525"]}, + {"sku": "PE-C4140", "name": "PowerEdge C4140", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["c4140"]}, + # --- PowerEdge 2U --- + {"sku": "PE-R740", "name": "PowerEdge R740", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740"]}, + {"sku": "PE-R740xd", "name": "PowerEdge R740xd", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd"]}, + {"sku": "PE-R740xd2", "name": "PowerEdge R740xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r740xd2"]}, + {"sku": "PE-R7425", "name": "PowerEdge R7425", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7425"]}, + {"sku": "PE-R750", "name": "PowerEdge R750", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750"]}, + {"sku": "PE-R750xa", "name": "PowerEdge R750xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r750xa"]}, + {"sku": "PE-R7515", "name": "PowerEdge R7515", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7515"]}, + {"sku": "PE-R7525", "name": "PowerEdge R7525", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r7525"]}, + {"sku": "PE-R760", "name": "PowerEdge R760", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760"]}, + {"sku": "PE-R760xd2", "name": "PowerEdge R760xd2", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xd2"]}, + {"sku": "PE-R760xa", "name": "PowerEdge R760xa", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r760xa"]}, + {"sku": "PE-R770", "name": "PowerEdge R770", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r770"]}, + {"sku": "PE-R840", "name": "PowerEdge R840", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r840"]}, + {"sku": "PE-R940", "name": "PowerEdge R940", "family": "PowerEdge", "category": "server", "u_height": 3, "face": "server_4u", "match": ["r940"]}, + {"sku": "PE-R940xa", "name": "PowerEdge R940xa", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["r940xa"]}, + {"sku": "PE-R340", "name": "PowerEdge R340", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r340"]}, + {"sku": "PE-R450", "name": "PowerEdge R450", "family": "PowerEdge", "category": "server", "u_height": 1, "face": "server_1u", "match": ["r450"]}, + {"sku": "PE-R550", "name": "PowerEdge R550", "family": "PowerEdge", "category": "server", "u_height": 2, "face": "server_2u", "match": ["r550"]}, + {"sku": "PE-T640", "name": "PowerEdge T640", "family": "PowerEdge", "category": "server", "u_height": 5, "face": "server_4u", "match": ["t640"]}, + {"sku": "PE-DSS8440", "name": "DSS8440", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["dss8440"]}, + {"sku": "PE-XE9680", "name": "PowerEdge XE9680", "family": "PowerEdge", "category": "server", "u_height": 6, "face": "server_4u", "match": ["xe9680"]}, + {"sku": "PE-XE8640", "name": "PowerEdge XE8640", "family": "PowerEdge", "category": "server", "u_height": 4, "face": "server_4u", "match": ["xe8640"]}, + # --- MX / modular --- + {"sku": "PE-MX7000", "name": "PowerEdge MX7000", "family": "PowerEdge MX", "category": "chassis", "u_height": 7, "face": "chassis_mx", "match": ["mx7000"]}, + {"sku": "PE-MX740c", "name": "PowerEdge MX740c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx740c"]}, + {"sku": "PE-MX750c", "name": "PowerEdge MX750c", "family": "PowerEdge MX", "category": "blade", "u_height": 0, "face": "blade", "match": ["mx750c"]}, + {"sku": "PE-M640", "name": "PowerEdge M640", "family": "PowerEdge", "category": "blade", "u_height": 0, "face": "blade", "match": ["m640"]}, + # --- Networking / PowerSwitch / Force10 --- + {"sku": "PS-S4048", "name": "PowerSwitch S4048-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4048", "force10"]}, + {"sku": "PS-S4148", "name": "PowerSwitch S4148-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s4148"]}, + {"sku": "PS-S5248", "name": "PowerSwitch S5248F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5248"]}, + {"sku": "PS-S5296", "name": "PowerSwitch S5296F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5296"]}, + {"sku": "PS-N3248", "name": "PowerSwitch N3248TE-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3248"]}, + {"sku": "PS-Z9264", "name": "PowerSwitch Z9264F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 2, "face": "switch_2u", "match": ["z9264"]}, + {"sku": "PS-Z9432", "name": "PowerSwitch Z9432F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9432"]}, + {"sku": "PC-5548", "name": "PowerConnect 5548", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 5548", "pc5548"]}, + {"sku": "PC-6248", "name": "PowerConnect 6248", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 6248", "pc6248"]}, + {"sku": "PC-7048", "name": "PowerConnect 7048", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7048", "pc7048"]}, + {"sku": "PC-8024", "name": "PowerConnect 8024F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8024", "pc8024"]}, + {"sku": "PC-8132", "name": "PowerConnect 8132F", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8132", "pc8132"]}, + {"sku": "MX-9116n", "name": "MX9116n Fabric Switching Engine", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx9116"]}, + {"sku": "MX-5108n", "name": "MX5108n Ethernet Switch", "family": "PowerEdge MX", "category": "switch", "u_height": 0, "face": "fabric", "match": ["mx5108"]}, + # --- Storage --- + {"sku": "ME-4012", "name": "PowerVault ME4012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4012"]}, + {"sku": "ME-4024", "name": "PowerVault ME4024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me4024"]}, + {"sku": "ME-5012", "name": "PowerVault ME5012", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5012"]}, + {"sku": "ME-5024", "name": "PowerVault ME5024", "family": "PowerVault", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["me5024"]}, + {"sku": "SC-AllFlash", "name": "SC Series Array", "family": "PowerStore/SC", "category": "storage", "u_height": 3, "face": "storage_3u", "match": ["sc "]}, + {"sku": "PST-5000T", "name": "PowerStore 5000T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 5000"]}, + {"sku": "PST-1200T", "name": "PowerStore 1200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 1200"]}, + {"sku": "PST-3200T", "name": "PowerStore 3200T", "family": "PowerStore/SC", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["powerstore 3200"]}, + {"sku": "DD-6900", "name": "PowerProtect DD6900", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd6900"]}, + {"sku": "DD-9400", "name": "PowerProtect DD9400", "family": "PowerProtect", "category": "storage", "u_height": 4, "face": "storage_3u", "match": ["dd9400"]}, + {"sku": "UNITY-380", "name": "Unity XT 380", "family": "Unity", "category": "storage", "u_height": 2, "face": "storage_2u", "match": ["unity"]}, + # --- VxRail appliances --- + {"sku": "VX-E460F", "name": "VxRail E460F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["e460f", "vxrail e"]}, + {"sku": "VX-V470F", "name": "VxRail V470F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["v470f"]}, + {"sku": "VX-P570F", "name": "VxRail P570F", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["p570f"]}, + {"sku": "VX-V670F", "name": "VxRail V670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["v670f"]}, + {"sku": "VX-P670F", "name": "VxRail P670F", "family": "VxRail", "category": "server", "u_height": 2, "face": "server_2u", "match": ["p670f"]}, + {"sku": "VX-S570", "name": "VxRail S570", "family": "VxRail", "category": "server", "u_height": 1, "face": "server_1u", "match": ["s570"]}, + # --- Networking extras --- + {"sku": "PS-S5232", "name": "PowerSwitch S5232F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5232"]}, + {"sku": "PS-S5212", "name": "PowerSwitch S5212F-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["s5212"]}, + {"sku": "PS-N3224", "name": "PowerSwitch N3224T-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["n3224"]}, + {"sku": "PS-Z9100", "name": "PowerSwitch Z9100-ON", "family": "PowerSwitch", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["z9100"]}, + {"sku": "PC-7024", "name": "PowerConnect 7024", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 7024"]}, + {"sku": "PC-8100", "name": "PowerConnect 8100", "family": "PowerConnect", "category": "switch", "u_height": 1, "face": "switch_1u", "match": ["powerconnect 8100"]}, + # --- PDU / misc --- + {"sku": "PDU-Geist-0U", "name": "Geist Rack PDU (0U vertical)", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["geist", "mg03", "mgs3"]}, + {"sku": "PDU-1U", "name": "Rack PDU 1U horizontal", "family": "PDU", "category": "pdu", "u_height": 1, "face": "pdu_1u", "match": []}, + {"sku": "PDU-APC-0U", "name": "APC Rack PDU 0U", "family": "PDU", "category": "pdu", "u_height": 0, "face": "pdu_0u", "match": ["apc"]}, + {"sku": "PANEL-1U", "name": "Blanking panel 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []}, + {"sku": "PANEL-2U", "name": "Blanking panel 2U", "family": "Accessory", "category": "blank", "u_height": 2, "face": "blank_2u", "match": []}, + {"sku": "CABLE-MGMT-1U", "name": "Cable management arm 1U", "family": "Accessory", "category": "blank", "u_height": 1, "face": "blank_1u", "match": []}, + {"sku": "KVM-1U", "name": "Rack KVM 1U", "family": "Accessory", "category": "custom", "u_height": 1, "face": "custom", "match": ["kvm"]}, + {"sku": "CUSTOM", "name": "Custom device", "family": "Custom", "category": "custom", "u_height": 1, "face": "custom", "match": []}, +] + + +FACE_IMAGE_MAP: dict[str, str] = { + "server_1u": "/assets/dell/face-server-1u.jpg?v=bezel2", + "server_2u": "/assets/dell/face-server-2u.jpg?v=bezel2", + "server_4u": "/assets/dell/face-server-4u.jpg?v=bezel2", + "switch_1u": "/assets/dell/face-switch-1u.jpg?v=bezel2", + "switch_2u": "/assets/dell/face-switch-1u.jpg?v=bezel2", + "storage_2u": "/assets/dell/face-storage-2u.jpg?v=bezel2", + "storage_3u": "/assets/dell/face-storage-2u.jpg?v=bezel2", + "chassis_mx": "/assets/dell/face-chassis-mx.jpg?v=bezel2", + "pdu_1u": "/assets/dell/face-pdu-1u.jpg?v=bezel2", + "pdu_0u": "/assets/dell/face-pdu-1u.jpg?v=bezel2", + "blank_1u": "/assets/dell/face-blank-1u.jpg?v=bezel2", + "blank_2u": "/assets/dell/face-blank-1u.jpg?v=bezel2", + "blade": "/assets/dell/face-chassis-mx.jpg?v=bezel2", + "fabric": "/assets/dell/face-switch-1u.jpg?v=bezel2", + "custom": "/assets/dell/face-server-1u.jpg?v=bezel2", +} + +# Official Dell product shots by SKU (processed to dark background to match faces) +SKU_IMAGE_MAP: dict[str, str] = { + # Prefer shared dark face art for visual consistency across all PowerEdge 1U/2U +} + + +def catalog_image_for(item: dict) -> str: + sku = item.get("sku") or "" + if sku in SKU_IMAGE_MAP: + return SKU_IMAGE_MAP[sku] + face = item.get("face") or "custom" + return FACE_IMAGE_MAP.get(face, FACE_IMAGE_MAP["custom"]) + + +def enrich_catalog_images() -> None: + for item in DELL_RACK_CATALOG: + item["image"] = catalog_image_for(item) + + +enrich_catalog_images() + + +def match_rack_catalog(model: str | None, role: str | None = None) -> dict | None: + m = (model or "").lower() + best = None + for item in DELL_RACK_CATALOG: + for token in item.get("match") or []: + if token and token in m: + best = item + break + if best: + break + if best: + return best + if role == "switch": + return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PS-S4048"), None) + if role == "storage": + return next((x for x in DELL_RACK_CATALOG if x["sku"] == "ME-4024"), None) + if role == "chassis": + return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-MX7000"), None) + if role == "pdu": + return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PDU-Geist-0U"), None) + if role == "server": + return next((x for x in DELL_RACK_CATALOG if x["sku"] == "PE-R740"), None) + return None + +MODEL_EXPANSION: dict[str, dict[str, Any]] = { + "PowerEdge R740xd": { + "generation": "14G", + "dimm_slots": 24, + "dimms_per_cpu": 12, + "channels_per_cpu": 6, + "max_memory_tb": 3.0, + "max_memory_note": "Up to 3 TB with LRDIMM (+ DCPMM configs); typical RDIMM max lower per CPU SKU", + "memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 8, + "pcie_note": "Up to 8× PCIe 3.0 via riser options; slot count depends on riser kit + CPU count", + "drive_bays_max": 24, + "drive_note": "Chassis-dependent: up to 24× 2.5\" (or mixed 3.5\"/NVMe mid-tray options on xd)", + "cpu_sockets": 2, + "best_practices": [ + "Populate identical DIMMs across all six channels per CPU (6 or 12 DIMMs/CPU) for best bandwidth.", + "White release-tab sockets first, then black (channel rank 1 then rank 2).", + "Do not mix RDIMM and LRDIMM in the same system.", + "Unbalanced channel population reduces memory bandwidth / can lower effective speed.", + "Performance Optimized (4 DIMMs/CPU): slots 1,2,4,5 — (8 DIMMs/CPU): 1,2,4,5,7,8,10,11 (Dell KB 000141539).", + ], + "performance_notes": [ + "Peak memory bandwidth needs one matched DIMM in every channel (6 per CPU) before adding the second rank (slots 7–12).", + "Mixing capacities: place largest DIMMs in the first populated sockets of each channel.", + "DCPMM: up to 6 PMem + 6 DRAM per CPU; full 12-slot population recommended for App Direct / Memory Mode.", + ], + "docs": [ + {"title": "R740xd Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740xd"], "kind": "spec"}, + {"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"}, + {"title": "R740xd Owner's Manual (PDF)", "url": _DELL_LINKS["own_r740xd"], "kind": "manual"}, + {"title": "Dell Support · R740xd docs", "url": _DELL_LINKS["docs_r740xd"], "kind": "support"}, + ], + "channel_map": _CH14, + "slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)], + }, + "PowerEdge R740": { + "generation": "14G", + "dimm_slots": 24, + "dimms_per_cpu": 12, + "channels_per_cpu": 6, + "max_memory_tb": 3.0, + "max_memory_note": "Up to 3 TB with LRDIMM/DCPMM depending on CPU", + "memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "NVDIMM-N", "DCPMM/PMem"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 8, + "pcie_note": "Up to 8× PCIe 3.0 (riser dependent)", + "drive_bays_max": 16, + "drive_note": "Up to 16× 2.5\" or 8× 3.5\" depending on chassis kit", + "cpu_sockets": 2, + "best_practices": [ + "Same 14G rules as R740xd: balance channels; white tabs before black.", + "Performance Optimized 4/8 DIMM/CPU populations per Dell KB 000141539.", + ], + "performance_notes": [ + "Populate 6 identical DIMMs per CPU before doubling up channels for max throughput.", + ], + "docs": [ + {"title": "R740 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r740"], "kind": "spec"}, + {"title": "R740/R740xd Technical Guide (PDF)", "url": _DELL_LINKS["tech_r740"], "kind": "tech"}, + {"title": "Dell Support · R740 docs", "url": _DELL_LINKS["docs_r740"], "kind": "support"}, + ], + "channel_map": _CH14, + "slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)], + }, + "PowerEdge R640": { + "generation": "14G", + "dimm_slots": 24, + "dimms_per_cpu": 12, + "channels_per_cpu": 6, + "max_memory_tb": 3.0, + "memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM", "DCPMM/PMem"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 3, + "pcie_note": "1U: up to 3× PCIe 3.0 (+ rNDC)", + "drive_bays_max": 10, + "drive_note": "Up to 10× 2.5\" (NVMe options vary)", + "cpu_sockets": 2, + "best_practices": [ + "14G dual-socket population: identical DIMMs across 6 channels per CPU.", + "White-tab sockets first; avoid RDIMM+LRDIMM mix.", + ], + "performance_notes": [ + "1U thermal/airflow limits denser GPU/NVMe kits — verify riser/backplane BOM.", + ], + "docs": [ + {"title": "R640 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r640"], "kind": "spec"}, + {"title": "R640 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r640"], "kind": "manual"}, + {"title": "Dell Support · R640 docs", "url": _DELL_LINKS["docs_r640"], "kind": "support"}, + ], + "channel_map": _CH14, + "slot_labels": [f"A{i}" for i in range(1, 13)] + [f"B{i}" for i in range(1, 13)], + }, + "PowerEdge R440": { + "generation": "14G", + "dimm_slots": 16, + "dimms_per_cpu": 8, + "channels_per_cpu": 6, + "max_memory_tb": 0.512, + "memory_types": ["DDR4 RDIMM"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 3, + "drive_bays_max": 10, + "cpu_sockets": 2, + "best_practices": ["Populate channels evenly; match DIMM size/speed/rank per Dell 14G guidance."], + "performance_notes": ["Entry 14G — keep DIMM speeds matched to CPU QPI/UPI memory controller limits."], + "docs": [ + {"title": "R440 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r440"], "kind": "spec"}, + {"title": "Dell Support · R440 docs", "url": _DELL_LINKS["docs_r440"], "kind": "support"}, + ], + "slot_labels": [f"A{i}" for i in range(1, 9)] + [f"B{i}" for i in range(1, 9)], + }, + "PowerEdge R6415": { + "generation": "14G-AMD", + "dimm_slots": 16, + "dimms_per_cpu": 16, + "channels_per_cpu": 8, + "max_memory_tb": 1.0, + "memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 3, + "drive_bays_max": 10, + "cpu_sockets": 1, + "best_practices": [ + "AMD EPYC Naples/Rome 14G: for 4 DIMMs use slots 1,3,5,7 (Dell KB 000141539).", + "Populate evenly across memory channels for Infinity Fabric bandwidth.", + ], + "performance_notes": [ + "EPYC memory performance scales strongly with channels populated — prefer 8 identical DIMMs when possible.", + ], + "docs": [ + {"title": "R6415 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r6415"], "kind": "spec"}, + {"title": "R6415 Tech Specs / Reference (PDF)", "url": _DELL_LINKS["ref_r6415"], "kind": "tech"}, + {"title": "R6415 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r6415"], "kind": "manual"}, + {"title": "Dell Support · R6415 docs", "url": _DELL_LINKS["docs_r6415"], "kind": "support"}, + ], + "slot_labels": [f"A{i}" for i in range(1, 17)], + }, + "PowerEdge R7425": { + "generation": "14G-AMD", + "dimm_slots": 32, + "dimms_per_cpu": 16, + "channels_per_cpu": 8, + "max_memory_tb": 2.0, + "memory_types": ["DDR4 RDIMM", "DDR4 LRDIMM"], + "max_dimm_speed_mts": 2666, + "pcie_slots_max": 8, + "drive_bays_max": 24, + "cpu_sockets": 2, + "best_practices": [ + "Dual EPYC: mirror CPU1 and CPU2 DIMM populations.", + "4 DIMMs/CPU → slots 1,3,5,7 per Dell AMD 14G guidance.", + ], + "performance_notes": ["Balance both sockets identically to avoid NUMA imbalance."], + "docs": [ + {"title": "R7425 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r7425"], "kind": "spec"}, + {"title": "R7425 Owner's Manual (PDF)", "url": _DELL_LINKS["own_r7425"], "kind": "manual"}, + {"title": "Dell Support · R7425 docs", "url": _DELL_LINKS["docs_r7425"], "kind": "support"}, + ], + "slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)], + }, + "PowerEdge R660": { + "generation": "16G", + "dimm_slots": 32, + "dimms_per_cpu": 16, + "channels_per_cpu": 8, + "max_memory_tb": 8.0, + "memory_types": ["DDR5 RDIMM"], + "max_dimm_speed_mts": 4800, + "pcie_slots_max": 3, + "drive_bays_max": 10, + "cpu_sockets": 2, + "best_practices": [ + "16G DDR5: populate all memory channels evenly; follow white/black tab order in the ISM for the SKU.", + "Use matched DDR5 RDIMM kits; mixing ranks/speeds can downclock the entire domain.", + ], + "performance_notes": [ + "Sapphire Rapids / Emerald Rapids memory bandwidth benefits strongly from full channel population.", + ], + "docs": [ + {"title": "R660 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_r660"], "kind": "spec"}, + {"title": "R660 Technical Guide (PDF)", "url": _DELL_LINKS["tech_r660"], "kind": "tech"}, + {"title": "Dell Support · R660 docs", "url": _DELL_LINKS["docs_r660"], "kind": "support"}, + ], + "slot_labels": [f"A{i}" for i in range(1, 17)] + [f"B{i}" for i in range(1, 17)], + }, + "PowerEdge XR5610": { + "generation": "16G-XR", + "dimm_slots": 8, + "dimms_per_cpu": 8, + "channels_per_cpu": 8, + "max_memory_tb": 1.0, + "max_memory_note": "Dell Spec Sheet: 8× DDR5 RDIMM, up to 1 TB, up to 5600 MT/s (CPU/SKU dependent)", + "memory_types": ["DDR5 RDIMM"], + "max_dimm_speed_mts": 5600, + "pcie_slots_max": 2, + "pcie_note": "Spec Sheet: up to 2× PCIe Gen5 x16 (+ OCP 3.0) — riser/GPU option dependent", + "drive_bays_max": 4, + "drive_note": "Spec Sheet: up to 4× 2.5\" SAS/SATA/NVMe front; BOSS-N1 M.2 is separate boot — not front-bay count", + "cpu_sockets": 1, + "best_practices": [ + "Populate DDR5 channels evenly with matched RDIMMs.", + "Confirm exact DIMM map in the XR5610 Technical Guide / ISM for this BOM.", + ], + "performance_notes": [ + "Single-socket edge platform: memory bandwidth scales with channels populated.", + ], + "docs": [ + {"title": "XR5610 Spec Sheet (PDF)", "url": _DELL_LINKS["spec_xr5610"], "kind": "spec"}, + {"title": "XR5610 Technical Guide (PDF)", "url": _DELL_LINKS["tech_xr5610"], "kind": "tech"}, + {"title": "Dell Support · XR5610 docs", "url": _DELL_LINKS["docs_xr5610"], "kind": "support"}, + ], + "slot_labels": [f"A{i}" for i in range(1, 9)], + }, +} + + +def _normalize_model_key(model: str | None) -> str: + m = (model or "").strip() + if not m: + return "" + if m in MODEL_EXPANSION: + return m + # fuzzy family match + for key in MODEL_EXPANSION: + if key.lower() in m.lower() or m.lower() in key.lower(): + return key + # strip suffixes like "vSAN Ready Node" + for key in MODEL_EXPANSION: + short = key.replace("PowerEdge ", "") + if short.lower() in m.lower(): + return key + return m + + +def _parse_dimm_socket(name: str | None) -> tuple[str | None, int | None]: + import re + + s = name or "" + m = re.search(r"(?:DIMM\.Socket\.|DIMM\s*)([A-D])(\d{1,2})", s, re.I) + if not m: + m = re.search(r"\b([A-D])(\d{1,2})\b", s) + if not m: + return None, None + return m.group(1).upper(), int(m.group(2)) + + +def _disk_size_gb(raw: Any) -> float | None: + """OME serverArrayDisks.Size is already capacity in GB (e.g. '446.62').""" + if raw is None or raw == "": + return None + try: + return round(float(str(raw).strip()), 2) + except Exception: + return None + + +def _mem_size_gb(raw: Any) -> float | None: + """OME serverMemoryDevices.Size is capacity in MB.""" + if raw is None or raw == "": + return None + try: + return round(float(raw) / 1024.0, 1) + except Exception: + return None + + +def build_expansion_report(model: str | None, inventory: dict) -> dict: + """Split live OME facts (current) from Dell catalog max expansion (recommendations).""" + key = _normalize_model_key(model) + catalog_known = bool(MODEL_EXPANSION.get(key)) + spec = dict(MODEL_EXPANSION.get(key) or {}) + if not catalog_known: + spec = { + "generation": None, + "dimm_slots": None, + "max_memory_tb": None, + "pcie_slots_max": None, + "drive_bays_max": None, + "cpu_sockets": None, + "best_practices": [ + "No Cockpit catalog entry for this model — verify DIMM/CPU/drive maxima in the Dell Tech Specs / ISM for the exact Service Tag BOM.", + ], + "performance_notes": [], + "docs": [{"title": "Dell Support home", "url": _DELL_LINKS["support_home"]}], + "slot_labels": [], + "channel_map": {}, + "memory_types": [], + "max_memory_note": "Maximum not asserted — catalog unknown for this model.", + "drive_note": "Bay maximum not asserted — catalog unknown for this model.", + "pcie_note": "PCIe maximum not asserted — catalog unknown for this model.", + } + + mem_raw = inventory.get("serverMemoryDevices") or [] + cpu_raw = inventory.get("serverProcessors") or [] + disk_raw = inventory.get("serverArrayDisks") or [] + cards_raw = inventory.get("serverDeviceCards") or [] + raid_raw = inventory.get("serverRaidControllers") or [] + os_raw = inventory.get("serverOperatingSystems") or [] + + dimm_modules: list[dict] = [] + for dimm in mem_raw: + bank, num = _parse_dimm_socket( + dimm.get("Name") or dimm.get("InstanceId") or dimm.get("DeviceDescription") + ) + if not bank or not num: + continue + label = f"{bank}{num}" + size_gb = _mem_size_gb(dimm.get("Size")) + dimm_modules.append( + { + "slot": label, + "cpu_bank": bank, + "slot_num": num, + "size_gb": size_gb, + "size_mb": dimm.get("Size"), + "rated_mts": dimm.get("Speed"), + "operating_mts": dimm.get("CurrentOperatingSpeed"), + "rank": dimm.get("Rank"), + "type": dimm.get("TypeDetails") or dimm.get("MemoryType") or "DDR", + "manufacturer": dimm.get("Manufacturer"), + "part_number": dimm.get("PartNumber"), + "serial": dimm.get("SerialNumber"), + "status": dimm.get("Status"), + } + ) + dimm_modules.sort(key=lambda s: (s.get("cpu_bank") or "", s.get("slot_num") or 0)) + mem_total_gb = round(sum(s.get("size_gb") or 0 for s in dimm_modules), 1) + rated_speeds = sorted({s["rated_mts"] for s in dimm_modules if s.get("rated_mts") is not None}) + op_speeds = sorted({s["operating_mts"] for s in dimm_modules if s.get("operating_mts") is not None}) + + cpus: list[dict] = [] + for p in cpu_raw: + cpus.append( + { + "slot": p.get("SlotNumber") or p.get("InstanceId"), + "model": p.get("ModelName") or p.get("BrandName"), + "cores": p.get("NumberOfEnabledCores") or p.get("NumberOfCores"), + "threads": p.get("NumberOfEnabledThreads") or p.get("NumberOfThreads"), + "current_mhz": p.get("CurrentSpeed"), + "max_mhz": p.get("MaxSpeed"), + "status": p.get("Status"), + } + ) + cpu_cores_total = sum(int(c.get("cores") or 0) for c in cpus) + + disks: list[dict] = [] + for d in disk_raw: + disks.append( + { + "bay": d.get("SlotNumber"), + "model": d.get("ModelNumber"), + "vendor": d.get("VendorName"), + "media": d.get("MediaType"), + "size_gb": _disk_size_gb(d.get("Size")), + "bus": d.get("BusType"), + "status": d.get("StatusString") or d.get("Status"), + "serial": d.get("SerialNumber"), + "enclosure": d.get("EnclosureId"), + "form_factor": d.get("FormFactor"), + } + ) + disks.sort(key=lambda x: (str(x.get("bus") or ""), x.get("bay") is None, x.get("bay") or 0)) + disk_total_gb = round(sum(d.get("size_gb") or 0 for d in disks), 2) + + adapters: list[dict] = [] + for c in cards_raw: + adapters.append( + { + "slot": str(c.get("SlotNumber") or ""), + "description": c.get("Description"), + "manufacturer": c.get("Manufacturer"), + "slot_type": c.get("SlotType"), + "width": c.get("DatabusWidth"), + } + ) + for r in raid_raw: + adapters.append( + { + "slot": str(r.get("PciSlot") or r.get("Fqdd") or "RAID"), + "description": r.get("Name") or r.get("DeviceDescription"), + "manufacturer": "Dell PERC", + "slot_type": "RAID", + "firmware": r.get("FirmwareVersion"), + "cache_mb": r.get("CacheSizeInMb"), + } + ) + + os_info = [] + for o in os_raw: + os_info.append( + { + "os_name": (o.get("OsName") or "").strip() or None, + "os_version": (o.get("OsVersion") or "").strip() or None, + "hostname": (o.get("Hostname") or "").strip() or None, + } + ) + + current = { + "source": "OME DeviceService inventory (live)", + "memory": { + "installed_gb": mem_total_gb, + "dimm_count": len(dimm_modules), + "modules": dimm_modules, + "rated_mts": rated_speeds, + "operating_mts": op_speeds, + "banks_seen": sorted({m["cpu_bank"] for m in dimm_modules if m.get("cpu_bank")}), + }, + "cpu": { + "sockets_populated": len(cpus), + "cores_total": cpu_cores_total, + "processors": cpus, + }, + "disks": { + "count": len(disks), + "capacity_gb": disk_total_gb, + "items": disks, + }, + "adapters": adapters, + "os": os_info, + } + + populated_map = {m["slot"]: m for m in dimm_modules} + labels = list(spec.get("slot_labels") or []) + slot_map: list[dict] = [] + for label in labels: + bank = label[0] + num = int(label[1:] or 0) + ch_map = (spec.get("channel_map") or {}).get(bank) or {} + if label in populated_map: + row = dict(populated_map[label]) + row["populated"] = True + row["channel"] = ch_map.get(num) + row["rank_color"] = "white" if num <= 6 else "black" + row["speed_mts"] = row.get("operating_mts") or row.get("rated_mts") + slot_map.append(row) + else: + slot_map.append( + { + "slot": label, + "cpu_bank": bank, + "slot_num": num, + "channel": ch_map.get(num), + "rank_color": "white" if num <= 6 else "black", + "populated": False, + } + ) + for label, row in populated_map.items(): + if label not in labels: + extra = dict(row) + extra["populated"] = True + extra["speed_mts"] = extra.get("operating_mts") or extra.get("rated_mts") + slot_map.append(extra) + + empty_n = sum(1 for s in slot_map if not s.get("populated")) if labels else None + dimm_slots_catalog = spec.get("dimm_slots") + max_tb = spec.get("max_memory_tb") + headroom_gb = None + if max_tb is not None: + try: + headroom_gb = round(float(max_tb) * 1024 - mem_total_gb, 1) + except Exception: + headroom_gb = None + + drive_max = spec.get("drive_bays_max") + free_bays = (int(drive_max) - len(disks)) if catalog_known and isinstance(drive_max, int) else None + cpu_max = spec.get("cpu_sockets") + free_cpu = (int(cpu_max) - len(cpus)) if catalog_known and isinstance(cpu_max, int) else None + + findings: list[dict] = [] + if catalog_known and labels: + by_bank: dict[str, list] = {} + for s in slot_map: + by_bank.setdefault(s.get("cpu_bank") or "?", []).append(s) + for bank, bank_slots in by_bank.items(): + ch_counts: dict[int, int] = {} + for s in bank_slots: + if s.get("populated") and s.get("channel") is not None: + ch_counts[int(s["channel"])] = ch_counts.get(int(s["channel"]), 0) + 1 + if ch_counts and spec.get("channels_per_cpu"): + vals = list(ch_counts.values()) + if max(vals) - min(vals) > 0 or len(ch_counts) < int(spec["channels_per_cpu"]): + findings.append( + { + "severity": "warning", + "code": "unbalanced_channels", + "text": f"Bank {bank}: channel population {dict(sorted(ch_counts.items()))} — Dell guidance prefers even channel fill for bandwidth.", + } + ) + if len(op_speeds) > 1: + findings.append( + { + "severity": "warning", + "code": "mixed_operating_speeds", + "text": f"OME reports mixed operating speeds {op_speeds} MT/s — controller usually clocks all DIMMs to the lowest.", + } + ) + if rated_speeds and op_speeds and max(op_speeds) < max(rated_speeds): + findings.append( + { + "severity": "info", + "code": "downclocked", + "text": f"DIMMs rated {rated_speeds} MT/s, operating at {op_speeds} MT/s (CPU/population limited).", + } + ) + + expansion = { + "source": "Dell model catalog in Cockpit (not live measurement)" if catalog_known else "Catalog unknown", + "catalog_known": catalog_known, + "catalog_model": key if catalog_known else None, + "generation": spec.get("generation"), + "memory": { + "max_capacity_tb": max_tb, + "max_capacity_gb": round(float(max_tb) * 1024, 1) if max_tb is not None else None, + "dimm_slots_max": dimm_slots_catalog, + "dimms_installed": len(dimm_modules), + "empty_slots_estimate": empty_n if catalog_known else None, + "headroom_gb": headroom_gb if catalog_known else None, + "supported_types": spec.get("memory_types") or [], + "max_rated_mts": spec.get("max_dimm_speed_mts"), + "note": spec.get("max_memory_note") + or ( + f"Catalog max {max_tb} TB / {dimm_slots_catalog} DIMM slots" + if catalog_known + else "No catalog max — do not treat empty map as measured free slots." + ), + "slot_map": slot_map if catalog_known else [], + }, + "cpu": { + "sockets_max": cpu_max, + "sockets_populated": len(cpus), + "sockets_free_estimate": free_cpu, + "note": ( + f"Chassis supports up to {cpu_max} CPU socket(s); OME sees {len(cpus)} populated." + if catalog_known and cpu_max is not None + else "CPU socket maximum not in Cockpit catalog for this model." + ), + }, + "disks": { + "bays_max": drive_max if catalog_known else None, + "drives_installed": len(disks), + "bays_free_estimate": free_bays, + "note": spec.get("drive_note") + or ( + "Bay maximum from Dell chassis options — verify BOM; M.2/BOSS may not count as front bays." + if catalog_known + else "Drive bay maximum unknown for this model." + ), + }, + "pcie": { + "slots_max": spec.get("pcie_slots_max") if catalog_known else None, + "note": spec.get("pcie_note") + or ("PCIe max depends on riser kit." if catalog_known else "PCIe maximum unknown for this model."), + }, + "best_practices": spec.get("best_practices") or [], + "performance_notes": spec.get("performance_notes") or [], + "docs": spec.get("docs") or [], + } + + return { + "model": model, + "catalog_key": key if catalog_known else None, + "generic": not catalog_known, + "current": current, + "expansion": expansion, + "findings": findings, + "spec": { + "generation": expansion.get("generation"), + "dimm_slots": expansion["memory"].get("dimm_slots_max"), + "max_memory_tb": expansion["memory"].get("max_capacity_tb"), + "max_memory_note": expansion["memory"].get("note"), + "memory_types": expansion["memory"].get("supported_types"), + "max_dimm_speed_mts": expansion["memory"].get("max_rated_mts"), + "pcie_slots_max": expansion["pcie"].get("slots_max"), + "pcie_note": expansion["pcie"].get("note"), + "drive_bays_max": expansion["disks"].get("bays_max"), + "drive_note": expansion["disks"].get("note"), + "cpu_sockets": expansion["cpu"].get("sockets_max"), + "best_practices": expansion.get("best_practices") or [], + "performance_notes": expansion.get("performance_notes") or [], + "docs": expansion.get("docs") or [], + }, + "memory": { + "total_gb": mem_total_gb, + "populated": len(dimm_modules), + "empty": empty_n, + "headroom_gb": headroom_gb, + "slots": slot_map if slot_map else [{**m, "populated": True, "speed_mts": m.get("operating_mts") or m.get("rated_mts")} for m in dimm_modules], + "banks": current["memory"]["banks_seen"], + }, + "processors": cpus, + "disks": { + "installed": len(disks), + "max_bays": drive_max if catalog_known else None, + "free_bays_estimate": free_bays, + "items": [{**d, "slot": d.get("bay"), "name": d.get("model")} for d in disks], + }, + "pcie": { + "max_slots": expansion["pcie"].get("slots_max"), + "items": adapters, + "note": expansion["pcie"].get("note"), + }, + } + + + def subnet_of(ip: str | None) -> str: if not ip or ip.count(".") != 3: return "unknown" @@ -170,6 +985,83 @@ def pick_ip(device: dict) -> str | None: return None +def pick_management(device: dict) -> dict: + """Extract iDRAC/management IP plus OS hostname hints from OME DeviceManagement.""" + mgmt = (device.get("DeviceManagement") or [{}])[0] or {} + idrac_ip = None + for m in device.get("DeviceManagement") or []: + addr = m.get("NetworkAddress") + if addr and str(addr).count(".") == 3: + idrac_ip = addr + mgmt = m + break + os_hostname = (mgmt.get("InstrumentationName") or "").strip() or None + dns_name = (mgmt.get("DnsName") or "").strip() or None + return { + "idrac_ip": idrac_ip, + "os_hostname": os_hostname, + "mgmt_dns_name": dns_name, + "mac": mgmt.get("MacAddress"), + } + + +async def resolve_host_ips(hostname: str | None) -> list[str]: + if not hostname or hostname.count(".") == 3: + return [] + loop = asyncio.get_running_loop() + try: + infos = await loop.getaddrinfo(hostname, None, family=socket.AF_INET) + return sorted({i[4][0] for i in infos}) + except Exception: + return [] + + +async def enrich_rdp_targets(devices: list[dict]) -> None: + """DNS-resolve OS hostnames from OME InstrumentationName → RDP candidate IPs.""" + hostnames: set[str] = set() + for n in devices: + h = (n.get("os_hostname") or "").strip() + if h and h.count(".") != 3: + hostnames.add(h) + # also try short name + if "." in h: + hostnames.add(h.split(".", 1)[0]) + + resolved: dict[str, list[str]] = {} + sem = asyncio.Semaphore(20) + + async def one(h: str): + async with sem: + resolved[h] = await resolve_host_ips(h) + + if hostnames: + await asyncio.gather(*[one(h) for h in hostnames]) + + for n in devices: + idrac = n.get("idrac_ip") or n.get("ip") + h = (n.get("os_hostname") or "").strip() + ips: list[str] = [] + seen: set[str] = set() + for candidate in (h, h.split(".", 1)[0] if h and "." in h else None): + if not candidate: + continue + for ip in resolved.get(candidate) or []: + if ip not in seen: + seen.add(ip) + ips.append(ip) + # Prefer OS IPs that differ from iDRAC + os_ips = [ip for ip in ips if ip != idrac] + rdp_ips = os_ips or ips + n["rdp_ips"] = rdp_ips + n["rdp_host"] = rdp_ips[0] if rdp_ips else None + n["os_ips"] = os_ips + # Windows hint from name / later OS inventory + blob = f"{n.get('name') or ''} {n.get('os_hostname') or ''} {n.get('os_name') or ''}".lower() + n["is_windows"] = any(k in blob for k in ("windows", "hyperv", "hyper-v", "-hv", "hv0")) or bool( + n.get("os_name") and "windows" in str(n.get("os_name")).lower() + ) + + async def fetch_power(client: httpx.AsyncClient, base: str, headers: dict, device_id: int) -> dict: try: r = await client.get(f"{base}/api/DeviceService/Devices({device_id})/Power", headers=headers) @@ -270,26 +1162,52 @@ async def ome_fetch() -> dict: subnet_map: dict[str, list] = defaultdict(list) model_counts: dict[str, int] = defaultdict(int) idrac_count = server_count = connected = powered = 0 + switch_count = chassis_count = pdu_count = storage_count = 0 total_watts = 0.0 watts_samples = 0 status_counts: dict[str, int] = defaultdict(int) for d in devices_raw: - ip = pick_ip(d) + mgmt = pick_management(d) + ip = mgmt.get("idrac_ip") or pick_ip(d) subnet = subnet_of(ip) dtype = d.get("Type") sub = d.get("SubDeviceType") or "" + model = d.get("Model") or "Unknown" is_server = dtype == 1000 is_idrac = sub == "iDRAC" + is_switch = dtype == 7000 or "switch" in (sub or "").lower() or "switch" in model.lower() + is_chassis = dtype == 2000 or "chassis" in (sub or "").lower() + is_pdu = dtype == 100 + is_storage = dtype == 5000 or "storage" in (sub or "").lower() + if is_switch: + role = "switch" + elif is_chassis: + role = "chassis" + elif is_pdu: + role = "pdu" + elif is_storage: + role = "storage" + elif is_server: + role = "server" + else: + role = "other" conn = bool(d.get("ConnectionState")) power_state = d.get("PowerState") status = str(d.get("Status") or "") status_counts[status] += 1 - model = d.get("Model") or "Unknown" if is_server: model_counts[model] += 1 if is_idrac: idrac_count += 1 + if is_switch: + switch_count += 1 + if is_chassis: + chassis_count += 1 + if is_pdu: + pdu_count += 1 + if is_storage: + storage_count += 1 if is_server: server_count += 1 if conn: @@ -317,9 +1235,22 @@ async def ome_fetch() -> dict: "powered_on": power_state == 17, "status": status, "ip": ip, + "idrac_ip": mgmt.get("idrac_ip") or ip, + "os_hostname": mgmt.get("os_hostname"), + "mgmt_dns_name": mgmt.get("mgmt_dns_name"), + "rdp_host": None, + "rdp_ips": [], + "os_ips": [], + "is_windows": False, "subnet": subnet, + "role": role, "is_server": is_server, "is_idrac": is_idrac, + "is_switch": is_switch, + "is_chassis": is_chassis, + "is_pdu": is_pdu, + "is_storage": is_storage, + "chassis_service_tag": d.get("ChassisServiceTag"), "idrac_url": f"https://{ip}" if ip else None, "watts": watts, "avg_watts": pwr.get("avg_watts"), @@ -331,16 +1262,24 @@ async def ome_fetch() -> dict: devices.append(node) subnet_map[subnet].append(node["id"]) + await enrich_rdp_targets(devices) + subnets = [] for cidr, ids in sorted(subnet_map.items(), key=lambda x: -len(x[1])): members = [n for n in devices if n["id"] in ids] sub_watts = sum(n["watts"] or 0 for n in members if n.get("watts") is not None) + switches = [n for n in members if n.get("is_switch")] subnets.append({ "cidr": cidr, "count": len(members), "connected": sum(1 for n in members if n["connected"]), "watts": round(sub_watts, 1) if sub_watts else None, "device_ids": ids, + "has_switch": bool(switches), + "switches": [ + {"id": s["id"], "name": s["name"], "service_tag": s.get("service_tag"), "ip": s.get("ip")} + for s in switches + ], }) groups = [ @@ -362,11 +1301,19 @@ async def ome_fetch() -> dict: current_ids: set[int] = set() def _role(n: dict) -> str: + if n.get("is_switch"): + return "switch" + if n.get("is_chassis"): + return "chassis" if n.get("is_idrac"): return "iDRAC" if n.get("is_server"): return "server" - return "device" + if n.get("is_storage"): + return "storage" + if n.get("is_pdu"): + return "PDU" + return n.get("role") or "device" for n in devices: did = n.get("id") @@ -535,6 +1482,10 @@ async def ome_fetch() -> dict: "total": len(devices), "idracs": idrac_count, "servers": server_count, + "switches": switch_count, + "chassis": chassis_count, + "pdus": pdu_count, + "storage": storage_count, "connected": connected, "powered_on": powered, "offline": max(0, server_count - connected), @@ -569,6 +1520,509 @@ async def broadcast(payload: dict): CLIENTS.discard(ws) + +def _seed_atc_racks() -> None: + """Idempotent seed: ATC1 (8×42U) + ATC2 (6×42U).""" + now = time.time() + with _db() as conn: + n = conn.execute("SELECT COUNT(*) AS c FROM racks").fetchone()["c"] + if n > 0: + return + conn.execute( + "INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)", + ("ATC1", "ATC1", 1), + ) + conn.execute( + "INSERT OR IGNORE INTO sites(id, name, sort_order) VALUES (?, ?, ?)", + ("ATC2", "ATC2", 2), + ) + for i in range(1, 9): + conn.execute( + """ + INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at) + VALUES (?, ?, 42, ?, ?, ?) + """, + ("ATC1", f"ATC1-R{i:02d}", i, now, now), + ) + for i in range(1, 7): + conn.execute( + """ + INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at) + VALUES (?, ?, 42, ?, ?, ?) + """, + ("ATC2", f"ATC2-R{i:02d}", i, now, now), + ) + log.info("Seeded ATC racks: ATC1×8 + ATC2×6 (42U)") + + +def build_network_fabric(devices: list[dict] | None = None) -> dict: + """Inferred switch↔server / chassis↔blade edges (no LLDP).""" + devices = list(devices if devices is not None else (STATE.get("devices") or [])) + by_id = {d.get("id"): d for d in devices if d.get("id") is not None} + edges: list[dict] = [] + seen: set[tuple] = set() + + def add_edge(a: int, b: int, kind: str): + if a is None or b is None or a == b: + return + key = (min(a, b), max(a, b), kind) + if key in seen: + return + seen.add(key) + da, db = by_id.get(a) or {}, by_id.get(b) or {} + edges.append( + { + "from_id": a, + "to_id": b, + "kind": kind, + "confidence": "inferred", + "note": ( + "management subnet" + if kind == "subnet" + else "MX chassis / blade" + ), + "live": bool(da.get("connected") and db.get("connected")), + } + ) + + switches = [d for d in devices if d.get("is_switch")] + for sw in switches: + cidr = sw.get("subnet") or "unknown" + if cidr == "unknown": + continue + for d in devices: + if d.get("id") == sw.get("id"): + continue + if d.get("subnet") != cidr: + continue + if d.get("is_switch") or d.get("is_pdu"): + continue + # Prefer linking servers/storage/chassis endpoints + if d.get("is_server") or d.get("is_storage") or d.get("is_chassis") or d.get("is_idrac"): + add_edge(sw["id"], d["id"], "subnet") + + chassis_list = [d for d in devices if d.get("is_chassis")] + for ch in chassis_list: + cst = (ch.get("service_tag") or "").upper() + for d in devices: + if not d.get("is_server"): + continue + model = (d.get("model") or "").upper() + blade_cst = (d.get("chassis_service_tag") or "").upper() + if cst and blade_cst and blade_cst == cst: + add_edge(ch["id"], d["id"], "chassis") + elif "MX" in model and "MX7000" not in model: + # fallback: all MX blades to MX chassis in same environment + add_edge(ch["id"], d["id"], "chassis") + + return { + "edges": edges, + "note": "Edges inferred from management subnet / MX chassis — not LLDP.", + "switches": [ + { + "id": s.get("id"), + "name": s.get("name"), + "service_tag": s.get("service_tag"), + "ip": s.get("ip") or s.get("idrac_ip"), + "subnet": s.get("subnet"), + "connected": s.get("connected"), + "model": s.get("model"), + "connected_device_ids": [ + e["to_id"] if e["from_id"] == s.get("id") else e["from_id"] + for e in edges + if e["kind"] == "subnet" + and (e["from_id"] == s.get("id") or e["to_id"] == s.get("id")) + ], + } + for s in switches + ], + "summary": { + "devices": len(devices), + "edges": len(edges), + "switches": len(switches), + "chassis": sum(1 for d in devices if d.get("is_chassis")), + }, + } + + +def _default_u_height(device: dict | None) -> int: + if not device: + return 1 + cat = match_rack_catalog(device.get("model"), device.get("role")) + if cat is not None: + uh = int(cat.get("u_height") or 0) + return uh if uh > 0 else 1 # 0U vertical PDUs place as 1U marker unless side=left/right + if device.get("is_chassis"): + return 7 + if device.get("is_storage"): + return 2 + if device.get("is_switch"): + return 1 + return 1 + + +def _rack_overlap( + conn: sqlite3.Connection, + rack_id: int, + u_start: int, + u_height: int, + side: str = "front", + exclude_item_id: int | None = None, + exclude_device_id: int | None = None, +) -> bool: + if side in ("left", "right"): + return False # side PDUs do not consume front/rear U map + u_end = u_start + max(u_height, 1) - 1 + rows = conn.execute( + "SELECT id, device_id, u_start, u_height, side FROM rack_items WHERE rack_id=?", + (rack_id,), + ).fetchall() + for r in rows: + if exclude_item_id is not None and r["id"] == exclude_item_id: + continue + if exclude_device_id is not None and r["device_id"] == exclude_device_id: + continue + rside = (r["side"] or "front") + if rside in ("left", "right"): + continue + if rside != side: + continue + a, b = r["u_start"], r["u_start"] + r["u_height"] - 1 + if not (u_end < a or u_start > b): + return True + return False + + +def _enrich_rack_item(item: dict, devices: dict) -> dict: + out = dict(item) + dev = devices.get(item.get("device_id")) if item.get("device_id") is not None else None + sku = item.get("catalog_sku") + cat = None + if sku: + cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None) + if not cat and dev: + cat = match_rack_catalog(dev.get("model"), dev.get("role")) + if cat: + out["catalog"] = {k: cat[k] for k in ("sku", "name", "family", "category", "u_height", "face", "image") if k in cat} + out["face"] = cat.get("face") + out["image"] = cat.get("image") or catalog_image_for(cat) + out["category"] = cat.get("category") + else: + out["catalog"] = None + out["face"] = "custom" + out["image"] = FACE_IMAGE_MAP["custom"] + out["category"] = (dev or {}).get("role") or "custom" + if dev: + out["device"] = { + "id": dev.get("id"), + "name": dev.get("name"), + "model": dev.get("model"), + "service_tag": dev.get("service_tag"), + "role": dev.get("role"), + "ip": dev.get("ip") or dev.get("idrac_ip"), + "connected": dev.get("connected"), + "is_switch": dev.get("is_switch"), + "is_server": dev.get("is_server"), + "is_storage": dev.get("is_storage"), + "is_chassis": dev.get("is_chassis"), + "is_pdu": dev.get("is_pdu"), + } + out["label"] = item.get("label") or dev.get("name") + out["model"] = dev.get("model") + else: + out["device"] = None + out["label"] = item.get("label") or ((cat or {}).get("name") if cat else "Custom") + out["model"] = (cat or {}).get("name") + return out + + +def _racks_payload() -> dict: + _migrate_rack_items() + devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None} + with _db() as conn: + sites = [dict(r) for r in conn.execute("SELECT * FROM sites ORDER BY sort_order").fetchall()] + racks = [dict(r) for r in conn.execute("SELECT * FROM racks ORDER BY site_id, sort_order").fetchall()] + items = [dict(r) for r in conn.execute("SELECT * FROM rack_items ORDER BY u_start DESC").fetchall()] + placed_ids = {i["device_id"] for i in items if i.get("device_id") is not None} + by_site: dict[str, list] = {} + for rk in racks: + card = dict(rk) + card["items"] = [] + card["placements"] = [] # backward compat alias + for it in items: + if it["rack_id"] != rk["id"]: + continue + enriched = _enrich_rack_item(it, devices) + card["items"].append(enriched) + # compat shape for older UI bits + card["placements"].append( + { + "device_id": it.get("device_id"), + "rack_id": it["rack_id"], + "u_start": it["u_start"], + "u_height": it["u_height"], + "item_id": it["id"], + "side": it.get("side") or "front", + "face": enriched.get("face"), + "catalog": enriched.get("catalog"), + "label": enriched.get("label"), + "device": enriched.get("device") or { + "id": it.get("device_id"), + "name": enriched.get("label"), + "model": enriched.get("model"), + "role": enriched.get("category"), + "service_tag": None, + }, + } + ) + by_site.setdefault(rk["site_id"], []).append(card) + site_out = [{**s, "racks": by_site.get(s["id"], [])} for s in sites] + unplaced = [] + for d in STATE.get("devices") or []: + if d.get("id") in placed_ids: + continue + if not (d.get("is_server") or d.get("is_switch") or d.get("is_chassis") or d.get("is_storage") or d.get("is_pdu")): + continue + cat = match_rack_catalog(d.get("model"), d.get("role")) + unplaced.append( + { + "id": d.get("id"), + "name": d.get("name"), + "model": d.get("model"), + "service_tag": d.get("service_tag"), + "role": d.get("role"), + "ip": d.get("ip") or d.get("idrac_ip"), + "connected": d.get("connected"), + "default_u_height": _default_u_height(d), + "catalog_sku": (cat or {}).get("sku"), + "face": (cat or {}).get("face") or "custom", + "image": (cat or {}).get("image") or catalog_image_for(cat or {"face": "custom"}), + "is_switch": d.get("is_switch"), + "is_server": d.get("is_server"), + "is_storage": d.get("is_storage"), + "is_chassis": d.get("is_chassis"), + "is_pdu": d.get("is_pdu"), + } + ) + unplaced.sort(key=lambda x: ((x.get("role") or ""), (x.get("name") or "").lower())) + return { + "sites": site_out, + "items": items, + "placements": items, + "unplaced": unplaced, + "catalog": DELL_RACK_CATALOG, + } + + +def _switch_port_count(device: dict | None) -> int: + """Default front-panel port count from model (OME has no LLDP ports here).""" + model = ((device or {}).get("model") or "").lower() + if "s4048" in model: + return 48 + if "s5248" in model or "s5296" in model: + return 48 + if "s4148" in model: + return 48 + if "mx9116" in model: + return 16 + if "mx5108" in model: + return 8 + return 48 + + +def _port_label(port_num: int, total: int = 48) -> str: + return f"Eth1/{port_num}" + + +def build_switch_portmap(switch_id: int) -> dict: + devices = {d.get("id"): d for d in (STATE.get("devices") or []) if d.get("id") is not None} + sw = devices.get(switch_id) + if not sw or not sw.get("is_switch"): + # still allow if type 7000 missing flag somehow + sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None) + if not sw: + raise HTTPException(404, "Switch not found in fleet") + nports = _switch_port_count(sw) + with _db() as conn: + rows = conn.execute( + "SELECT * FROM port_links WHERE switch_id=? ORDER BY switch_port", + (switch_id,), + ).fetchall() + links = [dict(r) for r in rows] + by_port = {int(l["switch_port"]): l for l in links} + ports = [] + for i in range(1, nports + 1): + link = by_port.get(i) + peer = devices.get(link["device_id"]) if link else None + ports.append( + { + "port": i, + "label": _port_label(i, nports), + "wired": bool(link), + "link": ( + { + "id": link["id"], + "device_id": link["device_id"], + "device_port": link.get("device_port") or "", + "note": link.get("note") or "", + "device": { + "id": peer.get("id"), + "name": peer.get("name"), + "service_tag": peer.get("service_tag"), + "ip": peer.get("ip") or peer.get("idrac_ip"), + "model": peer.get("model"), + "role": peer.get("role"), + "connected": peer.get("connected"), + } + if peer + else None, + } + if link + else None + ), + } + ) + return { + "switch": { + "id": sw.get("id"), + "name": sw.get("name"), + "model": sw.get("model"), + "service_tag": sw.get("service_tag"), + "ip": sw.get("ip") or sw.get("idrac_ip"), + "subnet": sw.get("subnet"), + "connected": sw.get("connected"), + "port_count": nports, + }, + "ports": ports, + "wired_count": sum(1 for p in ports if p["wired"]), + "note": "Port wiring is manual (OME has no LLDP port map for this switch). Map switch port → server NIC/port.", + } + + +def _seed_atc_vlans() -> None: + """Seed known ATC lab VLANs/subnets (editable).""" + now = time.time() + defaults = [ + (40, "iDRAC / OOB A", "10.0.40.0/24", "Out-of-band management (has S4048)", "ATC1", "#00a8e8", 1), + (41, "iDRAC / OOB B", "10.0.41.0/24", "Out-of-band management", "ATC1", "#3dffe0", 2), + (42, "iDRAC / OOB C", "10.0.42.0/24", "Out-of-band management", "ATC2", "#7c5cff", 3), + (10, "Production / OS", "10.0.10.0/24", "Hypervisor & Windows OS / RDP", "ATC1", "#ff9a3c", 4), + (11, "Lab / OS", "10.0.11.0/24", "Lab OS network", "ATC2", "#ff5c5c", 5), + (120, "Secondary OS", "10.0.120.0/24", "Secondary OS / RDP path", "ATC1", "#f0d060", 6), + ] + with _db() as conn: + n = conn.execute("SELECT COUNT(*) AS c FROM vlans").fetchone()["c"] + if n > 0: + return + for vlan_id, name, cidr, purpose, site, color, so in defaults: + conn.execute( + """ + INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (vlan_id, name, cidr, purpose, site, color, so, now, now), + ) + log.info("Seeded ATC VLAN catalog (%s entries)", len(defaults)) + + +def build_vlan_map() -> dict: + """VLANs from catalog + live membership from fleet IPs (mgmt + RDP).""" + _seed_atc_vlans() + _migrate_rack_items() + devices = STATE.get("devices") or [] + with _db() as conn: + rows = [dict(r) for r in conn.execute("SELECT * FROM vlans ORDER BY sort_order, vlan_id").fetchall()] + + def ips_for(d: dict) -> list[str]: + out = [] + for k in ("ip", "idrac_ip"): + v = d.get(k) + if v and v not in out: + out.append(v) + for v in d.get("rdp_ips") or []: + if v and v not in out: + out.append(v) + return out + + def in_cidr(ip: str, cidr: str) -> bool: + try: + return ipaddress.ip_address(ip) in ipaddress.ip_network(cidr, strict=False) + except Exception: + return False + + vlans = [] + for v in rows: + members = [] + for d in devices: + matched = [ip for ip in ips_for(d) if in_cidr(ip, v["cidr"])] + if not matched: + continue + members.append( + { + "id": d.get("id"), + "name": d.get("name"), + "service_tag": d.get("service_tag"), + "role": d.get("role"), + "connected": d.get("connected"), + "ips": matched, + "model": d.get("model"), + } + ) + members.sort(key=lambda m: (m.get("name") or "").lower()) + vlans.append( + { + **v, + "member_count": len(members), + "connected_count": sum(1 for m in members if m.get("connected")), + "members": members[:80], + } + ) + + # Also surface discovered /24s not in catalog + seen = {v["cidr"] for v in rows} + discovered = [] + buckets: dict[str, int] = {} + for d in devices: + for ip in ips_for(d): + if ip.count(".") != 3: + continue + cidr = ".".join(ip.split(".")[:3]) + ".0/24" + if cidr.startswith("127."): + continue + buckets[cidr] = buckets.get(cidr, 0) + 1 + for cidr, cnt in sorted(buckets.items(), key=lambda x: -x[1]): + if cidr not in seen: + discovered.append({"cidr": cidr, "device_ips": cnt, "in_catalog": False}) + + return { + "vlans": vlans, + "discovered_subnets": discovered, + "note": "VLAN names are an ATC catalog (editable). Membership is live from OME mgmt/RDP IPs.", + } + + +def _migrate_rack_items() -> None: + """Copy legacy rack_placements into flexible rack_items once.""" + with _db() as conn: + n = conn.execute("SELECT COUNT(*) AS c FROM rack_items").fetchone()["c"] + if n > 0: + return + try: + rows = conn.execute("SELECT * FROM rack_placements").fetchall() + except Exception: + return + now = time.time() + for r in rows: + conn.execute( + """ + INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at) + VALUES (?, ?, NULL, NULL, ?, ?, 'front', NULL, ?) + """, + (r["rack_id"], r["device_id"], r["u_start"], r["u_height"], now), + ) + if rows: + log.info("Migrated %s rack_placements → rack_items", len(rows)) + ADMINS = [ {"id": "jody", "name": "Jody van Dongen", "role": "ATC Datacenter Admin", "email": "jody.van.dongen@dell.com"}, {"id": "laurens", "name": "Laurens Rammers", "role": "ATC Datacenter Admin", "email": "laurens.rammers@dell.com"}, @@ -586,7 +2040,66 @@ def _db(): return conn +def _backup_ops_db(reason: str = "manual") -> Path | None: + """Snapshot ops.db so rebuilds / wipes do not silently lose ticket history.""" + try: + if not DB_PATH.exists() or DB_PATH.stat().st_size < 100: + return None + bak_dir = DATA_DIR / "backups" + bak_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + dest = bak_dir / f"ops-{stamp}-{reason}-{int(time.time()*1000)%100000}.db" + import shutil + shutil.copy2(DB_PATH, dest) + # keep last 20 backups + olds = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True) + for stale in olds[20:]: + try: + stale.unlink() + except Exception: + pass + return dest + except Exception as e: + log.warning("ops.db backup failed: %s", e) + return None + + +def _restore_ops_db_if_empty() -> bool: + """If tickets table is empty but a backup exists, restore the newest backup with tickets.""" + try: + with _db() as conn: + n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"] + if n > 0: + return False + bak_dir = DATA_DIR / "backups" + if not bak_dir.exists(): + return False + candidates = sorted(bak_dir.glob("ops-*.db"), key=lambda x: x.stat().st_mtime, reverse=True) + for cand in candidates: + try: + c2 = sqlite3.connect(cand) + c2.row_factory = sqlite3.Row + cnt = c2.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"] + c2.close() + if cnt <= 0: + continue + import shutil + # preserve empty current as .empty + if DB_PATH.exists(): + shutil.copy2(DB_PATH, bak_dir / f"ops-empty-before-restore-{int(time.time())}.db") + shutil.copy2(cand, DB_PATH) + log.warning("Restored ops.db from backup %s (%s tickets)", cand.name, cnt) + return True + except Exception as e: + log.debug("skip backup %s: %s", cand, e) + return False + except Exception as e: + log.warning("ops.db restore check failed: %s", e) + return False + + def init_db(): + DATA_DIR.mkdir(parents=True, exist_ok=True) with _db() as conn: conn.executescript( """ @@ -614,6 +2127,75 @@ def init_db(): cols = {r[1] for r in conn.execute("PRAGMA table_info(tickets)").fetchall()} if "accepted_by" not in cols: conn.execute("ALTER TABLE tickets ADD COLUMN accepted_by TEXT") + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS sites ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + sort_order INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS racks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site_id TEXT NOT NULL REFERENCES sites(id), + name TEXT NOT NULL, + units INTEGER NOT NULL DEFAULT 42, + sort_order INTEGER NOT NULL, + created_at REAL NOT NULL, + updated_at REAL NOT NULL + ); + CREATE TABLE IF NOT EXISTS rack_placements ( + device_id INTEGER PRIMARY KEY, + rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE, + u_start INTEGER NOT NULL, + u_height INTEGER NOT NULL, + updated_at REAL NOT NULL + ); + CREATE TABLE IF NOT EXISTS rack_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rack_id INTEGER NOT NULL REFERENCES racks(id) ON DELETE CASCADE, + device_id INTEGER, + catalog_sku TEXT, + label TEXT, + u_start INTEGER NOT NULL, + u_height INTEGER NOT NULL, + side TEXT NOT NULL DEFAULT 'front', + notes TEXT, + updated_at REAL NOT NULL + ); + CREATE TABLE IF NOT EXISTS port_links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + switch_id INTEGER NOT NULL, + switch_port INTEGER NOT NULL, + device_id INTEGER NOT NULL, + device_port TEXT, + note TEXT, + updated_at REAL NOT NULL, + UNIQUE(switch_id, switch_port) + ); + CREATE TABLE IF NOT EXISTS vlans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + vlan_id INTEGER, + name TEXT NOT NULL, + cidr TEXT NOT NULL, + purpose TEXT, + site_id TEXT, + color TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + created_at REAL NOT NULL, + updated_at REAL NOT NULL + ); + """ + ) + _restore_ops_db_if_empty() + _seed_atc_racks() + _seed_atc_vlans() + n = 0 + try: + with _db() as conn: + n = conn.execute("SELECT COUNT(*) AS c FROM tickets").fetchone()["c"] + except Exception: + pass + log.info("Ops tickets DB ready at %s (%s tickets)", DB_PATH, n) async def fetch_gpu() -> dict: @@ -673,13 +2255,14 @@ async def gpu_loop(): @app.on_event("startup") async def startup(): init_db() + _backup_ops_db("startup") asyncio.create_task(poll_loop()) asyncio.create_task(gpu_loop()) asyncio.create_task(warm_reports_cache()) async def warm_reports_cache(): - """Background warm of warranty/compliance for chat + inspector.""" + """Background warm of warranty/compliance/memory for chat + inspector.""" await asyncio.sleep(8) try: await fetch_warranties() @@ -691,6 +2274,20 @@ async def warm_reports_cache(): ) except Exception as e: log.warning("Reports cache warm failed: %s", e) + # Wait until fleet snapshot exists, then warm memory inventory for chat tools + for _ in range(40): + if STATE.get("devices"): + break + await asyncio.sleep(2) + try: + mem = await fetch_fleet_memory(force=False) + log.info( + "Memory cache warmed: %s/%s servers", + mem.get("with_memory"), + mem.get("count"), + ) + except Exception as e: + log.warning("Memory cache warm failed: %s", e) @app.get("/api/health") @@ -813,6 +2410,7 @@ async def device_detail(device_id: int): "drivers": drivers, "applications": applications, } + detail["expansion"] = build_expansion_report(node.get("model"), detail.get("inventory") or {}) finally: if sid: try: @@ -827,6 +2425,18 @@ async def device_detail(device_id: int): return detail +@app.get("/api/devices/{device_id}/expansion") +async def device_expansion(device_id: int, force: bool = False): + if force and device_id in DETAIL_CACHE: + DETAIL_CACHE.pop(device_id, None) + detail = await device_detail(device_id) + return { + "device": detail.get("device"), + "expansion": detail.get("expansion") + or build_expansion_report((detail.get("device") or {}).get("model"), detail.get("inventory") or {}), + } + + @app.websocket("/ws/fleet") async def ws_fleet(ws: WebSocket): await ws.accept() @@ -868,6 +2478,205 @@ class TicketPatch(BaseModel): accepted_by: str | None = None +def _device_by_id(device_id: int | None) -> dict | None: + if device_id is None: + return None + return next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None) + + +def _device_by_service_tag(st: str) -> dict | None: + needle = (st or "").strip().upper() + if not needle: + return None + for d in STATE.get("devices") or []: + if (d.get("service_tag") or "").strip().upper() == needle: + return d + return None + + +def _find_devices_fuzzy(query: str, limit: int = 8) -> list[dict]: + q = (query or "").strip().lower() + if not q: + return [] + hits = [] + for d in STATE.get("devices") or []: + blob = " ".join( + str(d.get(k) or "").lower() + for k in ("name", "service_tag", "ip", "idrac_ip", "rdp_host", "os_hostname", "model") + ) + if q in blob: + hits.append(d) + hits.sort(key=lambda d: (0 if (d.get("service_tag") or "").lower() == q else 1, d.get("name") or "")) + return hits[:limit] + + +def _format_device_card(d: dict) -> str: + mem = d.get("memory_gb") + mem_s = f"{mem}GB/{d.get('dimm_count') or '?'}DIMM" if mem is not None else "unknown" + return ( + "ST={st} | name={name} | model={model} | idrac={idrac} | " + "rdp={rdp} | os_host={osh} | ram={ram} | connected={conn} | status={status} | watts={watts}" + ).format( + st=(d.get("service_tag") or "NONE"), + name=(d.get("name") or "")[:48], + model=(d.get("model") or "")[:36], + idrac=d.get("idrac_ip") or d.get("ip") or "—", + rdp=(",".join(d.get("rdp_ips") or []) or d.get("rdp_host") or "unresolved"), + osh=d.get("os_hostname") or "—", + ram=mem_s, + conn="yes" if d.get("connected") else "no", + status=d.get("status"), + watts=d.get("watts"), + ) + + +async def ome_fetch_inventory_types(device_id: int, types: list[str]) -> dict[str, list]: + """Fast parallel inventory fetch for chat tools (not full device landscape).""" + inventory: dict[str, list] = {} + async with httpx.AsyncClient(verify=False, timeout=45.0) as client: + base, headers, sid = await ome_session(client) + try: + + async def one(inv_type: str): + try: + ir = await client.get( + f"{base}/api/DeviceService/Devices({device_id})/InventoryDetails('{inv_type}')", + headers=headers, + ) + if ir.status_code == 200: + info = ir.json().get("InventoryInfo") or [] + if info: + return inv_type, info + except Exception as e: + log.debug("chat inv %s failed: %s", inv_type, e) + return inv_type, [] + + results = await asyncio.gather(*[one(t) for t in types]) + for inv_type, info in results: + if info: + inventory[inv_type] = info + finally: + await ome_session_delete(client, base, headers, sid) + return inventory + + + +def _memory_from_dimms(mem_raw: list) -> dict: + """Compact CURRENT memory summary from OME serverMemoryDevices.""" + dimms = 0 + total_gb = 0.0 + for dimm in mem_raw or []: + gb = _mem_size_gb(dimm.get("Size")) + if gb is None: + continue + dimms += 1 + total_gb += gb + return { + "memory_gb": round(total_gb, 1) if dimms else None, + "dimm_count": dimms or None, + } + + +async def fetch_fleet_memory(force: bool = False) -> dict: + """Warm/cache CURRENT installed RAM for all managed servers (OME inventory).""" + cached = None if force else _cache_get("fleet_memory") + if cached is not None: + return cached + + devices = [ + d + for d in (STATE.get("devices") or []) + if d.get("is_server") or d.get("is_idrac") or (d.get("type") == 1000) + ] + # Prefer unique device ids + seen: set[int] = set() + targets: list[dict] = [] + for d in devices: + did = d.get("id") + if did is None or did in seen: + continue + seen.add(int(did)) + targets.append(d) + + rows: list[dict] = [] + errors = 0 + sem = asyncio.Semaphore(10) + + async with httpx.AsyncClient(verify=False, timeout=60.0) as client: + base, headers, sid = await ome_session(client) + try: + + async def one(node: dict): + nonlocal errors + did = int(node["id"]) + async with sem: + try: + ir = await client.get( + f"{base}/api/DeviceService/Devices({did})/InventoryDetails('serverMemoryDevices')", + headers=headers, + ) + mem_raw = [] + if ir.status_code == 200: + mem_raw = ir.json().get("InventoryInfo") or [] + stats = _memory_from_dimms(mem_raw) + return { + "id": did, + "service_tag": node.get("service_tag"), + "name": node.get("name"), + "model": node.get("model"), + "idrac_ip": node.get("idrac_ip") or node.get("ip"), + "connected": bool(node.get("connected")), + "memory_gb": stats.get("memory_gb"), + "dimm_count": stats.get("dimm_count"), + } + except Exception as e: + errors += 1 + log.debug("fleet memory %s: %s", did, e) + return { + "id": did, + "service_tag": node.get("service_tag"), + "name": node.get("name"), + "model": node.get("model"), + "idrac_ip": node.get("idrac_ip") or node.get("ip"), + "connected": bool(node.get("connected")), + "memory_gb": None, + "dimm_count": None, + "error": str(e)[:80], + } + + rows = list(await asyncio.gather(*[one(n) for n in targets])) + finally: + await ome_session_delete(client, base, headers, sid) + + rows.sort(key=lambda r: ((r.get("name") or "").lower(), r.get("service_tag") or "")) + known = [r for r in rows if r.get("memory_gb") is not None] + payload = { + "source": "OME DeviceService InventoryDetails(serverMemoryDevices)", + "count": len(rows), + "with_memory": len(known), + "errors": errors, + "total_memory_gb": round(sum(r.get("memory_gb") or 0 for r in known), 1), + "servers": rows, + "updated_at": time.time(), + } + _cache_set("fleet_memory", payload) + # also stamp onto fleet nodes for UI/context + by_id = {r["id"]: r for r in rows if r.get("id") is not None} + for d in STATE.get("devices") or []: + hit = by_id.get(d.get("id")) + if hit: + d["memory_gb"] = hit.get("memory_gb") + d["dimm_count"] = hit.get("dimm_count") + log.info( + "Fleet memory cache: %s/%s servers with RAM (%.0f GB total)", + len(known), + len(rows), + payload["total_memory_gb"], + ) + return payload + + + def build_fleet_context(focus_device_id: int | None = None, max_chars: int | None = None) -> str: summary = STATE.get("summary") or {} gpu = STATE.get("gpu") or {} @@ -885,19 +2694,24 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non hottest = (ctx.get("hottest") or [])[:5] compliance = _cache_get("compliance") or REPORT_CACHE.get("compliance") or {} comp_sum = (compliance or {}).get("summary") or {} + by_id = {d.get("id"): d for d in devices if d.get("id") is not None} - # SERVICE TAG INDEX first — must survive truncation st_lines = [ - "SERVICE TAG INDEX (use these Service Tags in every node answer):", + "SERVICE TAG INDEX (canonical). idrac=BMC management IP. rdp=OS/Windows RDP candidates (DNS). Never call idrac the RDP IP.", ] for d in sorted(devices, key=lambda x: (x.get("name") or "").lower()): st = (d.get("service_tag") or "").strip() or "NONE" + rdp = ",".join((d.get("rdp_ips") or [])[:2]) or (d.get("rdp_host") or "—") + mem = d.get("memory_gb") + ram = f"{mem}GB" if mem is not None else "?" st_lines.append( - "- ST={st} | {name} | ip={ip} | model={model} | connected={conn}".format( + "- ST={st} | {name} | idrac={idrac} | rdp={rdp} | model={model} | ram={ram} | connected={conn}".format( st=st, - name=(d.get("name") or "")[:40], - ip=d.get("ip") or "—", - model=(d.get("model") or "")[:28], + name=(d.get("name") or "")[:36], + idrac=d.get("idrac_ip") or d.get("ip") or "—", + rdp=rdp[:40], + model=(d.get("model") or "")[:26], + ram=ram, conn="yes" if d.get("connected") else "no", ) ) @@ -906,9 +2720,12 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non lines = [ "You are OpenManage Cockpit Copilot for Dell ATC. Be concise and operational.", "Admins: Jody van Dongen, Laurens Rammers, Mohamed El Kadi.", - "Use ONLY this live snapshot. Unknown => say unknown.", - "ALWAYS include Service Tag (ST=...), model, and IP when discussing any system.", - "If a Service Tag is missing in the index, say so explicitly.", + "ACCURACY RULES (mandatory):", + "1) Use ONLY facts from this snapshot and any TOOL FACTS block. If missing => say unknown. Never invent ST, IPs, DIMM counts, firmware versions, or RDP targets.", + "2) ALWAYS cite Service Tag (ST=...), model, idrac IP, and rdp IP (or 'rdp unresolved') when discussing a system.", + "3) Management/iDRAC IP is NOT the Windows RDP IP. Never recommend RDP to an iDRAC address.", + "4) Hardware CURRENT (OME inventory) is NOT Dell CATALOG MAX. Never state catalog maxima as installed capacity.", + "5) Prefer TOOL FACTS over the summary index when both are present.", "", "FLEET: total={t} connected={c} offline={o} watts={w} alerts_total={a}".format( t=summary.get("total"), @@ -920,25 +2737,48 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non ] if comp_sum: lines.append( - "FIRMWARE COMPLIANCE (Dell catalog baseline): outdated_devices={od} critical_components={cc} baseline={bn}".format( + "FIRMWARE COMPLIANCE summary (Dell baseline): outdated_devices={od} critical_components={cc} baseline={bn}".format( od=comp_sum.get("outdated_devices"), cc=comp_sum.get("critical_components"), bn=(comp_sum.get("baseline_name") or "")[:40], ) ) - lines.append("CONNECTED:") - for d in connected[:18]: + mem_cache = _cache_get("fleet_memory") or REPORT_CACHE.get("fleet_memory") or {} + mem_servers = mem_cache.get("servers") or [] + if mem_servers: lines.append( - "- ST={st} {name} ip={ip} model={model} W={watts} status={status}".format( - st=(d.get("service_tag") or "NONE"), - name=(d.get("name") or "")[:36], - ip=d.get("ip"), - model=(d.get("model") or "")[:28], - watts=d.get("watts"), - status=d.get("status"), + "MEMORY INDEX (CURRENT OME inventory, GB installed): known={k}/{n} total_gb={tg}".format( + k=mem_cache.get("with_memory"), + n=mem_cache.get("count"), + tg=mem_cache.get("total_memory_gb"), ) ) + ranked = sorted( + [r for r in mem_servers if r.get("memory_gb") is not None], + key=lambda r: (0 if r.get("connected") else 1, (r.get("name") or "").lower()), + ) + for r in ranked[:55]: + lines.append( + "- ST={st} {name} model={model} ram={gb}GB dimms={dc} idrac={ip}".format( + st=r.get("service_tag") or "NONE", + name=(r.get("name") or "")[:28], + model=(r.get("model") or "")[:22], + gb=r.get("memory_gb"), + dc=r.get("dimm_count") or "?", + ip=r.get("idrac_ip") or "—", + ) + ) + if len(ranked) > 55: + lines.append(f"- … +{len(ranked) - 55} more in TOOL FACTS list_fleet_memory") + else: + lines.append( + "MEMORY INDEX: not warmed yet — call list_fleet_memory tool / wait for background warm." + ) + + lines.append("CONNECTED (sample):") + for d in connected[:14]: + lines.append("- " + _format_device_card(d)) lines.append("SUBNETS:") for s in subnets[:8]: @@ -951,27 +2791,25 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non ) ) + def _alert_line(a: dict, msg_len: int) -> str: + node = by_id.get(a.get("device_id")) or {} + st = node.get("service_tag") or a.get("service_tag") or "NONE" + return "- ST={st} {dev} idrac={ip}: {msg}".format( + st=st, + dev=(a.get("device") or node.get("name") or "")[:28], + ip=a.get("ip") or node.get("idrac_ip") or node.get("ip") or "—", + msg=(a.get("message") or "")[:msg_len], + ) + lines.append("CRITICAL:") if not critical: lines.append("(none)") for a in critical: - lines.append( - "- {dev} ({ip}): {msg}".format( - dev=(a.get("device") or "")[:32], - ip=a.get("ip"), - msg=(a.get("message") or "")[:110], - ) - ) + lines.append(_alert_line(a, 100)) lines.append("WARNINGS:") for a in warnings: - lines.append( - "- {dev} ({ip}): {msg}".format( - dev=(a.get("device") or "")[:32], - ip=a.get("ip"), - msg=(a.get("message") or "")[:90], - ) - ) + lines.append(_alert_line(a, 80)) lines.append("DELTAS:") if not events: @@ -981,7 +2819,14 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non lines.append("HOT POWER:") for h in hottest: - lines.append("- {name}: {watts}W".format(name=(h.get("name") or "")[:28], watts=h.get("watts"))) + node = by_id.get(h.get("id")) or {} + lines.append( + "- ST={st} {name}: {watts}W".format( + st=(node.get("service_tag") or h.get("service_tag") or "NONE"), + name=(h.get("name") or node.get("name") or "")[:28], + watts=h.get("watts"), + ) + ) lines.append( "GPU atc-gpu-prod: avg_util={u}% power={p}W mem={mu}/{mt}MB".format( @@ -1002,31 +2847,20 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non ) if focus_device_id is not None: - node = next((d for d in devices if d.get("id") == focus_device_id), None) - lines.append("FOCUS:") + node = _device_by_id(focus_device_id) + lines.append("FOCUS DEVICE:") if node: - lines.append( - "ST={st} {name} id={id} ip={ip} model={model} W={watts} connected={conn} status={stt}".format( - st=(node.get("service_tag") or "NONE"), - name=node.get("name"), - id=node.get("id"), - ip=node.get("ip"), - model=node.get("model"), - watts=node.get("watts"), - conn=node.get("connected"), - stt=node.get("status"), - ) - ) + lines.append(_format_device_card(node)) related = [a for a in alerts if a.get("device_id") == focus_device_id][:5] for a in related: lines.append( "alert {sev}: {msg}".format(sev=a.get("severity"), msg=(a.get("message") or "")[:100]) ) else: - lines.append("device_id=%s missing" % focus_device_id) + lines.append("device_id=%s missing from fleet snapshot" % focus_device_id) lines.append( - "Reply with concrete Service Tags, names, IPs, models, and next actions." + "Reply with concrete Service Tags, idrac/rdp IPs, models, and next actions. Quote TOOL FACTS verbatim for hardware numbers." ) body = "\n".join(lines) budget = max(800, limit - len(st_block) - 40) @@ -1038,6 +2872,396 @@ def build_fleet_context(focus_device_id: int | None = None, max_chars: int | Non return out +def plan_chat_tools(message: str, focus_device_id: int | None = None) -> list[tuple[str, dict]]: + """Server-side tool planner — Llama3-GPTQ does not emit native tool_calls reliably.""" + import re + + msg = message or "" + low = msg.lower() + planned: list[tuple[str, dict]] = [] + seen: set[str] = set() + + def add(name: str, args: dict): + key = name + json.dumps(args, sort_keys=True, default=str) + if key in seen: + return + seen.add(key) + planned.append((name, args)) + + for st in re.findall(r"\b([A-Z0-9]{7})\b", msg.upper()): + if st in {"OMEPROD", "WINDOWS", "POWERED", "UNKNOWN", "SERVICE", "CONNECTED"}: + continue + if _device_by_service_tag(st): + add("lookup_device", {"query": st}) + + for m in re.finditer(r"\b(?:for|on|about|host|server|node)\s+([A-Za-z0-9._-]{4,40})", msg, re.I): + add("lookup_device", {"query": m.group(1)}) + + wants_hw = bool( + re.search( + r"\b(memory|ram|dimm|disk|drive|ssd|hdd|pcie|expansion|cpu|processor|core|slot|bay|installed|capacity)\b", + low, + ) + ) + wants_rdp = bool(re.search(r"\b(rdp|remote\s*desktop|windows|hyper-?v|os\s*ip|hostname)\b", low)) + wants_fw = bool(re.search(r"\b(firmware|bios\s*version|outdated|compliance|firmware\s*catalog|update\s*catalog|baseline)\b", low)) + wants_war = bool(re.search(r"\b(warrant|prosupport|support\s*end|days?\s*left)\b", low)) + wants_alerts = bool(re.search(r"\b(alert|critical|warning|fault|health)\b", low)) + + target_id = focus_device_id + target_st = None + if planned: + q0 = planned[0][1].get("query") + node = _device_by_service_tag(str(q0)) or ((_find_devices_fuzzy(str(q0), 1) or [None])[0]) + if node: + target_id = node.get("id") + target_st = node.get("service_tag") + + if wants_rdp and not any(n == "lookup_device" for n, _ in planned): + if focus_device_id: + node = _device_by_id(focus_device_id) + if node: + add("lookup_device", {"query": node.get("service_tag") or node.get("name") or str(focus_device_id)}) + else: + n_lookups = 0 + for d in STATE.get("devices") or []: + if d.get("is_windows") or d.get("rdp_host"): + add("lookup_device", {"query": d.get("service_tag") or d.get("name")}) + n_lookups += 1 + if n_lookups >= 4: + break + + wants_fleet_mem = bool( + re.search( + r"\b(each|every|all|per\s+server|fleet|overview|elk|alle|iedere|elke)\b", + low, + ) + ) or bool(re.search(r"hoeveel\s+(memory|ram|geheugen)", low)) + + if wants_hw: + args: dict = {} + if target_id: + args["device_id"] = int(target_id) + if target_st: + args["service_tag"] = target_st + if args: + add("get_hardware", args) + if wants_fleet_mem: + add("list_fleet_memory", {}) + elif wants_fleet_mem or not focus_device_id: + add("list_fleet_memory", {}) + elif focus_device_id: + add("get_hardware", {"device_id": int(focus_device_id)}) + + if wants_fw: + args = {"outdated_only": True} + if target_id: + args["device_id"] = int(target_id) + if target_st: + args["service_tag"] = target_st + add("get_compliance", args) + + if wants_war: + args = {} + if target_id: + args["device_id"] = int(target_id) + if target_st: + args["service_tag"] = target_st + if args: + add("get_warranty", args) + elif focus_device_id: + add("get_warranty", {"device_id": int(focus_device_id)}) + + if wants_alerts: + args = {} + if target_id: + args["device_id"] = int(target_id) + if target_st: + args["service_tag"] = target_st + add("list_alerts", args) + + if focus_device_id and not planned: + node = _device_by_id(focus_device_id) + if node: + add("lookup_device", {"query": node.get("service_tag") or str(focus_device_id)}) + + return planned[:6] + + +async def run_chat_tool(name: str, args: dict) -> dict: + args = args or {} + try: + if name == "lookup_device": + q = str(args.get("query") or "").strip() + exact = _device_by_service_tag(q) + hits = [exact] if exact else _find_devices_fuzzy(q, 8) + return { + "tool": name, + "query": q, + "count": len(hits), + "devices": [ + { + "id": d.get("id"), + "service_tag": d.get("service_tag"), + "name": d.get("name"), + "model": d.get("model"), + "idrac_ip": d.get("idrac_ip") or d.get("ip"), + "rdp_host": d.get("rdp_host"), + "rdp_ips": d.get("rdp_ips") or [], + "os_hostname": d.get("os_hostname"), + "is_windows": d.get("is_windows"), + "connected": d.get("connected"), + "status": d.get("status"), + "watts": d.get("watts"), + } + for d in hits + if d + ], + } + + if name == "get_hardware": + node = None + if args.get("device_id") is not None: + node = _device_by_id(int(args["device_id"])) + if not node and args.get("service_tag"): + node = _device_by_service_tag(str(args["service_tag"])) + if not node: + return {"tool": name, "error": "device not found in fleet snapshot"} + did = int(node["id"]) + inv = await ome_fetch_inventory_types( + did, + [ + "serverMemoryDevices", + "serverProcessors", + "serverArrayDisks", + "serverRaidControllers", + "serverDeviceCards", + "serverOperatingSystems", + ], + ) + report = build_expansion_report(node.get("model"), inv) + cur = report.get("current") or {} + exp = report.get("expansion") or {} + return { + "tool": name, + "device": { + "id": did, + "service_tag": node.get("service_tag"), + "name": node.get("name"), + "model": node.get("model"), + "idrac_ip": node.get("idrac_ip") or node.get("ip"), + "rdp_ips": node.get("rdp_ips") or [], + "os_hostname": node.get("os_hostname"), + }, + "current_ome_inventory": { + "memory_installed_gb": (cur.get("memory") or {}).get("installed_gb"), + "dimm_count": (cur.get("memory") or {}).get("dimm_count"), + "dimm_modules": [ + { + "slot": m.get("slot"), + "size_gb": m.get("size_gb"), + "rated_mts": m.get("rated_mts"), + "operating_mts": m.get("operating_mts"), + } + for m in ((cur.get("memory") or {}).get("modules") or [])[:24] + ], + "cpu_sockets_populated": (cur.get("cpu") or {}).get("sockets_populated"), + "cpu_cores_total": (cur.get("cpu") or {}).get("cores_total"), + "processors": (cur.get("cpu") or {}).get("processors") or [], + "disk_count": (cur.get("disks") or {}).get("count"), + "disk_capacity_gb": (cur.get("disks") or {}).get("capacity_gb"), + "disks": [ + { + "bay": d.get("bay"), + "model": d.get("model"), + "size_gb": d.get("size_gb"), + "bus": d.get("bus"), + "media": d.get("media"), + } + for d in ((cur.get("disks") or {}).get("items") or [])[:20] + ], + "os": cur.get("os") or [], + }, + "catalog_max_not_measured": { + "source": exp.get("source"), + "catalog_known": exp.get("catalog_known"), + "catalog_model": exp.get("catalog_model"), + "memory_max_tb": (exp.get("memory") or {}).get("max_capacity_tb"), + "dimm_slots_max": (exp.get("memory") or {}).get("dimm_slots_max"), + "cpu_sockets_max": (exp.get("cpu") or {}).get("sockets_max"), + "drive_bays_max": (exp.get("disks") or {}).get("bays_max"), + "pcie_slots_max": (exp.get("pcie") or {}).get("slots_max"), + }, + "findings": report.get("findings") or [], + } + + if name == "list_fleet_memory": + mem = await fetch_fleet_memory(force=bool(args.get("force"))) + servers = mem.get("servers") or [] + # Compact rows for the LLM — include unknown so the model does not invent + slim = [ + { + "service_tag": s.get("service_tag"), + "name": s.get("name"), + "model": s.get("model"), + "idrac_ip": s.get("idrac_ip"), + "connected": s.get("connected"), + "memory_gb": s.get("memory_gb"), + "dimm_count": s.get("dimm_count"), + } + for s in servers + ] + return { + "tool": name, + "source": mem.get("source"), + "count": mem.get("count"), + "with_memory": mem.get("with_memory"), + "total_memory_gb": mem.get("total_memory_gb"), + "note": "memory_gb is CURRENT installed RAM from OME inventory (not catalog max).", + "servers": slim, + } + + if name == "get_compliance": + compliance = await fetch_compliance(force=False) + comps = compliance.get("components") or [] + node = None + if args.get("device_id") is not None: + node = _device_by_id(int(args["device_id"])) + if not node and args.get("service_tag"): + node = _device_by_service_tag(str(args["service_tag"])) + outdated_only = bool(args.get("outdated_only", True)) + if node: + did = node.get("id") + st = (node.get("service_tag") or "").upper() + rows = [ + c + for c in comps + if c.get("device_id") == did + or (st and (c.get("service_tag") or "").upper() == st) + ] + else: + rows = list(comps) + if outdated_only: + rows = [ + c + for c in rows + if str(c.get("compliance_status") or "").lower() + in ("critical", "warning", "downgrade", "non-compliant", "outdated") + or ( + str(c.get("update_action") or "").lower() + not in ("", "equal", "compliant", "ok", "none") + and str(c.get("current_version") or "") != str(c.get("catalog_version") or "") + ) + ] + slim = [ + { + "service_tag": c.get("service_tag"), + "device_name": c.get("device_name"), + "component": c.get("component"), + "current_version": c.get("current_version"), + "catalog_version": c.get("catalog_version"), + "compliance_status": c.get("compliance_status"), + "update_action": c.get("update_action"), + } + for c in rows[:40] + ] + return {"tool": name, "summary": compliance.get("summary"), "count": len(slim), "components": slim} + + if name == "get_warranty": + warranties = await fetch_warranties(force=False) + items = warranties.get("items") or warranties.get("warranties") or [] + node = None + if args.get("device_id") is not None: + node = _device_by_id(int(args["device_id"])) + if not node and args.get("service_tag"): + node = _device_by_service_tag(str(args["service_tag"])) + if not node: + return {"tool": name, "error": "device not found"} + st = (node.get("service_tag") or "").upper() + rows = [ + w + for w in items + if w.get("device_id") == node.get("id") + or (st and (w.get("service_tag") or "").upper() == st) + ] + return { + "tool": name, + "service_tag": node.get("service_tag"), + "name": node.get("name"), + "warranties": rows[:8], + } + + if name == "list_alerts": + alerts = STATE.get("alerts") or [] + by_id = {d.get("id"): d for d in (STATE.get("devices") or [])} + node = None + if args.get("device_id") is not None: + node = _device_by_id(int(args["device_id"])) + if not node and args.get("service_tag"): + node = _device_by_service_tag(str(args["service_tag"])) + sev = (args.get("severity") or "").strip().lower() + out = [] + for a in alerts: + if node and a.get("device_id") != node.get("id"): + continue + if sev and str(a.get("severity") or "").lower() != sev: + continue + d = by_id.get(a.get("device_id")) or {} + out.append( + { + "severity": a.get("severity"), + "service_tag": d.get("service_tag"), + "device": a.get("device") or d.get("name"), + "idrac_ip": a.get("ip") or d.get("idrac_ip") or d.get("ip"), + "message": (a.get("message") or "")[:180], + } + ) + if len(out) >= 25: + break + return {"tool": name, "count": len(out), "alerts": out} + + return {"tool": name, "error": f"unknown tool {name}"} + except Exception as e: + log.exception("chat tool %s failed", name) + return {"tool": name, "error": str(e)} + + +def format_tool_facts(results: list[dict]) -> str: + if not results: + return "" + parts = ["TOOL FACTS (live lookups — authoritative; do not contradict):"] + for r in results: + if r.get("tool") == "list_fleet_memory" and r.get("servers"): + lines = [ + "list_fleet_memory CURRENT RAM:", + f"known={r.get('with_memory')}/{r.get('count')} total_gb={r.get('total_memory_gb')}", + "ST\tname\tmodel\tmemory_gb\tdimms\tidrac", + ] + for s in r.get("servers") or []: + lines.append( + "{st}\t{name}\t{model}\t{gb}\t{dc}\t{ip}".format( + st=s.get("service_tag") or "NONE", + name=(s.get("name") or "")[:36], + model=(s.get("model") or "")[:24], + gb=s.get("memory_gb") if s.get("memory_gb") is not None else "unknown", + dc=s.get("dimm_count") if s.get("dimm_count") is not None else "unknown", + ip=s.get("idrac_ip") or "—", + ) + ) + parts.append("\n".join(lines)[:14000]) + else: + parts.append(json.dumps(r, ensure_ascii=False, default=str)[:14000]) + return "\n".join(parts) + + + + + +@app.get("/api/fleet-memory") +async def api_fleet_memory(force: bool = False): + """CURRENT installed RAM per server (OME inventory cache).""" + return await fetch_fleet_memory(force=force) + @app.get("/api/gpu") async def api_gpu(): @@ -1147,20 +3371,126 @@ async def api_models(): return {"models": models, "default": settings.vllm_model} + +def format_fleet_memory_reply(result: dict) -> str: + """Deterministic full table — LLMs truncate long fleet lists.""" + servers = result.get("servers") or [] + lines = [ + "CURRENT installed RAM from OME inventory (not Dell catalog max).", + "known={k}/{n} · total_installed={tg} GB".format( + k=result.get("with_memory"), + n=result.get("count"), + tg=result.get("total_memory_gb"), + ), + "", + "ST | name | model | memory_gb | dimms | idrac | connected", + ] + for s in servers: + gb = s.get("memory_gb") + gb_s = "unknown" if gb is None else str(gb) + dc = s.get("dimm_count") + dc_s = "unknown" if dc is None else str(dc) + lines.append( + "{st} | {name} | {model} | {gb} | {dc} | {ip} | {conn}".format( + st=s.get("service_tag") or "NONE", + name=(s.get("name") or "")[:40], + model=(s.get("model") or "")[:28], + gb=gb_s, + dc=dc_s, + ip=s.get("idrac_ip") or "—", + conn="yes" if s.get("connected") else "no", + ) + ) + return "\n".join(lines) + + +def should_use_deterministic_fleet_memory(message: str, planned: list, tool_results: list[dict]) -> dict | None: + if not any(n == "list_fleet_memory" for n, _ in planned): + return None + # Prefer deterministic when the ask is clearly a full memory inventory + low = (message or "").lower() + inventoryish = bool( + re.search(r"\b(memory|ram|geheugen|dimm)\b", low) + and re.search(r"\b(elk|elke|iedere|alle|each|every|all|per\s+server|hoeveel|list|overview)\b", low) + ) + # Or when list_fleet_memory is the only tool + only_mem = len(planned) == 1 and planned[0][0] == "list_fleet_memory" + if not (inventoryish or only_mem): + return None + for r in tool_results: + if r.get("tool") == "list_fleet_memory" and not r.get("error"): + return r + return None + + @app.post("/api/chat") async def api_chat(payload: ChatIn): gpu = STATE.get("gpu") or {} gsum = gpu.get("summary") or {} model = (payload.model or settings.vllm_model or "").strip() or settings.vllm_model + # 1) Plan + execute live tools BEFORE the LLM (accurate facts; Llama3-GPTQ lacks native tool_calls) + planned = plan_chat_tools(payload.message, payload.focus_device_id) + tool_results: list[dict] = [] + if planned: + tool_results = list( + await asyncio.gather(*[run_chat_tool(name, args) for name, args in planned]) + ) + + max_tokens = settings.vllm_max_tokens + if any(n == "list_fleet_memory" for n, _ in planned): + max_tokens = max(max_tokens, 1800) + + # Full fleet memory lists: answer from TOOL FACTS directly (accurate, complete) + det = should_use_deterministic_fleet_memory(payload.message, planned, tool_results) + if det is not None: + gpu = STATE.get("gpu") or {} + gsum = gpu.get("summary") or {} + return { + "reply": format_fleet_memory_reply(det), + "model": model, + "backend": "ome-tools", + "gpu": gsum, + "context_bytes": 0, + "tools_used": [{"name": n, "args": a} for n, a in planned], + "tool_count": len(tool_results), + "deterministic": True, + } + system = build_fleet_context(payload.focus_device_id, max_chars=settings.chat_system_chars) + facts = format_tool_facts(tool_results) + if facts: + # Prefer tool facts; keep within budget (vLLM context is tight) + budget = max(1800, settings.chat_system_chars - 100) + # Accuracy rules + short fleet head, then authoritative TOOL FACTS + head = system + if len(system) > 2800: + head = system[:2800] + "\n…[fleet truncated for tools]" + combined = head + "\n\n" + facts + if len(combined) > budget: + # Keep TOOL FACTS intact; shrink head further + max_facts = min(len(facts), budget - 900) + facts_trim = facts[:max_facts] + head = system[: max(600, budget - len(facts_trim) - 40)] + combined = head + "\n…\n\n" + facts_trim + system = combined[:budget] + messages = [{"role": "system", "content": system}] for h in (payload.history or [])[-4:]: role = h.get("role") content = h.get("content") if role in ("user", "assistant") and content: messages.append({"role": role, "content": str(content)[:1200]}) - messages.append({"role": "user", "content": payload.message[:2500]}) + user_msg = payload.message[:2500] + if tool_results: + user_msg += ( + "\n\n[System note: live TOOL FACTS were fetched for this question. " + "Treat TOOL FACTS as ground truth. Quote CURRENT vs CATALOG separately. " + "Do not hedge or invent. Say unknown only if a field is absent from TOOL FACTS. " + "For fleet lists: output EVERY row from TOOL FACTS in a compact table " + "(ST | name | model | memory_gb | dimms | idrac) — do not summarize or omit.]" + ) + messages.append({"role": "user", "content": user_msg}) owui_models = {"ome-copilot", "arena-model"} use_owui = model in owui_models @@ -1181,8 +3511,8 @@ async def api_chat(payload: ChatIn): json={ "model": model, "messages": messages, - "temperature": 0.2, - "max_tokens": settings.vllm_max_tokens, + "temperature": 0.1, + "max_tokens": max_tokens, "stream": False, }, ) @@ -1198,21 +3528,23 @@ async def api_chat(payload: ChatIn): json={ "model": model, "messages": messages, - "temperature": 0.2, - "max_tokens": settings.vllm_max_tokens, + "temperature": 0.1, + "max_tokens": max_tokens, }, ) if r.status_code >= 400: detail = r.text if r.status_code == 400 and "context length" in detail.lower(): - tight = build_fleet_context(payload.focus_device_id, max_chars=3200) + tight = build_fleet_context(payload.focus_device_id, max_chars=2800) + if facts: + tight = (tight + "\n\n" + facts)[:3800] messages = [{"role": "system", "content": tight}, messages[-1]] r = await client.post( url, json={ "model": model, "messages": messages, - "temperature": 0.2, + "temperature": 0.1, "max_tokens": 500, }, ) @@ -1232,6 +3564,8 @@ async def api_chat(payload: ChatIn): "backend": backend, "gpu": gsum, "context_bytes": len(system), + "tools_used": [{"name": n, "args": a} for n, a in planned], + "tool_count": len(tool_results), } except HTTPException: raise @@ -1285,6 +3619,7 @@ async def create_ticket(payload: TicketIn): (tid, payload.created_by, payload.body.strip()[:5000], now), ) row = conn.execute("SELECT * FROM tickets WHERE id=?", (tid,)).fetchone() + _backup_ops_db("create") return dict(row) @@ -1317,6 +3652,7 @@ async def add_ticket_message(ticket_id: int, payload: TicketMsgIn): "SELECT * FROM ticket_messages WHERE ticket_id=? ORDER BY created_at ASC", (ticket_id,), ).fetchall() + _backup_ops_db("message") return {"messages": [dict(m) for m in msgs]} @@ -1351,20 +3687,465 @@ async def patch_ticket(ticket_id: int, payload: TicketPatch): ), ) row = conn.execute("SELECT * FROM tickets WHERE id=?", (ticket_id,)).fetchone() + _backup_ops_db("patch") return dict(row) @app.delete("/api/tickets/{ticket_id}") async def delete_ticket(ticket_id: int): + _backup_ops_db("pre-delete") with _db() as conn: row = conn.execute("SELECT id FROM tickets WHERE id=?", (ticket_id,)).fetchone() if not row: raise HTTPException(404, "Ticket not found") conn.execute("DELETE FROM ticket_messages WHERE ticket_id=?", (ticket_id,)) conn.execute("DELETE FROM tickets WHERE id=?", (ticket_id,)) + _backup_ops_db("delete") return {"ok": True, "deleted": ticket_id} +class RackIn(BaseModel): + site_id: str + name: str + units: int = 42 + sort_order: int | None = None + + +class RackPatch(BaseModel): + name: str | None = None + sort_order: int | None = None + + +class PlacementIn(BaseModel): + rack_id: int + u_start: int + u_height: int | None = None + + +@app.get("/api/network/fabric") +async def api_network_fabric(): + return build_network_fabric() + + +class VlanIn(BaseModel): + vlan_id: int | None = None + name: str + cidr: str + purpose: str = "" + site_id: str | None = None + color: str = "#00a8e8" + sort_order: int = 0 + + +@app.get("/api/network/vlans") +async def api_network_vlans(): + return build_vlan_map() + + +@app.post("/api/network/vlans") +async def api_create_vlan(payload: VlanIn): + now = time.time() + try: + ipaddress.ip_network(payload.cidr, strict=False) + except Exception as e: + raise HTTPException(400, f"Invalid cidr: {e}") from e + with _db() as conn: + cur = conn.execute( + """ + INSERT INTO vlans(vlan_id, name, cidr, purpose, site_id, color, sort_order, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + payload.vlan_id, + payload.name.strip()[:80], + payload.cidr.strip(), + (payload.purpose or "")[:200], + payload.site_id, + payload.color or "#00a8e8", + payload.sort_order, + now, + now, + ), + ) + vid = cur.lastrowid + _backup_ops_db("vlan-create") + return {"id": vid, **build_vlan_map()} + + + +class PortLinkIn(BaseModel): + switch_port: int + device_id: int + device_port: str = "" + note: str = "" + + +@app.get("/api/network/switches/{switch_id}/ports") +async def api_switch_ports(switch_id: int): + return build_switch_portmap(switch_id) + + +@app.put("/api/network/switches/{switch_id}/ports/{port_num}") +async def api_put_port_link(switch_id: int, port_num: int, payload: PortLinkIn): + sw = next((d for d in (STATE.get("devices") or []) if d.get("id") == switch_id), None) + if not sw: + raise HTTPException(404, "Switch not found") + nports = _switch_port_count(sw) + if port_num < 1 or port_num > nports: + raise HTTPException(400, f"Port must be 1..{nports}") + peer = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None) + if not peer: + raise HTTPException(404, "Target device not found in fleet") + now = time.time() + with _db() as conn: + conn.execute( + """ + INSERT INTO port_links(switch_id, switch_port, device_id, device_port, note, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(switch_id, switch_port) DO UPDATE SET + device_id=excluded.device_id, + device_port=excluded.device_port, + note=excluded.note, + updated_at=excluded.updated_at + """, + ( + switch_id, + port_num, + payload.device_id, + (payload.device_port or "")[:80], + (payload.note or "")[:200], + now, + ), + ) + _backup_ops_db("port-link") + return build_switch_portmap(switch_id) + + +@app.delete("/api/network/switches/{switch_id}/ports/{port_num}") +async def api_delete_port_link(switch_id: int, port_num: int): + with _db() as conn: + conn.execute( + "DELETE FROM port_links WHERE switch_id=? AND switch_port=?", + (switch_id, port_num), + ) + _backup_ops_db("port-unlink") + return build_switch_portmap(switch_id) + + +@app.get("/api/devices/{device_id}/nics") +async def api_device_nics(device_id: int): + """OME serverNetworkInterfaces for wiring UI (best-effort).""" + node = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None) + if not node: + raise HTTPException(404, "Device not found") + nics = [] + try: + inv = await ome_fetch_inventory_types(device_id, ["serverNetworkInterfaces"]) + for nic in inv.get("serverNetworkInterfaces") or []: + nics.append( + { + "name": nic.get("ProductName") or nic.get("DeviceDescription") or nic.get("Fqdd") or "NIC", + "fqdd": nic.get("Fqdd") or nic.get("InstanceId"), + "mac": nic.get("PermanentMACAddress") or nic.get("CurrentMACAddress"), + "speed": nic.get("LinkSpeed") or nic.get("Speed"), + "vendor": nic.get("Manufacturer") or nic.get("VendorName"), + } + ) + except Exception as e: + log.debug("nic fetch %s: %s", device_id, e) + return {"device_id": device_id, "nics": nics, "count": len(nics)} + + + +@app.get("/api/racks") +async def api_racks(): + _seed_atc_racks() + return _racks_payload() + + +@app.post("/api/racks") +async def api_create_rack(payload: RackIn): + now = time.time() + site = (payload.site_id or "").upper() + if site not in ("ATC1", "ATC2"): + raise HTTPException(400, "site_id must be ATC1 or ATC2") + with _db() as conn: + row = conn.execute("SELECT id FROM sites WHERE id=?", (site,)).fetchone() + if not row: + conn.execute( + "INSERT INTO sites(id, name, sort_order) VALUES (?, ?, ?)", + (site, site, 1 if site == "ATC1" else 2), + ) + so = payload.sort_order + if so is None: + mx = conn.execute( + "SELECT COALESCE(MAX(sort_order), 0) AS m FROM racks WHERE site_id=?", + (site,), + ).fetchone()["m"] + so = int(mx) + 1 + cur = conn.execute( + """ + INSERT INTO racks(site_id, name, units, sort_order, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + (site, payload.name.strip()[:64], max(1, min(48, payload.units)), so, now, now), + ) + rid = cur.lastrowid + _backup_ops_db("rack-create") + return {"id": rid, **_racks_payload()} + + +@app.patch("/api/racks/{rack_id}") +async def api_patch_rack(rack_id: int, payload: RackPatch): + now = time.time() + with _db() as conn: + row = conn.execute("SELECT * FROM racks WHERE id=?", (rack_id,)).fetchone() + if not row: + raise HTTPException(404, "Rack not found") + name = payload.name.strip()[:64] if payload.name else row["name"] + so = payload.sort_order if payload.sort_order is not None else row["sort_order"] + conn.execute( + "UPDATE racks SET name=?, sort_order=?, updated_at=? WHERE id=?", + (name, so, now, rack_id), + ) + _backup_ops_db("rack-patch") + return _racks_payload() + + + +class RackItemIn(BaseModel): + rack_id: int + u_start: int = 1 + u_height: int | None = None + side: str = "front" # front|rear|left|right + device_id: int | None = None + catalog_sku: str | None = None + label: str | None = None + notes: str | None = None + + +class RackItemPatch(BaseModel): + rack_id: int | None = None + u_start: int | None = None + u_height: int | None = None + side: str | None = None + catalog_sku: str | None = None + label: str | None = None + notes: str | None = None + + +@app.get("/api/rack-catalog") +async def api_rack_catalog(): + families: dict[str, list] = {} + for item in DELL_RACK_CATALOG: + families.setdefault(item["family"], []).append(item) + return {"items": DELL_RACK_CATALOG, "families": families, "count": len(DELL_RACK_CATALOG)} + + +@app.put("/api/racks/placements/{device_id}") +async def api_put_placement(device_id: int, payload: PlacementIn): + """Place/move an OME device (compat). Writes rack_items.""" + device = next((d for d in (STATE.get("devices") or []) if d.get("id") == device_id), None) + cat = match_rack_catalog((device or {}).get("model"), (device or {}).get("role")) + body = RackItemIn( + rack_id=payload.rack_id, + u_start=payload.u_start, + u_height=payload.u_height if payload.u_height is not None else _default_u_height(device), + side="front", + device_id=device_id, + catalog_sku=(cat or {}).get("sku"), + label=(device or {}).get("name"), + ) + return await api_upsert_rack_item(body) + + +@app.delete("/api/racks/placements/{device_id}") +async def api_delete_placement(device_id: int): + with _db() as conn: + conn.execute("DELETE FROM rack_items WHERE device_id=?", (device_id,)) + try: + conn.execute("DELETE FROM rack_placements WHERE device_id=?", (device_id,)) + except Exception: + pass + _backup_ops_db("unplace") + return _racks_payload() + + +@app.post("/api/racks/items") +async def api_upsert_rack_item(payload: RackItemIn): + now = time.time() + side = (payload.side or "front").lower() + if side not in ("front", "rear", "left", "right"): + raise HTTPException(400, "side must be front|rear|left|right") + device = None + if payload.device_id is not None: + device = next((d for d in (STATE.get("devices") or []) if d.get("id") == payload.device_id), None) + if not device: + raise HTTPException(404, "OME device not found") + sku = payload.catalog_sku + cat = next((c for c in DELL_RACK_CATALOG if c["sku"] == sku), None) if sku else None + if not cat and device: + cat = match_rack_catalog(device.get("model"), device.get("role")) + sku = (cat or {}).get("sku") + if not device and not cat and not sku: + raise HTTPException(400, "Provide device_id or catalog_sku") + height = payload.u_height + # Prefer catalog U-height for known SKUs so 2U/3U/… always place correctly + if cat is not None: + cat_h = int(cat.get("u_height") or 0) + if cat_h > 0: + height = cat_h + elif height is None: + height = _default_u_height(device) + elif height is None: + height = _default_u_height(device) + if side in ("left", "right"): + u_start, height = 1, 42 + else: + height = max(1, min(42, int(height))) + u_start = int(payload.u_start) + if u_start < 1 or u_start + height - 1 > 42: + raise HTTPException(400, f"Must fit U1–U42 (u_start={u_start} height={height})") + label = (payload.label or (device or {}).get("name") or (cat or {}).get("name") or "Device")[:120] + with _db() as conn: + rack = conn.execute("SELECT * FROM racks WHERE id=?", (payload.rack_id,)).fetchone() + if not rack: + raise HTTPException(404, "Rack not found") + if _rack_overlap( + conn, + payload.rack_id, + u_start if side not in ("left", "right") else 1, + height, + side=side, + exclude_device_id=payload.device_id, + ): + raise HTTPException(409, "U range overlaps another item on this side") + existing = None + if payload.device_id is not None: + existing = conn.execute( + "SELECT id FROM rack_items WHERE device_id=?", (payload.device_id,) + ).fetchone() + if existing: + conn.execute( + """ + UPDATE rack_items + SET rack_id=?, catalog_sku=?, label=?, u_start=?, u_height=?, side=?, notes=?, updated_at=? + WHERE id=? + """, + ( + payload.rack_id, + sku, + label, + u_start if side not in ("left", "right") else 1, + height, + side, + (payload.notes or "")[:200], + now, + existing["id"], + ), + ) + item_id = existing["id"] + else: + cur = conn.execute( + """ + INSERT INTO rack_items(rack_id, device_id, catalog_sku, label, u_start, u_height, side, notes, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + payload.rack_id, + payload.device_id, + sku, + label, + u_start if side not in ("left", "right") else 1, + height, + side, + (payload.notes or "")[:200], + now, + ), + ) + item_id = cur.lastrowid + # keep legacy table in sync for OME devices + if payload.device_id is not None and side == "front": + try: + conn.execute( + """ + INSERT INTO rack_placements(device_id, rack_id, u_start, u_height, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(device_id) DO UPDATE SET + rack_id=excluded.rack_id, u_start=excluded.u_start, + u_height=excluded.u_height, updated_at=excluded.updated_at + """, + (payload.device_id, payload.rack_id, u_start, height, now), + ) + except Exception: + pass + _backup_ops_db("rack-item") + payload_out = _racks_payload() + payload_out["item_id"] = item_id + return payload_out + + +@app.patch("/api/racks/items/{item_id}") +async def api_patch_rack_item(item_id: int, payload: RackItemPatch): + now = time.time() + with _db() as conn: + row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone() + if not row: + raise HTTPException(404, "Item not found") + rack_id = payload.rack_id if payload.rack_id is not None else row["rack_id"] + side = (payload.side or row["side"] or "front").lower() + u_start = payload.u_start if payload.u_start is not None else row["u_start"] + height = payload.u_height if payload.u_height is not None else row["u_height"] + if side in ("left", "right"): + u_start, height = 1, 42 + else: + height = max(1, min(42, int(height))) + u_start = int(u_start) + if u_start < 1 or u_start + height - 1 > 42: + raise HTTPException(400, "Must fit U1–U42") + if _rack_overlap(conn, rack_id, u_start, height, side=side, exclude_item_id=item_id): + raise HTTPException(409, "U range overlaps another item on this side") + conn.execute( + """ + UPDATE rack_items + SET rack_id=?, catalog_sku=COALESCE(?, catalog_sku), label=COALESCE(?, label), + u_start=?, u_height=?, side=?, notes=COALESCE(?, notes), updated_at=? + WHERE id=? + """, + ( + rack_id, + payload.catalog_sku, + payload.label, + u_start, + height, + side, + payload.notes, + now, + item_id, + ), + ) + _backup_ops_db("rack-item-patch") + return _racks_payload() + + +@app.delete("/api/racks/items/{item_id}") +async def api_delete_rack_item(item_id: int): + with _db() as conn: + row = conn.execute("SELECT * FROM rack_items WHERE id=?", (item_id,)).fetchone() + if not row: + raise HTTPException(404, "Item not found") + conn.execute("DELETE FROM rack_items WHERE id=?", (item_id,)) + if row["device_id"] is not None: + try: + conn.execute("DELETE FROM rack_placements WHERE device_id=?", (row["device_id"],)) + except Exception: + pass + _backup_ops_db("rack-item-delete") + return _racks_payload() + + + + def _get_ssh_sem() -> asyncio.Semaphore: global _ssh_sem @@ -1966,6 +4747,12 @@ def build_fleet_report_rows(warranties: dict | None = None) -> list[dict]: "service_tag": d.get("service_tag"), "model": d.get("model"), "ip": d.get("ip"), + "idrac_ip": d.get("idrac_ip") or d.get("ip"), + "os_hostname": d.get("os_hostname"), + "rdp_host": d.get("rdp_host"), + "rdp_ips": d.get("rdp_ips") or [], + "os_ips": d.get("os_ips") or [], + "is_windows": d.get("is_windows"), "subnet": d.get("subnet"), "type": d.get("type"), "sub_type": d.get("sub_type"), @@ -2233,11 +5020,80 @@ async def ops_js(): return FileResponse(STATIC_DIR / "ops.js", media_type="application/javascript") +@app.get("/network.js") +async def network_js(): + return FileResponse(STATIC_DIR / "network.js", media_type="application/javascript") + + @app.get("/ssh.js") async def ssh_js(): return FileResponse(STATIC_DIR / "ssh.js", media_type="application/javascript") +@app.get("/rdp.js") +async def rdp_js(): + return FileResponse(STATIC_DIR / "rdp.js", media_type="application/javascript") + + +@app.get("/rdp-popout.html") +async def rdp_popout_html(): + resp = FileResponse(STATIC_DIR / "rdp-popout.html") + resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" + return resp + + +@app.get("/api/rdp.rdp") +async def api_rdp_file(host: str, port: int = 3389, username: str = ""): + """Download a standard .rdp file for the local Remote Desktop client.""" + host = (host or "").strip() + if not host or len(host) > 253 or any(c in host for c in "\r\n\x00"): + raise HTTPException(400, "Invalid host") + if port < 1 or port > 65535: + raise HTTPException(400, "Invalid port") + username = (username or "").strip() + if len(username) > 256 or any(c in username for c in "\r\n\x00"): + raise HTTPException(400, "Invalid username") + full = f"{host}:{port}" if port != 3389 else host + lines = [ + "screen mode id:i:2", + "use multimon:i:0", + "desktopwidth:i:1920", + "desktopheight:i:1080", + "session bpp:i:32", + "compression:i:1", + "keyboardhook:i:2", + "audiocapturemode:i:0", + "videoplaybackmode:i:1", + "connection type:i:7", + "networkautodetect:i:1", + "bandwidthautodetect:i:1", + "displayconnectionbar:i:1", + "bitmapcachepersistenable:i:1", + f"full address:s:{full}", + "audiomode:i:0", + "redirectclipboard:i:1", + "autoreconnection enabled:i:1", + "authentication level:i:2", + "prompt for credentials:i:1", + "negotiate security layer:i:1", + "gatewayusagemethod:i:4", + "gatewaycredentialssource:i:4", + "gatewayprofileusagemethod:i:0", + ] + if username: + lines.insert(15, f"username:s:{username}") + body = "\r\n".join(lines) + "\r\n" + safe = "".join(c if c.isalnum() or c in ".-_" else "_" for c in host)[:80] or "session" + return Response( + content=body, + media_type="application/x-rdp", + headers={ + "Content-Disposition": f'attachment; filename="{safe}.rdp"', + "Cache-Control": "no-store", + }, + ) + + @app.get("/vendor/xterm/xterm.css") async def vendor_xterm_css(): return FileResponse(STATIC_DIR / "vendor/xterm/xterm.css", media_type="text/css") @@ -2277,3 +5133,18 @@ async def dell_mark(): @app.get("/dell.svg") async def dell_svg(): return FileResponse(STATIC_DIR / "dell.png", media_type="image/png") + + +@app.get("/assets/dell/{filename}") +async def dell_asset(filename: str): + """Serve Dell product face photos for rack Design Studio.""" + safe = Path(filename).name + if not safe or safe != filename or ".." in filename: + raise HTTPException(400, "invalid filename") + path = STATIC_DIR / "assets" / "dell" / safe + if not path.is_file(): + raise HTTPException(404, "asset not found") + media = "image/jpeg" if safe.lower().endswith((".jpg", ".jpeg")) else "image/png" + resp = FileResponse(path, media_type=media) + resp.headers["Cache-Control"] = "public, max-age=86400" + return resp diff --git a/ui/app.js b/ui/app.js index 53879f6..cdadebb 100644 --- a/ui/app.js +++ b/ui/app.js @@ -757,6 +757,7 @@
+ @@ -1480,7 +1481,13 @@ SSH terminal Login with your username · ${escapeHtml(ip)} ` : ""} + ${ip ? `` : ""}
`; + mount.appendChild(box); + $("#btn-open-hw")?.addEventListener("click", () => { + window.cockpitReports?.open?.(); + // slight delay for drawer open + setTimeout(() => { + if (window.cockpitReports) { + window.cockpitReports.state = window.cockpitReports.state || {}; + } + const st = document.querySelector('#reports-tabs [data-tab="hardware"]'); + st?.click(); + // set device after tab loads + setTimeout(() => { + const btn = document.querySelector(`[data-hw-id="${deviceId}"]`); + btn?.click(); + }, 400); + }, 200); + }); + } catch (_) { + /* ignore */ + } + } + + async function loadWarrantyCompliance(deviceId) { const mount = $("#warranty-compliance-mount"); if (!mount) return; try { @@ -1681,6 +1733,7 @@ ${node.idrac_url ? `iDRAC Web` : ""} ${node.ip ? `` : ""} + ${node.ip ? `` : ""} @@ -1715,8 +1768,10 @@ updateFocusContext(); $("#btn-quick-connect")?.addEventListener("click", () => openConnect(node)); $("#btn-ssh-term")?.addEventListener("click", () => window.cockpitSsh?.open(node)); + $("#btn-rdp-term")?.addEventListener("click", () => window.cockpitRdp?.open(node)); $("#btn-ask-ai")?.addEventListener("click", () => openAi(node)); loadWarrantyCompliance(node.id); + loadExpansionPreview(node.id); const cachedInv = state.inventoryCache[node.id]; if (cachedInv) { const mount = $("#inv-mount"); @@ -2061,6 +2116,8 @@ showToast("Focused " + (node.name || "")); } else if (a === "ssh") { window.cockpitSsh?.open(node); + } else if (a === "rdp") { + window.cockpitRdp?.open(node); } else if (a === "connect") { closeKpiPopup(); openConnect(node); @@ -2238,6 +2295,25 @@ } }); + function applyTheme(theme) { + const t = theme === "dark" ? "dark" : "light"; + document.documentElement.setAttribute("data-theme", t); + try { + localStorage.setItem("cockpit_theme", t); + } catch (_) {} + const btn = $("#btn-theme"); + if (btn) { + btn.dataset.theme = t; + btn.title = t === "light" ? "Switch to dark theme" : "Switch to light theme"; + btn.setAttribute("aria-label", btn.title); + } + } + applyTheme(localStorage.getItem("cockpit_theme") || "light"); + $("#btn-theme")?.addEventListener("click", () => { + const cur = document.documentElement.getAttribute("data-theme") || "light"; + applyTheme(cur === "light" ? "dark" : "light"); + }); + $("#btn-reset-view").addEventListener("click", (e) => { e.preventDefault(); e.stopPropagation(); diff --git a/ui/assets/dell/face-blank-1u.jpg b/ui/assets/dell/face-blank-1u.jpg new file mode 100644 index 0000000..f02a485 Binary files /dev/null and b/ui/assets/dell/face-blank-1u.jpg differ diff --git a/ui/assets/dell/face-chassis-mx.jpg b/ui/assets/dell/face-chassis-mx.jpg new file mode 100644 index 0000000..80534e7 Binary files /dev/null and b/ui/assets/dell/face-chassis-mx.jpg differ diff --git a/ui/assets/dell/face-pdu-1u.jpg b/ui/assets/dell/face-pdu-1u.jpg new file mode 100644 index 0000000..cfe14bc Binary files /dev/null and b/ui/assets/dell/face-pdu-1u.jpg differ diff --git a/ui/assets/dell/face-server-1u.jpg b/ui/assets/dell/face-server-1u.jpg new file mode 100644 index 0000000..c4f1fd8 Binary files /dev/null and b/ui/assets/dell/face-server-1u.jpg differ diff --git a/ui/assets/dell/face-server-2u.jpg b/ui/assets/dell/face-server-2u.jpg new file mode 100644 index 0000000..0ac8317 Binary files /dev/null and b/ui/assets/dell/face-server-2u.jpg differ diff --git a/ui/assets/dell/face-server-4u.jpg b/ui/assets/dell/face-server-4u.jpg new file mode 100644 index 0000000..c52916e Binary files /dev/null and b/ui/assets/dell/face-server-4u.jpg differ diff --git a/ui/assets/dell/face-storage-2u.jpg b/ui/assets/dell/face-storage-2u.jpg new file mode 100644 index 0000000..7639a16 Binary files /dev/null and b/ui/assets/dell/face-storage-2u.jpg differ diff --git a/ui/assets/dell/face-switch-1u.jpg b/ui/assets/dell/face-switch-1u.jpg new file mode 100644 index 0000000..b194ea6 Binary files /dev/null and b/ui/assets/dell/face-switch-1u.jpg differ diff --git a/ui/index.html b/ui/index.html index 43d1be2..a1fa677 100644 --- a/ui/index.html +++ b/ui/index.html @@ -7,9 +7,19 @@ - + +
@@ -28,9 +38,10 @@
+ - +
@@ -96,18 +107,6 @@ -
- -
-

Live atc-gpu-prod · llama3-70b-gptq

-
Waiting for GPU telemetry…
-
-
-
-
+ + + @@ -278,6 +296,30 @@ + + +