feat: Authentik login + switchable GPU prod target

Add OIDC auth for Command Center and runtime GPU endpoint selection
pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
This commit is contained in:
mo
2026-07-21 23:20:24 +00:00
parent f36c8906bc
commit 9008fbd512
31 changed files with 4667 additions and 139 deletions
+349 -42
View File
@@ -15,6 +15,7 @@ import httpx
import redis.asyncio as aioredis
from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
import auth as cockpit_auth
from agent_terminal import (
get_all_terminals,
get_terminal_lines,
@@ -77,6 +78,14 @@ from db import SessionLocal, db_health, init_database
from supervisor import mirror_terminal_line, mirror_to_supervisors
from workload import build_workload_payload
from gpu_config import (
get_gpu_config,
get_gpu_config_payload,
get_gpu_urls,
reset_gpu_config,
save_gpu_config,
test_gpu_target,
)
_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
@@ -89,12 +98,130 @@ from sqlalchemy.orm import DeclarativeBase
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
def _dockhand_headers() -> dict[str, str]:
if DOCKHAND_API_TOKEN:
return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
return {}
GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1")
LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))
# Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget.
LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096"))
# Conservative estimate: Llama tokenizers often use ~2.22.8 chars/token on English+lab text.
LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4"))
LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160"))
LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256"))
LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500"))
def _estimate_tokens(text: str) -> int:
# Slightly inflate so we never underestimate vs vLLM's tokenizer.
return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32)
def _compact_lab_context(context: str) -> str:
"""Keep primary section + short per-domain summaries; drop verbose inventory lines."""
lines = context.splitlines()
out: list[str] = []
in_full = False
detail = 0
max_detail = 4
for line in lines:
if line.startswith("=== FULL LAB"):
in_full = True
out.append(line)
continue
if line.startswith("=== PRIMARY"):
in_full = False
detail = 0
out.append(line)
continue
if line.startswith("--- "):
detail = 0
out.append(line)
continue
# Drop long platform-capabilities essays if present — keep a one-liner marker
if line.startswith("=== PLATFORM CAPABILITIES"):
out.append(line)
out.append(" (see Command Center UI for full feature list)")
continue
if out and out[-1].startswith(" (see Command Center"):
if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
pass
else:
continue
# Always keep masking / PII evidence sections in full (demo-critical)
if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
# flush remaining lines of this section without detail limits by marking
out.append(line)
continue
# Limit bullet detail in general lab dump, but keep masking evidence intact
keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING"))
if (line.startswith(" - ") or line.startswith(" - ")) and not keep_full:
detail += 1
if detail > max_detail:
if detail == max_detail + 1:
out.append("")
continue
out.append(line)
return "\n".join(out)
def _truncate_for_llm(context: str, max_chars: int) -> str:
context = _compact_lab_context(context)
if len(context) <= max_chars:
return context
# Prefer keeping PRIMARY section; cut FULL LAB first
primary_end = context.find("=== FULL LAB")
if primary_end > 200:
head = context[:primary_end].rstrip()
tail_budget = max(400, max_chars - len(head) - 80)
tail = context[primary_end: primary_end + tail_budget]
trimmed = head + "\n" + tail
else:
trimmed = context[:max_chars]
if len(trimmed) > max_chars:
trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0]
if len(context) > len(trimmed):
trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]"
return trimmed
def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]:
"""Fit prompt+completion into the served model length with a safety margin."""
user_tok = _estimate_tokens(user_message)
rules_tok = _estimate_tokens(system_rules)
budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN
max_out = min(LLM_MAX_OUTPUT, 256)
# Absolute char cap first (independent of estimate errors)
context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS)
for _ in range(6):
ctx_budget_tok = budget - user_tok - rules_tok - max_out
if ctx_budget_tok < 200:
max_out = max(64, max_out // 2)
continue
ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85))
fitted = _truncate_for_llm(context, ctx_max_chars)
total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out
if total <= budget:
return fitted, max_out
# Still too big — shrink context harder, then output
context = fitted
LLM_MAX = max(800, int(len(fitted) * 0.7))
context = _truncate_for_llm(context, LLM_MAX)
max_out = max(64, max_out - 32)
return _truncate_for_llm(context, 800), 64
AGENTS = [
{
@@ -247,7 +374,11 @@ ZONES = [
]
INTENT_KEYWORDS: dict[str, list[str]] = {
"data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "],
"data-custodian": [
"database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ",
"pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer",
"employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id",
],
"lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
"hadoop-ranger": [
"hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
@@ -327,10 +458,24 @@ class ApprovalDecision(BaseModel):
def route_agent(message: str) -> str:
lower = message.lower()
# PII / privacy questions always go to Data Custodian (masking demo path)
pii_words = (
"pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address",
"phone", "telefoon", "iban", "address", "adres", "customer name", "employee",
"gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig",
)
if any(w in lower for w in pii_words):
return "data-custodian"
# Storage/data questions default to Hadoop unless clearly about databases
if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")):
if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")):
if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")):
if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")):
return "hadoop-ranger"
if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")):
return "infra-sentinel"
if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")):
return "etl-guardian"
if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")):
return "lakehouse-ops"
scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
best = max(scores, key=scores.get)
if scores[best] == 0:
@@ -338,13 +483,34 @@ def route_agent(message: str) -> str:
return best
def _is_pii_question(message: str) -> bool:
lower = message.lower()
return any(w in lower for w in (
"pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone",
"iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive",
"personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id",
"ssn", "bsn", "geboorte", "birth",
))
async def _pii_evidence_block(message: str, log: Any | None = None) -> str:
"""Live policy + samples synced with Data Flow masking toggles."""
try:
from pii_catalog import build_policy_evidence
return build_policy_evidence()
except Exception as exc:
return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})"
async def gather_agent_context(
agent_id: str,
status: dict[str, Any],
log: Any | None = None,
message: str | None = None,
) -> str:
"""Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log)
snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False)
snapshot["domains_summary"] = status.get("domains", {})
ctx = format_context_for_agent(agent_id, snapshot)
agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
@@ -353,10 +519,24 @@ async def gather_agent_context(
agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}")
ctx = ctx + "\n".join(agent_lines)
try:
from platform_context import build_llm_addendum
ctx = ctx + "\n\n" + build_llm_addendum()
from platform_context import build_masking_section
# Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog).
ctx = ctx + "\n\n" + build_masking_section(fresh=True)
except Exception:
pass
try:
from platform_context import build_llm_addendum
ctx = ctx + "\n\n" + build_llm_addendum()
except Exception:
pass
if message and _is_pii_question(message):
try:
evidence = await _pii_evidence_block(message, log=log)
ctx = ctx + "\n\n" + evidence
if log:
await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)")
except Exception as exc:
if log:
await log("warn", "fetch", f"▸ PII evidence skipped: {exc}")
if log:
await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
return ctx
@@ -369,34 +549,44 @@ async def ask_llm(
log: Any | None = None,
) -> str | None:
agent = next(a for a in AGENTS if a["id"] == agent_id)
system = f"""You are {agent['name']}, an autonomous ops agent in the Dell ATC data lab.
Specialization: {agent['role']}.
Motto: {agent.get('motto', '')}.
# Deterministic PII path: always mirror Data Flow masked vs visible toggles.
if _is_pii_question(message):
try:
from pii_catalog import format_pii_chat_answer
answer = format_pii_chat_answer(message)
if log:
await log("ok", "pii", "▸ Returning Data Flowsynced masking answer (masked + visible)")
return answer
except Exception as exc:
if log:
await log("warn", "pii", f"▸ PII answer builder failed: {exc}")
rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab.
Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers.
If data is missing or DOWN, say so.
You respond on behalf of your domain but have visibility into the FULL lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, and GPU/vLLM.
PII / masking rules (critical — synced with Data Flow tab):
- MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present.
- VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy.
- Never invent emails, phones, names, IBANs, or addresses that are not in the live samples.
- If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections.
Rules:
- Always respond in English.
- You have full visibility into the entire cluster: all VMs, zones, connectors, GPU, Hadoop, ObjectScale and Command Center.
- Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
- Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
- If something is DOWN or 0 GB, say so honestly.
- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
- Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
--- LIVE LAB DATA (primary domain first, then full stack) ---
{context}
"""
--- LIVE LAB DATA ---"""
fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message)
system = rules + "\n" + fitted_ctx
urls = get_gpu_urls()
llm_url = urls["llm_url"]
if log:
await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}")
await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions")
est = _estimate_tokens(system) + _estimate_tokens(message)
await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)")
if len(context) > len(fitted_ctx):
await log("warn", "llm", f" context trimmed {len(context)}{len(fitted_ctx)} chars")
await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions")
await log("info", "llm", f" user: {message[:160]}{'' if len(message) > 160 else ''}")
try:
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
t0 = time.monotonic()
r = await client.post(
f"{LLM_URL.rstrip('/')}/chat/completions",
f"{llm_url.rstrip('/')}/chat/completions",
headers={
"Authorization": f"Bearer {LLM_API_KEY}",
"Content-Type": "application/json",
@@ -407,7 +597,7 @@ Rules:
{"role": "system", "content": system},
{"role": "user", "content": message},
],
"max_tokens": 800,
"max_tokens": max_tokens,
"temperature": 0.25,
},
)
@@ -422,15 +612,78 @@ Rules:
return content
if log:
await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
except httpx.HTTPStatusError as exc:
detail = exc.response.text[:200] if exc.response is not None else str(exc)
if log:
await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}")
# One hard retry with a minimal context if we blew the window.
if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail:
tiny = _truncate_for_llm(context, 1200)
system2 = rules + "\n" + tiny
max2 = 128
if log:
await log("warn", "llm", f" retry with tiny context ({len(tiny)} chars, max_tokens={max2})")
try:
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
r2 = await client.post(
f"{llm_url.rstrip('/')}/chat/completions",
headers={
"Authorization": f"Bearer {LLM_API_KEY}",
"Content-Type": "application/json",
},
json={
"model": LLM_MODEL,
"messages": [
{"role": "system", "content": system2},
{"role": "user", "content": message},
],
"max_tokens": max2,
"temperature": 0.25,
},
)
r2.raise_for_status()
content2 = r2.json()["choices"][0]["message"]["content"].strip()
if content2:
if log:
await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars")
return content2
except Exception as exc2:
if log:
await log("err", "llm", f"✗ vLLM retry failed: {exc2}")
except Exception as exc:
if log:
await log("err", "llm", f"✗ vLLM error: {exc}")
return None
def fallback_answer(agent_id: str, context: str) -> str:
def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str:
agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}"
if user_message and _is_pii_question(user_message):
try:
from pii_catalog import format_pii_chat_answer
return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message)
except Exception:
marker = "=== PII MASKING EVIDENCE"
if marker in context:
return (
f"**{agent_name}** — masking policy (synced with Data Flow):\n\n"
+ context[context.index(marker):].strip()
)
preview_lines: list[str] = []
for line in context.splitlines():
if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")):
preview_lines.append(line)
if len(preview_lines) >= 14:
break
hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable."
q = f"\n\nYour question: _{user_message[:200]}_" if user_message else ""
return (
f"**{agent_name}** — I could not get a reply from the GPU LLM "
f"(context window or vLLM error).{q}\n\n"
"Try a short, specific question "
"(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n"
f"Quick snapshot:\n{hint}"
)
async def publish_event(event: dict[str, Any]) -> None:
@@ -480,7 +733,11 @@ def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
async def dockhand_env_containers(env_id: int) -> list[dict]:
try:
async with httpx.AsyncClient(timeout=8.0) as client:
r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id})
r = await client.get(
f"{DOCKHAND_URL}/api/containers",
params={"env": env_id},
headers=_dockhand_headers(),
)
r.raise_for_status()
return r.json()
except Exception:
@@ -497,15 +754,26 @@ async def probe_url(url: str) -> bool:
async def collect_gpu() -> dict[str, Any]:
host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0]
base = {"ok": False, "host": host, "ui_url": GPU_UI_URL}
urls = get_gpu_urls()
cfg = get_gpu_config()
gpu_url = urls["gpu_url"]
host = urls["host"]
base = {
"ok": False,
"host": host,
"ip": host,
"ui_url": urls["gpu_ui_url"],
"config_source": cfg.get("source", "env"),
"preset_id": cfg.get("preset_id"),
"config_label": cfg.get("label"),
}
try:
async with httpx.AsyncClient(timeout=6.0) as client:
metrics_r, model_r, integration_r = await asyncio.gather(
client.get(f"{GPU_URL}/api/gpu/metrics"),
client.get(f"{GPU_URL}/api/active-model"),
client.get(f"{GPU_URL}/api/integration"),
client.get(f"{gpu_url}/api/gpu/metrics"),
client.get(f"{gpu_url}/api/active-model"),
client.get(f"{gpu_url}/api/integration"),
return_exceptions=True,
)
@@ -667,12 +935,12 @@ async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
status = await collect_status()
context = await gather_agent_context(agent_id, status, log=log)
context = await gather_agent_context(agent_id, status, log=log, message=message)
answer = await ask_llm(agent_id, message, context, log=log)
if not answer:
await log("warn", "llm", "LLM fallback — returning raw context")
answer = fallback_answer(agent_id, context)
answer = fallback_answer(agent_id, context, message)
if not approval_created:
proposed = detect_agent_proposed_action(answer, message)
@@ -784,6 +1052,10 @@ app.add_middleware(
allow_headers=["*"],
)
# Authentik OIDC session + API guard
cockpit_auth.init_auth_middleware(app)
cockpit_auth.setup_auth(app)
@app.get("/api/health")
async def health():
@@ -983,6 +1255,41 @@ async def get_status():
async def get_gpu():
return await collect_gpu()
@app.get("/api/gpu/config")
async def get_gpu_config_endpoint():
return get_gpu_config_payload()
@app.post("/api/gpu/config")
async def post_gpu_config(body: dict[str, Any]):
try:
saved = save_gpu_config(
preset_id=body.get("preset_id"),
host=body.get("host"),
gpu_ui_port=body.get("gpu_ui_port"),
llm_port=body.get("llm_port"),
)
return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]}
except ValueError as exc:
return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400)
@app.post("/api/gpu/config/test")
async def post_gpu_config_test(body: dict[str, Any]):
return await test_gpu_target(
preset_id=body.get("preset_id"),
host=body.get("host"),
gpu_ui_port=body.get("gpu_ui_port"),
llm_port=body.get("llm_port"),
)
@app.delete("/api/gpu/config")
async def delete_gpu_config():
active = reset_gpu_config()
return {"ok": True, "active": active}
def agent_stats() -> dict[str, dict[str, Any]]:
stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
@@ -1053,11 +1360,11 @@ async def run_node_ask_task(node_id: str, message: str) -> None:
await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
log = make_logger(node_id)
status = await collect_status()
context = await gather_agent_context(agent_id, status, log=log)
context = await gather_agent_context(agent_id, status, log=log, message=message)
node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
if not answer:
answer = fallback_answer(agent_id, context)
answer = fallback_answer(agent_id, context, message)
await terminal_log(node_id, f"{answer}", level="llm", phase="answer")
await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})