feat: Authentik login + switchable GPU prod target
Add OIDC auth for Command Center and runtime GPU endpoint selection pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
This commit is contained in:
+349
-42
@@ -15,6 +15,7 @@ import httpx
|
||||
import redis.asyncio as aioredis
|
||||
from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
import auth as cockpit_auth
|
||||
from agent_terminal import (
|
||||
get_all_terminals,
|
||||
get_terminal_lines,
|
||||
@@ -77,6 +78,14 @@ from db import SessionLocal, db_health, init_database
|
||||
from supervisor import mirror_terminal_line, mirror_to_supervisors
|
||||
|
||||
from workload import build_workload_payload
|
||||
from gpu_config import (
|
||||
get_gpu_config,
|
||||
get_gpu_config_payload,
|
||||
get_gpu_urls,
|
||||
reset_gpu_config,
|
||||
save_gpu_config,
|
||||
test_gpu_target,
|
||||
)
|
||||
|
||||
_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
@@ -89,12 +98,130 @@ from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
|
||||
GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
|
||||
DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
|
||||
|
||||
|
||||
def _dockhand_headers() -> dict[str, str]:
|
||||
if DOCKHAND_API_TOKEN:
|
||||
return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
|
||||
return {}
|
||||
|
||||
GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
|
||||
GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
|
||||
LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1")
|
||||
LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
|
||||
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
|
||||
LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
|
||||
LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))
|
||||
# Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget.
|
||||
LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096"))
|
||||
# Conservative estimate: Llama tokenizers often use ~2.2–2.8 chars/token on English+lab text.
|
||||
LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4"))
|
||||
LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160"))
|
||||
LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256"))
|
||||
LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500"))
|
||||
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
# Slightly inflate so we never underestimate vs vLLM's tokenizer.
|
||||
return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32)
|
||||
|
||||
|
||||
def _compact_lab_context(context: str) -> str:
|
||||
"""Keep primary section + short per-domain summaries; drop verbose inventory lines."""
|
||||
lines = context.splitlines()
|
||||
out: list[str] = []
|
||||
in_full = False
|
||||
detail = 0
|
||||
max_detail = 4
|
||||
for line in lines:
|
||||
if line.startswith("=== FULL LAB"):
|
||||
in_full = True
|
||||
out.append(line)
|
||||
continue
|
||||
if line.startswith("=== PRIMARY"):
|
||||
in_full = False
|
||||
detail = 0
|
||||
out.append(line)
|
||||
continue
|
||||
if line.startswith("--- "):
|
||||
detail = 0
|
||||
out.append(line)
|
||||
continue
|
||||
# Drop long platform-capabilities essays if present — keep a one-liner marker
|
||||
if line.startswith("=== PLATFORM CAPABILITIES"):
|
||||
out.append(line)
|
||||
out.append(" (see Command Center UI for full feature list)")
|
||||
continue
|
||||
if out and out[-1].startswith(" (see Command Center"):
|
||||
if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
|
||||
pass
|
||||
else:
|
||||
continue
|
||||
# Always keep masking / PII evidence sections in full (demo-critical)
|
||||
if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
|
||||
# flush remaining lines of this section without detail limits by marking
|
||||
out.append(line)
|
||||
continue
|
||||
# Limit bullet detail in general lab dump, but keep masking evidence intact
|
||||
keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING"))
|
||||
if (line.startswith(" - ") or line.startswith(" - ")) and not keep_full:
|
||||
detail += 1
|
||||
if detail > max_detail:
|
||||
if detail == max_detail + 1:
|
||||
out.append(" …")
|
||||
continue
|
||||
out.append(line)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _truncate_for_llm(context: str, max_chars: int) -> str:
|
||||
context = _compact_lab_context(context)
|
||||
if len(context) <= max_chars:
|
||||
return context
|
||||
# Prefer keeping PRIMARY section; cut FULL LAB first
|
||||
primary_end = context.find("=== FULL LAB")
|
||||
if primary_end > 200:
|
||||
head = context[:primary_end].rstrip()
|
||||
tail_budget = max(400, max_chars - len(head) - 80)
|
||||
tail = context[primary_end: primary_end + tail_budget]
|
||||
trimmed = head + "\n" + tail
|
||||
else:
|
||||
trimmed = context[:max_chars]
|
||||
if len(trimmed) > max_chars:
|
||||
trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0]
|
||||
if len(context) > len(trimmed):
|
||||
trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]"
|
||||
return trimmed
|
||||
|
||||
|
||||
def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]:
|
||||
"""Fit prompt+completion into the served model length with a safety margin."""
|
||||
user_tok = _estimate_tokens(user_message)
|
||||
rules_tok = _estimate_tokens(system_rules)
|
||||
budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN
|
||||
max_out = min(LLM_MAX_OUTPUT, 256)
|
||||
|
||||
# Absolute char cap first (independent of estimate errors)
|
||||
context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS)
|
||||
|
||||
for _ in range(6):
|
||||
ctx_budget_tok = budget - user_tok - rules_tok - max_out
|
||||
if ctx_budget_tok < 200:
|
||||
max_out = max(64, max_out // 2)
|
||||
continue
|
||||
ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85))
|
||||
fitted = _truncate_for_llm(context, ctx_max_chars)
|
||||
total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out
|
||||
if total <= budget:
|
||||
return fitted, max_out
|
||||
# Still too big — shrink context harder, then output
|
||||
context = fitted
|
||||
LLM_MAX = max(800, int(len(fitted) * 0.7))
|
||||
context = _truncate_for_llm(context, LLM_MAX)
|
||||
max_out = max(64, max_out - 32)
|
||||
|
||||
return _truncate_for_llm(context, 800), 64
|
||||
|
||||
|
||||
AGENTS = [
|
||||
{
|
||||
@@ -247,7 +374,11 @@ ZONES = [
|
||||
]
|
||||
|
||||
INTENT_KEYWORDS: dict[str, list[str]] = {
|
||||
"data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "],
|
||||
"data-custodian": [
|
||||
"database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ",
|
||||
"pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer",
|
||||
"employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id",
|
||||
],
|
||||
"lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
|
||||
"hadoop-ranger": [
|
||||
"hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
|
||||
@@ -327,10 +458,24 @@ class ApprovalDecision(BaseModel):
|
||||
|
||||
def route_agent(message: str) -> str:
|
||||
lower = message.lower()
|
||||
# PII / privacy questions always go to Data Custodian (masking demo path)
|
||||
pii_words = (
|
||||
"pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address",
|
||||
"phone", "telefoon", "iban", "address", "adres", "customer name", "employee",
|
||||
"gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig",
|
||||
)
|
||||
if any(w in lower for w in pii_words):
|
||||
return "data-custodian"
|
||||
# Storage/data questions default to Hadoop unless clearly about databases
|
||||
if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")):
|
||||
if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")):
|
||||
if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")):
|
||||
if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")):
|
||||
return "hadoop-ranger"
|
||||
if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")):
|
||||
return "infra-sentinel"
|
||||
if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")):
|
||||
return "etl-guardian"
|
||||
if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")):
|
||||
return "lakehouse-ops"
|
||||
scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
|
||||
best = max(scores, key=scores.get)
|
||||
if scores[best] == 0:
|
||||
@@ -338,13 +483,34 @@ def route_agent(message: str) -> str:
|
||||
return best
|
||||
|
||||
|
||||
|
||||
def _is_pii_question(message: str) -> bool:
|
||||
lower = message.lower()
|
||||
return any(w in lower for w in (
|
||||
"pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone",
|
||||
"iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive",
|
||||
"personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id",
|
||||
"ssn", "bsn", "geboorte", "birth",
|
||||
))
|
||||
|
||||
|
||||
async def _pii_evidence_block(message: str, log: Any | None = None) -> str:
|
||||
"""Live policy + samples synced with Data Flow masking toggles."""
|
||||
try:
|
||||
from pii_catalog import build_policy_evidence
|
||||
return build_policy_evidence()
|
||||
except Exception as exc:
|
||||
return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})"
|
||||
|
||||
|
||||
async def gather_agent_context(
|
||||
agent_id: str,
|
||||
status: dict[str, Any],
|
||||
log: Any | None = None,
|
||||
message: str | None = None,
|
||||
) -> str:
|
||||
"""Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
|
||||
snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log)
|
||||
snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False)
|
||||
snapshot["domains_summary"] = status.get("domains", {})
|
||||
ctx = format_context_for_agent(agent_id, snapshot)
|
||||
agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
|
||||
@@ -353,10 +519,24 @@ async def gather_agent_context(
|
||||
agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}")
|
||||
ctx = ctx + "\n".join(agent_lines)
|
||||
try:
|
||||
from platform_context import build_llm_addendum
|
||||
ctx = ctx + "\n\n" + build_llm_addendum()
|
||||
from platform_context import build_masking_section
|
||||
# Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog).
|
||||
ctx = ctx + "\n\n" + build_masking_section(fresh=True)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from platform_context import build_llm_addendum
|
||||
ctx = ctx + "\n\n" + build_llm_addendum()
|
||||
except Exception:
|
||||
pass
|
||||
if message and _is_pii_question(message):
|
||||
try:
|
||||
evidence = await _pii_evidence_block(message, log=log)
|
||||
ctx = ctx + "\n\n" + evidence
|
||||
if log:
|
||||
await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)")
|
||||
except Exception as exc:
|
||||
if log:
|
||||
await log("warn", "fetch", f"▸ PII evidence skipped: {exc}")
|
||||
if log:
|
||||
await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
|
||||
return ctx
|
||||
@@ -369,34 +549,44 @@ async def ask_llm(
|
||||
log: Any | None = None,
|
||||
) -> str | None:
|
||||
agent = next(a for a in AGENTS if a["id"] == agent_id)
|
||||
system = f"""You are {agent['name']}, an autonomous ops agent in the Dell ATC data lab.
|
||||
Specialization: {agent['role']}.
|
||||
Motto: {agent.get('motto', '')}.
|
||||
# Deterministic PII path: always mirror Data Flow masked vs visible toggles.
|
||||
if _is_pii_question(message):
|
||||
try:
|
||||
from pii_catalog import format_pii_chat_answer
|
||||
answer = format_pii_chat_answer(message)
|
||||
if log:
|
||||
await log("ok", "pii", "▸ Returning Data Flow–synced masking answer (masked + visible)")
|
||||
return answer
|
||||
except Exception as exc:
|
||||
if log:
|
||||
await log("warn", "pii", f"▸ PII answer builder failed: {exc}")
|
||||
rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab.
|
||||
Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers.
|
||||
If data is missing or DOWN, say so.
|
||||
|
||||
You respond on behalf of your domain but have visibility into the FULL lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, and GPU/vLLM.
|
||||
PII / masking rules (critical — synced with Data Flow tab):
|
||||
- MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present.
|
||||
- VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy.
|
||||
- Never invent emails, phones, names, IBANs, or addresses that are not in the live samples.
|
||||
- If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections.
|
||||
|
||||
Rules:
|
||||
- Always respond in English.
|
||||
- You have full visibility into the entire cluster: all VMs, zones, connectors, GPU, Hadoop, ObjectScale and Command Center.
|
||||
- Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
|
||||
- Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
|
||||
- If something is DOWN or 0 GB, say so honestly.
|
||||
- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
|
||||
- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
|
||||
- Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
|
||||
|
||||
--- LIVE LAB DATA (primary domain first, then full stack) ---
|
||||
{context}
|
||||
"""
|
||||
--- LIVE LAB DATA ---"""
|
||||
fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message)
|
||||
system = rules + "\n" + fitted_ctx
|
||||
urls = get_gpu_urls()
|
||||
llm_url = urls["llm_url"]
|
||||
if log:
|
||||
await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}")
|
||||
await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions")
|
||||
est = _estimate_tokens(system) + _estimate_tokens(message)
|
||||
await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)")
|
||||
if len(context) > len(fitted_ctx):
|
||||
await log("warn", "llm", f" context trimmed {len(context)} → {len(fitted_ctx)} chars")
|
||||
await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions")
|
||||
await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
|
||||
t0 = time.monotonic()
|
||||
r = await client.post(
|
||||
f"{LLM_URL.rstrip('/')}/chat/completions",
|
||||
f"{llm_url.rstrip('/')}/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {LLM_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
@@ -407,7 +597,7 @@ Rules:
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": message},
|
||||
],
|
||||
"max_tokens": 800,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.25,
|
||||
},
|
||||
)
|
||||
@@ -422,15 +612,78 @@ Rules:
|
||||
return content
|
||||
if log:
|
||||
await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
|
||||
except httpx.HTTPStatusError as exc:
|
||||
detail = exc.response.text[:200] if exc.response is not None else str(exc)
|
||||
if log:
|
||||
await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}")
|
||||
# One hard retry with a minimal context if we blew the window.
|
||||
if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail:
|
||||
tiny = _truncate_for_llm(context, 1200)
|
||||
system2 = rules + "\n" + tiny
|
||||
max2 = 128
|
||||
if log:
|
||||
await log("warn", "llm", f" retry with tiny context ({len(tiny)} chars, max_tokens={max2})")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
|
||||
r2 = await client.post(
|
||||
f"{llm_url.rstrip('/')}/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {LLM_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": LLM_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system2},
|
||||
{"role": "user", "content": message},
|
||||
],
|
||||
"max_tokens": max2,
|
||||
"temperature": 0.25,
|
||||
},
|
||||
)
|
||||
r2.raise_for_status()
|
||||
content2 = r2.json()["choices"][0]["message"]["content"].strip()
|
||||
if content2:
|
||||
if log:
|
||||
await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars")
|
||||
return content2
|
||||
except Exception as exc2:
|
||||
if log:
|
||||
await log("err", "llm", f"✗ vLLM retry failed: {exc2}")
|
||||
except Exception as exc:
|
||||
if log:
|
||||
await log("err", "llm", f"✗ vLLM error: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def fallback_answer(agent_id: str, context: str) -> str:
|
||||
def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str:
|
||||
agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
|
||||
return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}"
|
||||
if user_message and _is_pii_question(user_message):
|
||||
try:
|
||||
from pii_catalog import format_pii_chat_answer
|
||||
return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message)
|
||||
except Exception:
|
||||
marker = "=== PII MASKING EVIDENCE"
|
||||
if marker in context:
|
||||
return (
|
||||
f"**{agent_name}** — masking policy (synced with Data Flow):\n\n"
|
||||
+ context[context.index(marker):].strip()
|
||||
)
|
||||
preview_lines: list[str] = []
|
||||
for line in context.splitlines():
|
||||
if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")):
|
||||
preview_lines.append(line)
|
||||
if len(preview_lines) >= 14:
|
||||
break
|
||||
hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable."
|
||||
q = f"\n\nYour question: _{user_message[:200]}_" if user_message else ""
|
||||
return (
|
||||
f"**{agent_name}** — I could not get a reply from the GPU LLM "
|
||||
f"(context window or vLLM error).{q}\n\n"
|
||||
"Try a short, specific question "
|
||||
"(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n"
|
||||
f"Quick snapshot:\n{hint}"
|
||||
)
|
||||
|
||||
|
||||
async def publish_event(event: dict[str, Any]) -> None:
|
||||
@@ -480,7 +733,11 @@ def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
|
||||
async def dockhand_env_containers(env_id: int) -> list[dict]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id})
|
||||
r = await client.get(
|
||||
f"{DOCKHAND_URL}/api/containers",
|
||||
params={"env": env_id},
|
||||
headers=_dockhand_headers(),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception:
|
||||
@@ -497,15 +754,26 @@ async def probe_url(url: str) -> bool:
|
||||
|
||||
|
||||
async def collect_gpu() -> dict[str, Any]:
|
||||
host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0]
|
||||
base = {"ok": False, "host": host, "ui_url": GPU_UI_URL}
|
||||
urls = get_gpu_urls()
|
||||
cfg = get_gpu_config()
|
||||
gpu_url = urls["gpu_url"]
|
||||
host = urls["host"]
|
||||
base = {
|
||||
"ok": False,
|
||||
"host": host,
|
||||
"ip": host,
|
||||
"ui_url": urls["gpu_ui_url"],
|
||||
"config_source": cfg.get("source", "env"),
|
||||
"preset_id": cfg.get("preset_id"),
|
||||
"config_label": cfg.get("label"),
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||||
metrics_r, model_r, integration_r = await asyncio.gather(
|
||||
client.get(f"{GPU_URL}/api/gpu/metrics"),
|
||||
client.get(f"{GPU_URL}/api/active-model"),
|
||||
client.get(f"{GPU_URL}/api/integration"),
|
||||
client.get(f"{gpu_url}/api/gpu/metrics"),
|
||||
client.get(f"{gpu_url}/api/active-model"),
|
||||
client.get(f"{gpu_url}/api/integration"),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
@@ -667,12 +935,12 @@ async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
|
||||
|
||||
await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
|
||||
status = await collect_status()
|
||||
context = await gather_agent_context(agent_id, status, log=log)
|
||||
context = await gather_agent_context(agent_id, status, log=log, message=message)
|
||||
|
||||
answer = await ask_llm(agent_id, message, context, log=log)
|
||||
if not answer:
|
||||
await log("warn", "llm", "LLM fallback — returning raw context")
|
||||
answer = fallback_answer(agent_id, context)
|
||||
answer = fallback_answer(agent_id, context, message)
|
||||
|
||||
if not approval_created:
|
||||
proposed = detect_agent_proposed_action(answer, message)
|
||||
@@ -784,6 +1052,10 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Authentik OIDC session + API guard
|
||||
cockpit_auth.init_auth_middleware(app)
|
||||
cockpit_auth.setup_auth(app)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
@@ -983,6 +1255,41 @@ async def get_status():
|
||||
async def get_gpu():
|
||||
return await collect_gpu()
|
||||
|
||||
@app.get("/api/gpu/config")
|
||||
async def get_gpu_config_endpoint():
|
||||
return get_gpu_config_payload()
|
||||
|
||||
|
||||
@app.post("/api/gpu/config")
|
||||
async def post_gpu_config(body: dict[str, Any]):
|
||||
try:
|
||||
saved = save_gpu_config(
|
||||
preset_id=body.get("preset_id"),
|
||||
host=body.get("host"),
|
||||
gpu_ui_port=body.get("gpu_ui_port"),
|
||||
llm_port=body.get("llm_port"),
|
||||
)
|
||||
return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]}
|
||||
except ValueError as exc:
|
||||
return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400)
|
||||
|
||||
|
||||
@app.post("/api/gpu/config/test")
|
||||
async def post_gpu_config_test(body: dict[str, Any]):
|
||||
return await test_gpu_target(
|
||||
preset_id=body.get("preset_id"),
|
||||
host=body.get("host"),
|
||||
gpu_ui_port=body.get("gpu_ui_port"),
|
||||
llm_port=body.get("llm_port"),
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/api/gpu/config")
|
||||
async def delete_gpu_config():
|
||||
active = reset_gpu_config()
|
||||
return {"ok": True, "active": active}
|
||||
|
||||
|
||||
|
||||
def agent_stats() -> dict[str, dict[str, Any]]:
|
||||
stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
|
||||
@@ -1053,11 +1360,11 @@ async def run_node_ask_task(node_id: str, message: str) -> None:
|
||||
await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
|
||||
log = make_logger(node_id)
|
||||
status = await collect_status()
|
||||
context = await gather_agent_context(agent_id, status, log=log)
|
||||
context = await gather_agent_context(agent_id, status, log=log, message=message)
|
||||
node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
|
||||
answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
|
||||
if not answer:
|
||||
answer = fallback_answer(agent_id, context)
|
||||
answer = fallback_answer(agent_id, context, message)
|
||||
await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer")
|
||||
await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user