sysops
486 lines
17 KiB
Python
486 lines
17 KiB
Python
"""Unified LLM router — Ollama, DeepSeek, Gemini, Groq, OpenRouter, custom OpenAI-compatible."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.config import settings
|
|
from app.db import execute, fetch_all, fetch_one
|
|
from app.services import ollama
|
|
|
|
# Preset catalog for Settings UI (signup links + default models)
|
|
LLM_PRESETS: dict[str, dict[str, Any]] = {
|
|
"ollama": {
|
|
"label": "Ollama (lokaal)",
|
|
"api_base_url": "",
|
|
"models": ["qwen3:8b", "gemma3:12b", "llama3.2", "mistral"],
|
|
"needs_key": False,
|
|
"hint": "Geen API key — draait op je Ollama server.",
|
|
},
|
|
"deepseek": {
|
|
"label": "DeepSeek",
|
|
"api_base_url": "https://api.deepseek.com/v1",
|
|
"models": ["deepseek-chat", "deepseek-reasoner"],
|
|
"needs_key": True,
|
|
"signup_url": "https://platform.deepseek.com/",
|
|
"hint": "Goedkoop · sterk voor code en analyse.",
|
|
},
|
|
"gemini": {
|
|
"label": "Google Gemini",
|
|
"api_base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
"models": ["gemini-2.0-flash", "gemini-1.5-flash", "gemini-1.5-pro"],
|
|
"needs_key": True,
|
|
"signup_url": "https://aistudio.google.com/apikey",
|
|
"hint": "Gratis tier via Google AI Studio.",
|
|
},
|
|
"groq": {
|
|
"label": "Groq (snel · gratis tier)",
|
|
"api_base_url": "https://api.groq.com/openai/v1",
|
|
"models": ["llama-3.3-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"],
|
|
"needs_key": True,
|
|
"signup_url": "https://console.groq.com/",
|
|
"hint": "Zeer snelle inference · gratis limiet.",
|
|
},
|
|
"openrouter": {
|
|
"label": "OpenRouter",
|
|
"api_base_url": "https://openrouter.ai/api/v1",
|
|
"models": [
|
|
"deepseek/deepseek-v4-flash",
|
|
"deepseek/deepseek-v4-flash:free",
|
|
"deepseek/deepseek-chat",
|
|
"deepseek/deepseek-r1",
|
|
"google/gemini-2.0-flash-exp:free",
|
|
"meta-llama/llama-3.3-70b-instruct:free",
|
|
],
|
|
"default_model": "deepseek/deepseek-v4-flash",
|
|
"needs_key": True,
|
|
"signup_url": "https://openrouter.ai/keys",
|
|
"hint": "Eén API key · DeepSeek V4 Flash via openrouter.ai. Vul key in Instellingen → AI.",
|
|
},
|
|
"mistral": {
|
|
"label": "Mistral AI",
|
|
"api_base_url": "https://api.mistral.ai/v1",
|
|
"models": ["mistral-small-latest", "open-mistral-nemo"],
|
|
"needs_key": True,
|
|
"signup_url": "https://console.mistral.ai/",
|
|
"hint": "EU-hosted · gratis proef tier.",
|
|
},
|
|
"hermes": {
|
|
"label": "Hermes Agent (Herman)",
|
|
"api_base_url": "http://10.4.7.10:8642/v1",
|
|
"models": ["hermes-agent"],
|
|
"default_model": "hermes-agent",
|
|
"needs_key": True,
|
|
"hint": "OpenAI-compatible API van Hermes Agent LXC — OpenCode eronder.",
|
|
},
|
|
"custom_openai": {
|
|
"label": "Custom OpenAI-compatible",
|
|
"api_base_url": "",
|
|
"models": [],
|
|
"needs_key": True,
|
|
"hint": "Elke API die /v1/chat/completions ondersteunt.",
|
|
},
|
|
}
|
|
|
|
|
|
|
|
OPENROUTER_PLACEHOLDER_LABEL = "OpenRouter · DeepSeek V4 Flash"
|
|
|
|
|
|
def ensure_openrouter_placeholder() -> None:
|
|
"""Seed inactive OpenRouter row so user can fill API key later in Settings."""
|
|
row = fetch_one(
|
|
"SELECT id FROM llm_providers WHERE provider_type = 'openrouter' LIMIT 1"
|
|
)
|
|
if row:
|
|
return
|
|
execute(
|
|
"""INSERT INTO llm_providers (
|
|
label, provider_type, api_base_url, api_key, model,
|
|
is_active, is_default, extra_config, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, NOW())""",
|
|
(
|
|
OPENROUTER_PLACEHOLDER_LABEL,
|
|
"openrouter",
|
|
"https://openrouter.ai/api/v1",
|
|
"",
|
|
"deepseek/deepseek-v4-flash",
|
|
False,
|
|
False,
|
|
json.dumps({"referer": "https://foodlinkk.local", "title": "Foodlinkk Command Center"}),
|
|
),
|
|
)
|
|
|
|
|
|
def _env_openrouter_provider() -> dict[str, Any] | None:
|
|
from app.config import settings as app_settings
|
|
|
|
key = (app_settings.OPENROUTER_API_KEY or "").strip()
|
|
if not key:
|
|
return None
|
|
return {
|
|
"id": -1,
|
|
"label": "OpenRouter (environment)",
|
|
"provider_type": "openrouter",
|
|
"api_base_url": (app_settings.OPENROUTER_BASE_URL or "https://openrouter.ai/api/v1").rstrip("/"),
|
|
"api_key": key,
|
|
"model": app_settings.OPENROUTER_MODEL or "deepseek/deepseek-v4-flash",
|
|
"extra_config": {"referer": "https://foodlinkk.local", "title": "Foodlinkk Command Center"},
|
|
}
|
|
|
|
|
|
|
|
|
|
def _env_hermes_provider() -> dict[str, Any] | None:
|
|
from app.config import settings as app_settings
|
|
|
|
key = (app_settings.HERMES_API_KEY or "").strip()
|
|
base = (app_settings.HERMES_API_BASE or "").strip()
|
|
if not key or not base:
|
|
return None
|
|
return {
|
|
"id": -2,
|
|
"label": "Hermes Agent (Herman)",
|
|
"provider_type": "hermes",
|
|
"api_base_url": base.rstrip("/"),
|
|
"api_key": key,
|
|
"model": app_settings.HERMES_API_MODEL or "hermes-agent",
|
|
"extra_config": {"temperature": 0.35, "max_tokens": 4096},
|
|
}
|
|
|
|
|
|
def list_presets() -> list[dict[str, Any]]:
|
|
out = []
|
|
for key, meta in LLM_PRESETS.items():
|
|
row = dict(meta)
|
|
row["id"] = key
|
|
out.append(row)
|
|
return out
|
|
|
|
|
|
def _mask_provider(row: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
if not row:
|
|
return None
|
|
out = dict(row)
|
|
for k, v in list(out.items()):
|
|
if hasattr(v, "isoformat"):
|
|
out[k] = v.isoformat()
|
|
if isinstance(out.get("extra_config"), str):
|
|
try:
|
|
out["extra_config"] = json.loads(out["extra_config"])
|
|
except Exception:
|
|
out["extra_config"] = {}
|
|
out["api_key_set"] = bool(row.get("api_key"))
|
|
out.pop("api_key", None)
|
|
preset = LLM_PRESETS.get(out.get("provider_type") or "", {})
|
|
out["preset_label"] = preset.get("label", out.get("provider_type"))
|
|
out["needs_key"] = preset.get("needs_key", True)
|
|
return out
|
|
|
|
|
|
def list_providers() -> list[dict[str, Any]]:
|
|
rows = fetch_all("SELECT * FROM llm_providers ORDER BY is_default DESC, is_active DESC, id ASC")
|
|
return [_mask_provider(r) for r in rows if r]
|
|
|
|
|
|
def get_provider(provider_id: int | None = None) -> dict[str, Any] | None:
|
|
if provider_id:
|
|
return fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
|
|
row = fetch_one(
|
|
"SELECT * FROM llm_providers WHERE is_default = TRUE ORDER BY id LIMIT 1"
|
|
)
|
|
if row:
|
|
return row
|
|
row = fetch_one(
|
|
"SELECT * FROM llm_providers WHERE is_active = TRUE ORDER BY id LIMIT 1"
|
|
)
|
|
if row:
|
|
return row
|
|
return fetch_one("SELECT * FROM llm_providers ORDER BY id LIMIT 1")
|
|
|
|
|
|
def resolve_provider(provider_id: int | None = None) -> dict[str, Any]:
|
|
from app.config import settings as app_settings
|
|
|
|
backend = (app_settings.HERMAN_LLM_BACKEND or "auto").lower()
|
|
hermes = _env_hermes_provider()
|
|
if hermes and backend in ("hermes", "auto"):
|
|
# Prefer Hermes when configured (Ollama is often offline)
|
|
if backend == "hermes" or provider_id is None:
|
|
row = get_provider(provider_id) if provider_id else None
|
|
# Explicit provider_id still wins when set
|
|
if provider_id and row:
|
|
pass
|
|
else:
|
|
return hermes
|
|
|
|
row = get_provider(provider_id)
|
|
if not row:
|
|
if hermes:
|
|
return hermes
|
|
env_or = _env_openrouter_provider()
|
|
if env_or:
|
|
return env_or
|
|
return {
|
|
"id": 0,
|
|
"label": "Ollama lokaal",
|
|
"provider_type": "ollama",
|
|
"api_base_url": settings.OLLAMA_URL,
|
|
"api_key": "",
|
|
"model": settings.OLLAMA_MODEL,
|
|
"extra_config": {},
|
|
}
|
|
env_or = _env_openrouter_provider()
|
|
ptype = (row.get("provider_type") or "").lower()
|
|
if hermes and ptype == "ollama" and row.get("is_default") and not (row.get("api_key") or "").strip():
|
|
return hermes
|
|
if env_or and ptype == "openrouter" and not (row.get("api_key") or "").strip():
|
|
merged = dict(row)
|
|
merged["api_key"] = env_or["api_key"]
|
|
if not (merged.get("model") or "").strip():
|
|
merged["model"] = env_or["model"]
|
|
return merged
|
|
if env_or and ptype == "ollama" and row.get("is_default") and not (row.get("api_key") or "").strip():
|
|
if backend in ("router", "openrouter"):
|
|
return env_or
|
|
return row
|
|
|
|
|
|
async def chat_messages(
|
|
messages: list[dict[str, str]],
|
|
*,
|
|
provider_id: int | None = None,
|
|
model: str | None = None,
|
|
timeout: float = 120.0,
|
|
user: str | None = None,
|
|
max_tokens: int | None = None,
|
|
temperature: float | None = None,
|
|
) -> tuple[str, dict[str, Any]]:
|
|
"""Returns (reply_text, meta dict with provider info)."""
|
|
prov = resolve_provider(provider_id)
|
|
ptype = (prov.get("provider_type") or "ollama").lower()
|
|
use_model = model or prov.get("model") or settings.OLLAMA_MODEL
|
|
ollama_timeout = min(timeout, 85.0) if ptype == "ollama" else timeout
|
|
|
|
if ptype == "ollama":
|
|
try:
|
|
reply = await ollama.chat_messages(messages, timeout=ollama_timeout, model=use_model)
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"Ollama timeout/ offline ({exc}). "
|
|
"Voeg DeepSeek of Gemini toe via Instellingen → AI / LLM voor snelle cloud-chat."
|
|
) from exc
|
|
return reply, {
|
|
"provider_id": prov.get("id"),
|
|
"provider_type": "ollama",
|
|
"provider_label": prov.get("label") or "Ollama",
|
|
"model": use_model,
|
|
}
|
|
|
|
api_key = (prov.get("api_key") or "").strip()
|
|
if not api_key:
|
|
raise RuntimeError(
|
|
f"Geen API key voor {prov.get('label') or ptype} — voeg key toe in Instellingen → AI / LLM"
|
|
)
|
|
|
|
base = (prov.get("api_base_url") or "").strip().rstrip("/")
|
|
if not base:
|
|
preset = LLM_PRESETS.get(ptype, {})
|
|
base = (preset.get("api_base_url") or "").rstrip("/")
|
|
if not base:
|
|
raise RuntimeError(f"Geen API URL voor provider {prov.get('label')}")
|
|
|
|
extra = prov.get("extra_config") or {}
|
|
if isinstance(extra, str):
|
|
try:
|
|
extra = json.loads(extra)
|
|
except Exception:
|
|
extra = {}
|
|
|
|
url = f"{base}/chat/completions"
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
if ptype == "openrouter":
|
|
headers["HTTP-Referer"] = extra.get("referer", "https://foodlinkk.local")
|
|
headers["X-Title"] = extra.get("title", "Foodlinkk Command Center")
|
|
|
|
payload: dict[str, Any] = {
|
|
"model": use_model,
|
|
"messages": messages,
|
|
"temperature": float(temperature if temperature is not None else extra.get("temperature", 0.4)),
|
|
"max_tokens": int(max_tokens if max_tokens is not None else extra.get("max_tokens", 2048)),
|
|
}
|
|
# Hermes: `user` = Telegram chat_id → koppelt API-chat aan bestaande memory/sessie
|
|
if user:
|
|
payload["user"] = str(user)
|
|
elif ptype == "hermes":
|
|
payload["user"] = str(getattr(settings, "HERMES_CEO_CHAT_ID", None) or "8859782446")
|
|
|
|
async with httpx.AsyncClient(timeout=timeout) as client:
|
|
resp = await client.post(url, headers=headers, json=payload)
|
|
if resp.status_code >= 400:
|
|
detail = resp.text[:500]
|
|
try:
|
|
detail = resp.json().get("error", {}).get("message", detail)
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError(f"{prov.get('label')}: {detail}")
|
|
data = resp.json()
|
|
choices = data.get("choices") or []
|
|
if not choices:
|
|
raise RuntimeError(f"{prov.get('label')}: leeg antwoord")
|
|
content = (choices[0].get("message") or {}).get("content") or ""
|
|
return content.strip(), {
|
|
"provider_id": prov.get("id"),
|
|
"provider_type": ptype,
|
|
"provider_label": prov.get("label"),
|
|
"model": use_model,
|
|
"hermes_user": payload.get("user"),
|
|
}
|
|
|
|
|
|
|
|
async def stream_chat_messages(
|
|
messages: list[dict[str, str]],
|
|
*,
|
|
provider_id: int | None = None,
|
|
model: str | None = None,
|
|
timeout: float = 90.0,
|
|
user: str | None = None,
|
|
max_tokens: int | None = None,
|
|
temperature: float | None = None,
|
|
):
|
|
"""Yield text deltas from Hermes/OpenAI-compatible streaming chat."""
|
|
prov = resolve_provider(provider_id)
|
|
ptype = (prov.get("provider_type") or "ollama").lower()
|
|
use_model = model or prov.get("model") or settings.OLLAMA_MODEL
|
|
if ptype != "hermes":
|
|
# non-stream fallback: one chunk
|
|
reply, _meta = await chat_messages(
|
|
messages,
|
|
provider_id=provider_id,
|
|
model=model,
|
|
timeout=timeout,
|
|
user=user,
|
|
max_tokens=max_tokens,
|
|
temperature=temperature,
|
|
)
|
|
if reply:
|
|
yield reply
|
|
return
|
|
|
|
api_key = (prov.get("api_key") or "").strip()
|
|
if not api_key:
|
|
raise RuntimeError("Geen Hermes API key")
|
|
base = (prov.get("api_base_url") or "").strip().rstrip("/")
|
|
if not base:
|
|
raise RuntimeError("Geen Hermes API URL")
|
|
extra = prov.get("extra_config") or {}
|
|
if isinstance(extra, str):
|
|
try:
|
|
extra = json.loads(extra)
|
|
except Exception:
|
|
extra = {}
|
|
url = f"{base}/chat/completions"
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
"Accept": "text/event-stream",
|
|
}
|
|
payload: dict[str, Any] = {
|
|
"model": use_model,
|
|
"messages": messages,
|
|
"temperature": float(temperature if temperature is not None else extra.get("temperature", 0.35)),
|
|
"max_tokens": int(max_tokens if max_tokens is not None else extra.get("max_tokens", 512)),
|
|
"stream": True,
|
|
}
|
|
if user:
|
|
payload["user"] = str(user)
|
|
else:
|
|
payload["user"] = str(getattr(settings, "HERMES_CEO_CHAT_ID", None) or "8859782446")
|
|
|
|
async with httpx.AsyncClient(timeout=timeout) as client:
|
|
async with client.stream("POST", url, headers=headers, json=payload) as resp:
|
|
if resp.status_code >= 400:
|
|
body = (await resp.aread()).decode("utf-8", errors="replace")[:500]
|
|
raise RuntimeError(f"Hermes stream fout ({resp.status_code}): {body}")
|
|
async for line in resp.aiter_lines():
|
|
if not line:
|
|
continue
|
|
if line.startswith(":"):
|
|
continue
|
|
if not line.startswith("data:"):
|
|
continue
|
|
data = line[5:].strip()
|
|
if not data or data == "[DONE]":
|
|
if data == "[DONE]":
|
|
break
|
|
continue
|
|
try:
|
|
obj = json.loads(data)
|
|
except Exception:
|
|
continue
|
|
choices = obj.get("choices") or []
|
|
if not choices:
|
|
continue
|
|
delta = (choices[0].get("delta") or {}).get("content")
|
|
if delta:
|
|
yield delta
|
|
|
|
|
|
async def generate(
|
|
prompt: str,
|
|
system: str | None = None,
|
|
*,
|
|
provider_id: int | None = None,
|
|
model: str | None = None,
|
|
timeout: float = 120.0,
|
|
) -> str:
|
|
messages: list[dict[str, str]] = []
|
|
if system:
|
|
messages.append({"role": "system", "content": system})
|
|
messages.append({"role": "user", "content": prompt})
|
|
reply, _meta = await chat_messages(
|
|
messages, provider_id=provider_id, model=model, timeout=timeout
|
|
)
|
|
return reply
|
|
|
|
|
|
async def test_provider(provider_id: int) -> tuple[bool, str]:
|
|
prov = fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
|
|
if not prov:
|
|
return False, "Provider niet gevonden"
|
|
try:
|
|
reply, meta = await chat_messages(
|
|
[{"role": "user", "content": "Antwoord met exact één woord: OK"}],
|
|
provider_id=provider_id,
|
|
timeout=60.0,
|
|
)
|
|
msg = f"{meta.get('provider_label')} · {meta.get('model')} — {reply[:80]}"
|
|
execute(
|
|
"""UPDATE llm_providers SET last_test_status = %s, last_test_message = %s,
|
|
last_test_at = NOW(), updated_at = NOW() WHERE id = %s""",
|
|
("ok", msg, provider_id),
|
|
)
|
|
return True, msg
|
|
except Exception as exc:
|
|
execute(
|
|
"""UPDATE llm_providers SET last_test_status = %s, last_test_message = %s,
|
|
last_test_at = NOW(), updated_at = NOW() WHERE id = %s""",
|
|
("error", str(exc)[:500], provider_id),
|
|
)
|
|
return False, str(exc)
|
|
|
|
|
|
def set_default(provider_id: int) -> None:
|
|
execute("UPDATE llm_providers SET is_default = FALSE, updated_at = NOW()")
|
|
execute(
|
|
"UPDATE llm_providers SET is_default = TRUE, is_active = TRUE, updated_at = NOW() WHERE id = %s",
|
|
(provider_id,),
|
|
)
|