SysOps: voice-agy-webbuilder-backup — 2026-06-23 10:04 UTC

This commit is contained in:
sysops
2026-06-23 10:04:23 +00:00
parent 3bf15c4850
commit 26fe76afdd
165 changed files with 47427 additions and 1264 deletions
+4
View File
@@ -10,6 +10,10 @@ class Settings:
OLLAMA_MODEL: str = os.getenv("OLLAMA_MODEL", "qwen3:8b")
TOOLS_API_URL: str = os.getenv("TOOLS_API_URL", "http://tools-api:8700")
HERMAN_ORCHESTRATOR_URL: str = os.getenv("HERMAN_ORCHESTRATOR_URL", "http://10.4.7.19:8090")
WHISPER_API_URL: str = os.getenv("WHISPER_API_URL", "http://10.4.7.19:8877/v1/audio/transcriptions")
WHISPER_MODEL: str = os.getenv("WHISPER_MODEL", "Systran/faster-whisper-base")
WHISPER_LANGUAGE: str = os.getenv("WHISPER_LANGUAGE", "nl")
HERMES_BUILD_URL: str = os.getenv("HERMES_BUILD_URL", "http://10.4.7.27:8798")
CHROMA_HOST: str = os.getenv("CHROMA_HOST", "chroma")
CHROMA_PORT: int = int(os.getenv("CHROMA_PORT", "8000"))
MINIO_ENDPOINT: str = os.getenv("MINIO_ENDPOINT", "minio:9000")
+21 -1
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.responses import Response
from app.db import close_pool, fetch_all, init_pool
from app.routes import (
@@ -31,7 +32,10 @@ from app.routes import (
packaging,
ops,
ops_api,
revenue_cockpit,
export_intel,
)
from app.routes.revenue_cockpit import api as revenue_cockpit_api
from app.routes.admin_api import admin_router, ai_router, herman_api, voice_api
from app.routes.settings_api import settings_router
from app.routes.agents_api import router as agents_api_router
@@ -41,8 +45,21 @@ from app.routes.projects_api import router as projects_api_router
BASE_DIR = Path(__file__).resolve().parent.parent
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
class NoCacheStaticFiles(StaticFiles):
"""Serve static assets without browser/SW long-lived caching."""
async def get_response(self, path: str, scope) -> Response:
response = await super().get_response(path, scope)
if path.endswith((".css", ".js", ".html")) or "/css/" in path or "/js/" in path:
response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
response.headers["Pragma"] = "no-cache"
response.headers["Expires"] = "0"
return response
app = FastAPI(title="Foodlinkk Command Center", version="2.5.0")
app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static")
app.mount("/static", NoCacheStaticFiles(directory=str(BASE_DIR / "static")), name="static")
for r in (
dashboard.router,
@@ -66,6 +83,9 @@ for r in (
hermes.router,
packaging.router,
ops.router,
revenue_cockpit.router,
export_intel.router,
revenue_cockpit_api,
api.router,
ops_api.router,
admin_router,
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -22,7 +22,7 @@ AGENT_STATUSES = [
@router.get("")
async def agents_page(request: Request):
active_tab = request.query_params.get("tab", "souls")
if active_tab not in {"souls", "mesh", "approvals"}:
if active_tab not in {"souls", "mesh", "terminals", "approvals"}:
active_tab = "souls"
events: list = []
try:
+355 -34
View File
@@ -1,13 +1,18 @@
"""Agents API — souls & activity."""
from __future__ import annotations
import asyncio
import json
from typing import Any, Optional
from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from app.db import fetch_all
from app.services import agent_approvals, agent_souls
from app.db import fetch_all, fetch_one
from app.services import agent_approvals, agent_integration, agent_souls
from app.services.agent_names import normalize_agent_key
from app.services.agent_terminal import execute_terminal_command
router = APIRouter(prefix="/api/agents", tags=["agents-api"])
@@ -35,6 +40,236 @@ class ExecuteBody(BaseModel):
result: dict[str, Any] = Field(default_factory=dict)
class HandoffBody(BaseModel):
from_agent: str
to_agent: str
handoff_type: str = "partner"
payload: dict[str, Any] = Field(default_factory=dict)
correlation_id: Optional[str] = None
class TerminalCommandBody(BaseModel):
command: str = Field(..., min_length=1, max_length=2000)
def _aggregate_agent_stats() -> dict[str, dict[str, Any]]:
from datetime import datetime, timedelta, timezone
rows = fetch_all(
"""
SELECT agent_name, title, status, created_at
FROM agent_events
WHERE created_at >= NOW() - INTERVAL '7 days'
ORDER BY created_at DESC
"""
)
now = datetime.now(timezone.utc)
by_key: dict[str, dict[str, Any]] = {}
for r in rows:
key = normalize_agent_key(str(r.get("agent_name") or ""))
if not key:
continue
bucket = by_key.setdefault(
key,
{"events_6h": 0, "errors_24h": 0, "last_event_at": None, "last_event_title": None},
)
created = r.get("created_at")
if bucket["last_event_at"] is None and created is not None:
bucket["last_event_at"] = created
bucket["last_event_title"] = r.get("title")
if created is not None:
if hasattr(created, "tzinfo") and created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
if created >= now - timedelta(hours=6):
bucket["events_6h"] += 1
if created >= now - timedelta(hours=24) and str(r.get("status") or "") in ("error", "rejected"):
bucket["errors_24h"] += 1
return by_key
def _node_health(events_6h: int, errors_24h: int, event_count: int) -> str:
if events_6h > 0 and errors_24h == 0:
return "healthy"
if events_6h > 0:
return "warn"
if event_count > 0:
return "idle"
return "offline"
@router.get("/collaboration")
def api_collaboration() -> dict[str, Any]:
items = agent_integration.list_collaboration()
return {"items": items, "count": len(items)}
@router.post("/handoff")
def api_create_handoff(body: HandoffBody) -> dict[str, Any]:
try:
handoff = agent_integration.create_handoff(
body.from_agent,
body.to_agent,
handoff_type=body.handoff_type,
payload=body.payload,
correlation_id=body.correlation_id,
)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
return {"ok": True, "handoff": handoff}
def _terminal_payload(ev: dict[str, Any]) -> dict[str, Any]:
if ev.get("created_at") and hasattr(ev["created_at"], "isoformat"):
ev["created_at"] = ev["created_at"].isoformat()
meta = ev.get("metadata")
if isinstance(meta, str):
try:
meta = json.loads(meta)
except Exception:
meta = {}
ev["metadata"] = meta or {}
line_type = "handoff" if "handoff" in str(ev.get("event_type") or "") else "action"
if str(ev.get("event_type") or "").startswith("terminal_"):
line_type = "command" if ev.get("event_type") == "terminal_in" else "output"
if ev["metadata"].get("target_agent"):
line_type = "handoff_out"
if ev["metadata"].get("source_agent"):
line_type = "handoff_in"
if str(ev.get("status") or "") in ("error", "rejected"):
line_type = "error"
return {
"type": line_type,
"id": ev.get("id"),
"agent": normalize_agent_key(ev.get("agent_name")),
"message": ev.get("title") or ev.get("event_type"),
"detail": (ev.get("body") or "")[:500],
"status": ev.get("status"),
"correlation_id": ev["metadata"].get("correlation_id"),
"at": ev.get("created_at"),
}
@router.get("/souls/{agent_key}/terminal/stream")
async def api_agent_terminal_stream(agent_key: str, correlation_id: Optional[str] = None):
key = normalize_agent_key(agent_key)
soul = agent_souls.get_soul(key)
if not soul:
raise HTTPException(404, "Agent not found")
async def event_gen():
last_id = 0
try:
hist = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, metadata, created_at
FROM agent_events
WHERE LOWER(agent_name) = %s
ORDER BY id DESC
LIMIT 30
""",
(key,),
)
for row in reversed(hist):
last_id = max(last_id, int(row["id"]))
payload = _terminal_payload(dict(row))
yield f"data: {json.dumps(payload, default=str)}\n\n"
except Exception:
pass
while True:
try:
if correlation_id:
rows = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, metadata, created_at
FROM agent_events
WHERE metadata->>'correlation_id' = %s
AND id > %s
ORDER BY id ASC
LIMIT 50
""",
(correlation_id, last_id),
)
else:
rows = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, metadata, created_at
FROM agent_events
WHERE LOWER(agent_name) = %s AND id > %s
ORDER BY id ASC
LIMIT 50
""",
(key, last_id),
)
for row in rows:
last_id = max(last_id, int(row["id"]))
payload = _terminal_payload(dict(row))
yield f"data: {json.dumps(payload, default=str)}\n\n"
except Exception as exc:
err = {"type": "error", "message": str(exc)}
yield f"data: {json.dumps(err)}\n\n"
await asyncio.sleep(0.4)
return StreamingResponse(
event_gen(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.get("/live/stream")
async def api_agents_live_stream():
"""Single realtime stream for all agent terminals."""
async def event_gen():
last_id = 0
try:
recent = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, metadata, created_at
FROM agent_events
WHERE created_at >= NOW() - INTERVAL '30 minutes'
ORDER BY id ASC
LIMIT 80
"""
)
for row in recent:
last_id = max(last_id, int(row["id"]))
payload = _terminal_payload(dict(row))
payload["replay"] = True
yield f"data: {json.dumps(payload, default=str)}\n\n"
except Exception:
pass
yield f"data: {json.dumps({'type': 'connected', 'last_id': last_id})}\n\n"
while True:
try:
rows = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, metadata, created_at
FROM agent_events
WHERE id > %s
ORDER BY id ASC
LIMIT 100
""",
(last_id,),
)
for row in rows:
last_id = max(last_id, int(row["id"]))
payload = _terminal_payload(dict(row))
yield f"data: {json.dumps(payload, default=str)}\n\n"
except Exception as exc:
err = {"type": "error", "message": str(exc)}
yield f"data: {json.dumps(err)}\n\n"
await asyncio.sleep(0.25)
return StreamingResponse(
event_gen(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.get("/souls")
def api_list_souls() -> dict[str, Any]:
return {"items": agent_souls.list_souls(), "count": len(agent_souls.list_souls())}
@@ -57,6 +292,16 @@ def api_list_agent_events(agent_key: str, limit: int = 50) -> dict[str, Any]:
return {"agent_key": agent_key.lower(), "items": items, "count": len(items)}
@router.post("/souls/{agent_key}/terminal/command")
async def api_terminal_command(agent_key: str, body: TerminalCommandBody) -> dict[str, Any]:
try:
return await execute_terminal_command(agent_key, body.command.strip())
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
@router.put("/souls/{agent_key}")
def api_update_soul(agent_key: str, body: SoulUpdate) -> dict[str, Any]:
try:
@@ -69,20 +314,7 @@ def api_update_soul(agent_key: str, body: SoulUpdate) -> dict[str, Any]:
@router.get("/mesh")
def api_agents_mesh() -> dict[str, Any]:
souls = agent_souls.list_souls()
stats_rows = fetch_all(
"""
SELECT LOWER(agent_name) AS agent_key,
MAX(created_at) AS last_event_at,
COUNT(*) FILTER (WHERE created_at >= NOW() - INTERVAL '6 hours') AS events_6h,
COUNT(*) FILTER (
WHERE created_at >= NOW() - INTERVAL '24 hours'
AND status IN ('error', 'rejected')
) AS errors_24h
FROM agent_events
GROUP BY LOWER(agent_name)
"""
)
by_key = {str(r["agent_key"]): dict(r) for r in stats_rows}
by_key = _aggregate_agent_stats()
nodes: list[dict[str, Any]] = []
for soul in souls:
@@ -90,33 +322,112 @@ def api_agents_mesh() -> dict[str, Any]:
row = by_key.get(key, {})
events_6h = int(row.get("events_6h") or 0)
errors_24h = int(row.get("errors_24h") or 0)
health = "offline"
if events_6h > 0 and errors_24h == 0:
health = "healthy"
elif events_6h > 0:
health = "warn"
elif int(soul.get("event_count") or 0) > 0:
health = "idle"
health = _node_health(events_6h, errors_24h, int(soul.get("event_count") or 0))
is_active = health == "healthy" and events_6h > 0
node = dict(soul)
node["health"] = health
node["events_6h"] = events_6h
node["errors_24h"] = errors_24h
if row.get("last_event_at") is not None and hasattr(row["last_event_at"], "isoformat"):
node["last_event_at"] = row["last_event_at"].isoformat()
node["is_active"] = is_active
node["last_event_title"] = row.get("last_event_title")
lat = row.get("last_event_at")
if lat is not None and hasattr(lat, "isoformat"):
node["last_event_at"] = lat.isoformat()
nodes.append(node)
edge_rows = fetch_all(
report_edges: list[dict[str, Any]] = []
for node in nodes:
key = str(node.get("agent_key") or "").lower()
if key == "herman" or not node.get("is_active"):
continue
report_edges.append(
{
"source": key,
"target": "herman",
"type": "report",
"active": True,
"weight": int(node.get("events_6h") or 1),
}
)
delegate_rows = fetch_all(
"""
SELECT LOWER(agent_name) AS source_key, COUNT(*) AS weight
SELECT metadata, created_at
FROM agent_events
WHERE created_at >= NOW() - INTERVAL '24 hours'
AND LOWER(agent_name) <> 'herman'
GROUP BY LOWER(agent_name)
ORDER BY weight DESC
WHERE agent_name = 'herman'
AND created_at >= NOW() - INTERVAL '6 hours'
AND (
event_type IN ('openswarm_delegation', 'delegate', 'packaging_delivered', 'telegram_delegation')
OR metadata ? 'delegated'
)
ORDER BY created_at DESC
LIMIT 100
"""
)
edges = [{"source": str(r["source_key"]), "target": "herman", "weight": int(r["weight"])} for r in edge_rows]
return {"nodes": nodes, "edges": edges, "executives": [{"id": "ceo", "label": "CEO", "role": "Aissa"}, {"id": "cto", "label": "CTO", "role": "Platform"}]}
delegate_edges: list[dict[str, Any]] = []
seen_delegate: set[tuple[str, str]] = set()
for r in delegate_rows:
meta = r.get("metadata") or {}
if isinstance(meta, str):
try:
meta = json.loads(meta)
except Exception:
meta = {}
delegated = meta.get("delegated") or meta.get("delegated_agents") or []
if isinstance(delegated, str):
delegated = [delegated]
channel = meta.get("channel") or "herman"
for agent in delegated:
tgt = normalize_agent_key(str(agent))
if not tgt or tgt == "herman":
continue
pair = ("herman", tgt)
if pair in seen_delegate:
continue
seen_delegate.add(pair)
delegate_edges.append(
{
"source": "herman",
"target": tgt,
"type": "delegate",
"active": True,
"channel": channel,
}
)
peer_static = [
{
"source": str(r["from_agent"]),
"target": str(r["to_agent"]),
"type": "static",
"handoff_type": r.get("handoff_type"),
}
for r in agent_integration.list_collaboration()
]
peer_live = agent_integration.peer_edges_live(hours=6)
peer_edges = peer_static + peer_live
herman_active = any(n.get("agent_key") == "herman" and n.get("is_active") for n in nodes)
herman_active = herman_active or bool(delegate_edges) or bool(report_edges)
executive_edges = []
if herman_active:
executive_edges = [
{"source": "herman", "target": "ceo", "type": "executive", "active": True},
{"source": "herman", "target": "cto", "type": "executive", "active": True},
]
return {
"nodes": nodes,
"report_edges": report_edges,
"delegate_edges": delegate_edges,
"peer_edges": peer_edges,
"executive_edges": executive_edges,
"edges": report_edges,
"executives": [
{"id": "ceo", "label": "CEO", "role": "Aissa"},
{"id": "cto", "label": "CTO", "role": "Platform"},
],
}
@router.get("/approvals")
@@ -141,7 +452,17 @@ def api_create_action_request(body: ActionRequestBody) -> dict[str, Any]:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {"ok": True, "request": req, "message": "Wacht op goedkeuring voordat query uitgevoerd mag worden"}
auto = None
rid = (req or {}).get("id")
if rid:
try:
auto = agent_approvals.try_auto_approve_and_execute(int(rid))
except Exception:
auto = None
msg = "Wacht op goedkeuring voordat query uitgevoerd mag worden"
if auto:
msg = "Auto-goedgekeurd en uitgevoerd (SysOps policy)"
return {"ok": True, "request": req, "auto_executed": bool(auto), "message": msg}
@router.post("/approvals/{request_id}/approve")
+59 -1
View File
@@ -145,7 +145,7 @@ async def dashboard(request: Request):
"dashboard.html",
{
"request": request,
"page_title": "Herman · Command Center",
"page_title": "CEO Dashboard",
"kpis": kpis,
"briefing": briefing,
"briefing_payload": _briefing_payload(briefing),
@@ -158,6 +158,64 @@ async def dashboard(request: Request):
)
@router.get("/cto")
async def cto_redirect():
return RedirectResponse(url="/ops", status_code=302)
@router.get("/cto/dashboard")
async def cto_dashboard_legacy(request: Request):
kpis = {
"pending_approvals": _safe_count("agent_events", "status = 'needs_approval'"),
"browser_sessions_24h": 0,
}
try:
kpis["browser_sessions_24h"] = _safe_count(
"browser_sessions", "created_at >= NOW() - INTERVAL '24 hours'"
)
except Exception:
pass
agent_feed: list = []
try:
agent_feed = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, created_at
FROM agent_events ORDER BY created_at DESC LIMIT 25
"""
)
for ev in agent_feed:
if ev.get("created_at"):
ev["created_at"] = ev["created_at"].isoformat()
except Exception:
agent_feed = []
browser_sessions: list = []
try:
browser_sessions = fetch_all(
"""
SELECT id, url, final_url, title, task, status, created_at
FROM browser_sessions ORDER BY created_at DESC LIMIT 8
"""
)
for s in browser_sessions:
if s.get("created_at"):
s["created_at"] = s["created_at"].isoformat()
except Exception:
browser_sessions = []
return templates.TemplateResponse(
"cto-dashboard.html",
{
"request": request,
"page_title": "CTO Dashboard",
"kpis": kpis,
"agent_feed": agent_feed,
"browser_sessions": browser_sessions,
},
)
@router.get("/marketing-redirect")
async def marketing_redirect():
return RedirectResponse(url="/marketing", status_code=302)
+2 -1
View File
@@ -73,7 +73,8 @@ async def documents_page(request: Request):
SELECT filename, storage_path, doc_type, language, word_count,
unique_lemmas, sentiment_label, sentiment_compound,
sentiment_positive, sentiment_negative, sentiment_neutral,
extraction_method, analyzed_at
extraction_method, analyzed_at,
COALESCE(user_labels, '{}') AS user_labels, label_notes, labeled_at
FROM document_analytics
ORDER BY analyzed_at DESC
LIMIT 50
+209
View File
@@ -0,0 +1,209 @@
"""Export Intel Cockpit page + API proxy."""
from __future__ import annotations
import os
from typing import Any, Optional
import httpx
from fastapi import APIRouter, Query, Request
from fastapi.responses import RedirectResponse, StreamingResponse
from fastapi.templating import Jinja2Templates
from pathlib import Path
router = APIRouter()
BASE = Path(__file__).resolve().parent.parent.parent
templates = Jinja2Templates(directory=str(BASE / "templates"))
TOOLS = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/")
async def _proxy(method: str, path: str, **kwargs) -> Any:
url = f"{TOOLS}/export-intel{path}"
async with httpx.AsyncClient(timeout=60.0) as client:
r = await client.request(method, url, **kwargs)
if r.status_code >= 400:
return {"error": r.text, "status": r.status_code}
if "text/csv" in r.headers.get("content-type", ""):
return r
return r.json()
@router.get("/foodlinkk")
def foodlinkk_home():
return RedirectResponse(url="/export-intel", status_code=302)
@router.get("/export-intel")
def export_intel_page(request: Request):
return templates.TemplateResponse(
"export_intel.html",
{"request": request, "page_title": "Wereldexport"},
)
@router.get("/api/export-intel/stats")
async def api_stats(country: Optional[str] = None, region: Optional[str] = None):
params = {k: v for k, v in {"country": country, "region": region}.items() if v}
return await _proxy("GET", "/stats", params=params)
@router.get("/api/export-intel/regions")
async def api_regions():
return await _proxy("GET", "/regions")
@router.get("/api/export-intel/territories")
async def api_territories(region: Optional[str] = None):
params = {"region": region} if region else {}
return await _proxy("GET", "/territories", params=params)
@router.get("/api/export-intel/entities")
async def api_entities(
country: Optional[str] = None,
region: Optional[str] = None,
entity_type: Optional[str] = None,
entity_types: Optional[str] = None,
q: Optional[str] = None,
favorite_only: bool = False,
crm_linked: Optional[bool] = None,
limit: int = 200,
offset: int = 0,
):
params = {k: v for k, v in {
"country": country, "region": region, "entity_type": entity_type,
"entity_types": entity_types, "q": q, "limit": limit, "offset": offset,
"favorite_only": favorite_only, "crm_linked": crm_linked,
}.items() if v is not None and v is not False}
return await _proxy("GET", "/entities", params=params)
@router.get("/api/export-intel/entities/{entity_id}")
async def api_entity(entity_id: int):
return await _proxy("GET", f"/entities/{entity_id}")
@router.get("/api/export-intel/contacts")
async def api_contacts(
country: Optional[str] = None,
entity_type: Optional[str] = None,
has_email: Optional[bool] = None,
q: Optional[str] = None,
limit: int = 200,
offset: int = 0,
):
params: dict[str, Any] = {"limit": limit, "offset": offset}
if country:
params["country"] = country
if entity_type:
params["entity_type"] = entity_type
if has_email is not None:
params["has_email"] = has_email
if q:
params["q"] = q
return await _proxy("GET", "/contacts", params=params)
@router.get("/api/export-intel/contacts/export.csv")
async def api_contacts_export(country: Optional[str] = None, entity_type: Optional[str] = None):
params = {k: v for k, v in {"country": country, "entity_type": entity_type}.items() if v}
url = f"{TOOLS}/export-intel/contacts/export.csv"
async with httpx.AsyncClient(timeout=60.0) as client:
r = await client.get(url, params=params)
return StreamingResponse(
iter([r.content]),
media_type="text/csv",
headers={"Content-Disposition": "attachment; filename=export-intel-contacts.csv"},
)
@router.get("/api/export-intel/caterers/brands")
async def api_caterer_brands():
return await _proxy("GET", "/caterers/brands")
@router.get("/api/export-intel/caterers/presence")
async def api_caterer_presence(country: Optional[str] = None):
params = {"country": country} if country else {}
return await _proxy("GET", "/caterers/presence", params=params)
@router.get("/api/export-intel/gov-sources")
async def api_gov_sources(country: Optional[str] = None):
params = {"country": country} if country else {}
return await _proxy("GET", "/gov-sources", params=params)
@router.get("/api/export-intel/map/bundle")
async def api_map_bundle(
country: Optional[str] = None,
region: Optional[str] = None,
entity_type: Optional[str] = None,
entity_types: Optional[str] = None,
q: Optional[str] = None,
halal_min: Optional[float] = None,
favorite_only: bool = False,
crm_linked: Optional[bool] = None,
):
params = {k: v for k, v in {
"country": country, "region": region, "entity_type": entity_type,
"entity_types": entity_types, "q": q, "halal_min": halal_min,
"favorite_only": favorite_only, "crm_linked": crm_linked,
}.items() if v is not None and v is not False}
return await _proxy("GET", "/map/bundle", params=params)
@router.post("/api/export-intel/entities/favorites")
async def api_set_favorites(request: Request):
body = await request.json()
return await _proxy("POST", "/entities/favorites", json=body)
@router.get("/api/export-intel/entities/favorites")
async def api_list_favorites(country: Optional[str] = None, region: Optional[str] = None, limit: int = 200):
params = {k: v for k, v in {"country": country, "region": region, "limit": limit}.items() if v is not None}
return await _proxy("GET", "/entities/favorites", params=params)
@router.post("/api/export-intel/crm/push")
async def api_crm_push(request: Request):
body = await request.json()
return await _proxy("POST", "/crm/push", json=body)
@router.get("/api/export-intel/crm/pipeline")
async def api_crm_pipeline(country: Optional[str] = None, region: Optional[str] = None, limit: int = 100):
params = {k: v for k, v in {"country": country, "region": region, "limit": limit}.items() if v is not None}
return await _proxy("GET", "/crm/pipeline", params=params)
@router.get("/api/export-intel/halal/markets")
async def api_halal_markets(
region: Optional[str] = None,
country: Optional[str] = None,
limit: int = 30,
):
params = {k: v for k, v in {"region": region, "country": country, "limit": limit}.items() if v is not None}
return await _proxy("GET", "/halal/markets", params=params)
@router.get("/api/export-intel/tenders")
async def api_tenders(country: Optional[str] = None, status: Optional[str] = "open", limit: int = 100):
params = {k: v for k, v in {"country": country, "status": status, "limit": limit}.items() if v is not None}
return await _proxy("GET", "/tenders", params=params)
@router.post("/api/export-intel/sync/{kind}")
async def api_sync(kind: str, request: Request):
body = {}
if request.headers.get("content-type", "").startswith("application/json"):
try:
body = await request.json()
except Exception:
body = {}
if kind not in ("contacts", "caterers", "distributors", "customers", "tenders", "all", "world", "region"):
return {"error": "unknown sync kind"}
timeout = 7200.0 if kind in ("world", "region", "all") else 600.0
async with httpx.AsyncClient(timeout=timeout) as client:
url = f"{TOOLS}/export-intel/sync/{kind}"
r = await client.post(url, json=body)
return r.json()
+3 -3
View File
@@ -22,7 +22,7 @@ async def herman_page(request: Request):
try:
history = _iso_rows(fetch_all(
"""SELECT id, agent_name, title, body, metadata, created_at FROM agent_events
WHERE channel IN ('herman','dashboard') ORDER BY created_at DESC LIMIT 40"""
WHERE channel IN ('herman','dashboard','browser','voice') ORDER BY created_at DESC LIMIT 40"""
))
except Exception:
history = []
@@ -38,7 +38,7 @@ async def herman_chat(request: Request, message: str = Form(...)):
try:
history = _iso_rows(fetch_all(
"""SELECT id, agent_name, title, body, metadata, created_at FROM agent_events
WHERE channel IN ('herman','dashboard') ORDER BY created_at DESC LIMIT 40"""
WHERE channel IN ('herman','dashboard','browser','voice') ORDER BY created_at DESC LIMIT 40"""
))
except Exception:
history = []
@@ -62,7 +62,7 @@ async def herman_briefing_page(request: Request):
try:
history = _iso_rows(fetch_all(
"""SELECT id, agent_name, title, body, metadata, created_at FROM agent_events
WHERE channel IN ('herman','dashboard') ORDER BY created_at DESC LIMIT 40"""
WHERE channel IN ('herman','dashboard','browser','voice') ORDER BY created_at DESC LIMIT 40"""
))
except Exception:
history = []
+230
View File
@@ -0,0 +1,230 @@
"""Revenue Cockpit — page + API."""
from __future__ import annotations
from pathlib import Path
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from pydantic import BaseModel, Field
from app.services import revenue_cockpit as svc
BASE_DIR = Path(__file__).resolve().parent.parent.parent
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
router = APIRouter(tags=["revenue-cockpit"])
api = APIRouter(prefix="/api/revenue-cockpit", tags=["revenue-cockpit-api"])
class ProjectUpdateBody(BaseModel):
name: Optional[str] = None
category: Optional[str] = None
margin_month: Optional[float] = None
margin_year: Optional[float] = None
target_revenue: Optional[float] = None
next_steps: Optional[str] = None
status: Optional[str] = None
priority: Optional[str] = None
row_style: Optional[str] = None
class GoalsUpdateBody(BaseModel):
vision_text: Optional[str] = None
horizon_text: Optional[str] = None
mid_text: Optional[str] = None
tagline: Optional[str] = None
class ObjectiveBody(BaseModel):
title: str = Field(..., min_length=1)
description: Optional[str] = None
priority: str = "normal"
due_date: Optional[str] = None
class ObjectiveUpdateBody(BaseModel):
title: Optional[str] = None
description: Optional[str] = None
status: Optional[str] = None
priority: Optional[str] = None
due_date: Optional[str] = None
class AssignTaskBody(BaseModel):
agent_name: str = Field(..., min_length=1)
title: str = Field(..., min_length=1)
description: Optional[str] = None
objective_id: Optional[int] = None
priority: str = "normal"
delegate_herman: bool = True
class DelegateBody(BaseModel):
message: str = Field(..., min_length=1)
class ImportBody(BaseModel):
path: str = "Succes Sheet .xlsx"
sheet: Optional[str] = "Projects next steps revenue"
replace: bool = True
@router.get("/revenue-cockpit", response_class=HTMLResponse)
async def revenue_cockpit_page(request: Request):
return templates.TemplateResponse(
"revenue_cockpit.html",
{"request": request, "page_title": "Revenue Cockpit"},
)
@api.get("/live")
async def live_from_excel(path: str = "Succes Sheet .xlsx", sheet: Optional[str] = "Projects next steps revenue"):
"""Leading data source: fresh parse from NAS Excel."""
try:
parsed = await svc.fetch_excel_parse_async(path, sheet)
except Exception as exc:
raise HTTPException(502, str(exc)) from exc
projects = parsed.get("projects") or []
margin_month = sum(p.get("margin_month") or 0 for p in projects)
margin_year = sum(p.get("margin_year") or 0 for p in projects)
by_style: dict[str, int] = {}
by_category: dict[str, int] = {}
for p in projects:
by_style[p.get("row_style") or "white"] = by_style.get(p.get("row_style") or "white", 0) + 1
by_category[p.get("category") or "deal"] = by_category.get(p.get("category") or "deal", 0) + 1
return {
"ok": True,
"source": "excel",
**parsed,
"aggregates": {
"total_margin_month": margin_month,
"total_margin_year": margin_year,
"with_margin": sum(1 for p in projects if p.get("margin_month")),
"by_style": by_style,
"by_category": by_category,
},
}
@api.get("/dashboard")
async def dashboard():
return {"ok": True, "stats": svc.dashboard_stats(), "projects": svc.list_projects()}
@api.get("/projects")
def list_projects(status: Optional[str] = None):
return {"ok": True, "items": svc.list_projects(status)}
@api.get("/projects/{project_id}")
def get_project(project_id: int):
p = svc.get_project(project_id)
if not p:
raise HTTPException(404, "Project not found")
return {"ok": True, "project": p}
@api.patch("/projects/{project_id}")
def patch_project(project_id: int, body: ProjectUpdateBody):
data = body.model_dump(exclude_unset=True)
row_style = data.pop("row_style", None)
if row_style is not None:
svc.set_project_row_style(project_id, row_style)
p = svc.update_project(project_id, data)
if not p:
raise HTTPException(404, "Project not found")
svc.take_snapshot()
return {"ok": True, "project": p}
@api.patch("/goals")
def patch_goals(body: GoalsUpdateBody):
data = body.model_dump(exclude_unset=True)
g = svc.update_goals(data)
if not g:
raise HTTPException(404, "Goals not found")
return {"ok": True, "goals": g}
@api.post("/projects/{project_id}/objectives")
def add_objective(project_id: int, body: ObjectiveBody):
try:
obj = svc.create_objective(project_id, body.title, body.description, body.priority)
except Exception as exc:
raise HTTPException(400, str(exc)) from exc
svc.take_snapshot()
return {"ok": True, "objective": obj}
@api.patch("/objectives/{objective_id}")
def patch_objective(objective_id: int, body: ObjectiveUpdateBody):
data = body.model_dump(exclude_unset=True)
obj = svc.update_objective(objective_id, data)
if not obj:
raise HTTPException(404, "Objective not found")
svc.take_snapshot()
return {"ok": True, "objective": obj}
@api.post("/projects/{project_id}/assign-task")
def assign_task(project_id: int, body: AssignTaskBody):
try:
task = svc.assign_agent_task(
project_id,
body.agent_name,
body.title,
body.description,
body.objective_id,
body.priority,
body.delegate_herman,
)
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
svc.take_snapshot()
return {"ok": True, "task": task}
@api.post("/projects/{project_id}/delegate")
async def delegate_project(project_id: int, body: DelegateBody):
try:
result = await svc.delegate_via_herman(project_id, body.message)
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
return result
@api.get("/tasks")
def list_tasks(limit: int = 50):
return {"ok": True, "items": svc.list_agent_tasks(limit)}
@api.get("/snapshots")
def snapshots(limit: int = 90):
return {"ok": True, "items": svc.list_snapshots(limit)}
@api.post("/snapshot")
def create_snapshot():
snap = svc.take_snapshot()
return {"ok": True, "snapshot": snap}
@api.post("/import-from-excel")
async def import_from_excel(body: ImportBody):
try:
parsed = await svc.fetch_excel_parse_async(body.path, body.sheet)
result = svc.import_from_parsed(parsed, imported_by="ceo", replace=body.replace)
return {"ok": True, **result, "preview": {"project_count": parsed.get("project_count"), "sheet": parsed.get("sheet_name")}}
except Exception as exc:
raise HTTPException(502, f"Import failed: {exc}") from exc
@api.get("/preview-excel")
async def preview_excel(path: str = "Succes Sheet .xlsx", sheet: Optional[str] = "Projects next steps revenue"):
try:
parsed = await svc.fetch_excel_parse_async(path, sheet)
return {"ok": True, **parsed}
except Exception as exc:
raise HTTPException(502, str(exc)) from exc
+1 -1
View File
@@ -10,7 +10,7 @@ templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
@router.get("/settings")
async def settings_page(request: Request, tab: str = "email"):
allowed = ("email", "general", "permissions", "social")
allowed = ("email", "general", "permissions", "social", "ai")
return templates.TemplateResponse(
"settings.html",
{
+115
View File
@@ -381,3 +381,118 @@ def update_permission(module_key: str, body: PermissionBody) -> dict[str, Any]:
def grant_all_permissions() -> dict[str, Any]:
n = agent_souls.grant_all_permissions()
return {"ok": True, "granted_count": n, "message": f"Herman heeft nu {n} module-rechten"}
# ── LLM providers ──────────────────────────────────────────────────────────
from app.services import llm_router
class LlmProviderBody(BaseModel):
label: str = Field(..., max_length=128)
provider_type: str = Field(default="deepseek", max_length=48)
api_base_url: Optional[str] = None
api_key: Optional[str] = None
model: str = Field(default="deepseek-chat", max_length=128)
is_active: bool = True
is_default: bool = False
extra_config: dict[str, Any] = Field(default_factory=dict)
@settings_router.get("/llm/presets")
def llm_presets() -> dict[str, Any]:
return {"presets": llm_router.list_presets()}
@settings_router.get("/llm")
def llm_list() -> dict[str, Any]:
items = llm_router.list_providers()
default = next((i for i in items if i.get("is_default")), items[0] if items else None)
return {"providers": items, "default": default, "presets": llm_router.list_presets()}
@settings_router.post("/llm")
def llm_create(body: LlmProviderBody) -> dict[str, Any]:
preset = llm_router.LLM_PRESETS.get(body.provider_type, {})
base_url = (body.api_base_url or preset.get("api_base_url") or "").strip() or None
model = body.model or (preset.get("models") or ["deepseek-chat"])[0]
if body.is_default:
execute("UPDATE llm_providers SET is_default = FALSE, updated_at = NOW()")
row = fetch_one(
"""INSERT INTO llm_providers (
label, provider_type, api_base_url, api_key, model,
is_active, is_default, extra_config, updated_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, NOW())
RETURNING *""",
(
body.label,
body.provider_type,
base_url,
body.api_key or "",
model,
body.is_active,
body.is_default,
json.dumps(body.extra_config or {}),
),
)
return {"provider": llm_router._mask_provider(row)}
@settings_router.put("/llm/{provider_id}")
def llm_update(provider_id: int, body: LlmProviderBody) -> dict[str, Any]:
existing = fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
if not existing:
raise HTTPException(404, "Provider niet gevonden")
preset = llm_router.LLM_PRESETS.get(body.provider_type, {})
base_url = (body.api_base_url or preset.get("api_base_url") or existing.get("api_base_url") or "").strip() or None
api_key = body.api_key if body.api_key else existing.get("api_key") or ""
if body.is_default:
execute("UPDATE llm_providers SET is_default = FALSE, updated_at = NOW()")
row = fetch_one(
"""UPDATE llm_providers SET
label = %s, provider_type = %s, api_base_url = %s, api_key = %s, model = %s,
is_active = %s, is_default = %s, extra_config = %s::jsonb, updated_at = NOW()
WHERE id = %s RETURNING *""",
(
body.label,
body.provider_type,
base_url,
api_key,
body.model,
body.is_active,
body.is_default,
json.dumps(body.extra_config or {}),
provider_id,
),
)
return {"provider": llm_router._mask_provider(row)}
@settings_router.delete("/llm/{provider_id}")
def llm_delete(provider_id: int) -> dict[str, Any]:
row = fetch_one("SELECT is_default FROM llm_providers WHERE id = %s", (provider_id,))
if not row:
raise HTTPException(404, "Provider niet gevonden")
execute("DELETE FROM llm_providers WHERE id = %s", (provider_id,))
if row.get("is_default"):
execute(
"""UPDATE llm_providers SET is_default = TRUE, updated_at = NOW()
WHERE id = (SELECT id FROM llm_providers ORDER BY id LIMIT 1)"""
)
return {"ok": True}
@settings_router.post("/llm/{provider_id}/activate")
def llm_activate(provider_id: int) -> dict[str, Any]:
llm_router.set_default(provider_id)
row = fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
return {"ok": True, "provider": llm_router._mask_provider(row)}
@settings_router.post("/llm/{provider_id}/test")
async def llm_test(provider_id: int) -> dict[str, Any]:
ok, msg = await llm_router.test_provider(provider_id)
if not ok:
raise HTTPException(502, msg)
return {"ok": True, "message": msg}
File diff suppressed because it is too large Load Diff
+45
View File
@@ -191,3 +191,48 @@ def require_approved(request_id: int) -> dict[str, Any]:
if req.get("status") != "approved":
raise PermissionError(f"Request status is {req.get('status')}, approval required")
return req
def _auto_approve_policy() -> dict[str, bool]:
try:
row = fetch_one("SELECT value FROM app_settings WHERE key = 'sysops_auto_approve'")
if row and row.get("value"):
val = row["value"]
if isinstance(val, str):
return json.loads(val)
return dict(val)
except Exception:
pass
return {"maintenance_scan": True, "config_backup": True}
def try_auto_approve_and_execute(request_id: int) -> dict[str, Any] | None:
"""Auto-approve low-risk SysOps requests when policy allows."""
req = get_request(request_id)
if not req or req.get("status") != "pending":
return None
key = str(req.get("agent_key") or "").lower()
action = str(req.get("action_type") or "")
policy = _auto_approve_policy()
if key != "sysops" or not policy.get(action):
return None
approved = approve_request(request_id, approved_by="auto_policy")
import os
import httpx
tools_url = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/")
result_payload: dict[str, Any] = {}
try:
if action == "config_backup":
with httpx.Client(timeout=180.0) as client:
resp = client.post(f"{tools_url}/ops/backup/run", json={"approval_request_id": request_id})
result_payload = resp.json() if resp.status_code < 500 else {"ok": False, "detail": resp.text}
elif action == "maintenance_scan":
with httpx.Client(timeout=120.0) as client:
resp = client.post(f"{tools_url}/ops/maintenance/scan")
result_payload = resp.json() if resp.status_code < 500 else {"ok": False, "detail": resp.text}
except Exception as exc:
result_payload = {"ok": False, "detail": str(exc)}
executed = mark_executed(request_id, result_payload)
return {"approved": approved, "executed": executed, "auto": True}
+48
View File
@@ -0,0 +1,48 @@
"""Log agent activity to agent_events for terminals and audit."""
from __future__ import annotations
import json
from typing import Any, Optional
from app.db import fetch_one
from app.services.agent_names import normalize_agent_key
def log_agent_event(
agent_name: str,
event_type: str,
title: str,
body: str = "",
*,
agent_type: Optional[str] = None,
status: str = "completed",
channel: str = "cockpit",
metadata: Optional[dict[str, Any]] = None,
) -> Optional[dict[str, Any]]:
key = normalize_agent_key(agent_name)
if not key:
return None
meta = json.dumps(metadata or {})
row = fetch_one(
"""
INSERT INTO agent_events (agent_name, agent_type, event_type, title, body, status, channel, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id, agent_name, event_type, title, body, status, channel, metadata, created_at
""",
(
key,
agent_type or key,
event_type,
title[:255],
(body or "")[:8000],
status,
channel,
meta,
),
)
if not row:
return None
out = dict(row)
if out.get("created_at") and hasattr(out["created_at"], "isoformat"):
out["created_at"] = out["created_at"].isoformat()
return out
+140
View File
@@ -0,0 +1,140 @@
"""Agent handoffs and collaboration matrix."""
from __future__ import annotations
import json
import uuid
from typing import Any, Optional
from app.db import execute, fetch_all, fetch_one
from app.services.agent_names import normalize_agent_key
def _serialize(row: dict[str, Any] | None) -> dict[str, Any] | None:
if not row:
return None
out = dict(row)
for k, v in list(out.items()):
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
elif k == "correlation_id" and v is not None:
out[k] = str(v)
return out
def list_collaboration() -> list[dict[str, Any]]:
rows = fetch_all(
"SELECT from_agent, to_agent, handoff_type, description FROM agent_collaboration ORDER BY from_agent, to_agent"
)
return [dict(r) for r in rows]
def create_handoff(
from_agent: str,
to_agent: str,
*,
handoff_type: str = "partner",
payload: dict[str, Any] | None = None,
correlation_id: str | None = None,
status: str = "completed",
) -> dict[str, Any]:
src = normalize_agent_key(from_agent)
dst = normalize_agent_key(to_agent)
if not src or not dst:
raise ValueError("from_agent and to_agent required")
cid = correlation_id or str(uuid.uuid4())
try:
uuid.UUID(str(cid))
except ValueError:
cid = str(uuid.uuid4())
row = fetch_one(
"""
INSERT INTO agent_handoffs (correlation_id, from_agent, to_agent, handoff_type, payload, status, completed_at)
VALUES (%s::uuid, %s, %s, %s, %s::jsonb, %s, CASE WHEN %s = 'completed' THEN NOW() ELSE NULL END)
RETURNING *
""",
(cid, src, dst, handoff_type, json.dumps(payload or {}), status, status),
)
handoff = _serialize(row) or {}
meta = {
"correlation_id": cid,
"handoff_id": handoff.get("id"),
"target_agent": dst,
"source_agent": src,
"handoff_type": handoff_type,
}
_log_handoff_events(src, dst, handoff_type, payload or {}, meta, cid)
return handoff
def _log_handoff_events(
src: str,
dst: str,
handoff_type: str,
payload: dict[str, Any],
meta: dict[str, Any],
cid: str,
) -> None:
title_out = f"{src}{dst}: {handoff_type}"
title_in = f"Handoff van {src}: {handoff_type}"
body = json.dumps(payload)[:2000] if payload else ""
for agent, etype, title, extra in (
(src, "handoff_out", title_out, {"target_agent": dst}),
(dst, "handoff_in", title_in, {"source_agent": src}),
):
try:
execute(
"""
INSERT INTO agent_events (agent_name, agent_type, event_type, title, body, status, channel, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb)
""",
(
agent,
"agent_handoff",
etype,
title[:255],
body,
"completed",
"agents",
json.dumps({**meta, **extra}),
),
)
except Exception:
pass
def recent_handoffs(hours: int = 6) -> list[dict[str, Any]]:
rows = fetch_all(
"""
SELECT * FROM agent_handoffs
WHERE created_at >= NOW() - make_interval(hours => %s)
ORDER BY created_at DESC
LIMIT 200
""",
(max(1, min(hours, 168)),),
)
return [_serialize(r) for r in rows if r]
def peer_edges_live(hours: int = 6) -> list[dict[str, Any]]:
rows = fetch_all(
"""
SELECT from_agent, to_agent, handoff_type, COUNT(*) AS weight,
MAX(correlation_id::text) AS correlation_id
FROM agent_handoffs
WHERE created_at >= NOW() - make_interval(hours => %s)
AND status = 'completed'
GROUP BY from_agent, to_agent, handoff_type
""",
(max(1, min(hours, 168)),),
)
return [
{
"source": str(r["from_agent"]),
"target": str(r["to_agent"]),
"type": "live",
"handoff_type": r.get("handoff_type"),
"weight": int(r["weight"] or 1),
"correlation_id": r.get("correlation_id"),
}
for r in rows
]
+22
View File
@@ -0,0 +1,22 @@
"""Normalize legacy agent_name values to agent_souls keys."""
from __future__ import annotations
AGENT_NAME_MAP: dict[str, str] = {
"retail_360": "retail",
"retail_scraper": "retail",
"retail_crm": "retail",
"retail_intel": "retail",
"rss_feeds": "marketing",
"wholesale_scraper": "sourcing",
"halal_registry": "halal",
"branch_scraper": "sourcing",
"hermes": "herman",
"herman_delegate": "herman",
}
def normalize_agent_key(name: str | None) -> str:
key = (name or "").strip().lower()
if not key:
return ""
return AGENT_NAME_MAP.get(key, key)
+456
View File
@@ -0,0 +1,456 @@
"""Interactive agent terminal — parse and execute whitelisted commands."""
from __future__ import annotations
import json
import os
from typing import Any, Optional
import httpx
from app.db import fetch_all, fetch_one
from app.services import agent_integration, agent_souls, herman, webbuilder_agent
from app.services.agent_events_log import log_agent_event
from app.services.agent_names import normalize_agent_key
TOOLS_URL = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/")
BROWSER_URL = os.getenv("BROWSER_AGENT_URL", "http://browser-agent:7790").rstrip("/")
Line = dict[str, Any]
GLOBAL_HELP = [
"help — dit overzicht",
"status — huidige status & taak",
"history — laatste events (in terminal)",
"handoff <agent> <type> [notitie]",
"say <bericht> — log een notitie",
"ask <vraag> — AI-antwoord in rol van deze agent",
]
AGENT_HELP: dict[str, list[str]] = {
"herman": [
"briefing — dagrapport genereren",
"<tekst> — vraag / opdracht aan Herman",
],
"browser": [
"monitor add <url> — site toevoegen aan monitor",
"monitor list — actieve sites",
"browse <url> — pagina openen",
"crawl — alle sites crawlen + parse-overzicht",
"parse — laatste parse-resultaten",
"parse <id> — parse van specifieke site",
"intel — hype & trend analyse (samenvatting)",
"intel <zoekterm> — filter analyse",
],
"research": ["run — research pipeline starten", "briefs — recente briefs"],
"retail": ["rss — retail RSS verversen", "scores — halal opportunity scores"],
"sysops": [
"backup — config backup (approval)",
"scan — maintenance scan",
"topology — infra overzicht",
"status — sysops status",
],
"marketing": ["reco — aanbevelingen genereren"],
"packaging": ["design <brief> — packaging opdracht (via Herman)"],
"webbuilder": [
"build <opdracht> — website bouwen (agy op Hermes)",
"projects — bestaande website-projecten",
"status [project] — build-status",
"preview [project] — preview-URL",
],
"email": ["sync — inbox sync trigger"],
"finance": ["margins — marge-overzicht (placeholder)"],
}
def _line(msg: str, *, detail: str = "", line_type: str = "output") -> Line:
return {"type": line_type, "message": msg, "detail": detail}
def _help_lines(key: str) -> list[Line]:
lines = [_line("Beschikbare commando's:", line_type="output")]
for h in GLOBAL_HELP:
lines.append(_line(" " + h, line_type="output"))
extra = AGENT_HELP.get(key, [])
if extra:
lines.append(_line(f"{key}", line_type="output"))
for h in extra:
lines.append(_line(" " + h, line_type="output"))
return lines
def _status_lines(key: str, soul: dict[str, Any]) -> list[Line]:
recent = soul.get("recent_events") or []
last = recent[0] if recent else {}
lines = [
_line(f"{soul.get('display_name') or key} · {soul.get('role_title') or 'agent'}"),
_line(f"Status: {last.get('status') or soul.get('current_status') or 'standby'}"),
_line(f"Taak: {last.get('title') or soul.get('current_task') or ''}"),
_line(f"Events totaal: {soul.get('event_count') or 0}"),
]
resp = (soul.get("responsibilities") or "").strip()
if resp:
lines.append(_line(resp[:220], detail=resp if len(resp) > 220 else ""))
return lines
async def _tools_post(path: str, payload: dict | None = None, timeout: float = 180.0) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(f"{TOOLS_URL}{path}", json=payload or {})
try:
data = resp.json()
except Exception:
data = {"ok": False, "detail": resp.text[:500]}
if resp.status_code >= 400:
data.setdefault("ok", False)
data.setdefault("detail", resp.text[:500])
return data
async def _tools_get(path: str, timeout: float = 60.0) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.get(f"{TOOLS_URL}{path}")
try:
return resp.json()
except Exception:
return {"ok": False, "detail": resp.text[:500]}
async def _browser_post(path: str, payload: dict, timeout: float = 120.0) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(f"{BROWSER_URL}{path}", json=payload)
try:
return resp.json()
except Exception:
return {"ok": False, "detail": resp.text[:500]}
def _handoff(src: str, dst_raw: str, htype: str, note: str) -> list[Line]:
dst = normalize_agent_key(dst_raw)
if not dst:
return [_line("Onbekende agent: " + dst_raw, line_type="error")]
payload = {"note": note} if note else {}
try:
agent_integration.create_handoff(src, dst, handoff_type=htype, payload=payload)
return [_line(f"Handoff → {dst} ({htype})" + (f": {note}" if note else ""))]
except Exception as exc:
return [_line("Handoff mislukt: " + str(exc), line_type="error")]
async def _agent_ask(key: str, soul: dict[str, Any], question: str) -> list[Line]:
name = soul.get("display_name") or key
role = soul.get("role_title") or ""
prompt = (
f"Je bent {name} ({key}), {role}. "
f"Beantwoord kort en actiegericht in het Nederlands.\n\nOpdracht: {question}"
)
result = await herman.chat(prompt, channel=f"terminal:{key}")
reply = (result.get("reply") or "").strip()
delegated = result.get("delegated_agents") or []
lines = [_line(reply or "(geen antwoord)", detail=reply)]
if delegated:
lines.append(_line("Gedelegeerd: " + ", ".join(delegated), line_type="handoff_out"))
log_agent_event(
key,
"terminal_out",
reply[:255] if reply else "Antwoord",
reply[:4000],
channel="terminal",
metadata={"delegated": delegated, "question": question[:500]},
)
return lines
def _format_crawl_results(result: dict[str, Any]) -> list[Line]:
if result.get("message") and not result.get("results"):
return [_line(result["message"], line_type="output")]
lines: list[Line] = [
_line(
f"Crawl klaar — {result.get('sites', 0)} site(s), "
f"{result.get('changed', 0)} wijziging(en), {result.get('errors', 0)} fout(en)"
)
]
for r in result.get("results") or []:
if r.get("status") == "ERROR":
lines.append(_line(f"{r.get('name') or r.get('url')}: {r.get('error')}", line_type="error"))
continue
flag = "" if r.get("changed") else ""
lines.append(
_line(
f"{flag} {r.get('title') or r.get('name')}{r.get('word_count', 0)} woorden, "
f"{r.get('links_count', 0)} links",
detail=(r.get("excerpt") or "")[:200],
)
)
headings = r.get("headings") or []
if headings:
lines.append(_line(" H: " + " · ".join(headings[:4])[:180], line_type="output"))
for link in (r.get("links_sample") or [])[:3]:
lbl = link.get("label") or link.get("href") or ""
lines.append(_line("" + lbl[:70], line_type="output"))
return lines
def _format_parse_items(items: list[dict[str, Any]]) -> list[Line]:
if not items:
return [_line("Geen parse-resultaten — voer eerst crawl uit.")]
lines: list[Line] = [_line(f"Parse-overzicht ({len(items)} pagina's):")]
for p in items:
lines.append(
_line(
f"#{p.get('site_id') or ''} {p.get('title') or p.get('url')}"
f"{p.get('word_count', 0)} woorden",
detail=(p.get("excerpt") or "")[:200],
)
)
if p.get("meta_description"):
lines.append(_line(" desc: " + str(p["meta_description"])[:120], line_type="output"))
for h in (p.get("headings") or [])[:3]:
lines.append(_line(" · " + h[:90], line_type="output"))
return lines
async def _run_agent_command(key: str, soul: dict[str, Any], cmd: str, args: list[str], raw: str) -> list[Line]:
if key == "herman":
if cmd == "briefing":
try:
content = await herman.generate_briefing()
preview = (content or "")[:400]
log_agent_event("herman", "terminal_out", "Briefing gegenereerd", content[:4000], channel="terminal")
return [_line("CEO briefing gegenereerd.", detail=preview)]
except Exception as exc:
return [_line("Briefing mislukt: " + str(exc), line_type="error")]
result = await herman.chat(raw, channel="terminal")
reply = (result.get("reply") or "").strip()
delegated = result.get("delegated_agents") or []
lines = [_line(reply or "(geen antwoord)", detail=reply)]
if delegated:
lines.append(_line("" + ", ".join(delegated), line_type="handoff_out"))
log_agent_event("herman", "terminal_out", reply[:255] if reply else "Antwoord", reply[:4000], channel="terminal")
return lines
if key == "browser":
if cmd == "monitor" and args and args[0].lower() == "add" and len(args) >= 2:
url = args[1]
from app.services.monitor import add_site
site = add_site(url, name=url[:80])
log_agent_event(
"browser",
"monitor_site_added",
f"Terminal: site toegevoegd {url[:120]}",
"",
channel="terminal",
metadata={"url": url, "site_id": site.get("id")},
)
return [_line(f"Monitor: {url} toegevoegd (id {site.get('id')})")]
if cmd == "monitor" and args and args[0].lower() == "list":
rows = fetch_all(
"SELECT id, url, is_active FROM monitored_sites ORDER BY id DESC LIMIT 15"
)
if not rows:
return [_line("Geen monitor-sites.")]
return [_line(f"#{r['id']} {'' if r.get('is_active') else ''} {r['url']}") for r in rows]
if cmd == "browse" and args:
url = args[0]
data = await _browser_post("/browse", {"url": url, "purpose": "terminal"})
log_agent_event("browser", "browse", f"Terminal browse: {url[:120]}", json.dumps(data)[:500], channel="terminal")
title = data.get("title") or data.get("url") or url
return [_line(f"Browse OK: {title}")]
if cmd == "crawl":
from app.services.monitor import trigger_crawl
site_id = int(args[0]) if args and str(args[0]).isdigit() else None
result = trigger_crawl(site_id)
log_agent_event(
"browser",
"monitor_crawl",
f"Terminal crawl ({result.get('sites', 0)} sites)",
json.dumps([{k: r.get(k) for k in ("url", "title", "word_count", "status")} for r in result.get("results", [])])[:800],
channel="terminal",
)
return _format_crawl_results(result)
if cmd == "parse":
from app.services.monitor import list_parse_results
sid = int(args[0]) if args and str(args[0]).isdigit() else None
return _format_parse_items(list_parse_results(site_id=sid, limit=10))
if cmd == "intel":
from app.services.monitor import build_parse_intelligence
q = " ".join(args) if args else None
data = build_parse_intelligence(query=q, limit=20)
lines = [
_line(
f"Analyse: {data['summary']['pages']} pagina's · "
f"{data['summary']['total_words']} woorden · "
f"{data['summary']['themes_detected']} trend-termen"
),
_line("Open Browser → tab Parse Analyse voor volledig overzicht", line_type="output"),
]
for t in (data.get("hype_terms") or [])[:12]:
flag = "" if t.get("cross_site") else ""
lines.append(
_line(f"{flag}{t['term']} ({t['score']}) — {', '.join(t.get('sites') or [])[:3]}", line_type="output")
)
return lines
if key == "research":
if cmd == "run":
data = await _tools_post("/research/run")
ok = data.get("ok", True)
log_agent_event("research", "terminal_action", "Research run (terminal)", json.dumps(data)[:500], channel="terminal")
return [_line("Research pipeline: " + ("OK" if ok else "fout"), detail=json.dumps(data)[:300])]
if cmd == "briefs":
data = await _tools_get("/research/briefs")
items = data if isinstance(data, list) else data.get("items") or []
if not items:
return [_line("Geen briefs.")]
return [_line(str(b.get("title") or b.get("id") or b))[:120] for b in items[:8]]
if key == "retail":
if cmd == "rss":
data = await _tools_post("/retail/rss/refresh")
log_agent_event("retail", "terminal_action", "RSS refresh (terminal)", json.dumps(data)[:300], channel="terminal")
return [_line("RSS verversd.", detail=json.dumps(data)[:200])]
if cmd == "scores":
data = await _tools_post("/retail/compute-opportunities")
return [_line("Opportunity scores bijgewerkt.", detail=json.dumps(data)[:200])]
if key == "sysops":
if cmd == "backup":
data = await _tools_post("/ops/backup/run")
log_agent_event("sysops", "terminal_action", "Backup (terminal)", json.dumps(data)[:400], channel="terminal")
return [_line("Backup uitgevoerd.", detail=json.dumps(data)[:300])]
if cmd == "scan":
data = await _tools_post("/ops/maintenance/scan")
return [_line("Maintenance scan klaar.", detail=json.dumps(data)[:300])]
if cmd == "topology":
data = await _tools_get("/ops/topology")
return [_line("Topology geladen.", detail=json.dumps(data)[:400])]
if cmd == "status":
data = await _tools_get("/ops/status")
return [_line(json.dumps(data)[:350], detail=json.dumps(data)[:800])]
if key == "marketing" and cmd == "reco":
data = await _tools_post("/recommendations/generate")
return [_line("Aanbevelingen gegenereerd.", detail=json.dumps(data)[:300])]
if key == "packaging" and cmd == "design":
brief = " ".join(args) or raw
result = await herman.chat(f"Packaging design: {brief}", channel="terminal:packaging")
reply = (result.get("reply") or "").strip()
return [_line(reply, detail=reply)]
if key == "webbuilder":
if cmd == "build":
brief = rest or raw
if not brief:
return [_line("Gebruik: build <opdracht>", line_type="error")]
try:
outcome = await webbuilder_agent.generate_from_message(brief, channel="terminal", wait=False)
proj = outcome.get("project") or "?"
preview = outcome.get("preview_url") or ""
return [
_line(f"Build gestart: {proj}"),
_line(f"Preview: {preview}", line_type="output"),
_line("Volg voortgang live in dit terminal-venster.", line_type="output"),
]
except Exception as exc:
return [_line("Build mislukt: " + str(exc), line_type="error")]
if cmd == "projects":
data = await webbuilder_agent.list_projects()
items = data.get("items") or []
if not items:
return [_line("Geen projecten op Hermes/NAS.")]
return [
_line(
f"{p.get('slug')}{p.get('files', 0)} bestand(en)"
+ (" · index.html ✓" if p.get("has_index") else "")
)
for p in items
]
if cmd == "status":
slug = args[0] if args else ""
if not slug:
return [_line("Gebruik: status <project>", line_type="error")]
st = await webbuilder_agent.get_build_status(slug)
lines = [_line(f"{st.get('project')} · {st.get('status')}{st.get('message') or ''}")]
tail = (st.get("log_tail") or "")[:300]
if tail:
lines.append(_line(tail, line_type="output"))
return lines
if cmd == "preview":
slug = args[0] if args else ""
if slug:
st = await webbuilder_agent.get_build_status(slug)
url = st.get("preview_url") or webbuilder_agent.HERMES_PREVIEW_BASE
else:
url = webbuilder_agent.HERMES_PREVIEW_BASE
return [_line(f"Preview: {url}")]
return await _agent_ask(key, soul, raw)
async def execute_terminal_command(
agent_key: str,
command: str,
*,
issued_by: str = "ceo",
) -> dict[str, Any]:
key = normalize_agent_key(agent_key)
soul = agent_souls.get_soul(key)
if not soul:
raise ValueError("Agent niet gevonden")
raw = (command or "").strip()
if not raw:
raise ValueError("Leeg commando")
log_agent_event(
key,
"terminal_in",
raw[:255],
"",
channel="terminal",
metadata={"issued_by": issued_by, "command": raw[:500]},
)
parts = raw.split()
cmd = parts[0].lower()
args = parts[1:]
rest = " ".join(args)
if cmd in ("help", "?", "h"):
lines = _help_lines(key)
elif cmd == "status":
lines = _status_lines(key, soul)
elif cmd == "handoff":
if len(args) < 2:
lines = [_line("Gebruik: handoff <agent> <type> [notitie]", line_type="error")]
else:
note = " ".join(args[2:]) if len(args) > 2 else ""
lines = _handoff(key, args[0], args[1], note)
elif cmd == "say":
if not rest:
lines = [_line("Gebruik: say <bericht>", line_type="error")]
else:
log_agent_event(key, "terminal_note", rest[:255], rest, channel="terminal")
lines = [_line("Genoteerd.")]
elif cmd == "ask":
if not rest:
lines = [_line("Gebruik: ask <vraag>", line_type="error")]
else:
lines = await _agent_ask(key, soul, rest)
elif cmd == "history":
events = agent_souls.list_agent_events(key, limit=8)
if not events:
lines = [_line("Geen history.")]
else:
lines = []
for ev in reversed(events):
t = (ev.get("created_at") or "")[11:19] or "--:--"
lines.append(_line(f"{t} {ev.get('title') or ev.get('event_type')}"))
else:
lines = await _run_agent_command(key, soul, cmd, args, raw)
return {"ok": True, "agent_key": key, "command": raw, "lines": lines}
+130
View File
@@ -0,0 +1,130 @@
"""Automatische NAS + second-brain synchronisatie."""
from __future__ import annotations
import hashlib
import json
import logging
import os
from typing import Any
import httpx
from app.db import execute, fetch_all, fetch_one
log = logging.getLogger("cockpit.auto_ingest")
DOC_INGEST_URL = os.getenv("DOC_INGEST_URL", "http://10.4.7.19:8750")
TOOLS_API_URL = os.getenv("TOOLS_API_URL", "http://tools-api:8700")
def _log_run(source: str, status: str, details: dict[str, Any]) -> None:
try:
execute(
"INSERT INTO ingest_automation_log (source, status, details) VALUES (%s, %s, %s::jsonb)",
(source, status, json.dumps(details)),
)
except Exception as exc:
log.warning("ingest log failed: %s", exc)
def _brain_chat_id_for_client(client_id: int) -> int:
return -int(client_id)
async def sync_nas_to_brain(limit: int = 40) -> dict[str, Any]:
"""Indexeer geanalyseerde documenten in second brain (pgvector) per klant."""
rows = fetch_all(
"""
SELECT da.storage_path, da.filename, da.doc_type, da.sentiment_label,
da.word_count, dl.client_id, c.name AS client_name
FROM document_analytics da
LEFT JOIN document_links dl ON (
da.storage_path = dl.storage_path
OR da.storage_path LIKE dl.storage_path || '/%%'
) AND dl.is_folder = TRUE
LEFT JOIN document_links dl2 ON da.storage_path = dl2.storage_path AND dl2.is_folder = FALSE
LEFT JOIN clients c ON c.id = COALESCE(dl2.client_id, dl.client_id)
WHERE da.storage_path IS NOT NULL
ORDER BY da.analyzed_at DESC NULLS LAST
LIMIT %s
""",
(max(1, min(limit, 200)),),
)
synced = 0
errors = 0
async with httpx.AsyncClient(timeout=120.0) as client:
for row in rows:
path = row.get("storage_path") or ""
if not path:
continue
client_id = row.get("client_id")
chat_id = _brain_chat_id_for_client(client_id) if client_id else 888_000_001
text = (
f"NAS document: {row.get('filename') or path}\n"
f"Pad: {path}\nType: {row.get('doc_type')}\n"
f"Sentiment: {row.get('sentiment_label')}\nWoorden: {row.get('word_count')}"
)
msg_hash = int(hashlib.md5(path.encode()).hexdigest()[:8], 16)
try:
r = await client.post(
f"{TOOLS_API_URL.rstrip('/')}/brain/messages",
json={
"chat_id": chat_id,
"direction": "inbound",
"role": "system",
"content_type": "document",
"content_text": text[:4000],
"agent_name": "auto-ingest",
"embed": True,
"chat_type": "client_brain" if client_id else "nas_corpus",
"user_name": row.get("client_name") or "NAS",
"content_json": {"storage_path": path, "client_id": client_id},
"telegram_message_id": msg_hash,
},
)
if r.status_code < 400:
synced += 1
else:
errors += 1
except Exception as exc:
log.warning("brain sync %s: %s", path, exc)
errors += 1
return {"synced": synced, "errors": errors, "candidates": len(rows)}
async def run_full_auto_sync(force_scan: bool = False) -> dict[str, Any]:
"""Volledige pipeline: NAS scan → Chroma RAG → brain embeddings."""
out: dict[str, Any] = {"ok": True, "steps": {}}
try:
async with httpx.AsyncClient(timeout=300.0) as client:
r = await client.post(
f"{DOC_INGEST_URL.rstrip('/')}/ingest/scan",
params={"force": "true" if force_scan else "false"},
)
r.raise_for_status()
out["steps"]["nas_scan"] = r.json()
except Exception as exc:
out["steps"]["nas_scan"] = {"error": str(exc)}
brain = await sync_nas_to_brain()
out["steps"]["brain_sync"] = brain
status = "ok" if not out["steps"].get("nas_scan", {}).get("error") else "partial"
_log_run("auto_sync", status, out)
out["last_logged"] = True
return out
def last_auto_sync() -> dict[str, Any] | None:
row = fetch_one(
"SELECT source, status, details, created_at FROM ingest_automation_log ORDER BY created_at DESC LIMIT 1"
)
if not row:
return None
d = dict(row)
if hasattr(d.get("created_at"), "isoformat"):
d["created_at"] = d["created_at"].isoformat()
if isinstance(d.get("details"), str):
try:
d["details"] = json.loads(d["details"])
except Exception:
pass
return d
+28 -2
View File
@@ -7,7 +7,7 @@ from typing import Any
from app.config import settings
from app.db import execute, fetch_all, fetch_one
from app.services import market_stocks, ollama
from app.services import market_stocks, llm_router
def _safe_count(table: str, where: str = "", params: tuple = ()) -> int:
@@ -166,6 +166,16 @@ def _collect_pipeline_events(data: dict[str, Any]) -> None:
except Exception:
data["recent_events"] = []
try:
data["recent_handoffs"] = fetch_all(
"""SELECT from_agent, to_agent, handoff_type, status, created_at, correlation_id::text
FROM agent_handoffs
WHERE created_at >= NOW() - INTERVAL '24 hours'
ORDER BY created_at DESC LIMIT 20"""
)
except Exception:
data["recent_handoffs"] = []
try:
data["pending_items"] = fetch_all(
"""SELECT agent_name, title, event_type, created_at
@@ -565,6 +575,13 @@ def _build_activity_log(data: dict[str, Any]) -> list[str]:
ts_s = ts.isoformat()[11:16] if hasattr(ts, "isoformat") else ""
lines.append(f"{row.get('agent_name')}: {row.get('title')} ({ts_s})")
for row in (data.get("recent_handoffs") or [])[:8]:
ts = row.get("created_at")
ts_s = ts.isoformat()[11:16] if hasattr(ts, "isoformat") else ""
lines.append(
f"🔗 {row.get('from_agent')}{row.get('to_agent')}: {row.get('handoff_type')} ({ts_s})"
)
return lines[:25]
@@ -644,6 +661,15 @@ def build_template_report(data: dict[str, Any]) -> str:
for entry in data["activity_log"][:20]:
lines.append(f"- {entry}")
if data.get("recent_handoffs"):
lines.extend(["", "## Agent samenwerking (handoffs 24u)"])
for row in data["recent_handoffs"][:15]:
ts = row.get("created_at")
ts_s = ts.isoformat()[11:16] if hasattr(ts, "isoformat") else ""
lines.append(
f"- {row.get('from_agent')}{row.get('to_agent')} ({row.get('handoff_type')}) {ts_s}"
)
if data.get("pending_items"):
lines.extend(["", "## Legacy goedkeuringen"])
for row in data["pending_items"]:
@@ -683,7 +709,7 @@ async def _ai_executive_summary(data: dict[str, Any]) -> str:
"Schrijf warm, professioneel, actionable."
)
try:
return await ollama.generate(prompt, system=system, timeout=120.0)
return await llm_router.generate(prompt, system=system, timeout=120.0)
except Exception:
return ""
+147
View File
@@ -0,0 +1,147 @@
"""Klant 360° — geaggregeerde data uit NAS, CRM, retail, trends."""
from __future__ import annotations
from typing import Any
from app.db import fetch_all, fetch_one
from app.services import projects as projects_svc
def _serialize_row(row: dict | None) -> dict | None:
if not row:
return None
out = dict(row)
for k, v in list(out.items()):
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
return out
def _serialize_rows(rows: list) -> list[dict]:
return [_serialize_row(r) for r in rows if r]
def get_client_360(client_id: int) -> dict[str, Any]:
client = fetch_one("SELECT * FROM clients WHERE id = %s", (client_id,))
if not client:
return {"ok": False, "error": "Client not found"}
projs = projects_svc.list_projects(client_id=client_id, limit=50)
links = fetch_all(
"""
SELECT dl.*, p.name AS project_name
FROM document_links dl
LEFT JOIN cockpit_projects p ON p.id = dl.project_id
WHERE dl.client_id = %s
ORDER BY dl.created_at DESC
""",
(client_id,),
)
paths = [l["storage_path"] for l in links if l.get("storage_path")]
docs: list[dict] = []
if paths:
placeholders = ",".join(["%s"] * len(paths))
docs = fetch_all(
f"""
SELECT filename, storage_path, doc_type, word_count, sentiment_label,
sentiment_compound, analyzed_at, user_labels
FROM document_analytics
WHERE storage_path IN ({placeholders})
OR storage_path LIKE ANY (
SELECT dl.storage_path || '/%%' FROM document_links dl
WHERE dl.client_id = %s AND dl.is_folder = TRUE
)
ORDER BY analyzed_at DESC NULLS LAST
LIMIT 80
""",
tuple(paths + [client_id]),
)
sentiment_rows = fetch_all(
"""
SELECT sentiment_label, COUNT(*) AS n, AVG(sentiment_compound) AS avg_c
FROM document_analytics da
WHERE EXISTS (
SELECT 1 FROM document_links dl
WHERE dl.client_id = %s
AND (da.storage_path = dl.storage_path
OR (dl.is_folder AND da.storage_path LIKE dl.storage_path || '/%%'))
)
GROUP BY sentiment_label
""",
(client_id,),
)
top_words = fetch_all(
"""
SELECT dwc.lemma, SUM(dwc.count) AS total
FROM document_word_counts dwc
JOIN document_analytics da ON da.storage_path = dwc.storage_path
WHERE EXISTS (
SELECT 1 FROM document_links dl
WHERE dl.client_id = %s
AND (da.storage_path = dl.storage_path
OR (dl.is_folder AND da.storage_path LIKE dl.storage_path || '/%%'))
)
AND NOT dwc.is_stopword
GROUP BY dwc.lemma
ORDER BY total DESC
LIMIT 25
""",
(client_id,),
)
deals = fetch_all(
"SELECT id, title, value, stage, next_action, deadline FROM deals WHERE client_id = %s ORDER BY updated_at DESC LIMIT 15",
(client_id,),
)
stores = fetch_all(
"""
SELECT s.id, s.name, s.chain, s.city, s.partnership_status, s.halal_certified
FROM client_supermarket_links l
JOIN supermarkets s ON s.id = l.supermarket_id
WHERE l.client_id = %s
LIMIT 30
""",
(client_id,),
)
trends = fetch_all(
"""
SELECT DATE_TRUNC('month', da.analyzed_at) AS month,
COUNT(*) AS docs,
AVG(da.sentiment_compound) AS avg_sentiment,
SUM(da.word_count) AS words
FROM document_analytics da
WHERE da.analyzed_at IS NOT NULL
AND EXISTS (
SELECT 1 FROM document_links dl
WHERE dl.client_id = %s
AND (da.storage_path = dl.storage_path
OR (dl.is_folder AND da.storage_path LIKE dl.storage_path || '/%%'))
)
GROUP BY 1
ORDER BY 1 DESC
LIMIT 12
""",
(client_id,),
)
for t in trends:
if hasattr(t.get("month"), "isoformat"):
t["month"] = t["month"].isoformat()
return {
"ok": True,
"client": _serialize_row(client),
"projects": projs,
"links": _serialize_rows(links),
"documents": _serialize_rows(docs),
"sentiment_breakdown": _serialize_rows(sentiment_rows),
"top_words": _serialize_rows(top_words),
"deals": _serialize_rows(deals),
"stores": _serialize_rows(stores),
"monthly_trends": trends,
"stats": {
"linked_paths": len(links),
"documents": len(docs),
"projects": len(projs),
"stores": len(stores),
"deals": len(deals),
},
}
+152
View File
@@ -0,0 +1,152 @@
version: '3.8'
services:
redis:
image: redis:7-alpine
container_name: foodlinkk_redis
ports:
- "6379:6379"
restart: unless-stopped
tools-api:
build: ./tools-api
container_name: foodlinkk_tools_api
environment:
DB_HOST: foodlinkk_db
DB_USER: aissa
DB_PASSWORD: Foodlinkk#2026
DB_NAME: foodlinkk
BACKUP_ROOT: /data/backup-root
GITEA_BACKUP_REMOTE: http://sysops:Foodlinkk%23SysOps2026@gitea:3001/aissa/foodlinkk-command-center.git
GITEA_SYSOPS_USER: sysops
GITEA_SYSOPS_EMAIL: sysops@foodlinkk.local
GITEA_SYSOPS_NAME: sysops
GITEA_BACKUP_BRANCH: main
PROXMOX_HOST: 10.4.7.14
VM106_IP: 10.4.7.18
OLLAMA_URL: http://10.4.7.19:11434
OLLAMA_MODEL: qwen3:8b
CHROMA_HOST: 10.4.7.19
CHROMA_PORT: "8000"
DOC_INGEST_URL: http://10.4.7.19:8750
BROWSER_USE_URL: http://browser-agent:7790
BROWSER_AGENT_URL: http://browser-agent:7790
COMFYUI_URL: http://10.4.7.18:8188
SMTP_HOST: ${SMTP_HOST:-}
SMTP_PORT: ${SMTP_PORT:-587}
SMTP_USER: ${SMTP_USER:-}
SMTP_PASS: ${SMTP_PASS:-}
SMTP_FROM: ${SMTP_FROM:-}
ports:
- "8700:8700"
volumes:
- /home/aissa/foodlinkk-command-center:/data/backup-root:ro
extra_hosts:
- "foodlinkk_db:10.4.7.18"
restart: unless-stopped
cockpit:
build: ./cockpit
container_name: foodlinkk_cockpit
environment:
DB_HOST: foodlinkk_db
DB_USER: aissa
DB_PASSWORD: Foodlinkk#2026
DB_NAME: foodlinkk
VNC_CDP_URL: http://10.4.7.18:9223
OLLAMA_URL: http://10.4.7.19:11434
OLLAMA_MODEL: qwen3:8b
TOOLS_API_URL: http://tools-api:8700
CHROMA_HOST: 10.4.7.19
CHROMA_PORT: "8000"
DOC_INGEST_URL: http://10.4.7.19:8750
BROWSER_AGENT_URL: http://browser-agent:7790
COMFYUI_URL: http://10.4.7.18:8188
HERMAN_ORCHESTRATOR_URL: http://10.4.7.19:8090
NAS_CLIENTS_ROOT: /data/nas-clients
NAS_SHARE_ROOT: /data/nas-share
ports:
- "8600:8600"
volumes:
- /home/aissa/nas-clients:/data/nas-clients
- /mnt/synology:/data/nas-share:ro
depends_on:
- tools-api
extra_hosts:
- "foodlinkk_db:10.4.7.18"
restart: unless-stopped
browser-agent:
build: ./browser-agent
container_name: foodlinkk_browser_agent
environment:
DB_HOST: foodlinkk_db
DB_USER: aissa
DB_PASSWORD: Foodlinkk#2026
DB_NAME: foodlinkk
VNC_CDP_URL: http://10.4.7.18:9223
ports:
- "7790:7790"
extra_hosts:
- "foodlinkk_db:10.4.7.18"
restart: unless-stopped
shm_size: "1gb"
gitea:
image: gitea/gitea:1.21-rootless
container_name: foodlinkk_gitea
environment:
GITEA__database__DB_TYPE: sqlite3
GITEA__server__ROOT_URL: http://10.4.7.18:3001/
GITEA__server__HTTP_PORT: 3001
GITEA__security__INSTALL_LOCK: "true"
GITEA__security__SECRET_KEY: foodlinkk-gitea-secret-2026
GITEA__service__DISABLE_REGISTRATION: "true"
ports:
- "3001:3001"
volumes:
- gitea_data:/var/lib/gitea
restart: unless-stopped
email-agent:
build: ./email-agent
container_name: foodlinkk_email_agent
environment:
DB_HOST: foodlinkk_db
DB_USER: aissa
DB_PASSWORD: Foodlinkk#2026
DB_NAME: foodlinkk
TOOLS_API_URL: http://tools-api:8700
SYNC_INTERVAL_SEC: 300
ports:
- "8801:8801"
depends_on:
- tools-api
extra_hosts:
- "foodlinkk_db:10.4.7.18"
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.53.0
container_name: foodlinkk_prometheus
volumes:
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
ports:
- "9090:9090"
restart: unless-stopped
grafana:
image: grafana/grafana:11.1.0
container_name: foodlinkk_grafana
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: Foodlinkk#2026
GF_SERVER_ROOT_URL: http://10.4.7.18:3002
ports:
- "3002:3000"
depends_on:
- prometheus
restart: unless-stopped
volumes:
gitea_data:
File diff suppressed because it is too large Load Diff
+145
View File
@@ -0,0 +1,145 @@
"""Parse Succes Sheet revenue tab — runs in cockpit with NAS mount."""
from __future__ import annotations
import os
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
NAS_SHARE_ROOT = Path(os.getenv("NAS_SHARE_ROOT", "/data/nas-share"))
DEFAULT_FILE = os.getenv("CEO_EXCEL_PATH", "Succes Sheet .xlsx")
DEFAULT_SHEET = os.getenv("CEO_EXCEL_SHEET", "Projects next steps revenue")
_FILL_STYLE = {
"FF92D050": "green",
"FFFF0000": "red",
"FFFFC000": "orange",
"FFFFFF00": "yellow",
"FFBDD7EE": "blue",
"FFDDEBF7": "blue",
"FF0070C0": "blue",
}
def _num(val: Any) -> float | None:
if val is None or val == "":
return None
try:
return float(val)
except (TypeError, ValueError):
return None
def _txt(val: Any) -> str:
if val is None:
return ""
return str(val).strip()
def _cell_fill_style(cell) -> str:
try:
if not cell or not cell.fill or cell.fill.fill_type != "solid":
return "white"
rgb = cell.fill.fgColor.rgb if cell.fill.fgColor else None
if not rgb or rgb in ("00000000", "FFFFFFFF", "00FFFFFF"):
return "white"
key = rgb[-8:].upper() if len(rgb) >= 8 else rgb.upper()
if key in _FILL_STYLE:
return _FILL_STYLE[key]
short = key[-6:]
for k, v in _FILL_STYLE.items():
if k.endswith(short):
return v
return "white"
except Exception:
return "white"
def parse_revenue_sheet(
rel_path: str = DEFAULT_FILE,
sheet_name: str | None = DEFAULT_SHEET,
nas_root: Path | None = None,
) -> dict[str, Any]:
try:
from openpyxl import load_workbook
except ImportError as exc:
return {"ok": False, "error": f"openpyxl not installed: {exc}"}
root = nas_root or NAS_SHARE_ROOT
full = root / rel_path.lstrip("/")
if not full.is_file():
return {"ok": False, "error": f"File not found: {full}"}
st = full.stat()
wb = load_workbook(full, read_only=False, data_only=True)
names = wb.sheetnames
target_name = sheet_name
if not target_name:
for n in names:
if "project" in n.lower() and "revenue" in n.lower():
target_name = n
break
if not target_name and len(names) > 2:
target_name = names[2]
if not target_name:
return {"ok": False, "error": "Sheet not found", "sheetnames": names}
ws = wb[target_name]
rows = list(ws.iter_rows(values_only=False))
goals: dict[str, str] = {"vision_text": "", "horizon_text": "", "mid_text": "", "tagline": ""}
if rows:
r0 = rows[0]
goals["vision_text"] = _txt(r0[0].value if len(r0) > 0 else "")
goals["horizon_text"] = _txt(r0[1].value if len(r0) > 1 else "")
goals["mid_text"] = _txt(r0[2].value if len(r0) > 2 else "")
goals["tagline"] = _txt(r0[4].value if len(r0) > 4 else (_txt(r0[3].value if len(r0) > 3 else "")))
projects: list[dict[str, Any]] = []
sort_order = 0
for idx, row in enumerate(rows):
if idx <= 1:
continue
cells = list(row) if row else []
name = _txt(cells[1].value if len(cells) > 1 else "")
if not name:
continue
margin_month = _num(cells[2].value if len(cells) > 2 else None)
margin_year = _num(cells[3].value if len(cells) > 3 else None)
next_steps = _txt(cells[4].value if len(cells) > 4 else "")
target_extra = _num(cells[5].value if len(cells) > 5 else None)
row_style = _cell_fill_style(cells[1] if len(cells) > 1 else None)
category = "deal" if margin_month is not None or margin_year is not None else "initiative"
if row_style == "yellow" or re.search(r"foodlinkk|linknbit|subsid|total earnings|loonkosten", name, re.I):
category = "strategic"
projects.append(
{
"name": name,
"category": category,
"margin_month": margin_month,
"margin_year": margin_year,
"target_revenue": target_extra or margin_year,
"next_steps": next_steps,
"status": "active",
"sort_order": sort_order,
"source_row": idx,
"row_style": row_style,
}
)
sort_order += 1
wb.close()
return {
"ok": True,
"source_file": rel_path,
"sheet_name": target_name,
"sheetnames": names,
"file_mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
"file_size": st.st_size,
"goals": goals,
"projects": projects,
"parsed_at": datetime.now(timezone.utc).isoformat(),
"project_count": len(projects),
}
+141 -10
View File
@@ -6,6 +6,8 @@ from app.config import settings
from app.db import execute, fetch_one
from app.services import ollama
from app.services import packaging_agent
from app.services import webbuilder_agent
from app.services import voice_export_actions
AGENTS: dict[str, dict[str, str]] = {
"marketing": {"name": "Marketing", "persona": "Social, brand voice, campaigns for Foodlinkk."},
@@ -49,7 +51,14 @@ def _extract_image_prompt(raw: str) -> str:
return t
async def _log_event(agent_name: str, event_type: str, title: str, body: str, metadata: dict | None = None) -> None:
async def _log_event(
agent_name: str,
event_type: str,
title: str,
body: str,
metadata: dict | None = None,
channel: str = "herman",
) -> None:
payload = {
"agent_name": agent_name,
"agent_type": "herman_delegate",
@@ -58,7 +67,7 @@ async def _log_event(agent_name: str, event_type: str, title: str, body: str, me
"body": body,
"metadata": metadata or {},
"status": "completed",
"channel": "herman",
"channel": channel,
}
try:
async with httpx.AsyncClient(timeout=15.0) as client:
@@ -68,7 +77,7 @@ async def _log_event(agent_name: str, event_type: str, title: str, body: str, me
execute(
"""INSERT INTO agent_events (agent_name, agent_type, event_type, title, body, status, channel, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb)""",
(agent_name, "herman_delegate", event_type, title[:255], body, "completed", "herman", json.dumps(metadata or {})),
(agent_name, "herman_delegate", event_type, title[:255], body, "completed", channel, json.dumps(metadata or {})),
)
except Exception:
pass
@@ -83,7 +92,113 @@ def _pick_agent(raw: str) -> str:
return k
return "knowledge"
async def chat(message: str) -> dict[str, Any]:
def _agent_steps(delegated: list[str], routing_reason: str = "", extra: list[dict] | None = None) -> list[dict[str, str]]:
steps: list[dict[str, str]] = []
for a in delegated:
key = (a or "").strip().lower()
if key and key != "herman":
steps.append({"agent": key, "status": "delegated", "message": routing_reason or "Aangestuurd door Herman"})
if extra:
steps.extend(extra)
return steps
async def chat(
message: str,
channel: str = "cockpit",
session_id: str | None = None,
confirm_action_id: str | None = None,
) -> dict[str, Any]:
use_voice_router = (
channel in voice_export_actions.VOICE_CHANNELS
or bool(session_id)
or voice_export_actions.wants_export_search(message)
or webbuilder_agent.wants_website(message)
)
if use_voice_router:
routed = await voice_export_actions.handle_voice_command(
message,
session_id=session_id,
confirm_action_id=confirm_action_id,
channel=channel,
)
if routed is not None:
await _log_event(
"herman",
"voice_command" if channel == "voice" else "browser_command",
(routed.get("reply") or "")[:120],
message[:2000],
{
"pending_action": routed.get("pending_action"),
"ui_actions": routed.get("ui_actions"),
"channel": channel,
},
channel=channel,
)
return routed
if webbuilder_agent.wants_website(message):
try:
outcome = await webbuilder_agent.generate_from_message(message, channel=channel, wait=False)
project = outcome.get("project") or "website"
preview = outcome.get("preview_url") or ""
reply_lines = [
f"Web Builder is gestart voor project **{project}**.",
f"NAS: {outcome.get('nas_path') or ''}",
f"Preview (na afloop): {preview}",
"Volg de live terminal: Agents → Terminals → Web Builder.",
]
reply = "\n".join(reply_lines)
await _log_event(
"herman",
"website_delegation",
f"Herman → Web Builder: {project}",
message[:2000],
{
"delegated": ["webbuilder"],
"delegated_agents": ["webbuilder"],
"project": project,
"preview_url": preview,
"channel": channel,
},
channel=channel,
)
delegated = ["webbuilder"]
return {
"agent": "webbuilder",
"agent_label": "Web Builder → Herman",
"reply": reply,
"delegated_agents": delegated,
"routing_reason": "Website-opdracht — Web Builder gestart op Hermes/agy",
"agent_steps": _agent_steps(delegated, "Website build gestart"),
"webbuilder_project": project,
"webbuilder_preview_url": preview,
"ui_actions": [
{
"type": "open_webbuilder_build",
"title": f"Website build — {project}",
"project": project,
"preview_url": preview,
"nas_path": outcome.get("nas_path") or "",
"agents_url": "/agents",
}
],
}
except Exception as exc:
await _log_event(
"webbuilder",
"website_build_error",
"Website start mislukt",
str(exc)[:1500],
{"message": message[:500]},
)
return {
"agent": "webbuilder",
"agent_label": "Web Builder",
"reply": f"Web Builder kon niet starten: {exc}",
"delegated_agents": ["webbuilder"],
}
if packaging_agent.wants_packaging(message):
try:
outcome = await packaging_agent.generate_from_message(message)
@@ -112,7 +227,10 @@ async def chat(message: str) -> dict[str, Any]:
"pdf_url": outcome.get("pdf_url"),
"nas": outcome.get("nas"),
"for_herman": True,
"delegated": ["packaging"],
"channel": channel,
},
channel=channel,
)
await _log_event(
"herman",
@@ -123,14 +241,19 @@ async def chat(message: str) -> dict[str, Any]:
"source_agent": "packaging",
"packaging_id": outcome.get("packaging_id"),
"project_id": outcome.get("cockpit_project_id"),
"delegated": ["packaging"],
"channel": channel,
},
channel=channel,
)
delegated = ["packaging"]
return {
"agent": "packaging",
"agent_label": "Packaging → Herman",
"reply": reply,
"delegated_agents": ["packaging", "herman"],
"delegated_agents": delegated,
"routing_reason": "Packaging-opdracht gedetecteerd — design gegenereerd en aan Herman gerapporteerd",
"agent_steps": _agent_steps(delegated, "Packaging design gegenereerd"),
"packaging_id": outcome.get("packaging_id"),
"packaging_studio_url": outcome.get("studio_url"),
"packaging_pdf_url": outcome.get("pdf_url"),
@@ -165,13 +288,16 @@ async def chat(message: str) -> dict[str, Any]:
img_type = data.get("type", "output")
proxy = f"/api/ai/generated-image?filename={filename}&subfolder={subfolder}&type={img_type}"
reply = f"Afbeelding gegenereerd voor: {prompt}"
await _log_event("design", "image_generated", "ComfyUI via Herman", prompt[:500], {"filename": filename})
await _log_event("design", "image_generated", "ComfyUI via Herman", prompt[:500], {"filename": filename, "channel": channel}, channel=channel)
delegated = ["design"]
return {
"agent": "design",
"agent_label": "Design",
"reply": reply,
"image_url": proxy,
"prompt": prompt,
"delegated_agents": delegated,
"agent_steps": _agent_steps(delegated, "Afbeelding gegenereerd"),
}
except Exception as exc:
return {
@@ -184,24 +310,29 @@ async def chat(message: str) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=620.0) as client:
r = await client.post(
f"{settings.HERMAN_ORCHESTRATOR_URL.rstrip('/')}/chat",
json={"message": message, "agent": "default", "use_crm": True, "channel": "cockpit"},
json={"message": message, "agent": "default", "use_crm": True, "use_rag": True, "channel": channel},
)
r.raise_for_status()
data = r.json()
delegated = data.get("delegated_agents") or [data.get("agent", "herman")]
reason = data.get("routing_reason", "") or ""
await _log_event(
"herman",
"openswarm_delegation",
"voice_delegation" if channel == "voice" else ("telegram_delegation" if channel == "telegram" else "openswarm_delegation"),
f"Herman → {', '.join(delegated)}",
message[:2000],
{"delegated": delegated, "reason": data.get("routing_reason", "")},
{"delegated": delegated, "delegated_agents": delegated, "reason": reason, "channel": channel},
channel=channel,
)
return {
"agent": data.get("agent", "herman"),
"agent_label": data.get("agent_label", "Herman"),
"reply": data.get("reply", ""),
"delegated_agents": delegated,
"routing_reason": data.get("routing_reason", ""),
"routing_reason": reason,
"agent_steps": _agent_steps(delegated, reason),
"rag_sources": data.get("rag_sources") or [],
"crm_loaded": data.get("crm_loaded", False),
}
except Exception as exc:
return {
+277
View File
@@ -0,0 +1,277 @@
"""Unified LLM router — Ollama, DeepSeek, Gemini, Groq, OpenRouter, custom OpenAI-compatible."""
from __future__ import annotations
import json
from typing import Any
import httpx
from app.config import settings
from app.db import execute, fetch_all, fetch_one
from app.services import ollama
# Preset catalog for Settings UI (signup links + default models)
LLM_PRESETS: dict[str, dict[str, Any]] = {
"ollama": {
"label": "Ollama (lokaal)",
"api_base_url": "",
"models": ["qwen3:8b", "gemma3:12b", "llama3.2", "mistral"],
"needs_key": False,
"hint": "Geen API key — draait op je Ollama server.",
},
"deepseek": {
"label": "DeepSeek",
"api_base_url": "https://api.deepseek.com/v1",
"models": ["deepseek-chat", "deepseek-reasoner"],
"needs_key": True,
"signup_url": "https://platform.deepseek.com/",
"hint": "Goedkoop · sterk voor code en analyse.",
},
"gemini": {
"label": "Google Gemini",
"api_base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
"models": ["gemini-2.0-flash", "gemini-1.5-flash", "gemini-1.5-pro"],
"needs_key": True,
"signup_url": "https://aistudio.google.com/apikey",
"hint": "Gratis tier via Google AI Studio.",
},
"groq": {
"label": "Groq (snel · gratis tier)",
"api_base_url": "https://api.groq.com/openai/v1",
"models": ["llama-3.3-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"],
"needs_key": True,
"signup_url": "https://console.groq.com/",
"hint": "Zeer snelle inference · gratis limiet.",
},
"openrouter": {
"label": "OpenRouter",
"api_base_url": "https://openrouter.ai/api/v1",
"models": [
"google/gemini-2.0-flash-exp:free",
"deepseek/deepseek-r1:free",
"meta-llama/llama-3.3-70b-instruct:free",
],
"needs_key": True,
"signup_url": "https://openrouter.ai/",
"hint": "Veel gratis modellen via één API.",
},
"mistral": {
"label": "Mistral AI",
"api_base_url": "https://api.mistral.ai/v1",
"models": ["mistral-small-latest", "open-mistral-nemo"],
"needs_key": True,
"signup_url": "https://console.mistral.ai/",
"hint": "EU-hosted · gratis proef tier.",
},
"custom_openai": {
"label": "Custom OpenAI-compatible",
"api_base_url": "",
"models": [],
"needs_key": True,
"hint": "Elke API die /v1/chat/completions ondersteunt.",
},
}
def list_presets() -> list[dict[str, Any]]:
out = []
for key, meta in LLM_PRESETS.items():
row = dict(meta)
row["id"] = key
out.append(row)
return out
def _mask_provider(row: dict[str, Any] | None) -> dict[str, Any] | None:
if not row:
return None
out = dict(row)
for k, v in list(out.items()):
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
if isinstance(out.get("extra_config"), str):
try:
out["extra_config"] = json.loads(out["extra_config"])
except Exception:
out["extra_config"] = {}
out["api_key_set"] = bool(row.get("api_key"))
out.pop("api_key", None)
preset = LLM_PRESETS.get(out.get("provider_type") or "", {})
out["preset_label"] = preset.get("label", out.get("provider_type"))
out["needs_key"] = preset.get("needs_key", True)
return out
def list_providers() -> list[dict[str, Any]]:
rows = fetch_all("SELECT * FROM llm_providers ORDER BY is_default DESC, is_active DESC, id ASC")
return [_mask_provider(r) for r in rows if r]
def get_provider(provider_id: int | None = None) -> dict[str, Any] | None:
if provider_id:
return fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
row = fetch_one(
"SELECT * FROM llm_providers WHERE is_default = TRUE ORDER BY id LIMIT 1"
)
if row:
return row
row = fetch_one(
"SELECT * FROM llm_providers WHERE is_active = TRUE ORDER BY id LIMIT 1"
)
if row:
return row
return fetch_one("SELECT * FROM llm_providers ORDER BY id LIMIT 1")
def resolve_provider(provider_id: int | None = None) -> dict[str, Any]:
row = get_provider(provider_id)
if not row:
return {
"id": 0,
"label": "Ollama lokaal",
"provider_type": "ollama",
"api_base_url": settings.OLLAMA_URL,
"api_key": "",
"model": settings.OLLAMA_MODEL,
"extra_config": {},
}
return row
async def chat_messages(
messages: list[dict[str, str]],
*,
provider_id: int | None = None,
model: str | None = None,
timeout: float = 120.0,
) -> tuple[str, dict[str, Any]]:
"""Returns (reply_text, meta dict with provider info)."""
prov = resolve_provider(provider_id)
ptype = (prov.get("provider_type") or "ollama").lower()
use_model = model or prov.get("model") or settings.OLLAMA_MODEL
ollama_timeout = min(timeout, 85.0) if ptype == "ollama" else timeout
if ptype == "ollama":
try:
reply = await ollama.chat_messages(messages, timeout=ollama_timeout, model=use_model)
except Exception as exc:
raise RuntimeError(
f"Ollama timeout/ offline ({exc}). "
"Voeg DeepSeek of Gemini toe via Instellingen → AI / LLM voor snelle cloud-chat."
) from exc
return reply, {
"provider_id": prov.get("id"),
"provider_type": "ollama",
"provider_label": prov.get("label") or "Ollama",
"model": use_model,
}
api_key = (prov.get("api_key") or "").strip()
if not api_key:
raise RuntimeError(
f"Geen API key voor {prov.get('label') or ptype} — voeg key toe in Instellingen → AI / LLM"
)
base = (prov.get("api_base_url") or "").strip().rstrip("/")
if not base:
preset = LLM_PRESETS.get(ptype, {})
base = (preset.get("api_base_url") or "").rstrip("/")
if not base:
raise RuntimeError(f"Geen API URL voor provider {prov.get('label')}")
extra = prov.get("extra_config") or {}
if isinstance(extra, str):
try:
extra = json.loads(extra)
except Exception:
extra = {}
url = f"{base}/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
if ptype == "openrouter":
headers["HTTP-Referer"] = extra.get("referer", "https://foodlinkk.local")
headers["X-Title"] = extra.get("title", "Foodlinkk Command Center")
payload: dict[str, Any] = {
"model": use_model,
"messages": messages,
"temperature": float(extra.get("temperature", 0.4)),
"max_tokens": int(extra.get("max_tokens", 2048)),
}
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(url, headers=headers, json=payload)
if resp.status_code >= 400:
detail = resp.text[:500]
try:
detail = resp.json().get("error", {}).get("message", detail)
except Exception:
pass
raise RuntimeError(f"{prov.get('label')}: {detail}")
data = resp.json()
choices = data.get("choices") or []
if not choices:
raise RuntimeError(f"{prov.get('label')}: leeg antwoord")
content = (choices[0].get("message") or {}).get("content") or ""
return content.strip(), {
"provider_id": prov.get("id"),
"provider_type": ptype,
"provider_label": prov.get("label"),
"model": use_model,
}
async def generate(
prompt: str,
system: str | None = None,
*,
provider_id: int | None = None,
model: str | None = None,
timeout: float = 120.0,
) -> str:
messages: list[dict[str, str]] = []
if system:
messages.append({"role": "system", "content": system})
messages.append({"role": "user", "content": prompt})
reply, _meta = await chat_messages(
messages, provider_id=provider_id, model=model, timeout=timeout
)
return reply
async def test_provider(provider_id: int) -> tuple[bool, str]:
prov = fetch_one("SELECT * FROM llm_providers WHERE id = %s", (provider_id,))
if not prov:
return False, "Provider niet gevonden"
try:
reply, meta = await chat_messages(
[{"role": "user", "content": "Antwoord met exact één woord: OK"}],
provider_id=provider_id,
timeout=60.0,
)
msg = f"{meta.get('provider_label')} · {meta.get('model')}{reply[:80]}"
execute(
"""UPDATE llm_providers SET last_test_status = %s, last_test_message = %s,
last_test_at = NOW(), updated_at = NOW() WHERE id = %s""",
("ok", msg, provider_id),
)
return True, msg
except Exception as exc:
execute(
"""UPDATE llm_providers SET last_test_status = %s, last_test_message = %s,
last_test_at = NOW(), updated_at = NOW() WHERE id = %s""",
("error", str(exc)[:500], provider_id),
)
return False, str(exc)
def set_default(provider_id: int) -> None:
execute("UPDATE llm_providers SET is_default = FALSE, updated_at = NOW()")
execute(
"UPDATE llm_providers SET is_default = TRUE, is_active = TRUE, updated_at = NOW() WHERE id = %s",
(provider_id,),
)
+529 -62
View File
@@ -1,14 +1,15 @@
from __future__ import annotations
import hashlib
import json
import re
import subprocess
from urllib.parse import urlparse
from typing import Any
from urllib.parse import urljoin, urlparse
import httpx
from bs4 import BeautifulSoup
from app.db import execute, fetch_one, get_connection
from app.db import execute, fetch_all, fetch_one, get_connection
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 FoodlinkkBot/2.0"
@@ -45,6 +46,59 @@ def _fetch_page(url: str) -> tuple[str, str, str, str] | None:
return None
def _extract_parse_info(
html: str,
text: str,
title: str,
final_url: str,
base_url: str,
) -> dict[str, Any]:
soup = BeautifulSoup(html or "", "html.parser")
headings: list[str] = []
for tag in soup.find_all(["h1", "h2", "h3"])[:12]:
t = re.sub(r"\s+", " ", (tag.get_text() or "").strip())
if t:
headings.append(t[:140])
links_sample: list[dict[str, str]] = []
seen_hrefs: set[str] = set()
for a in soup.find_all("a", href=True):
href = (a.get("href") or "").strip()
if not href or href.startswith("#") or href.lower().startswith("javascript:"):
continue
if not href.startswith("http"):
href = urljoin(base_url or final_url, href)
if href in seen_hrefs:
continue
seen_hrefs.add(href)
label = re.sub(r"\s+", " ", (a.get_text() or "").strip())[:90]
links_sample.append({"href": href, "label": label or href})
if len(links_sample) >= 10:
break
words = len(text.split()) if text else 0
excerpt = ""
if text:
excerpt = text[:320] + ("" if len(text) > 320 else "")
meta_desc = ""
md = soup.find("meta", attrs={"name": "description"})
if md and md.get("content"):
meta_desc = str(md["content"]).strip()[:240]
return {
"title": title or "(geen titel)",
"final_url": final_url,
"word_count": words,
"char_count": len(text or ""),
"excerpt": excerpt,
"meta_description": meta_desc,
"headings": headings,
"links_count": len(seen_hrefs) if seen_hrefs else len(soup.find_all("a", href=True)),
"links_sample": links_sample,
}
def get_page_hash(url: str) -> str | None:
fetched = _fetch_page(url)
if not fetched:
@@ -53,20 +107,31 @@ def get_page_hash(url: str) -> str | None:
return hashlib.md5(text.encode("utf-8")).hexdigest()
def _save_snapshot(site_id: int, url: str, final_url: str, title: str, text: str, html: str) -> int | None:
import json
def _save_snapshot(
site_id: int,
url: str,
final_url: str,
title: str,
text: str,
html: str,
parse_info: dict[str, Any] | None = None,
) -> int | None:
parse_info = parse_info or _extract_parse_info(html, text, title, final_url, url)
metadata = {"source": "monitor", "parse": parse_info}
links_json = json.dumps(parse_info.get("links_sample") or [])
try:
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO crawled_pages (url, final_url, title, content, content_html, site_id, metadata, crawled_at)
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, NOW())
INSERT INTO crawled_pages (url, final_url, title, content, content_html, site_id, metadata, links, crawled_at)
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, NOW())
ON CONFLICT (url) DO UPDATE SET
final_url=EXCLUDED.final_url, title=EXCLUDED.title,
content=EXCLUDED.content, content_html=EXCLUDED.content_html,
site_id=EXCLUDED.site_id, crawled_at=NOW()
site_id=EXCLUDED.site_id, metadata=EXCLUDED.metadata,
links=EXCLUDED.links, crawled_at=NOW()
RETURNING id
""",
(
@@ -76,7 +141,8 @@ def _save_snapshot(site_id: int, url: str, final_url: str, title: str, text: str
text[:50000],
html[:100000],
site_id,
json.dumps({"source": "monitor"}),
json.dumps(metadata),
links_json,
),
)
page_id = cur.fetchone()[0]
@@ -125,7 +191,8 @@ def add_site(url: str, name: str) -> dict:
)
site_id = cur.fetchone()[0]
if fetched:
_save_snapshot(site_id, url, final_url, title, text, html)
parse_info = _extract_parse_info(html, text, title, final_url, url)
_save_snapshot(site_id, url, final_url, title, text, html, parse_info)
row = fetch_one(
"SELECT id, url, name, last_hash, last_crawled, last_title, is_active, last_snapshot_id FROM monitored_sites WHERE id = %s",
(site_id,),
@@ -142,61 +209,461 @@ def remove_site(site_id: int, soft: bool = True) -> None:
execute("DELETE FROM monitored_sites WHERE id = %s", (site_id,))
def trigger_crawl(site_id: int | None = None) -> dict:
def _crawl_one_site(site: dict[str, Any]) -> dict[str, Any]:
site_id = int(site["id"])
url = site["url"]
name = site.get("name") or url
result: dict[str, Any] = {
"site_id": site_id,
"url": url,
"name": name,
"status": "ERROR",
"changed": False,
"error": None,
}
fetched = _fetch_page(url)
if not fetched:
msg = f"Kan {url} niet bereiken"
execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site_id, "ERROR", msg),
)
result["error"] = msg
return result
final_url, title, text, html = fetched
parse_info = _extract_parse_info(html, text, title, final_url, url)
new_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
old_hash = site.get("last_hash")
changed = bool(old_hash and old_hash != new_hash)
if changed:
execute(
"INSERT INTO page_changes (site_id, old_hash, new_hash) VALUES (%s, %s, %s)",
(site_id, old_hash, new_hash),
)
execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site_id, "CHANGE", f"Wijziging op {url}{title}"),
)
result["status"] = "CHANGE"
else:
execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site_id, "OK", f"Crawl OK — {title} ({parse_info['word_count']} woorden)"),
)
result["status"] = "OK"
execute(
"""
UPDATE monitored_sites SET last_hash=%s, last_crawled=NOW(), last_title=%s WHERE id=%s
""",
(new_hash, title, site_id),
)
snapshot_id = _save_snapshot(site_id, url, final_url, title, text, html, parse_info)
result.update(parse_info)
result["changed"] = changed
result["snapshot_id"] = snapshot_id
return result
def trigger_crawl(site_id: int | None = None) -> dict[str, Any]:
if site_id:
row = fetch_one(
"SELECT id, url, name, last_hash, last_title FROM monitored_sites WHERE id = %s AND is_active = TRUE",
(site_id,),
)
sites = [dict(row)] if row else []
else:
sites = [
dict(r)
for r in fetch_all(
"SELECT id, url, name, last_hash, last_title FROM monitored_sites WHERE is_active = TRUE ORDER BY id"
)
]
if not sites:
return {
"ok": True,
"method": "inline",
"sites": 0,
"changed": 0,
"errors": 0,
"results": [],
"message": "Geen actieve monitor-sites — voeg eerst een URL toe.",
}
results: list[dict[str, Any]] = []
changed = 0
errors = 0
for site in sites:
row = _crawl_one_site(site)
results.append(row)
if row.get("status") == "ERROR":
errors += 1
if row.get("changed"):
changed += 1
return {
"ok": errors < len(sites),
"method": "inline",
"sites": len(sites),
"changed": changed,
"errors": errors,
"results": results,
}
def list_parse_results(site_id: int | None = None, limit: int = 20) -> list[dict[str, Any]]:
limit = max(1, min(limit, 50))
if site_id:
rows = fetch_all(
"""
SELECT cp.id, cp.url, cp.final_url, cp.title, cp.content, cp.metadata, cp.links,
cp.crawled_at, cp.site_id, ms.name AS site_name
FROM crawled_pages cp
LEFT JOIN monitored_sites ms ON ms.id = cp.site_id
WHERE cp.site_id = %s
ORDER BY cp.crawled_at DESC NULLS LAST
LIMIT %s
""",
(site_id, limit),
)
else:
rows = fetch_all(
"""
SELECT cp.id, cp.url, cp.final_url, cp.title, cp.content, cp.metadata, cp.links,
cp.crawled_at, cp.site_id, ms.name AS site_name
FROM crawled_pages cp
LEFT JOIN monitored_sites ms ON ms.id = cp.site_id
ORDER BY cp.crawled_at DESC NULLS LAST
LIMIT %s
""",
(limit,),
)
out: list[dict[str, Any]] = []
for row in rows:
item = dict(row)
meta = item.get("metadata") or {}
if isinstance(meta, str):
try:
meta = json.loads(meta)
except Exception:
meta = {}
parse = (meta or {}).get("parse") or {}
content = item.get("content") or ""
if not parse.get("excerpt") and content:
parse["excerpt"] = content[:320] + ("" if len(content) > 320 else "")
if not parse.get("word_count") and content:
parse["word_count"] = len(str(content).split())
links = item.get("links") or []
if isinstance(links, str):
try:
links = json.loads(links)
except Exception:
links = []
if not parse.get("links_sample") and links:
parse["links_sample"] = links
crawled = item.get("crawled_at")
if crawled is not None and hasattr(crawled, "isoformat"):
item["crawled_at"] = crawled.isoformat()
out.append(
{
"id": item.get("id"),
"site_id": item.get("site_id"),
"site_name": item.get("site_name"),
"url": item.get("url"),
"final_url": item.get("final_url"),
"title": item.get("title") or parse.get("title"),
"crawled_at": item.get("crawled_at"),
"word_count": parse.get("word_count", 0),
"excerpt": parse.get("excerpt", ""),
"meta_description": parse.get("meta_description", ""),
"headings": parse.get("headings") or [],
"links_count": parse.get("links_count", len(links)),
"links_sample": parse.get("links_sample") or links[:10],
}
)
return out
NL_STOPWORDS = frozenset(
"""
de het een en van in op te dat die dit voor met als zij ze er maar om ook al naar dan wel
kan zo nog uit over bij tot door na ons uw u uw je jij mij hem haar hun was zijn worden wordt
heb hebt heeft hebben had deed doen done the and or is are was were be been being a an to of in
for on at by from with about into through during before after above below between under again
further then once here there when where why how all each few more most other some such no nor
not only own same so than too very just don should now naar website home pagina menu contact
service cookie cookies privacy login inloggen registreren meer lees read click klik
""".split()
)
def _tokens(text: str, min_len: int = 4) -> list[str]:
if not text:
return []
raw = re.findall(r"[a-zA-Zà-üÀ-Ü0-9][a-zA-Zà-üÀ-Ü0-9\-]{2,}", text.lower())
return [t for t in raw if len(t) >= min_len and t not in NL_STOPWORDS and not t.isdigit()]
def _normalize_page_row(row: dict[str, Any], *, content_limit: int = 8000) -> dict[str, Any]:
item = dict(row)
meta = item.get("metadata") or {}
if isinstance(meta, str):
try:
meta = json.loads(meta)
except Exception:
meta = {}
parse = (meta or {}).get("parse") or {}
content = str(item.get("content") or "")
if not parse.get("excerpt") and content:
parse["excerpt"] = content[:320] + ("" if len(content) > 320 else "")
if not parse.get("word_count") and content:
parse["word_count"] = len(content.split())
links = item.get("links") or []
if isinstance(links, str):
try:
links = json.loads(links)
except Exception:
links = []
if not parse.get("links_sample") and links:
parse["links_sample"] = links
crawled = item.get("crawled_at")
if crawled is not None and hasattr(crawled, "isoformat"):
crawled = crawled.isoformat()
content_read = content[:content_limit]
if len(content) > content_limit:
content_read += "\n\n[… tekst ingekort — open volledige pagina voor alles …]"
return {
"id": item.get("id"),
"site_id": item.get("site_id"),
"site_name": item.get("site_name"),
"url": item.get("url"),
"final_url": item.get("final_url"),
"title": item.get("title") or parse.get("title"),
"crawled_at": crawled,
"word_count": parse.get("word_count", 0),
"char_count": parse.get("char_count", len(content)),
"excerpt": parse.get("excerpt", ""),
"meta_description": parse.get("meta_description", ""),
"headings": parse.get("headings") or [],
"links_count": parse.get("links_count", len(links)),
"links_sample": parse.get("links_sample") or links[:15],
"content_read": content_read,
"content_length": len(content),
"has_full_content": len(content) > 0,
}
def get_parse_page(page_id: int) -> dict[str, Any] | None:
row = fetch_one(
"""
SELECT cp.id, cp.url, cp.final_url, cp.title, cp.content, cp.metadata, cp.links,
cp.crawled_at, cp.site_id, ms.name AS site_name
FROM crawled_pages cp
LEFT JOIN monitored_sites ms ON ms.id = cp.site_id
WHERE cp.id = %s
""",
(page_id,),
)
if not row:
return None
page = _normalize_page_row(dict(row), content_limit=50000)
page["content_full"] = str(dict(row).get("content") or "")
return page
def build_parse_intelligence(
site_id: int | None = None,
query: str | None = None,
limit: int = 30,
) -> dict[str, Any]:
"""Aggregate parsed pages for analysis — hype terms, trends, readable content."""
limit = max(1, min(limit, 100))
if site_id:
rows = fetch_all(
"""
SELECT DISTINCT ON (cp.site_id)
cp.id, cp.url, cp.final_url, cp.title, cp.content, cp.metadata, cp.links,
cp.crawled_at, cp.site_id, ms.name AS site_name
FROM crawled_pages cp
LEFT JOIN monitored_sites ms ON ms.id = cp.site_id
WHERE cp.site_id = %s
ORDER BY cp.site_id, cp.crawled_at DESC NULLS LAST
""",
(site_id,),
)
else:
rows = fetch_all(
"""
SELECT DISTINCT ON (cp.site_id)
cp.id, cp.url, cp.final_url, cp.title, cp.content, cp.metadata, cp.links,
cp.crawled_at, cp.site_id, ms.name AS site_name
FROM crawled_pages cp
LEFT JOIN monitored_sites ms ON ms.id = cp.site_id
WHERE cp.site_id IS NOT NULL
ORDER BY cp.site_id, cp.crawled_at DESC NULLS LAST
LIMIT %s
""",
(limit,),
)
pages = [_normalize_page_row(dict(r)) for r in rows]
q = (query or "").strip().lower()
if q:
pages = [
p
for p in pages
if q in (p.get("title") or "").lower()
or q in (p.get("content_read") or "").lower()
or q in (p.get("excerpt") or "").lower()
or any(q in h.lower() for h in p.get("headings") or [])
]
changed_site_ids: set[int] = set()
recent_changes: list[dict[str, Any]] = []
try:
cmd = ["docker", "exec", "foodlinkk_worker", "python", "-c", "import trigger"]
subprocess.run(cmd, capture_output=True, timeout=120, check=False)
return {"ok": True, "method": "worker"}
change_rows = fetch_all(
"""
SELECT pc.site_id, pc.changed_at, ms.name, ms.url
FROM page_changes pc
JOIN monitored_sites ms ON ms.id = pc.site_id
WHERE pc.changed_at >= NOW() - INTERVAL '7 days'
ORDER BY pc.changed_at DESC
LIMIT 30
"""
)
for cr in change_rows:
sid = int(cr["site_id"])
changed_site_ids.add(sid)
ts = cr.get("changed_at")
if ts is not None and hasattr(ts, "isoformat"):
ts = ts.isoformat()
recent_changes.append(
{
"site_id": sid,
"site_name": cr.get("name"),
"url": cr.get("url"),
"changed_at": ts,
}
)
except Exception:
pass
from app.db import fetch_all
term_scores: dict[str, dict[str, Any]] = {}
heading_counts: dict[str, dict[str, Any]] = {}
if site_id:
row = fetch_one(
"SELECT id, url, last_hash, last_title FROM monitored_sites WHERE id = %s AND is_active = TRUE",
(site_id,),
)
sites = [row] if row else []
else:
sites = fetch_all(
"SELECT id, url, last_hash, last_title FROM monitored_sites WHERE is_active = TRUE"
)
def bump_term(term: str, site_name: str, weight: int = 1) -> None:
if len(term) < 3:
return
bucket = term_scores.setdefault(term, {"term": term, "score": 0, "sites": set()})
bucket["score"] += weight
if site_name:
bucket["sites"].add(site_name)
changed = 0
with get_connection() as conn:
with conn.cursor() as cur:
for site in sites:
fetched = _fetch_page(site["url"])
if not fetched:
cur.execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site["id"], "ERROR", f"Cannot reach {site['url']}"),
)
continue
final_url, title, text, html = fetched
new_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
old_hash = site.get("last_hash")
if old_hash and old_hash != new_hash:
cur.execute(
"INSERT INTO page_changes (site_id, old_hash, new_hash) VALUES (%s, %s, %s)",
(site["id"], old_hash, new_hash),
)
cur.execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site["id"], "CHANGE", f"Change detected on {site['url']}{title}"),
)
changed += 1
else:
cur.execute(
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
(site["id"], "OK", f"Crawl OK — {title}"),
)
cur.execute(
"""
UPDATE monitored_sites SET last_hash=%s, last_crawled=NOW(), last_title=%s WHERE id=%s
""",
(new_hash, title, site["id"]),
)
_save_snapshot(site["id"], site["url"], final_url, title, text, html)
return {"ok": True, "method": "inline", "changes": changed, "sites": len(sites)}
for page in pages:
site_name = page.get("site_name") or str(page.get("site_id") or "")
for tok in _tokens(page.get("title") or "", min_len=3):
bump_term(tok, site_name, 3)
for h in page.get("headings") or []:
hnorm = re.sub(r"\s+", " ", h.strip())[:80]
if len(hnorm) < 3:
continue
hc = heading_counts.setdefault(hnorm.lower(), {"label": hnorm, "count": 0, "sites": set()})
hc["count"] += 1
hc["sites"].add(site_name)
for tok in _tokens(h, min_len=3):
bump_term(tok, site_name, 4)
for tok in _tokens(page.get("content_read") or ""):
bump_term(tok, site_name, 1)
for link in page.get("links_sample") or []:
for tok in _tokens(link.get("label") or "", min_len=3):
bump_term(tok, site_name, 2)
hype_terms: list[dict[str, Any]] = []
for term, data in term_scores.items():
if data["score"] < 4:
continue
sites_list = sorted(data["sites"])
hype_terms.append(
{
"term": term,
"score": data["score"],
"site_count": len(sites_list),
"sites": sites_list[:5],
"cross_site": len(sites_list) >= 2,
}
)
hype_terms.sort(key=lambda x: (-x["score"], -x["site_count"], x["term"]))
hype_terms = hype_terms[:40]
heading_trends = []
for _key, data in heading_counts.items():
if data["count"] < 1:
continue
heading_trends.append(
{
"label": data["label"],
"count": data["count"],
"sites": sorted(data["sites"])[:6],
"cross_site": len(data["sites"]) >= 2,
}
)
heading_trends.sort(key=lambda x: (-x["count"], -len(x["sites"]), x["label"]))
heading_trends = heading_trends[:25]
top_term_set = {t["term"] for t in hype_terms[:15]}
food_signals = frozenset(
"halal vegan plantaardig biologisch bio trend nieuw actie aanbieding kip rund vlees vis "
"groente fruit snack curry kebab burger protein eiwit alternatief duurzaam premium "
"supermarkt retail assortiment prijs private label merk".split()
)
for page in pages:
signals: list[str] = []
wc = int(page.get("word_count") or 0)
sid = page.get("site_id")
if wc >= 1500:
signals.append("Rijke pagina — veel te analyseren")
elif wc >= 400:
signals.append("Normale pagina-dichtheid")
if sid in changed_site_ids:
signals.append("Recent gewijzigd — mogelijke hype/shift")
page_terms = set(_tokens((page.get("content_read") or "") + " " + " ".join(page.get("headings") or [])))
matched_hype = [t for t in top_term_set if t in page_terms]
food_hits = [t for t in page_terms if t in food_signals]
for t in matched_hype[:4]:
signals.append(f"Trend-term: {t}")
for t in food_hits[:3]:
if f"Trend-term: {t}" not in signals:
signals.append(f"Food-signaal: {t}")
if page.get("meta_description"):
signals.append("SEO meta beschikbaar")
page["signals"] = signals[:8]
page["hype_score"] = len(matched_hype) * 10 + len(food_hits) * 5 + (20 if sid in changed_site_ids else 0) + min(wc // 200, 15)
page["recently_changed"] = sid in changed_site_ids
pages.sort(key=lambda p: (-(p.get("hype_score") or 0), -(p.get("word_count") or 0)))
total_words = sum(int(p.get("word_count") or 0) for p in pages)
return {
"summary": {
"pages": len(pages),
"total_words": total_words,
"themes_detected": len(hype_terms),
"headings_unique": len(heading_trends),
"changes_7d": len(recent_changes),
"query": q or None,
},
"hype_terms": hype_terms,
"heading_trends": heading_trends,
"recent_changes": recent_changes[:12],
"pages": pages,
}
+2 -2
View File
@@ -13,10 +13,10 @@ async def generate(prompt: str, system: str | None = None, timeout: float = 300.
return await chat_messages(messages, timeout=timeout)
async def chat_messages(messages: list[dict[str, str]], timeout: float = 300.0) -> str:
async def chat_messages(messages: list[dict[str, str]], timeout: float = 300.0, model: str | None = None) -> str:
url = f"{settings.OLLAMA_URL.rstrip('/')}/api/chat"
payload = {
"model": settings.OLLAMA_MODEL,
"model": model or settings.OLLAMA_MODEL,
"messages": messages,
"think": False,
"stream": False,
+11
View File
@@ -0,0 +1,11 @@
fastapi==0.115.6
uvicorn[standard]==0.32.1
jinja2==3.1.4
python-multipart==0.0.12
psycopg2-binary==2.9.9
httpx==0.27.2
websockets==13.1
beautifulsoup4==4.12.3
textblob==0.18.0.post0
openpyxl>=3.1.0
tweepy==4.15.0
+424
View File
@@ -0,0 +1,424 @@
"""Revenue Cockpit — DB operations, import, tracking, agent tasks."""
from __future__ import annotations
import json
from datetime import date, datetime, timezone
from typing import Any, Optional
from app.db import execute, fetch_all, fetch_one
from app.services import agent_integration
from app.services.excel_import import DEFAULT_FILE, DEFAULT_SHEET, parse_revenue_sheet
def fetch_excel_parse(path: str = DEFAULT_FILE, sheet: str | None = DEFAULT_SHEET) -> dict[str, Any]:
result = parse_revenue_sheet(path, sheet)
if not result.get("ok"):
raise RuntimeError(result.get("error") or "Excel parse failed")
return result
async def fetch_excel_parse_async(path: str = DEFAULT_FILE, sheet: str | None = DEFAULT_SHEET) -> dict[str, Any]:
return fetch_excel_parse(path, sheet)
def _ser(row: dict[str, Any] | None) -> dict[str, Any] | None:
if not row:
return None
out = dict(row)
for k, v in list(out.items()):
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
return out
def _log_change(entity_type: str, entity_id: int, field: str, old: Any, new: Any, by: str = "ceo") -> None:
execute(
"""
INSERT INTO revenue_change_log (entity_type, entity_id, field_name, old_value, new_value, changed_by)
VALUES (%s, %s, %s, %s, %s, %s)
""",
(entity_type, entity_id, field, str(old) if old is not None else None, str(new) if new is not None else None, by),
)
def import_from_parsed(parsed: dict[str, Any], imported_by: str = "ceo", replace: bool = True) -> dict[str, Any]:
if replace:
execute("DELETE FROM revenue_objectives WHERE project_id IN (SELECT id FROM revenue_projects)")
execute("DELETE FROM revenue_projects")
execute("UPDATE revenue_cockpit_goals SET is_active = FALSE WHERE is_active = TRUE")
goals = parsed.get("goals") or {}
g = fetch_one(
"""
INSERT INTO revenue_cockpit_goals (vision_text, horizon_text, mid_text, tagline, is_active)
VALUES (%s, %s, %s, %s, TRUE)
RETURNING *
""",
(
goals.get("vision_text", ""),
goals.get("horizon_text", ""),
goals.get("mid_text", ""),
goals.get("tagline", ""),
),
)
count = 0
for p in parsed.get("projects") or []:
row = fetch_one(
"""
INSERT INTO revenue_projects
(name, category, margin_month, margin_year, target_revenue, next_steps, status, sort_order, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id
""",
(
p["name"],
p.get("category", "deal"),
p.get("margin_month"),
p.get("margin_year"),
p.get("target_revenue"),
p.get("next_steps"),
p.get("status", "active"),
p.get("sort_order", count),
json.dumps({
"source_row": p.get("source_row"),
"row_style": p.get("row_style", "white"),
"imported": True,
}),
),
)
pid = row["id"] if row else None
if pid and p.get("next_steps"):
for i, line in enumerate([ln.strip() for ln in p["next_steps"].split("\n") if ln.strip()][:5]):
execute(
"""
INSERT INTO revenue_objectives (project_id, title, description, status, sort_order)
VALUES (%s, %s, %s, 'open', %s)
""",
(pid, line[:255], p["next_steps"] if i == 0 else None, i),
)
count += 1
run = fetch_one(
"""
INSERT INTO revenue_import_runs (source_file, sheet_name, rows_imported, goals_imported, imported_by, metadata)
VALUES (%s, %s, %s, TRUE, %s, %s::jsonb)
RETURNING *
""",
(
parsed.get("source_file", ""),
parsed.get("sheet_name", ""),
count,
imported_by,
json.dumps({"parsed_at": parsed.get("parsed_at"), "file_mtime": parsed.get("file_mtime")}),
),
)
take_snapshot()
return {"ok": True, "projects_imported": count, "goals": _ser(g), "import_run": _ser(run)}
def get_active_goals() -> dict[str, Any] | None:
return _ser(fetch_one("SELECT * FROM revenue_cockpit_goals WHERE is_active = TRUE ORDER BY id DESC LIMIT 1"))
def update_goals(data: dict[str, Any], changed_by: str = "ceo") -> dict[str, Any] | None:
old = fetch_one("SELECT * FROM revenue_cockpit_goals WHERE is_active = TRUE ORDER BY id DESC LIMIT 1")
if not old:
return None
fields = ("vision_text", "horizon_text", "mid_text", "tagline")
sets, params = [], []
for k in fields:
if k in data:
sets.append(f"{k} = %s")
params.append(data[k])
if str(old.get(k)) != str(data[k]):
_log_change("goals", old["id"], k, old.get(k), data[k], changed_by)
if not sets:
return get_active_goals()
params.append(old["id"])
execute(
f"UPDATE revenue_cockpit_goals SET {', '.join(sets)}, updated_at = NOW() WHERE id = %s",
tuple(params),
)
return get_active_goals()
def list_projects(status: str | None = None) -> list[dict[str, Any]]:
clauses, params = [], []
if status:
clauses.append("status = %s")
params.append(status)
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
rows = fetch_all(
f"""
SELECT p.*,
(SELECT COUNT(*) FROM revenue_objectives o WHERE o.project_id = p.id AND o.status = 'open') AS open_objectives,
(SELECT COUNT(*) FROM agent_tasks t WHERE t.revenue_project_id = p.id AND t.status NOT IN ('completed','cancelled')) AS open_tasks
FROM revenue_projects p
{where}
ORDER BY p.sort_order ASC, p.name ASC
""",
tuple(params) if params else None,
)
return [_ser({**r, "row_style": _infer_row_style(r.get("name"), r.get("margin_month"), r.get("metadata"))}) for r in rows]
def get_project(project_id: int) -> dict[str, Any] | None:
p = _ser(fetch_one("SELECT * FROM revenue_projects WHERE id = %s", (project_id,)))
if not p:
return None
p["objectives"] = [_ser(r) for r in fetch_all(
"SELECT * FROM revenue_objectives WHERE project_id = %s ORDER BY sort_order, id",
(project_id,),
)]
p["tasks"] = [_ser(r) for r in fetch_all(
"""
SELECT * FROM agent_tasks WHERE revenue_project_id = %s
ORDER BY created_at DESC LIMIT 50
""",
(project_id,),
)]
p["history"] = [_ser(r) for r in fetch_all(
"""
SELECT * FROM revenue_change_log
WHERE (entity_type = 'project' AND entity_id = %s)
OR (entity_type = 'objective' AND entity_id IN (
SELECT id FROM revenue_objectives WHERE project_id = %s))
ORDER BY created_at DESC LIMIT 30
""",
(project_id, project_id),
)]
p["row_style"] = _infer_row_style(p.get("name"), p.get("margin_month"), p.get("metadata"))
return p
def set_project_row_style(project_id: int, row_style: str) -> None:
old = fetch_one("SELECT metadata FROM revenue_projects WHERE id = %s", (project_id,))
meta = dict(old.get("metadata") or {}) if old else {}
meta["row_style"] = row_style
execute(
"UPDATE revenue_projects SET metadata = %s::jsonb, updated_at = NOW() WHERE id = %s",
(json.dumps(meta), project_id),
)
def update_project(project_id: int, data: dict[str, Any], changed_by: str = "ceo") -> dict[str, Any] | None:
old = fetch_one("SELECT * FROM revenue_projects WHERE id = %s", (project_id,))
if not old:
return None
fields = {
"name": data.get("name"),
"category": data.get("category"),
"margin_month": data.get("margin_month"),
"margin_year": data.get("margin_year"),
"target_revenue": data.get("target_revenue"),
"next_steps": data.get("next_steps"),
"status": data.get("status"),
"priority": data.get("priority"),
}
sets, params = [], []
for k, v in fields.items():
if k in data:
sets.append(f"{k} = %s")
params.append(v)
if str(old.get(k)) != str(v):
_log_change("project", project_id, k, old.get(k), v, changed_by)
if not sets:
return get_project(project_id)
params.append(project_id)
execute(f"UPDATE revenue_projects SET {', '.join(sets)}, updated_at = NOW() WHERE id = %s", tuple(params))
return get_project(project_id)
def _infer_row_style(name: str, margin_month: Any, metadata: Any) -> str:
if isinstance(metadata, dict) and metadata.get("row_style"):
return str(metadata["row_style"])
n = (name or "").lower()
if "foodlinkk food marketing" in n or "total earnings" in n or "loonkosten aissa" in n:
return "yellow"
if "foodservice" in n or n.strip() == "dirk":
return "orange"
if margin_month is not None and margin_month > 0:
return "green"
if any(x in n for x in ("plus", "vomar", "hoogvliet", "deka", "spar", "doner palace", "zakat")):
return "red"
return "white"
def create_objective(project_id: int, title: str, description: str | None = None, priority: str = "normal") -> dict[str, Any]:
row = fetch_one(
"""
INSERT INTO revenue_objectives (project_id, title, description, priority)
VALUES (%s, %s, %s, %s)
RETURNING *
""",
(project_id, title.strip(), description, priority),
)
_log_change("objective", row["id"], "created", None, title, "ceo")
return _ser(row) or {}
def update_objective(objective_id: int, data: dict[str, Any]) -> dict[str, Any] | None:
old = fetch_one("SELECT * FROM revenue_objectives WHERE id = %s", (objective_id,))
if not old:
return None
for k in ("title", "description", "status", "priority", "due_date"):
if k in data:
execute(
f"UPDATE revenue_objectives SET {k} = %s, updated_at = NOW() WHERE id = %s",
(data[k], objective_id),
)
_log_change("objective", objective_id, k, old.get(k), data[k])
return _ser(fetch_one("SELECT * FROM revenue_objectives WHERE id = %s", (objective_id,)))
def dashboard_stats() -> dict[str, Any]:
goals = get_active_goals()
totals = fetch_one(
"""
SELECT
COUNT(*) FILTER (WHERE status = 'active') AS active_projects,
COALESCE(SUM(margin_year) FILTER (WHERE status = 'active'), 0) AS total_margin_year,
COALESCE(SUM(margin_month) FILTER (WHERE status = 'active'), 0) AS total_margin_month,
COUNT(*) FILTER (WHERE category = 'deal' AND status = 'active') AS active_deals
FROM revenue_projects
"""
) or {}
open_obj = fetch_one("SELECT COUNT(*) AS n FROM revenue_objectives WHERE status = 'open'") or {"n": 0}
open_tasks = fetch_one(
"SELECT COUNT(*) AS n FROM agent_tasks WHERE status NOT IN ('completed','cancelled') AND revenue_project_id IS NOT NULL"
) or {"n": 0}
prev = fetch_one(
"SELECT * FROM revenue_snapshots WHERE snapshot_date < CURRENT_DATE ORDER BY snapshot_date DESC LIMIT 1"
)
return {
"goals": goals,
"active_projects": int(totals.get("active_projects") or 0),
"active_deals": int(totals.get("active_deals") or 0),
"total_margin_year": float(totals.get("total_margin_year") or 0),
"total_margin_month": float(totals.get("total_margin_month") or 0),
"open_objectives": int(open_obj.get("n") or 0),
"open_agent_tasks": int(open_tasks.get("n") or 0),
"previous_snapshot": _ser(prev),
}
def take_snapshot() -> dict[str, Any]:
stats = dashboard_stats()
row = fetch_one(
"""
INSERT INTO revenue_snapshots
(snapshot_date, total_margin_year, total_margin_month, active_projects, open_objectives, open_agent_tasks, payload)
VALUES (CURRENT_DATE, %s, %s, %s, %s, %s, %s::jsonb)
ON CONFLICT (snapshot_date) DO UPDATE SET
total_margin_year = EXCLUDED.total_margin_year,
total_margin_month = EXCLUDED.total_margin_month,
active_projects = EXCLUDED.active_projects,
open_objectives = EXCLUDED.open_objectives,
open_agent_tasks = EXCLUDED.open_agent_tasks,
payload = EXCLUDED.payload,
created_at = NOW()
RETURNING *
""",
(
stats["total_margin_year"],
stats["total_margin_month"],
stats["active_projects"],
stats["open_objectives"],
stats["open_agent_tasks"],
json.dumps({"goals_id": (stats.get("goals") or {}).get("id")}),
),
)
return _ser(row) or {}
def list_snapshots(limit: int = 90) -> list[dict[str, Any]]:
rows = fetch_all(
"SELECT * FROM revenue_snapshots ORDER BY snapshot_date DESC LIMIT %s",
(max(1, min(limit, 365)),),
)
return [_ser(r) for r in rows]
def assign_agent_task(
project_id: int,
agent_name: str,
title: str,
description: str | None = None,
objective_id: int | None = None,
priority: str = "normal",
delegate_herman: bool = True,
) -> dict[str, Any]:
project = fetch_one("SELECT name FROM revenue_projects WHERE id = %s", (project_id,))
if not project:
raise ValueError("Project not found")
agent = agent_name.strip().lower()
desc = description or ""
row = fetch_one(
"""
INSERT INTO agent_tasks
(agent_name, title, description, status, priority, assigned_by, revenue_project_id, revenue_objective_id, source)
VALUES (%s, %s, %s, 'pending', %s, 'ceo', %s, %s, 'revenue_cockpit')
RETURNING *
""",
(agent, title.strip(), desc, priority, project_id, objective_id),
)
task = _ser(row) or {}
body = f"Revenue Cockpit · {project['name']}: {title}"
if desc:
body += f"\n{desc}"
try:
execute(
"""
INSERT INTO agent_events (agent_name, agent_type, event_type, title, body, status, channel, metadata, related_table, related_id)
VALUES (%s, 'revenue_cockpit', 'task_assigned', %s, %s, 'pending', 'revenue_cockpit', %s::jsonb, 'agent_tasks', %s)
""",
(
agent,
title.strip(),
body,
json.dumps({"project_id": project_id, "objective_id": objective_id, "task_id": task.get("id")}),
task.get("id"),
),
)
except Exception:
pass
if delegate_herman and agent != "herman":
try:
agent_integration.create_handoff(
"ceo",
agent,
handoff_type="task",
payload={"task_id": task.get("id"), "project_id": project_id, "title": title},
status="pending",
)
except Exception:
pass
_log_change("project", project_id, "agent_task", None, title, "ceo")
return task
def list_agent_tasks(limit: int = 50) -> list[dict[str, Any]]:
rows = fetch_all(
"""
SELECT t.*, p.name AS project_name
FROM agent_tasks t
LEFT JOIN revenue_projects p ON p.id = t.revenue_project_id
WHERE t.revenue_project_id IS NOT NULL
ORDER BY t.created_at DESC
LIMIT %s
""",
(max(1, min(limit, 200)),),
)
return [_ser(r) for r in rows]
async def delegate_via_herman(project_id: int, message: str) -> dict[str, Any]:
from app.services import herman as herman_service
project = get_project(project_id)
if not project:
raise ValueError("Project not found")
prompt = f"[Revenue Cockpit · {project['name']}] {message}"
result = await herman_service.chat(prompt)
return {"ok": True, "result": result, "project_id": project_id}
@@ -0,0 +1,594 @@
"""Voice/browser/Telegram command router: export intel, webbuilder/agy, bevestiging + UI."""
from __future__ import annotations
import re
import time
import uuid
from typing import Any
import httpx
from app.config import settings
from app.services import webbuilder_agent
SESSION_TTL_SEC = 3600
_pending: dict[str, dict[str, Any]] = {}
VOICE_CHANNELS = frozenset({"voice", "browser", "telegram"})
LOCATION_ALIASES: dict[str, dict[str, str]] = {
"dubai": {"country": "AE", "q": "Dubai", "label": "Dubai (VAE)"},
"abu dhabi": {"country": "AE", "q": "Abu Dhabi", "label": "Abu Dhabi (VAE)"},
"sharjah": {"country": "AE", "q": "Sharjah", "label": "Sharjah (VAE)"},
"vae": {"country": "AE", "q": "", "label": "Verenigde Arabische Emiraten"},
"uae": {"country": "AE", "q": "", "label": "Verenigde Arabische Emiraten"},
"emiraten": {"country": "AE", "q": "", "label": "VAE"},
"saudi": {"country": "SA", "q": "", "label": "Saoedi-Arabië"},
"riyadh": {"country": "SA", "q": "Riyadh", "label": "Riyadh (SA)"},
"jeddah": {"country": "SA", "q": "Jeddah", "label": "Jeddah (SA)"},
"qatar": {"country": "QA", "q": "", "label": "Qatar"},
"doha": {"country": "QA", "q": "Doha", "label": "Doha (Qatar)"},
"kuwait": {"country": "KW", "q": "", "label": "Koeweit"},
"nederland": {"country": "NL", "q": "", "label": "Nederland"},
"amsterdam": {"country": "NL", "q": "Amsterdam", "label": "Amsterdam"},
"rotterdam": {"country": "NL", "q": "Rotterdam", "label": "Rotterdam"},
"belgië": {"country": "BE", "q": "", "label": "België"},
"belgie": {"country": "BE", "q": "", "label": "België"},
"antwerpen": {"country": "BE", "q": "Antwerp", "label": "Antwerpen"},
"duitsland": {"country": "DE", "q": "", "label": "Duitsland"},
"berlijn": {"country": "DE", "q": "Berlin", "label": "Berlijn"},
"frankfurt": {"country": "DE", "q": "Frankfurt", "label": "Frankfurt"},
"turkije": {"country": "TR", "q": "", "label": "Turkije"},
"istanbul": {"country": "TR", "q": "Istanbul", "label": "Istanbul"},
"marokko": {"country": "MA", "q": "", "label": "Marokko"},
"casablanca": {"country": "MA", "q": "Casablanca", "label": "Casablanca"},
}
ENTITY_ALIASES: dict[str, str] = {
"distri": "distributor,wholesaler,importer,logistics",
"distributeur": "distributor,wholesaler,importer,logistics",
"distributeurs": "distributor,wholesaler,importer,logistics",
"groothandel": "wholesaler,distributor",
"groothandels": "wholesaler,distributor",
"importeur": "importer",
"importeurs": "importer",
"logistiek": "logistics",
"restaurant": "restaurant",
"restaurants": "restaurant",
"cateraar": "caterer",
"cateraars": "caterer",
"slager": "butcher",
"slagers": "butcher",
"döner": "doner",
"doner": "doner",
}
SEARCH_TRIGGERS = (
"zoek", "opzoeken", "vind", "zoeken", "toon", "laat zien", "lijst",
"geef me", "haal op", "export intel", "wereldexport",
)
YES_RE = re.compile(
r"^(ja|jawel|jep|yep|ok|oke|oké|okay|klopt|bevestig|doe maar|graag|precies|goed|akkoord|start|uitvoeren|doorgaan)\b",
re.I,
)
NO_RE = re.compile(r"^(nee|neen|stop|annuleer|niet|cancel|laat maar|wacht)\b", re.I)
def _cleanup_sessions() -> None:
now = time.time()
dead = [k for k, v in _pending.items() if now - float(v.get("_ts", 0)) > SESSION_TTL_SEC]
for k in dead:
_pending.pop(k, None)
def is_confirmation_yes(text: str) -> bool:
t = (text or "").strip()
return bool(t and YES_RE.search(t))
def is_confirmation_no(text: str) -> bool:
t = (text or "").strip()
return bool(t and NO_RE.search(t))
def wants_export_search(text: str) -> bool:
t = (text or "").lower()
if not any(k in t for k in SEARCH_TRIGGERS):
return False
has_loc = any(alias in t for alias in LOCATION_ALIASES)
has_ent = any(alias in t for alias in ENTITY_ALIASES)
return has_loc or has_ent
def _detect_location(text: str) -> dict[str, str] | None:
t = text.lower()
best: tuple[int, dict[str, str]] | None = None
for alias, loc in LOCATION_ALIASES.items():
if alias in t:
score = len(alias)
if best is None or score > best[0]:
best = (score, loc)
return best[1] if best else None
def _detect_entity_types(text: str) -> str:
t = text.lower()
found: list[str] = []
for alias, types in ENTITY_ALIASES.items():
if alias in t:
for et in types.split(","):
if et not in found:
found.append(et)
if not found and any(w in t for w in ("distri", "distributeur", "groothandel", "b2b", "leverancier")):
return "distributor,wholesaler,importer,logistics"
return ",".join(found) if found else "distributor,wholesaler,importer,logistics"
def _entity_label(entity_types: str) -> str:
labels = {
"distributor": "distributeurs",
"wholesaler": "groothandels",
"importer": "importeurs",
"logistics": "logistiek",
"restaurant": "restaurants",
"caterer": "cateraars",
"butcher": "slagers",
"doner": "dönerzaken",
}
parts = [labels.get(x.strip(), x.strip()) for x in entity_types.split(",") if x.strip()]
return ", ".join(parts) if parts else "bedrijven"
def parse_export_search(text: str) -> dict[str, Any]:
loc = _detect_location(text)
entity_types = _detect_entity_types(text)
missing: list[str] = []
if not loc:
missing.append("locatie")
params: dict[str, Any] = {
"country": (loc or {}).get("country", ""),
"q": (loc or {}).get("q", ""),
"entity_types": entity_types,
"limit": 50,
}
label_loc = (loc or {}).get("label", "")
return {
"params": params,
"location_label": label_loc,
"entity_label": _entity_label(entity_types),
"missing": missing,
"incomplete": bool(missing),
}
def store_pending(session_id: str, action: dict[str, Any]) -> str:
_cleanup_sessions()
action_id = str(uuid.uuid4())[:12]
action = dict(action)
action["id"] = action_id
action["_ts"] = time.time()
_pending[session_id] = action
return action_id
def get_pending(session_id: str) -> dict[str, Any] | None:
_cleanup_sessions()
row = _pending.get(session_id)
if not row:
return None
if time.time() - float(row.get("_ts", 0)) > SESSION_TTL_SEC:
_pending.pop(session_id, None)
return None
return row
def clear_pending(session_id: str) -> None:
_pending.pop(session_id, None)
def build_confirmation_question(parsed: dict[str, Any]) -> str:
loc = parsed.get("location_label") or "de geselecteerde regio"
ent = parsed.get("entity_label") or "bedrijven"
return (
f"Ik ga **{ent}** in **{loc}** voor je opzoeken in Export Intel.\n\n"
"Klopt dat? Zeg **ja** om te starten, **nee** om te annuleren, "
"of geef aan wat ik moet aanpassen (bijv. alleen distributeurs, of een andere stad)."
)
def build_clarification_question(parsed: dict[str, Any]) -> str:
missing = parsed.get("missing") or []
if "locatie" in missing:
return (
"In welke **stad of welk land** wil je zoeken? "
"Bijvoorbeeld: Dubai, VAE, Nederland, Frankfurt…"
)
return "Kun je iets specifieker zijn over wat je zoekt en waar?"
async def fetch_export_entities(params: dict[str, Any]) -> dict[str, Any]:
query = {k: v for k, v in params.items() if v not in (None, "")}
url = f"{settings.TOOLS_API_URL.rstrip('/')}/export-intel/entities"
async with httpx.AsyncClient(timeout=60.0) as client:
resp = await client.get(url, params=query)
resp.raise_for_status()
return resp.json()
def _open_url(params: dict[str, Any]) -> str:
qs = []
if params.get("country"):
qs.append(f"country={params['country']}")
if params.get("q"):
qs.append(f"q={params['q']}")
qs.append("tab=distributors")
return "/export-intel?" + "&".join(qs)
async def execute_search_action(action: dict[str, Any]) -> dict[str, Any]:
params = action.get("params") or {}
data = await fetch_export_entities(params)
items = data.get("items") or []
total = int(data.get("total") or len(items))
loc = action.get("location_label") or params.get("q") or params.get("country") or "markt"
ent = action.get("entity_label") or "bedrijven"
title = f"{ent.title()}{loc}"
if not items:
reply = (
f"Ik heb gezocht maar vond **geen** {ent} in {loc}. "
"Wil je dat ik een sync start of een bredere regio probeer?"
)
else:
reply = (
f"Gevonden: **{total}** {ent} in {loc}. "
f"Ik toon de eerste {min(len(items), 50)} in het resultatenvenster."
)
return {
"agent": "sourcing",
"agent_label": "Export Intel → Herman",
"reply": reply,
"delegated_agents": ["sourcing", "export_intel"],
"routing_reason": f"Export Intel zoekopdracht: {title}",
"agent_steps": [
{"agent": "export_intel", "status": "done", "message": f"{total} resultaten"},
{"agent": "sourcing", "status": "delegated", "message": "Marktdata opgehaald"},
],
"needs_confirmation": False,
"ui_actions": [
{
"type": "show_export_results",
"title": title,
"entities": items[:50],
"total": total,
"params": params,
"open_url": _open_url(params),
}
],
}
def _pending_summary(action: dict[str, Any]) -> dict[str, Any]:
atype = action.get("type", "")
base = {"id": action.get("id"), "type": atype}
if atype == "export_intel_search":
base["location_label"] = action.get("location_label")
base["entity_label"] = action.get("entity_label")
base["params"] = action.get("params")
elif atype == "webbuilder_build":
base["project"] = action.get("project")
base["entity_label"] = f"website {action.get('project', '')}"
base["location_label"] = "Agy · Antigravity"
return base
def build_webbuilder_confirmation(project: str, raw: str) -> str:
snippet = (raw or "")[:240]
return (
f"Ik stuur **Agy (Antigravity)** op Hermes aan om een website te bouwen.\n\n"
f"**Project:** {project}\n"
f"**Opdracht:** {snippet}{'' if len(raw or '') > 240 else ''}\n\n"
"Klopt dat? Zeg **ja** om te starten, **nee** om te annuleren."
)
async def execute_webbuilder_action(action: dict[str, Any], channel: str = "voice") -> dict[str, Any]:
raw = action.get("raw_message") or ""
project = action.get("project") or webbuilder_agent.extract_project_name(raw)
try:
outcome = await webbuilder_agent.generate_from_message(raw, channel=channel, wait=False)
except Exception as exc:
return {
"agent": "webbuilder",
"agent_label": "Web Builder",
"reply": f"Agy kon niet starten op Hermes: {exc}\n\nControleer VM107 webbuilder-api (:8798).",
"delegated_agents": ["webbuilder"],
"needs_confirmation": False,
}
project = outcome.get("project") or project
preview = outcome.get("preview_url") or webbuilder_agent.HERMES_PREVIEW_BASE
nas_path = outcome.get("nas_path") or ""
reply = (
f"Agy is gestart voor **{project}**.\n"
f"Preview (na build): {preview}\n"
f"NAS: {nas_path}\n"
"Volg live: Agents → Terminals → Web Builder."
)
return {
"agent": "webbuilder",
"agent_label": "Agy / Web Builder → Herman",
"reply": reply,
"delegated_agents": ["webbuilder"],
"routing_reason": "Website via Antigravity CLI (agy) op Hermes VM107",
"agent_steps": [
{"agent": "webbuilder", "status": "running", "message": f"agy bouwt {project}"},
{"agent": "herman", "status": "delegated", "message": "Voice → build gestart"},
],
"webbuilder_project": project,
"webbuilder_preview_url": preview,
"needs_confirmation": False,
"ui_actions": [
{
"type": "open_webbuilder_build",
"title": f"Website build — {project}",
"project": project,
"preview_url": preview,
"nas_path": nas_path,
"agents_url": "/agents",
"status_hint": "Build duurt enkele minuten — preview opent na afloop",
}
],
}
async def execute_pending_action(action: dict[str, Any], channel: str = "voice") -> dict[str, Any]:
atype = action.get("type")
if atype == "webbuilder_build":
return await execute_webbuilder_action(action, channel=channel)
return await execute_search_action(action)
def _confirmation_reminder(pending: dict[str, Any]) -> str:
if pending.get("type") == "webbuilder_build":
return build_webbuilder_confirmation(pending.get("project", "website"), pending.get("raw_message", ""))
return build_confirmation_question(pending)
async def handle_voice_command(
message: str,
session_id: str | None = None,
confirm_action_id: str | None = None,
channel: str = "voice",
) -> dict[str, Any] | None:
"""Unified voice/browser router: bevestiging + export + webbuilder/agy."""
sid = (session_id or "").strip() or "default"
text = (message or "").strip()
if not text:
return None
pending = get_pending(sid)
if confirm_action_id and pending and pending.get("id") == confirm_action_id:
clear_pending(sid)
return await execute_pending_action(pending, channel=channel)
if pending:
if is_confirmation_yes(text):
clear_pending(sid)
return await execute_pending_action(pending, channel=channel)
if is_confirmation_no(text):
clear_pending(sid)
return {
"agent": "herman",
"agent_label": "Herman",
"reply": "Oké, geannuleerd. Waar kan ik je verder mee helpen?",
"needs_confirmation": False,
}
if pending.get("awaiting") == "location" and pending.get("type") == "export_intel_search":
loc = _detect_location(text)
if loc:
merged = dict(pending)
merged["params"] = dict(pending.get("params") or {})
merged["params"]["country"] = loc["country"]
merged["params"]["q"] = loc.get("q", "")
merged["location_label"] = loc["label"]
merged.pop("awaiting", None)
merged.pop("incomplete", None)
merged.pop("missing", None)
action_id = store_pending(sid, merged)
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_confirmation_question(merged),
"needs_confirmation": True,
"pending_action": _pending_summary(merged),
}
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_clarification_question({"missing": ["locatie"]}),
"needs_confirmation": True,
"clarification": True,
}
if webbuilder_agent.wants_website(text):
project = webbuilder_agent.extract_project_name(text)
action_id = store_pending(sid, {
"type": "webbuilder_build",
"project": project,
"raw_message": text,
})
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_webbuilder_confirmation(project, text),
"needs_confirmation": True,
"pending_action": _pending_summary(get_pending(sid) or {}),
}
if wants_export_search(text):
parsed = parse_export_search(text)
if not parsed.get("incomplete"):
action_id = store_pending(sid, {**parsed, "type": "export_intel_search"})
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_confirmation_question(parsed),
"needs_confirmation": True,
"pending_action": _pending_summary(get_pending(sid) or {}),
}
return {
"agent": "herman",
"agent_label": "Herman",
"reply": (
f"Ik wacht nog op bevestiging.\n\n{_confirmation_reminder(pending)}\n\n"
"Zeg **ja** of **nee**."
),
"needs_confirmation": True,
"pending_action": _pending_summary(pending),
}
if webbuilder_agent.wants_website(text):
project = webbuilder_agent.extract_project_name(text)
store_pending(sid, {"type": "webbuilder_build", "project": project, "raw_message": text})
pa = get_pending(sid) or {}
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_webbuilder_confirmation(project, text),
"needs_confirmation": True,
"pending_action": _pending_summary(pa),
}
return await handle_export_intent(message, session_id=session_id, confirm_action_id=confirm_action_id)
async def handle_export_intent(
message: str,
session_id: str | None = None,
confirm_action_id: str | None = None,
) -> dict[str, Any] | None:
"""Return Herman-shaped dict when export flow applies, else None."""
sid = (session_id or "").strip() or "default"
text = (message or "").strip()
if not text:
return None
pending = get_pending(sid)
if confirm_action_id and pending and pending.get("id") == confirm_action_id:
clear_pending(sid)
return await execute_pending_action(pending)
if pending:
if is_confirmation_yes(text):
clear_pending(sid)
return await execute_pending_action(pending)
if is_confirmation_no(text):
clear_pending(sid)
return {
"agent": "herman",
"agent_label": "Herman",
"reply": "Oké, geannuleerd. Waar kan ik je verder mee helpen?",
"needs_confirmation": False,
}
if pending.get("awaiting") == "location":
loc = _detect_location(text)
if loc:
merged = dict(pending)
merged["params"] = dict(pending.get("params") or {})
merged["params"]["country"] = loc["country"]
merged["params"]["q"] = loc.get("q", "")
merged["location_label"] = loc["label"]
merged.pop("awaiting", None)
merged.pop("incomplete", None)
merged.pop("missing", None)
action_id = store_pending(sid, merged)
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_confirmation_question(merged),
"needs_confirmation": True,
"pending_action": {
"id": action_id,
"type": "export_intel_search",
"params": merged.get("params"),
"location_label": merged.get("location_label"),
"entity_label": merged.get("entity_label"),
},
}
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_clarification_question({"missing": ["locatie"]}),
"needs_confirmation": True,
"clarification": True,
}
if wants_export_search(text):
parsed = parse_export_search(text)
if parsed.get("incomplete"):
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_clarification_question(parsed),
"needs_confirmation": True,
"clarification": True,
}
action_id = store_pending(sid, parsed)
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_confirmation_question(parsed),
"needs_confirmation": True,
"pending_action": {
"id": action_id,
"type": "export_intel_search",
"params": parsed.get("params"),
"location_label": parsed.get("location_label"),
"entity_label": parsed.get("entity_label"),
},
}
return {
"agent": "herman",
"agent_label": "Herman",
"reply": (
f"Ik wacht nog op bevestiging: {build_confirmation_question(pending)}\n\n"
"Zeg **ja** of **nee**, of stel een nieuwe zoekopdracht."
),
"needs_confirmation": True,
"pending_action": {
"id": pending.get("id"),
"type": pending.get("type", "export_intel_search"),
"params": pending.get("params"),
"location_label": pending.get("location_label"),
"entity_label": pending.get("entity_label"),
},
}
if not wants_export_search(text):
return None
parsed = parse_export_search(text)
if parsed.get("incomplete"):
store_pending(sid, {**parsed, "type": "export_intel_search", "awaiting": "location"})
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_clarification_question(parsed),
"needs_confirmation": True,
"clarification": True,
}
action_id = store_pending(sid, {**parsed, "type": "export_intel_search"})
return {
"agent": "herman",
"agent_label": "Herman",
"reply": build_confirmation_question(parsed),
"needs_confirmation": True,
"pending_action": {
"id": action_id,
"type": "export_intel_search",
"params": parsed.get("params"),
"location_label": parsed.get("location_label"),
"entity_label": parsed.get("entity_label"),
},
}
+267
View File
@@ -0,0 +1,267 @@
"""Web Builder agent — websites bouwen via Hermes/agy API op VM107."""
from __future__ import annotations
import os
import re
from typing import Any
import httpx
from app.config import settings
from app.services.agent_events_log import log_agent_event
HERMES_BUILD_URL = os.getenv("HERMES_BUILD_URL", "http://10.4.7.27:8798").rstrip("/")
HERMES_PREVIEW_BASE = os.getenv("HERMES_PREVIEW_BASE", "http://10.4.7.27:8080").rstrip("/")
WEBSITE_KEYWORDS = (
"maak website",
"maak een website",
"bouw website",
"bouw een website",
"genereer website",
"website voor",
"website bouwen",
"landingspagina",
"landing page",
"webshop",
"webpagina",
"nieuwe site",
"nieuwe website",
"/website",
"web builder",
"webbuilder",
"agy",
"antigravity",
"anti gravity",
"anti-gravity",
"site maken",
"maak een site",
"bouw een site",
"website laten maken",
"laat agy",
"via agy",
)
PROJECT_RE = re.compile(
r"(?:website|site|project)\s+(?:voor\s+|called\s+|genaamd\s+)?['\"]?([a-z0-9][a-z0-9 _-]{1,48})['\"]?",
re.I,
)
FOR_RE = re.compile(
r"\bvoor\s+['\"]?([a-z0-9][a-z0-9 _-]{1,48})['\"]?",
re.I,
)
def wants_website(raw: str) -> bool:
t = (raw or "").strip().lower()
return any(k in t for k in WEBSITE_KEYWORDS)
def slugify(name: str) -> str:
s = (name or "").strip().lower()
s = re.sub(r"[^a-z0-9_-]+", "-", s)
s = re.sub(r"-+", "-", s).strip("-")
return s[:64] or "website"
def extract_project_name(raw: str) -> str:
t = (raw or "").strip()
m = PROJECT_RE.search(t)
if m:
return slugify(m.group(1))
m = FOR_RE.search(t)
if m:
name = m.group(1)
name = re.split(r"\s+(?:met|with|incl|including)\b", name, maxsplit=1, flags=re.I)[0]
return slugify(name)
for token in t.split():
clean = slugify(token)
if len(clean) >= 3 and clean not in (
"maak", "bouw", "website", "site", "voor", "een", "the", "and", "met",
"simpele", "landing", "page", "pagina", "landingspagina",
):
return clean
return slugify(t[:40]) or "website"
def extract_build_prompt(raw: str, project: str) -> str:
body = (raw or "").strip()
lower = body.lower()
for k in WEBSITE_KEYWORDS:
if lower.startswith(k):
rest = body[len(k) :].strip(" :,-")
if rest:
return rest
return (
f"Bouw een complete, moderne, responsive website voor project '{project}'. "
f"Opdracht: {body}. "
"Gebruik index.html, style.css, script.js en een assets/ map. "
"Foodlinkk halal kant-en-klaar food branding waar passend."
)
async def _hermes_get(path: str, timeout: float = 30.0) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.get(f"{HERMES_BUILD_URL}{path}")
try:
data = resp.json()
except Exception:
data = {"ok": False, "detail": resp.text[:500]}
if resp.status_code >= 400:
data.setdefault("ok", False)
return data
async def _hermes_post(path: str, payload: dict[str, Any], timeout: float = 60.0) -> dict[str, Any]:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.post(f"{HERMES_BUILD_URL}{path}", json=payload)
try:
data = resp.json()
except Exception:
data = {"ok": False, "detail": resp.text[:500]}
if resp.status_code >= 400:
data.setdefault("ok", False)
data.setdefault("detail", resp.text[:500])
return data
async def list_projects() -> dict[str, Any]:
return await _hermes_get("/api/projects")
async def get_build_status(project: str) -> dict[str, Any]:
slug = slugify(project)
return await _hermes_get(f"/api/build/{slug}/status")
async def build_from_message(raw: str, *, channel: str = "herman") -> dict[str, Any]:
project = extract_project_name(raw)
prompt = extract_build_prompt(raw, project)
log_agent_event(
"webbuilder",
"website_build_start",
f"Start build: {project}",
prompt[:4000],
channel=channel,
metadata={"project": project, "prompt": prompt[:500]},
status="running",
)
data = await _hermes_post(
"/api/build",
{"project": project, "prompt": prompt},
timeout=45.0,
)
if not data.get("ok", True) and data.get("detail"):
log_agent_event(
"webbuilder",
"website_build_error",
f"Build mislukt: {project}",
str(data.get("detail", data))[:2000],
channel=channel,
metadata={"project": project},
status="error",
)
raise RuntimeError(str(data.get("detail") or data))
job_id = data.get("job_id") or project
preview = data.get("preview_url") or f"{HERMES_PREVIEW_BASE}/"
nas_path = data.get("nas_path") or f"//10.4.7.11/share/Websites/{project}/"
log_agent_event(
"webbuilder",
"website_build_running",
f"agy bezig: {project}",
f"Job {job_id}\nPreview: {preview}\nNAS: {nas_path}",
channel=channel,
metadata={
"project": project,
"job_id": job_id,
"preview_url": preview,
"nas_path": nas_path,
"correlation_id": job_id,
},
status="running",
)
return {
"project": project,
"job_id": job_id,
"preview_url": preview,
"nas_path": nas_path,
"prompt": prompt,
"hermes_status": data.get("status", "running"),
}
async def poll_until_done(project: str, *, channel: str = "herman", timeout: float = 1800.0) -> dict[str, Any]:
"""Poll Hermes build status until completed or failed."""
import asyncio
slug = slugify(project)
elapsed = 0.0
interval = 5.0
last_log = ""
while elapsed < timeout:
st = await get_build_status(slug)
status = str(st.get("status") or "unknown")
msg = str(st.get("message") or status)
if msg != last_log:
log_agent_event(
"webbuilder",
"website_build_progress",
msg[:255],
(st.get("log_tail") or "")[:4000],
channel=channel,
metadata={"project": slug, "status": status},
status="running" if status in ("running", "queued") else status,
)
last_log = msg
if status == "completed":
files = st.get("files") or []
preview = st.get("preview_url") or f"{HERMES_PREVIEW_BASE}/"
log_agent_event(
"webbuilder",
"website_build_done",
f"Website klaar: {slug}",
f"Bestanden: {', '.join(files[:8])}\nPreview: {preview}",
channel=channel,
metadata={
"project": slug,
"preview_url": preview,
"files": files,
"for_herman": True,
"source_agent": "webbuilder",
},
status="completed",
)
return {**st, "project": slug, "preview_url": preview}
if status == "failed":
detail = str(st.get("error") or st.get("message") or "Build mislukt")
log_agent_event(
"webbuilder",
"website_build_error",
f"Build mislukt: {slug}",
detail[:2000],
channel=channel,
metadata={"project": slug},
status="error",
)
raise RuntimeError(detail)
await asyncio.sleep(interval)
elapsed += interval
raise TimeoutError(f"Build timeout voor {slug} na {int(timeout)}s")
async def generate_from_message(raw: str, *, channel: str = "herman", wait: bool = True) -> dict[str, Any]:
started = await build_from_message(raw, channel=channel)
if wait:
finished = await poll_until_done(started["project"], channel=channel)
started.update(finished)
return started