food-cockpit/foodlinkk-command-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Platform bundle: marketing publish, IT ops, packaging, agents mesh.

Browse Source 
Volledige Foodlinkk Command Center uitbreiding met social automatisering,
reclamefolder filters, Proxmox monitoring en documentatie.
This commit is contained in:
Aissa
2026-06-09 00:41:27 +00:00
commit 5d60d33db1
212 changed files with 30044 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools-api/Dockerfile
+7
View File
@@ -0,0 +1,7 @@
FROM python:3.11-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 && rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app ./app
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8700"]
tools-api/__init__.py
View File
tools-api/app/__init__.py
View File
tools-api/app/brain.py
+425
View File
@@ -0,0 +1,425 @@
"""PostgreSQL second brain — Telegram messages, graph edges, pgvector."""
from __future__ import annotations
import logging
import os
from typing import Any, Optional
import httpx
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
log = logging.getLogger("tools.brain")
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.4.7.19:11434").rstrip("/")
EMBED_MODEL = os.getenv("EMBED_MODEL", "nomic-embed-text")
EMBED_DIM = 768
async def embed_text(text: str) -> list[float]:
text = (text or "").strip()[:4000]
if not text:
return []
async with httpx.AsyncClient(timeout=120.0) as client:
r = await client.post(
f"{OLLAMA_URL}/api/embeddings",
json={"model": EMBED_MODEL, "prompt": text},
)
r.raise_for_status()
vec = r.json().get("embedding") or []
if len(vec) != EMBED_DIM:
raise ValueError(f"embedding dim {len(vec)} != {EMBED_DIM}")
return vec
def _vec_param(vec: list[float]) -> str:
return "[" + ",".join(f"{x:.8f}" for x in vec) + "]"
def upsert_conversation(
chat_id: int,
*,
chat_type: str = "private",
user_name: str | None = None,
user_role: str | None = None,
metadata: dict | None = None,
) -> dict[str, Any]:
row = execute_returning(
"""
INSERT INTO telegram_conversations (chat_id, chat_type, user_name, user_role, metadata, updated_at)
VALUES (%s, %s, %s, %s, %s, NOW())
ON CONFLICT (chat_id) DO UPDATE SET
user_name = COALESCE(EXCLUDED.user_name, telegram_conversations.user_name),
user_role = COALESCE(EXCLUDED.user_role, telegram_conversations.user_role),
metadata = telegram_conversations.metadata || EXCLUDED.metadata,
updated_at = NOW()
RETURNING *
""",
(chat_id, chat_type, user_name, user_role, json_param(metadata or {})),
)
return row or {}
def store_message(
chat_id: int,
*,
direction: str,
content_text: str | None = None,
content_type: str = "text",
role: str = "user",
telegram_message_id: int | None = None,
reply_to_db_id: int | None = None,
agent_name: str | None = None,
content_json: dict | None = None,
user_name: str | None = None,
user_role: str | None = None,
chat_type: str = "private",
) -> dict[str, Any]:
conv = upsert_conversation(
chat_id, chat_type=chat_type, user_name=user_name, user_role=user_role
)
row = execute_returning(
"""
INSERT INTO telegram_messages (
conversation_id, telegram_message_id, direction, role, content_type,
content_text, content_json, reply_to_message_id, agent_name
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
conv["id"],
telegram_message_id,
direction,
role,
content_type,
(content_text or "")[:8000] or None,
json_param(content_json or {}),
reply_to_db_id,
agent_name,
),
)
return row or {}
def store_edge(
source_message_id: int,
edge_type: str,
*,
target_message_id: int | None = None,
target_entity_type: str | None = None,
target_entity_id: int | None = None,
weight: float = 1.0,
metadata: dict | None = None,
) -> dict[str, Any]:
row = execute_returning(
"""
INSERT INTO telegram_message_edges (
source_message_id, target_message_id, target_entity_type,
target_entity_id, edge_type, weight, metadata
) VALUES (%s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
source_message_id,
target_message_id,
target_entity_type,
target_entity_id,
edge_type,
weight,
json_param(metadata or {}),
),
)
return row or {}
def store_embedding(message_id: int, vec: list[float], chunk_index: int = 0) -> None:
execute(
"""
INSERT INTO telegram_message_embeddings (message_id, chunk_index, embedding, model)
VALUES (%s, %s, %s::vector, %s)
ON CONFLICT (message_id, chunk_index) DO UPDATE SET
embedding = EXCLUDED.embedding,
model = EXCLUDED.model
""",
(message_id, chunk_index, _vec_param(vec), EMBED_MODEL),
)
async def store_message_with_embedding(
chat_id: int,
*,
direction: str,
content_text: str | None = None,
**kwargs: Any,
) -> dict[str, Any]:
msg = store_message(chat_id, direction=direction, content_text=content_text, **kwargs)
text = (content_text or "").strip()
if text and len(text) >= 8:
try:
vec = await embed_text(text)
if vec:
store_embedding(msg["id"], vec)
except Exception as exc:
log.warning("embed failed msg=%s: %s", msg.get("id"), exc)
return msg
def get_graph(chat_id: int, limit: int = 50) -> dict[str, Any]:
conv = fetch_one("SELECT * FROM telegram_conversations WHERE chat_id = %s", (chat_id,))
if not conv:
return {"chat_id": chat_id, "nodes": [], "edges": []}
messages = fetch_all(
"""
SELECT id, direction, role, content_type, content_text, agent_name,
reply_to_message_id, created_at
FROM telegram_messages
WHERE conversation_id = %s
ORDER BY created_at DESC
LIMIT %s
""",
(conv["id"], limit),
)
msg_ids = [m["id"] for m in messages]
edges: list[dict] = []
if msg_ids:
edges = fetch_all(
"""
SELECT e.*, sm.content_text AS source_preview, tm.content_text AS target_preview
FROM telegram_message_edges e
JOIN telegram_messages sm ON sm.id = e.source_message_id
LEFT JOIN telegram_messages tm ON tm.id = e.target_message_id
WHERE e.source_message_id = ANY(%s) OR e.target_message_id = ANY(%s)
ORDER BY e.created_at DESC
LIMIT %s
""",
(msg_ids, msg_ids, limit * 2),
)
nodes = [
{
"id": m["id"],
"label": (m.get("content_text") or "")[:80],
"direction": m.get("direction"),
"role": m.get("role"),
"agent": m.get("agent_name"),
"type": m.get("content_type"),
"created_at": m.get("created_at"),
}
for m in reversed(messages)
]
edge_list = [
{
"id": e["id"],
"from": e["source_message_id"],
"to": e.get("target_message_id"),
"type": e["edge_type"],
"entity_type": e.get("target_entity_type"),
"entity_id": e.get("target_entity_id"),
"weight": e.get("weight"),
}
for e in edges
]
return {
"chat_id": chat_id,
"conversation_id": conv["id"],
"nodes": nodes,
"edges": edge_list,
"stats": {"messages": len(nodes), "edges": len(edge_list)},
}
async def search_memory(
query: str,
*,
chat_id: int | None = None,
limit: int = 8,
) -> list[dict[str, Any]]:
query = query.strip()
if not query:
return []
results: list[dict[str, Any]] = []
try:
vec = await embed_text(query)
if vec:
v = _vec_param(vec)
if chat_id is not None:
rows = fetch_all(
"""
SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at,
c.chat_id,
1 - (e.embedding <=> %s::vector) AS score
FROM telegram_message_embeddings e
JOIN telegram_messages m ON m.id = e.message_id
JOIN telegram_conversations c ON c.id = m.conversation_id
WHERE m.content_text IS NOT NULL AND c.chat_id = %s
ORDER BY e.embedding <=> %s::vector
LIMIT %s
""",
(v, chat_id, v, limit),
)
else:
rows = fetch_all(
"""
SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at,
c.chat_id,
1 - (e.embedding <=> %s::vector) AS score
FROM telegram_message_embeddings e
JOIN telegram_messages m ON m.id = e.message_id
JOIN telegram_conversations c ON c.id = m.conversation_id
WHERE m.content_text IS NOT NULL
ORDER BY e.embedding <=> %s::vector
LIMIT %s
""",
(v, v, limit),
)
results.extend(rows)
except Exception as exc:
log.warning("vector search failed: %s", exc)
if len(results) < limit:
params2: list[Any] = [query, limit - len(results)]
chat_clause = ""
if chat_id is not None:
chat_clause = "AND c.chat_id = %s"
params2.append(chat_id)
fts = fetch_all(
f"""
SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at, c.chat_id,
ts_rank(to_tsvector('simple', coalesce(m.content_text, '')),
plainto_tsquery('simple', %s)) AS score
FROM telegram_messages m
JOIN telegram_conversations c ON c.id = m.conversation_id
WHERE to_tsvector('simple', coalesce(m.content_text, '')) @@ plainto_tsquery('simple', %s)
{chat_clause}
ORDER BY score DESC
LIMIT %s
""",
tuple([query, query, *([chat_id] if chat_id else []), limit - len(results)]),
)
seen = {r["id"] for r in results}
for row in fts:
if row["id"] not in seen:
results.append(row)
return results[:limit]
# Append to tools-api/app/brain.py
def list_conversations(limit: int = 50) -> list[dict[str, Any]]:
return fetch_all(
"""
SELECT c.*,
(SELECT COUNT(*) FROM telegram_messages m WHERE m.conversation_id = c.id) AS message_count,
(SELECT MAX(m2.created_at) FROM telegram_messages m2 WHERE m2.conversation_id = c.id) AS last_message_at
FROM telegram_conversations c
ORDER BY (SELECT MAX(m3.created_at) FROM telegram_messages m3 WHERE m3.conversation_id = c.id) DESC NULLS LAST
LIMIT %s
""",
(limit,),
)
def list_feed(*, chat_id: int | None = None, limit: int = 80, offset: int = 0) -> list[dict[str, Any]]:
if chat_id is not None:
return fetch_all(
"""
SELECT m.*, c.chat_id, c.user_name, c.user_role,
EXISTS(SELECT 1 FROM telegram_message_embeddings e WHERE e.message_id = m.id) AS has_embedding
FROM telegram_messages m
JOIN telegram_conversations c ON c.id = m.conversation_id
WHERE c.chat_id = %s
ORDER BY m.created_at DESC
LIMIT %s OFFSET %s
""",
(chat_id, limit, offset),
)
return fetch_all(
"""
SELECT m.*, c.chat_id, c.user_name, c.user_role,
EXISTS(SELECT 1 FROM telegram_message_embeddings e WHERE e.message_id = m.id) AS has_embedding
FROM telegram_messages m
JOIN telegram_conversations c ON c.id = m.conversation_id
ORDER BY m.created_at DESC
LIMIT %s OFFSET %s
""",
(limit, offset),
)
def get_dashboard_stats() -> dict[str, Any]:
stats = fetch_one(
"""
SELECT
(SELECT COUNT(*) FROM telegram_conversations) AS conversations,
(SELECT COUNT(*) FROM telegram_messages) AS messages,
(SELECT COUNT(*) FROM telegram_messages WHERE direction = 'in') AS inbound,
(SELECT COUNT(*) FROM telegram_messages WHERE direction = 'out') AS outbound,
(SELECT COUNT(*) FROM telegram_message_edges) AS edges,
(SELECT COUNT(*) FROM telegram_message_embeddings) AS embeddings
"""
) or {}
recent = list_feed(limit=15)
events = fetch_all(
"""
SELECT id, agent_name, event_type, title, body, status, channel, created_at
FROM agent_events
WHERE channel = 'telegram' OR agent_name IN ('hermes', 'herman', 'personal_pa')
ORDER BY created_at DESC
LIMIT 20
"""
)
return {"stats": stats, "recent_messages": recent, "agent_events": events}
def get_global_graph(limit: int = 100) -> dict[str, Any]:
messages = fetch_all(
"""
SELECT m.id, m.direction, m.role, m.content_type, m.content_text, m.agent_name,
m.created_at, c.chat_id, c.user_name
FROM telegram_messages m
JOIN telegram_conversations c ON c.id = m.conversation_id
ORDER BY m.created_at DESC
LIMIT %s
""",
(limit,),
)
if not messages:
return {"nodes": [], "edges": []}
ids = [m["id"] for m in messages]
edges = fetch_all(
"""
SELECT e.*
FROM telegram_message_edges e
WHERE e.source_message_id = ANY(%s) OR e.target_message_id = ANY(%s)
""",
(ids, ids),
)
nodes = [
{
"id": m["id"],
"label": (m.get("content_text") or "")[:100],
"direction": m.get("direction"),
"role": m.get("role"),
"agent": m.get("agent_name"),
"chat_id": m.get("chat_id"),
"user_name": m.get("user_name"),
"type": m.get("content_type"),
"created_at": m.get("created_at"),
}
for m in reversed(messages)
]
edge_list = [
{
"id": e["id"],
"from": e["source_message_id"],
"to": e.get("target_message_id"),
"type": e["edge_type"],
"entity_type": e.get("target_entity_type"),
"entity_id": e.get("target_entity_id"),
}
for e in edges
]
return {"nodes": nodes, "edges": edge_list, "stats": {"nodes": len(nodes), "edges": len(edge_list)}}
tools-api/app/briefing.py
+352
View File
@@ -0,0 +1,352 @@
from __future__ import annotations
import asyncio
import json
from datetime import date, datetime, timezone
from typing import Any
from app.config import settings
from app.db import execute, fetch_all, fetch_one
from app.services import ollama
def _safe_count(table: str, where: str = "", params: tuple = ()) -> int:
try:
clause = f" WHERE {where}" if where else ""
row = fetch_one(f"SELECT COUNT(*) AS c FROM {table}{clause}", params or None)
return int(row["c"]) if row else 0
except Exception:
return 0
def _safe_sum(table: str, column: str, where: str = "", params: tuple = ()) -> float:
try:
clause = f" WHERE {where}" if where else ""
row = fetch_one(f"SELECT COALESCE(SUM({column}), 0) AS total FROM {table}{clause}", params or None)
return float(row["total"]) if row else 0.0
except Exception:
return 0.0
def serialize_stats(data: dict[str, Any]) -> dict[str, Any]:
def _default(o: Any) -> Any:
if hasattr(o, "isoformat"):
return o.isoformat()
if hasattr(o, "__float__"):
try:
return float(o)
except (TypeError, ValueError):
pass
return str(o)
return json.loads(json.dumps(data, default=_default))
def collect_briefing_data() -> dict[str, Any]:
data: dict[str, Any] = {
"date": date.today().isoformat(),
"generated_at": datetime.now(timezone.utc).isoformat(),
}
data["clients"] = _safe_count("clients")
data["deals"] = _safe_count("deals")
data["products"] = _safe_count("products")
data["suppliers"] = _safe_count("suppliers")
data["pipeline_eur"] = _safe_sum("deals", "value", "stage NOT IN ('won', 'lost')")
data["pending_approvals"] = _safe_count("agent_events", "status = 'needs_approval'")
try:
data["deals_by_stage"] = fetch_all(
"SELECT stage, COUNT(*) AS cnt, COALESCE(SUM(value), 0) AS total FROM deals GROUP BY stage ORDER BY cnt DESC"
)
except Exception:
data["deals_by_stage"] = []
try:
data["recent_clients"] = fetch_all(
"SELECT name, stage, email, created_at FROM clients ORDER BY created_at DESC LIMIT 5"
)
except Exception:
data["recent_clients"] = []
try:
data["recent_events"] = fetch_all(
"""SELECT agent_name, event_type, title, status, created_at
FROM agent_events ORDER BY created_at DESC LIMIT 12"""
)
except Exception:
data["recent_events"] = []
try:
data["pending_items"] = fetch_all(
"""SELECT agent_name, title, event_type, created_at
FROM agent_events WHERE status = 'needs_approval'
ORDER BY created_at DESC LIMIT 8"""
)
except Exception:
data["pending_items"] = []
try:
row = fetch_one(
"""SELECT COUNT(*) AS docs, COALESCE(SUM(word_count), 0) AS words,
COALESCE(AVG(sentiment_compound), 0) AS avg_sentiment
FROM document_analytics"""
)
data["nas_docs"] = int(row["docs"] or 0) if row else 0
data["nas_words"] = int(row["words"] or 0) if row else 0
data["nas_sentiment"] = round(float(row["avg_sentiment"] or 0), 3) if row else 0.0
except Exception:
data["nas_docs"] = data["nas_words"] = 0
data["nas_sentiment"] = 0.0
try:
data["nas_files"] = fetch_all(
"""SELECT filename, doc_type, sentiment_label, word_count
FROM document_analytics ORDER BY analyzed_at DESC LIMIT 8"""
)
except Exception:
data["nas_files"] = []
try:
data["top_words"] = fetch_all(
"""SELECT lemma, SUM(count) AS total FROM document_word_counts
WHERE NOT is_stopword GROUP BY lemma ORDER BY total DESC LIMIT 10"""
)
except Exception:
data["top_words"] = []
try:
data["calendar_events"] = fetch_all(
"""SELECT ce.title, ce.starts_at, ce.ends_at, c.name AS client_name
FROM calendar_events ce
LEFT JOIN clients c ON c.id = ce.client_id
WHERE ce.starts_at >= NOW() - INTERVAL '1 day'
AND ce.starts_at <= NOW() + INTERVAL '7 days'
ORDER BY ce.starts_at ASC LIMIT 10"""
)
except Exception:
data["calendar_events"] = []
# Retail intelligence
data["supermarkets"] = _safe_count("supermarkets")
data["crm_partnerships"] = _safe_count("supermarkets", "partnership_status = 'active'")
data["wholesalers"] = _safe_count("wholesalers")
try:
data["top_opportunities"] = fetch_all(
"""SELECT s.name, s.chain, s.city, ros.halal_opportunity_score
FROM retail_opportunity_scores ros
JOIN supermarkets s ON s.id = ros.supermarket_id
ORDER BY ros.halal_opportunity_score DESC LIMIT 5"""
)
except Exception:
data["top_opportunities"] = []
try:
data["milestones_pending"] = fetch_all(
"""SELECT sm.title, sm.milestone_type, sm.status, sm.target_date, sm.value_eur,
s.name AS store_name, s.chain, c.name AS client_name
FROM sales_milestones sm
LEFT JOIN supermarkets s ON s.id = sm.supermarket_id
LEFT JOIN clients c ON c.id = sm.client_id
WHERE sm.status IN ('pending', 'in_progress')
ORDER BY sm.target_date ASC NULLS LAST, sm.created_at DESC LIMIT 8"""
)
except Exception:
data["milestones_pending"] = []
try:
data["milestones_recent"] = fetch_all(
"""SELECT sm.title, sm.milestone_type, sm.status, sm.completed_at, sm.value_eur,
s.name AS store_name, s.chain
FROM sales_milestones sm
LEFT JOIN supermarkets s ON s.id = sm.supermarket_id
ORDER BY sm.created_at DESC LIMIT 5"""
)
except Exception:
data["milestones_recent"] = []
try:
data["rss_highlights"] = fetch_all(
"""SELECT i.title, i.link, i.summary, f.name AS feed_name, f.url AS feed_url
FROM rss_items i JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
WHERE i.title ILIKE ANY (ARRAY['%kant%','%maaltijd%','%supermarkt%','%retail%','%halal%','%jumbo%','%meal%'])
ORDER BY i.published_at DESC NULLS LAST LIMIT 6"""
)
except Exception:
data["rss_highlights"] = []
try:
data["market_trends"] = fetch_all(
"SELECT trend_name, description, opportunity_score FROM market_trends ORDER BY updated_at DESC LIMIT 4"
)
except Exception:
data["market_trends"] = []
return data
def build_template_report(data: dict[str, Any]) -> str:
lines = [
f"# Foodlinkk Dagrapport — {data['date']}",
"",
f"*Gegenereerd: {data['generated_at'][:19]} UTC · Model: {settings.OLLAMA_MODEL}*",
"",
"## KPI's",
f"- **Klanten:** {data['clients']} · **Deals:** {data['deals']} · **Pipeline:** €{data['pipeline_eur']:,.0f}",
f"- **Supermarkten in DB:** {data.get('supermarkets', 0)} · **CRM partnerships:** {data.get('crm_partnerships', 0)}",
f"- **Groothandels:** {data.get('wholesalers', 0)} · **Goedkeuringen open:** {data['pending_approvals']}",
"",
]
if data.get("top_opportunities"):
lines.extend(["## Top halal-markt kansen (Retail 360)"])
for row in data["top_opportunities"]:
score = round(float(row.get("halal_opportunity_score") or 0))
lines.append(f"- **{row.get('chain')} · {row.get('name')}** ({row.get('city')}) — score {score}/100")
lines.append("")
if data.get("milestones_pending"):
lines.extend(["## Sales milestones — open"])
for row in data["milestones_pending"]:
td = row.get("target_date")
td_s = td.isoformat()[:10] if hasattr(td, "isoformat") else str(td or "—")[:10]
lines.append(f"- [{td_s}] **{row.get('title')}** · {row.get('chain') or ''} {row.get('store_name') or ''} · €{row.get('value_eur') or '—'}")
lines.append("")
if data.get("rss_highlights"):
lines.extend(["## Kant-en-klaar & supermarkt nieuws"])
for row in data["rss_highlights"]:
lines.append(f"- [{row.get('feed_name')}] {row.get('title')}")
lines.append("")
if data.get("market_trends"):
lines.extend(["## Markt trends"])
for row in data["market_trends"]:
pct = round(float(row.get("opportunity_score") or 0) * 100)
lines.append(f"- **{row.get('trend_name')}** ({pct}% kans) — {row.get('description') or ''}")
lines.append("")
lines.extend(["## Pipeline per stage"])
for row in data.get("deals_by_stage") or []:
lines.append(f"- **{row.get('stage')}:** {row.get('cnt')} deals · €{float(row.get('total') or 0):,.0f}")
if not data.get("deals_by_stage"):
lines.append("- Geen deals in database.")
if data.get("calendar_events"):
lines.extend(["", "## Agenda (7 dagen)"])
for row in data["calendar_events"]:
ts = row.get("starts_at")
ts_s = ts.isoformat()[:16] if hasattr(ts, "isoformat") else str(ts)[:16]
lines.append(f"- [{ts_s}] {row.get('title')} ({row.get('client_name') or '-'})")
if data.get("pending_items"):
lines.extend(["", "## ⚠️ Wacht op jouw goedkeuring"])
for row in data["pending_items"]:
lines.append(f"- {row.get('agent_name')}: {row.get('title')}")
return "\n".join(lines)
async def _ai_executive_summary(data: dict[str, Any]) -> str:
opp_lines = ""
for row in data.get("top_opportunities") or []:
opp_lines += f"- {row.get('chain')} {row.get('name')} ({row.get('city')}): score {round(float(row.get('halal_opportunity_score') or 0))}\n"
ms_lines = ""
for row in data.get("milestones_pending") or []:
ms_lines += f"- {row.get('title')} ({row.get('chain') or 'CRM'}) deadline {row.get('target_date') or '?'}\n"
prompt = (
"Schrijf in het Nederlands (markdown) voor CEO Aïssa van Foodlinkk (halal kant-en-klaar maaltijden):\n\n"
"## Samenvatting\n(5-7 zinnen: wat is vandaag belangrijk, pipeline, retail kansen, milestones)\n\n"
"## Actiepunten vandaag — korte termijn\n(minimaal 5 concrete bullets met CRM/retail acties)\n\n"
"## Lange termijn focus\n(3-5 bullets: groei supermarkt partnerships, halal markt, milestones komende weken)\n\n"
f"Data vandaag ({data['date']}):\n"
f"- Pipeline €{data['pipeline_eur']:,.0f}, {data['clients']} klanten, {data['deals']} deals\n"
f"- {data.get('supermarkets',0)} supermarkten, {data.get('crm_partnerships',0)} actieve CRM partnerships\n"
f"- {data['pending_approvals']} goedkeuringen open\n"
f"Top kansen:\n{opp_lines or '- geen data'}\n"
f"Milestones open:\n{ms_lines or '- geen milestones'}\n"
)
system = (
"Je bent Herman, AI co-CEO van Foodlinkk. Schrijf warm, professioneel en actionable. "
"Focus op halal kant-en-klaar retail groei in Nederland. Geen vage tekst — concrete namen en acties."
)
try:
return await ollama.generate(prompt, system=system, timeout=120.0)
except Exception:
return ""
def _fallback_summary(data: dict[str, Any]) -> str:
opp = data.get("top_opportunities") or []
ms = data.get("milestones_pending") or []
lines = [
"## Samenvatting",
f"Vandaag ({data['date']}) heb je **€{data['pipeline_eur']:,.0f}** in je pipeline en **{data.get('crm_partnerships',0)} actieve supermarkt-partnerships**. "
f"In Retail 360 staan **{data.get('supermarkets',0)} filialen** met live CBS-data.",
]
if opp:
top = opp[0]
lines.append(
f"De grootste halal-kans is **{top.get('chain')} · {top.get('name')}** in {top.get('city')} "
f"(score {round(float(top.get('halal_opportunity_score') or 0))}/100)."
)
lines.extend(["", "## Actiepunten vandaag — korte termijn"])
actions = [
"Open Retail 360 en benader top-3 halal-gap filialen via CRM koppeling",
f"Behandel {data['pending_approvals']} openstaande agent-goedkeuringen",
"Check Marketing Live Feed voor kant-en-klaar trends",
]
if ms:
actions.insert(0, f"Follow-up milestone: **{ms[0].get('title')}**")
for a in actions[:6]:
lines.append(f"- {a}")
lines.extend(["", "## Lange termijn focus"])
lines.extend([
"- Schaal CRM partnerships van proposal naar actief in top-10 kans-filialen",
"- Halal kant-en-klaar listing bij Jumbo/AH regio's met hoogste demografische vraag",
"- Wekelijks milestones review in Retail 360 sales tab",
])
return "\n".join(lines)
def _save_briefing(content: str, data: dict[str, Any]) -> None:
safe = serialize_stats(data)
metadata = {"stats": safe, "model": settings.OLLAMA_MODEL, "type": "daily_ceo_report"}
try:
execute(
"INSERT INTO daily_briefings (content, generated_by, metadata) VALUES (%s, %s, %s::jsonb)",
(content, "herman", json.dumps(metadata)),
)
except Exception:
pass
try:
execute(
"""INSERT INTO agent_events (agent_name, agent_type, event_type, title, body, status, channel, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb)""",
(
"herman", "herman_delegate", "briefing",
f"CEO dagrapport {data['date']}", content[:2000],
"completed", "dashboard", json.dumps({"stats": safe}),
),
)
except Exception:
pass
async def generate_daily_briefing() -> tuple[str, dict[str, Any]]:
data = collect_briefing_data()
template = build_template_report(data)
try:
ai_part = await asyncio.wait_for(_ai_executive_summary(data), timeout=25.0)
except (asyncio.TimeoutError, Exception):
ai_part = ""
if ai_part and len(ai_part.strip()) > 80:
content = ai_part.strip() + "\n\n---\n\n" + template
else:
content = _fallback_summary(data) + "\n\n---\n\n" + template
_save_briefing(content, data)
return content, serialize_stats(data)
tools-api/app/comfyui.py
+396
View File
@@ -0,0 +1,396 @@
"""ComfyUI client — HD generation with resilient progress tracking."""
from __future__ import annotations
import asyncio
import json
import logging
import os
import random
import uuid
from typing import Any, Optional
from urllib.parse import urlencode
import httpx
log = logging.getLogger("tools-api.comfyui")
COMFYUI_URL = os.getenv("COMFYUI_URL", "http://10.4.7.18:8188").rstrip("/")
CHECKPOINT = os.getenv("COMFYUI_CHECKPOINT", "v1-5-pruned-emaonly.safetensors")
DEFAULT_STEPS = int(os.getenv("COMFYUI_STEPS", "15"))
POLL_INTERVAL = float(os.getenv("COMFYUI_POLL_INTERVAL", "2.0"))
MAX_WAIT = float(os.getenv("COMFYUI_MAX_WAIT", "1800"))
WS_IDLE_TIMEOUT = float(os.getenv("COMFYUI_WS_IDLE", "90"))
QUALITY_PRESETS: dict[str, dict[str, Any]] = {
"fast": {"width": 512, "height": 512, "steps": 15, "label": "Snel (512px)"},
"hd": {"width": 1024, "height": 1024, "steps": 28, "label": "HD (1024px)"},
"ultra": {"width": 1024, "height": 1024, "steps": 35, "label": "Ultra HD (1024px, 35 steps)"},
}
_jobs: dict[str, dict[str, Any]] = {}
def resolve_quality(
quality: str | None = None,
width: int | None = None,
height: int | None = None,
steps: int | None = None,
) -> tuple[int, int, int, str]:
q = (quality or "hd").lower()
preset = QUALITY_PRESETS.get(q, QUALITY_PRESETS["hd"])
w = width or preset["width"]
h = height or preset["height"]
s = steps or preset["steps"]
label = preset["label"]
return w, h, s, label
def get_job(prompt_id: str) -> dict[str, Any] | None:
return _jobs.get(prompt_id)
def build_workflow(
prompt: str,
negative: str = "blurry, low quality, watermark, text, ugly, deformed",
width: int = 1024,
height: int = 1024,
steps: int = DEFAULT_STEPS,
seed: Optional[int] = None,
) -> dict[str, Any]:
seed = seed if seed is not None else random.randint(1, 2**31 - 1)
return {
"3": {
"class_type": "KSampler",
"inputs": {
"seed": seed,
"steps": steps,
"cfg": 7.5,
"sampler_name": "euler",
"scheduler": "normal",
"denoise": 1.0,
"model": ["4", 0],
"positive": ["6", 0],
"negative": ["7", 0],
"latent_image": ["5", 0],
},
},
"4": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CHECKPOINT}},
"5": {
"class_type": "EmptyLatentImage",
"inputs": {"width": width, "height": height, "batch_size": 1},
},
"6": {
"class_type": "CLIPTextEncode",
"inputs": {"text": prompt, "clip": ["4", 1]},
},
"7": {
"class_type": "CLIPTextEncode",
"inputs": {"text": negative, "clip": ["4", 1]},
},
"8": {
"class_type": "VAEDecode",
"inputs": {"samples": ["3", 0], "vae": ["4", 2]},
},
"9": {
"class_type": "SaveImage",
"inputs": {"filename_prefix": "foodlinkk", "images": ["8", 0]},
},
}
def view_url(filename: str, subfolder: str = "", img_type: str = "output") -> str:
params = urlencode({"filename": filename, "type": img_type, "subfolder": subfolder})
return f"{COMFYUI_URL}/view?{params}"
def _new_job(prompt_id: str, prompt: str, width: int, height: int, steps: int, quality: str) -> None:
_jobs[prompt_id] = {
"prompt_id": prompt_id,
"status": "queued",
"percent": 0,
"step": 0,
"max_step": steps,
"node": None,
"message": "In wachtrij bij ComfyUI…",
"prompt": prompt[:500],
"width": width,
"height": height,
"steps": steps,
"quality": quality,
"events": [],
"result": None,
"error": None,
}
def _append_event(prompt_id: str, message: str) -> None:
job = _jobs.get(prompt_id)
if not job:
return
job["message"] = message
events: list[str] = job.setdefault("events", [])
if not events or events[-1] != message:
events.append(message)
if len(events) > 50:
del events[: len(events) - 50]
async def submit_prompt(workflow: dict[str, Any]) -> tuple[str, str]:
client_id = str(uuid.uuid4())
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.post(
f"{COMFYUI_URL}/prompt",
json={"prompt": workflow, "client_id": client_id},
)
resp.raise_for_status()
data = resp.json()
if data.get("node_errors"):
raise RuntimeError(f"ComfyUI node errors: {data['node_errors']}")
return data["prompt_id"], client_id
async def _prompt_in_history(prompt_id: str) -> bool:
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.get(f"{COMFYUI_URL}/history/{prompt_id}")
if resp.status_code == 200 and prompt_id in resp.json():
return True
return False
async def _prompt_in_queue(prompt_id: str) -> bool:
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.get(f"{COMFYUI_URL}/queue")
if resp.status_code != 200:
return False
data = resp.json()
for bucket in ("queue_running", "queue_pending"):
for item in data.get(bucket) or []:
if isinstance(item, (list, tuple)) and len(item) > 1 and item[1] == prompt_id:
return True
return False
async def _is_still_running(prompt_id: str) -> bool:
if await _prompt_in_history(prompt_id):
return False
return await _prompt_in_queue(prompt_id)
async def wait_for_output(prompt_id: str) -> dict[str, Any]:
deadline = asyncio.get_event_loop().time() + MAX_WAIT
tick = 0
async with httpx.AsyncClient(timeout=30.0) as client:
while asyncio.get_event_loop().time() < deadline:
resp = await client.get(f"{COMFYUI_URL}/history/{prompt_id}")
if resp.status_code == 200:
hist = resp.json()
if prompt_id in hist:
outputs = hist[prompt_id].get("outputs") or {}
for node_out in outputs.values():
images = node_out.get("images") or []
if images:
img = images[0]
return {
"filename": img["filename"],
"subfolder": img.get("subfolder", ""),
"type": img.get("type", "output"),
}
tick += 1
if tick % 15 == 0:
_append_event(prompt_id, "ComfyUI CPU render duurt even — nog bezig…")
await asyncio.sleep(POLL_INTERVAL)
raise TimeoutError(f"ComfyUI generation timed out after {int(MAX_WAIT)}s")
async def _track_ws(client_id: str, prompt_id: str) -> None:
try:
import websockets
except ImportError:
_append_event(prompt_id, "Polling modus (geen websocket)")
return
ws_url = COMFYUI_URL.replace("https://", "wss://").replace("http://", "ws://") + f"/ws?clientId={client_id}"
try:
async with websockets.connect(ws_url, ping_interval=30, ping_timeout=60, close_timeout=10) as ws:
finished = False
while not finished:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=WS_IDLE_TIMEOUT)
except asyncio.TimeoutError:
if await _prompt_in_history(prompt_id):
finished = True
break
if await _is_still_running(prompt_id):
_append_event(prompt_id, "Nog bezig op CPU (geen WS update)…")
continue
break
data = json.loads(raw)
msg_type = data.get("type")
payload = data.get("data") or {}
pid = payload.get("prompt_id")
if pid not in (None, prompt_id):
continue
if msg_type == "execution_start":
_jobs[prompt_id]["status"] = "running"
_append_event(prompt_id, "ComfyUI gestart")
elif msg_type == "progress":
val = int(payload.get("value") or 0)
mx = int(payload.get("max") or 1)
pct = int(100 * val / mx) if mx else 0
_jobs[prompt_id].update(
status="running",
percent=pct,
step=val,
max_step=mx,
node=payload.get("node"),
)
_append_event(prompt_id, f"KSampler {val}/{mx} ({pct}%)")
elif msg_type == "executing":
node = payload.get("node")
if node is None:
_jobs[prompt_id]["status"] = "finishing"
_append_event(prompt_id, "Render klaar — opslaan…")
finished = True
else:
_jobs[prompt_id]["node"] = node
_append_event(prompt_id, f"Node {node}")
elif msg_type == "execution_error":
err = payload.get("exception_message") or "ComfyUI execution error"
raise RuntimeError(str(err))
except Exception as exc:
log.warning("WS tracking ended for %s: %s — falling back to poll", prompt_id, exc)
if await _prompt_in_history(prompt_id):
return
if await _is_still_running(prompt_id):
_append_event(prompt_id, "Voortgang via polling (WS verbroken)")
return
raise
async def _run_job(
prompt_id: str,
client_id: str,
prompt: str,
width: int,
height: int,
steps: int,
quality: str,
seed: Optional[int],
) -> None:
try:
try:
await _track_ws(client_id, prompt_id)
except Exception as ws_exc:
log.warning("WS phase issue %s: %s", prompt_id, ws_exc)
if not await _is_still_running(prompt_id) and not await _prompt_in_history(prompt_id):
raise
img = await wait_for_output(prompt_id)
result = {
"prompt_id": prompt_id,
"filename": img["filename"],
"subfolder": img.get("subfolder", ""),
"type": img.get("type", "output"),
"image_url": view_url(img["filename"], img.get("subfolder", ""), img.get("type", "output")),
"prompt": prompt,
"width": width,
"height": height,
"steps": steps,
"quality": quality,
}
_jobs[prompt_id].update(status="done", percent=100, result=result, message="Klaar!")
_append_event(prompt_id, "Afbeelding klaar")
log.info("ComfyUI done %s (%dx%d)", prompt_id, width, height)
except Exception as exc:
log.exception("ComfyUI job failed %s", prompt_id)
if await _prompt_in_history(prompt_id):
try:
img = await wait_for_output(prompt_id)
result = {
"prompt_id": prompt_id,
"filename": img["filename"],
"subfolder": img.get("subfolder", ""),
"type": img.get("type", "output"),
"image_url": view_url(img["filename"], img.get("subfolder", ""), img.get("type", "output")),
"prompt": prompt,
"width": width,
"height": height,
"steps": steps,
"quality": quality,
}
_jobs[prompt_id].update(status="done", percent=100, result=result, message="Klaar!")
return
except Exception:
pass
_jobs[prompt_id].update(status="error", error=str(exc), message=str(exc))
_append_event(prompt_id, f"Fout: {exc}")
async def start_generation(
prompt: str,
*,
negative: str = "blurry, low quality, watermark, text, ugly, deformed",
quality: str = "hd",
width: int | None = None,
height: int | None = None,
steps: int | None = None,
seed: Optional[int] = None,
) -> dict[str, Any]:
w, h, s, label = resolve_quality(quality, width, height, steps)
workflow = build_workflow(prompt, negative=negative, width=w, height=h, steps=s, seed=seed)
prompt_id, client_id = await submit_prompt(workflow)
_new_job(prompt_id, prompt, w, h, s, quality)
_append_event(prompt_id, f"Gestart — {label}")
asyncio.create_task(_run_job(prompt_id, client_id, prompt, w, h, s, quality, seed))
return {
"prompt_id": prompt_id,
"client_id": client_id,
"quality": quality,
"width": w,
"height": h,
"steps": s,
"quality_label": label,
}
async def generate_image(
prompt: str,
width: int = 1024,
height: int = 1024,
steps: int = DEFAULT_STEPS,
seed: Optional[int] = None,
quality: str = "hd",
) -> dict[str, Any]:
if quality and quality != "custom":
width, height, steps, _ = resolve_quality(quality, width, height, steps)
started = await start_generation(
prompt,
quality="custom",
width=width,
height=height,
steps=steps,
seed=seed,
)
prompt_id = started["prompt_id"]
deadline = asyncio.get_event_loop().time() + MAX_WAIT
while asyncio.get_event_loop().time() < deadline:
job = _jobs.get(prompt_id) or {}
if job.get("status") == "done" and job.get("result"):
return job["result"]
if job.get("status") == "error":
raise RuntimeError(job.get("error") or "Generation failed")
await asyncio.sleep(POLL_INTERVAL)
raise TimeoutError(f"ComfyUI generation timed out after {int(MAX_WAIT)}s")
async def fetch_image_bytes(filename: str, subfolder: str = "", img_type: str = "output") -> bytes:
params = {"filename": filename, "type": img_type, "subfolder": subfolder}
async with httpx.AsyncClient(timeout=120.0) as client:
resp = await client.get(f"{COMFYUI_URL}/view", params=params)
resp.raise_for_status()
return resp.content
tools-api/app/config.py
+20
View File
@@ -0,0 +1,20 @@
import os
class Settings:
DB_HOST: str = os.getenv("DB_HOST", "foodlinkk_db")
DB_PORT: int = int(os.getenv("DB_PORT", "5432"))
DB_USER: str = os.getenv("DB_USER", "aissa")
DB_PASSWORD: str = os.getenv("DB_PASSWORD", "Foodlinkk#2026")
DB_NAME: str = os.getenv("DB_NAME", "foodlinkk")
OLLAMA_URL: str = os.getenv("OLLAMA_URL", "http://10.4.7.19:11434")
@property
def database_dsn(self) -> str:
return (
f"host={self.DB_HOST} port={self.DB_PORT} dbname={self.DB_NAME} "
f"user={self.DB_USER} password={self.DB_PASSWORD}"
)
settings = Settings()
tools-api/app/connectors/__init__.py
View File
tools-api/app/connectors/cbs.py
+142
View File
@@ -0,0 +1,142 @@
"""CBS Open Data — gemeente demografie via OData."""
from __future__ import annotations
import json
import re
import urllib.parse
import urllib.request
from typing import Any, Optional
CBS_BASE = "https://opendata.cbs.nl/ODataApi/OData"
_GEMEENTE_CACHE: dict[str, dict[str, Any]] = {}
def _int_val(raw: Any) -> Optional[int]:
if raw is None:
return None
s = str(raw).strip().replace(".", "")
if not s or s == ".":
return None
try:
return int(s)
except ValueError:
return None
def _float_val(raw: Any) -> Optional[float]:
if raw is None:
return None
s = str(raw).strip()
if not s or s == ".":
return None
try:
return float(s)
except ValueError:
return None
def _fetch_untyped(dataset: str, filter_expr: str, top: int = 1) -> list[dict[str, Any]]:
params = urllib.parse.urlencode(
{"$filter": filter_expr, "$top": str(top), "$format": "json"},
quote_via=urllib.parse.quote,
)
url = f"{CBS_BASE}/{dataset}/UntypedDataSet?{params}"
with urllib.request.urlopen(url, timeout=45) as resp:
data = json.loads(resp.read().decode())
return data.get("value", [])
def _normalize_gm(code: str) -> str:
code = (code or "").strip().upper()
if code.startswith("GM"):
return code
digits = re.sub(r"\D", "", code)
return f"GM{digits}" if digits else code
def fetch_gemeente_stats(gemeente_code: str) -> Optional[dict[str, Any]]:
gm = _normalize_gm(gemeente_code)
if not gm:
return None
if gm in _GEMEENTE_CACHE:
return _GEMEENTE_CACHE[gm]
pop_rows = _fetch_untyped(
"03759ned",
f"RegioS eq '{gm}' and Leeftijd eq '10000' and Geslacht eq 'T001038' "
f"and BurgerlijkeStaat eq 'T001019' and substringof('2024',Perioden)",
)
income_rows = _fetch_untyped(
"86005NED",
f"RegioS eq '{gm}' and substringof('2023',Perioden) and Geslacht eq 'T001038'",
)
area_rows = _fetch_untyped(
"84583NED",
f"startswith(WijkenEnBuurten,'{gm}') and SoortRegio_2 eq 'Gemeente '",
)
population = _int_val(pop_rows[0].get("BevolkingOp1Januari_1")) if pop_rows else None
avg_income = None
median_income = None
if income_rows:
avg_income = _float_val(income_rows[0].get("GemiddeldPersoonlijkInkomen_6"))
median_income = _float_val(income_rows[0].get("MediaanPersoonlijkInkomen_7"))
if avg_income:
avg_income *= 1000
if median_income:
median_income *= 1000
area = area_rows[0] if area_rows else {}
pop_area = _int_val(area.get("AantalInwoners_5")) or population
households = _int_val(area.get("HuishoudensTotaal_28"))
niet_westers = _int_val(area.get("NietWestersTotaal_18"))
marokko = _int_val(area.get("Marokko_19"))
turkije = _int_val(area.get("Turkije_22"))
suriname = _int_val(area.get("Suriname_21"))
avg_hh_size = _float_val(area.get("GemiddeldeHuishoudensgrootte_32"))
income_per_inhabitant = _float_val(area.get("GemiddeldInkomenPerInwoner_72"))
if income_per_inhabitant and not avg_income:
avg_income = income_per_inhabitant * 1000
muslim_proxy_pct = None
niet_westers_pct = None
if pop_area and pop_area > 0:
if marokko is not None and turkije is not None:
muslim_proxy_pct = round((marokko + turkije) / pop_area * 100, 2)
if niet_westers is not None:
niet_westers_pct = round(niet_westers / pop_area * 100, 2)
stats = {
"gemeente_code": gm,
"population": pop_area,
"households": households,
"avg_household_size": avg_hh_size,
"avg_income": avg_income,
"median_income": median_income,
"unemployment_rate": None,
"ethnic_composition": {
"niet_westers_totaal": niet_westers,
"niet_westers_pct": niet_westers_pct,
"marokko": marokko,
"turkije": turkije,
"suriname": suriname,
},
"religious_composition": {
"muslim_proxy_pct": muslim_proxy_pct,
"note": "Indicatief: Marokko+Turkije / bevolking (CBS Kerncijfers wijken en buurten)",
},
"education_level": {
"laag": _int_val(area.get("OpleidingsniveauLaag_64")),
"middelbaar": _int_val(area.get("OpleidingsniveauMiddelbaar_65")),
"hoog": _int_val(area.get("OpleidingsniveauHoog_66")),
},
"housing_type": {
"koop_pct": _float_val(area.get("Koopwoningen_40")),
"huur_pct": _float_val(area.get("HuurwoningenTotaal_41")),
},
"car_ownership": _float_val(area.get("PersonenautoSPerHuishouden_102")),
"data_granularity": "gemeente",
"data_source": "cbs+pdok",
}
_GEMEENTE_CACHE[gm] = stats
return stats
tools-api/app/connectors/food_trends.py
+167
View File
@@ -0,0 +1,167 @@
"""Halal vlees trends, top gerechten en food concept seeds."""
from __future__ import annotations
import re
from datetime import datetime, timezone
from typing import Any
from app.db import fetch_all
HALAL_MEAT_KEYWORDS = (
"halal vlees", "halal meat", "halal kip", "halal lam", "halal rund",
"halal gehakt", "halal chicken", "halal beef", "halal slacht",
"halal certific", "vleesvervanger halal",
)
TOP_DISH_KEYWORDS = (
"kant-en-klaar", "kant en klaar", "ready meal", "maaltijd", "gerecht",
"curry", "stamppot", "biryani", "tagine", "lasagne", "schotel",
"meal prep", "microwave meal", "diepvries maaltijd",
)
CONCEPT_SEEDS = [
"Halal {dish} single-serve voor {chain} schappen in regio's met score >{score}",
"Premium halal {meat} maaltijdlijn — inspelen op trend: {trend}",
"Seizoens {dish} tray (4-portions) voor Plus/Jumbo non-listed partnership pitch",
"AH {dish} variant — benchmark tegen {competitor} koers momentum ({pct}%)",
"Halal-gap fill: {dish} + {meat} combo voor filialen zonder halal schap",
]
def _match_keywords(text: str, keywords: tuple[str, ...]) -> bool:
blob = (text or "").lower()
return any(k in blob for k in keywords)
def fetch_halal_meat_trends(limit: int = 15) -> list[dict[str, Any]]:
rows = fetch_all(
"""SELECT i.title, i.link, i.summary, i.published_at, f.name AS feed_name, f.url AS feed_url
FROM rss_items i JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
ORDER BY i.published_at DESC NULLS LAST LIMIT 200"""
)
out = []
for r in rows:
title = r.get("title") or ""
summary = r.get("summary") or ""
if not _match_keywords(f"{title} {summary}", HALAL_MEAT_KEYWORDS):
continue
out.append({
"title": title,
"link": r.get("link"),
"summary": (summary or "")[:280],
"feed_name": r.get("feed_name"),
"feed_url": r.get("feed_url"),
"published_at": r.get("published_at").isoformat() if r.get("published_at") else None,
"category": "halal_vlees",
"source_url": r.get("link"),
})
if len(out) >= limit:
break
return out
def fetch_top_dishes(limit: int = 12) -> list[dict[str, Any]]:
rows = fetch_all(
"""SELECT i.title, i.link, i.summary, i.published_at, f.name AS feed_name, f.url AS feed_url
FROM rss_items i JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
ORDER BY i.published_at DESC NULLS LAST LIMIT 250"""
)
scored: dict[str, dict[str, Any]] = {}
for r in rows:
title = (r.get("title") or "").lower()
summary = (r.get("summary") or "").lower()
blob = f"{title} {summary}"
if not _match_keywords(blob, TOP_DISH_KEYWORDS):
continue
for kw in TOP_DISH_KEYWORDS:
if kw in blob:
key = kw.strip()
if key not in scored:
scored[key] = {
"dish_keyword": key,
"mentions": 0,
"latest_title": r.get("title"),
"latest_link": r.get("link"),
"feed_name": r.get("feed_name"),
"source_url": r.get("link"),
}
scored[key]["mentions"] += 1
break
items = sorted(scored.values(), key=lambda x: x["mentions"], reverse=True)[:limit]
return items
def fetch_market_trend_rows(limit: int = 8) -> list[dict[str, Any]]:
rows = fetch_all(
"""SELECT trend_name, description, opportunity_score, source, category, updated_at
FROM market_trends ORDER BY updated_at DESC NULLS LAST LIMIT %s""",
(limit,),
)
out = []
for r in rows:
out.append({
"trend_name": r.get("trend_name"),
"description": r.get("description"),
"opportunity_score": float(r.get("opportunity_score") or 0),
"source": r.get("source") or "Foodlinkk trends",
"source_url": "/retail",
"category": r.get("category") or "markt",
})
return out
def generate_concepts(
dishes: list[dict[str, Any]] | None = None,
halal_items: list[dict[str, Any]] | None = None,
market_best: dict[str, Any] | None = None,
limit: int = 6,
) -> list[dict[str, Any]]:
dishes = dishes or fetch_top_dishes(5)
halal_items = halal_items or fetch_halal_meat_trends(5)
best_chain = (market_best or {}).get("chains", ["Albert Heijn"])[0]
pct = (market_best or {}).get("change_pct", 0)
competitor = (market_best or {}).get("name", "Ahold Delhaize")
concepts = []
for i, tmpl in enumerate(CONCEPT_SEEDS[:limit]):
dish = dishes[i % len(dishes)]["dish_keyword"] if dishes else "kant-en-klaar maaltijd"
meat = "halal kip" if halal_items else "halal vlees"
trend = halal_items[i % len(halal_items)]["title"][:60] if halal_items else "groei halal convenience"
text = tmpl.format(
dish=dish,
meat=meat,
chain=best_chain,
score=75,
trend=trend,
competitor=competitor,
pct=pct,
)
concepts.append({
"id": i + 1,
"concept": text,
"based_on": {
"dish": dish,
"halal_trend": trend,
"market_signal": f"{competitor} {pct:+.1f}%" if market_best else "retail DB",
},
"source_urls": [
u for u in [
dishes[i % len(dishes)].get("source_url") if dishes else None,
halal_items[i % len(halal_items)].get("source_url") if halal_items else None,
(market_best or {}).get("source_url"),
] if u
],
})
return concepts
def food_trends_dashboard() -> dict[str, Any]:
halal = fetch_halal_meat_trends()
dishes = fetch_top_dishes()
trends = fetch_market_trend_rows()
return {
"halal_meat_trends": halal,
"top_dishes": dishes,
"market_trends": trends,
"updated_at": datetime.now(timezone.utc).isoformat(),
}
tools-api/app/connectors/halal_registry.py
+155
View File
@@ -0,0 +1,155 @@
"""Halal certification registry sync and supermarket matching."""
from __future__ import annotations
import json
import re
import urllib.parse
import urllib.request
from typing import Any, Optional
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
OVERPASS_URL = "https://overpass.kumi.systems/api/interpreter"
# Known halal-friendly retail brands (indicative — verified via certifier when possible)
HALAL_FRIENDLY_CHAINS = {
"Spar": {"has_halal_section": True, "note": "chain policy varies by franchise"},
"Ekoplaza": {"halal_certified": False, "has_halal_section": True},
}
def _fetch_overpass(query: str) -> list[dict[str, Any]]:
data = urllib.parse.urlencode({"data": query}).encode()
req = urllib.request.Request(OVERPASS_URL, data=data, method="POST")
with urllib.request.urlopen(req, timeout=120) as resp:
payload = json.loads(resp.read().decode())
return payload.get("elements", [])
def sync_osm_halal_tags() -> dict[str, Any]:
"""Mark supermarkets with OSM diet:halal=yes and import certification records."""
query = (
'[out:json][timeout:120];area["ISO3166-1"="NL"]->.nl;'
'(node["shop"="supermarket"]["diet:halal"="yes"](area.nl);'
'way["shop"="supermarket"]["diet:halal"="yes"](area.nl););out tags center;'
)
elements = _fetch_overpass(query)
matched = created = 0
for el in elements:
tags = el.get("tags") or {}
external_id = f"osm:{el.get('type')}:{el.get('id')}"
store = fetch_one("SELECT id, name FROM supermarkets WHERE external_id = %s", (external_id,))
if not store:
name = tags.get("name") or tags.get("brand") or "Unknown"
store = fetch_one(
"SELECT id, name FROM supermarkets WHERE name ILIKE %s LIMIT 1",
(f"%{name[:40]}%",),
)
if not store:
continue
matched += 1
execute(
"""UPDATE supermarkets SET halal_certified = TRUE, has_halal_section = TRUE,
halal_certifier = COALESCE(halal_certifier, 'OSM diet:halal'),
last_updated = NOW() WHERE id = %s""",
(store["id"],),
)
existing = fetch_one(
"SELECT id FROM halal_certifications WHERE supermarket_id = %s AND registry_source = 'osm'",
(store["id"],),
)
if not existing:
execute_returning(
"""INSERT INTO halal_certifications (
supermarket_id, certifier, business_name, status, registry_source,
matched_confidence, raw_data
) VALUES (%s, 'OSM', %s, 'active', 'osm', 0.85, %s) RETURNING id""",
(store["id"], store["name"], json_param(tags)),
)
created += 1
return {"osm_halal_elements": len(elements), "stores_matched": matched, "certs_created": created}
def sync_osm_contact_tags(limit: int = 500) -> dict[str, Any]:
"""Pull phone/email/website/operator from OSM for existing stores."""
stores = fetch_all(
"""SELECT id, external_id, phone, email, website, manager_name
FROM supermarkets WHERE external_id LIKE %s
AND (phone IS NULL OR email IS NULL OR manager_name IS NULL)
LIMIT %s""",
("osm:%", limit),
)
updated = contacts = 0
for store in stores:
parts = (store.get("external_id") or "").split(":")
if len(parts) != 3:
continue
osm_type, osm_id = parts[1], parts[2]
query = f'[out:json][timeout:30];{osm_type}({osm_id});out tags;'
try:
elements = _fetch_overpass(query)
except Exception:
continue
if not elements:
continue
tags = elements[0].get("tags") or {}
phone = tags.get("phone") or tags.get("contact:phone")
email = tags.get("email") or tags.get("contact:email")
website = tags.get("website") or tags.get("contact:website")
operator = tags.get("operator") or tags.get("contact:name")
manager = tags.get("manager") or tags.get("contact:manager") or operator
sets, params = [], []
if phone and not store.get("phone"):
sets.append("phone = %s"); params.append(str(phone)[:20])
if email and not store.get("email"):
sets.append("email = %s"); params.append(str(email)[:255])
if website and not store.get("website"):
sets.append("website = %s"); params.append(str(website)[:255])
if manager and not store.get("manager_name"):
sets.append("manager_name = %s"); params.append(str(manager)[:255])
if sets:
params.append(store["id"])
execute(f"UPDATE supermarkets SET {', '.join(sets)}, last_updated = NOW() WHERE id = %s", tuple(params))
updated += 1
if manager or phone or email:
existing = fetch_one(
"SELECT id FROM supermarket_contacts WHERE supermarket_id = %s AND source = 'osm' LIMIT 1",
(store["id"],),
)
if not existing:
execute(
"""INSERT INTO supermarket_contacts (
supermarket_id, role, full_name, phone, email, source, confidence
) VALUES (%s, 'manager', %s, %s, %s, 'osm', 0.6)""",
(store["id"], manager, phone, email),
)
contacts += 1
execute(
"""INSERT INTO supermarket_profiles (supermarket_id, manager_name, manager_phone,
manager_email, web_data, last_scraped_at, data_completeness)
VALUES (%s,%s,%s,%s,%s,NOW(),0.4)
ON CONFLICT (supermarket_id) DO UPDATE SET
manager_name = COALESCE(EXCLUDED.manager_name, supermarket_profiles.manager_name),
manager_phone = COALESCE(EXCLUDED.manager_phone, supermarket_profiles.manager_phone),
manager_email = COALESCE(EXCLUDED.manager_email, supermarket_profiles.manager_email),
web_data = supermarket_profiles.web_data || EXCLUDED.web_data,
last_scraped_at = NOW()""",
(store["id"], manager, phone, email, json_param({"osm_tags": tags})),
)
return {"scanned": len(stores), "stores_updated": updated, "contacts_added": contacts}
def list_halal_certified(limit: int = 500) -> list[dict[str, Any]]:
return fetch_all(
"""
SELECT s.*, h.certifier, h.certificate_number, h.expiry_date, h.registry_source,
h.matched_confidence
FROM supermarkets s
LEFT JOIN halal_certifications h ON h.supermarket_id = s.id AND h.status = 'active'
WHERE s.halal_certified = TRUE OR s.has_halal_section = TRUE OR h.id IS NOT NULL
ORDER BY s.chain, s.city LIMIT %s
""",
(limit,),
)
tools-api/app/connectors/market_stocks.py
+273
View File
@@ -0,0 +1,273 @@
"""Live supermarket & food-retail stock quotes — Yahoo Finance."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any
from urllib.parse import quote
from urllib.request import Request, urlopen
USER_AGENT = "Foodlinkk-MarketIntel/1.0"
DATA_SOURCE = "Yahoo Finance"
SOURCE_BASE = "https://finance.yahoo.com/quote/"
# Beursgenoteerde supermarkt / food-retail ketens (geen FMCG zoals Unilever)
LISTED_SUPERMARKET_STOCKS = [
{
"symbol": "AD.AS",
"name": "Ahold Delhaize",
"chains": ["Albert Heijn", "Gall & Gall", "Etos", "Bol"],
"country": "NL/EU",
"exchange": "Euronext Amsterdam",
"listed": True,
"color": "#0066cc",
},
{
"symbol": "CAR.PA",
"name": "Carrefour",
"chains": ["Carrefour", "Carrefour Express"],
"country": "EU",
"exchange": "Euronext Paris",
"listed": True,
"color": "#005baa",
},
{
"symbol": "TSCO.L",
"name": "Tesco",
"chains": ["Tesco", "Tesco Express"],
"country": "UK",
"exchange": "London Stock Exchange",
"listed": True,
"color": "#0050aa",
},
{
"symbol": "SBRY.L",
"name": "Sainsbury's",
"chains": ["Sainsbury's", "Argos food"],
"country": "UK",
"exchange": "London Stock Exchange",
"listed": True,
"color": "#f06c00",
},
{
"symbol": "MRW.L",
"name": "Morrisons",
"chains": ["Morrisons"],
"country": "UK",
"exchange": "London Stock Exchange",
"listed": True,
"color": "#f5c518",
},
{
"symbol": "MKS.L",
"name": "Marks & Spencer",
"chains": ["M&S Food"],
"country": "UK",
"exchange": "London Stock Exchange",
"listed": True,
"color": "#00663d",
},
{
"symbol": "COLR.BR",
"name": "Colruyt Group",
"chains": ["Colruyt", "Bio-Planet", "OKay"],
"country": "BE/EU",
"exchange": "Euronext Brussels",
"listed": True,
"color": "#e30613",
},
{
"symbol": "ICA-B.ST",
"name": "ICA Gruppen",
"chains": ["ICA", "Maxi", "Rimi"],
"country": "Nordics",
"exchange": "Nasdaq Stockholm",
"listed": True,
"color": "#e30613",
},
{
"symbol": "KR",
"name": "Kroger",
"chains": ["Kroger", "Albertsons merger context"],
"country": "USA",
"exchange": "NYSE",
"listed": True,
"color": "#004b87",
},
{
"symbol": "WMT",
"name": "Walmart",
"chains": ["Walmart", "Sam's Club"],
"country": "USA",
"exchange": "NYSE",
"listed": True,
"color": "#0071ce",
},
{
"symbol": "COST",
"name": "Costco",
"chains": ["Costco Wholesale"],
"country": "USA/Global",
"exchange": "NASDAQ",
"listed": True,
"color": "#e31837",
},
]
# NL supermarkten — niet beursgenoteerd (transparantie voor CEO)
UNLISTED_NL_CHAINS = [
{
"symbol": None,
"name": "Jumbo",
"chains": ["Jumbo", "Jumbo City"],
"country": "NL",
"exchange": "Familiebedrijf · niet beursgenoteerd",
"listed": False,
"parent": "Van Eerd familie",
"color": "#ffcc00",
"info_url": "https://www.jumbo.com/over-jumbo",
},
{
"symbol": None,
"name": "Plus",
"chains": ["Plus", "Plus Compact"],
"country": "NL",
"exchange": "Coöperatief · niet beursgenoteerd",
"listed": False,
"parent": "Plus Retail (coöperatie)",
"color": "#008040",
"info_url": "https://www.plus.nl",
},
{
"symbol": None,
"name": "Dirk van den Broek",
"chains": ["Dirk", "Dekamarkt"],
"country": "NL",
"exchange": "Privé · niet beursgenoteerd",
"listed": False,
"parent": "Schuitema / Dirk van den Broek",
"color": "#e30613",
"info_url": "https://www.dirk.nl",
},
{
"symbol": None,
"name": "Lidl",
"chains": ["Lidl"],
"country": "NL/EU",
"exchange": "Schwarz Group · privé",
"listed": False,
"parent": "Schwarz Gruppe (DE)",
"color": "#0050aa",
"info_url": "https://www.lidl.nl",
},
{
"symbol": None,
"name": "ALDI",
"chains": ["ALDI", "ALDI Nord/Süd"],
"country": "NL/EU",
"exchange": "Privé · niet beursgenoteerd",
"listed": False,
"parent": "Aldi Süd / Aldi Nord",
"color": "#0066b3",
"info_url": "https://www.aldi.nl",
},
]
def _yahoo_url(symbol: str) -> str:
return f"{SOURCE_BASE}{quote(symbol, safe='')}"
def _fetch_chart(symbol: str) -> dict[str, Any]:
url = (
f"https://query1.finance.yahoo.com/v8/finance/chart/{quote(symbol, safe='')}"
f"?interval=1d&range=1mo&includePrePost=false"
)
req = Request(url, headers={"User-Agent": USER_AGENT})
with urlopen(req, timeout=14) as resp:
payload = json.loads(resp.read().decode())
result = (payload.get("chart") or {}).get("result") or []
if not result:
return {}
meta = result[0].get("meta") or {}
closes = (result[0].get("indicators") or {}).get("quote") or [{}]
close_series = closes[0].get("close") or []
valid = [c for c in close_series if c is not None]
sparkline = valid[-14:] if len(valid) >= 14 else valid
prev = valid[-2] if len(valid) >= 2 else None
last = valid[-1] if valid else meta.get("regularMarketPrice")
change_pct = meta.get("regularMarketChangePercent")
if change_pct is None and prev and last and prev:
change_pct = ((last - prev) / prev) * 100
return {
"price": meta.get("regularMarketPrice") or last,
"currency": meta.get("currency") or "EUR",
"change_pct": round(float(change_pct or 0), 2),
"change_abs": meta.get("regularMarketChange"),
"sparkline": [round(float(v), 2) for v in sparkline],
"market_state": meta.get("marketState") or "CLOSED",
"exchange_name": meta.get("exchangeName") or meta.get("fullExchangeName"),
"quote_time": meta.get("regularMarketTime"),
}
def fetch_supermarket_quotes() -> list[dict[str, Any]]:
now = datetime.now(timezone.utc).isoformat()
items: list[dict[str, Any]] = []
for stock in LISTED_SUPERMARKET_STOCKS:
row = dict(stock)
sym = stock["symbol"]
row["source"] = DATA_SOURCE
row["source_url"] = _yahoo_url(sym)
row["chart_api"] = f"https://query1.finance.yahoo.com/v8/finance/chart/{sym}"
row["fetched_at"] = now
try:
chart = _fetch_chart(sym)
row.update(chart)
row["trend"] = "up" if (row.get("change_pct") or 0) >= 0 else "down"
row["live"] = row.get("price") is not None
except Exception as exc: # noqa: BLE001
row["error"] = str(exc)[:100]
row["price"] = None
row["change_pct"] = 0
row["sparkline"] = []
row["trend"] = "flat"
row["live"] = False
items.append(row)
for chain in UNLISTED_NL_CHAINS:
row = dict(chain)
row["source"] = "Foodlinkk Intel"
row["source_url"] = chain.get("info_url")
row["fetched_at"] = now
row["live"] = False
row["price"] = None
row["change_pct"] = None
row["note"] = "Niet beursgenoteerd — geen live koers beschikbaar"
items.append(row)
return items
def fetch_retail_quotes() -> list[dict[str, Any]]:
"""Back-compat — alleen beursgenoteerde supermarkt-aandelen."""
return [q for q in fetch_supermarket_quotes() if q.get("listed")]
def market_summary(quotes: list[dict[str, Any]] | None = None) -> dict[str, Any]:
quotes = quotes or fetch_retail_quotes()
valid = [q for q in quotes if q.get("price") is not None]
avg_change = sum(float(q.get("change_pct") or 0) for q in valid) / len(valid) if valid else 0
best = max(valid, key=lambda q: float(q.get("change_pct") or 0), default=None)
worst = min(valid, key=lambda q: float(q.get("change_pct") or 0), default=None)
return {
"avg_change_pct": round(avg_change, 2),
"best_performer": best,
"worst_performer": worst,
"quote_count": len(valid),
"listed_count": len(LISTED_SUPERMARKET_STOCKS),
"unlisted_nl_count": len(UNLISTED_NL_CHAINS),
"data_source": DATA_SOURCE,
"updated_at": datetime.now(timezone.utc).isoformat(),
}
tools-api/app/connectors/pdok.py
+56
View File
@@ -0,0 +1,56 @@
"""PDOK Locatieserver — postcode geocoding."""
from __future__ import annotations
import json
import re
import urllib.parse
import urllib.request
from typing import Any, Optional
PDOK_URL = "https://api.pdok.nl/bzk/locatieserver/search/v3_1/free"
POSTCODE_RE = re.compile(r"^(\d{4})\s?([A-Za-z]{2})$")
def normalize_postcode(raw: str) -> str:
cleaned = (raw or "").strip().upper().replace(" ", "")
m = POSTCODE_RE.match(cleaned)
if m:
return f"{m.group(1)}{m.group(2)}"
return cleaned[:6] if cleaned else ""
def _parse_point(value: Optional[str]) -> tuple[Optional[float], Optional[float]]:
if not value or "POINT" not in value:
return None, None
nums = re.findall(r"[-+]?\d*\.?\d+", value)
if len(nums) >= 2:
return float(nums[1]), float(nums[0]) # lat, lon
return None, None
def lookup_postcode(postcode: str) -> Optional[dict[str, Any]]:
pc = normalize_postcode(postcode)
if len(pc) < 6:
return None
q = urllib.parse.urlencode({"q": pc, "rows": 1, "fq": "type:postcode"})
with urllib.request.urlopen(f"{PDOK_URL}?{q}", timeout=20) as resp:
data = json.loads(resp.read().decode())
docs = data.get("response", {}).get("docs", [])
if not docs:
return None
doc = docs[0]
lat, lon = _parse_point(doc.get("centroide_ll"))
gemeente_code = (doc.get("gemeentecode") or "").strip()
if gemeente_code and not gemeente_code.startswith("GM"):
gemeente_code = f"GM{gemeente_code}"
return {
"postcode": pc,
"city": (doc.get("woonplaatsnaam") or "").strip(),
"province": (doc.get("provincienaam") or "").strip(),
"province_code": (doc.get("provinciecode") or "").strip(),
"municipality": (doc.get("gemeentenaam") or "").strip(),
"municipality_code": gemeente_code,
"street": (doc.get("straatnaam") or "").strip(),
"latitude": lat,
"longitude": lon,
}
tools-api/app/connectors/proxmox.py
+381
View File
@@ -0,0 +1,381 @@
"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
from __future__ import annotations
import json
import os
import shlex
import socket
import ssl
import subprocess
import time
from datetime import datetime, timezone
from typing import Any
from urllib.error import URLError
from urllib.request import Request, urlopen
from app.db import execute, fetch_all, fetch_one
PROXMOX_HOST = "10.4.7.14"
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
SSH_USER = "aissa"
SSH_PASSWORD = "Foodlinkk#2026"
VM_105_IP = "10.4.7.19"
VM_106_IP = "10.4.7.18"
SERVICE_LAYOUT: list[dict[str, Any]] = [
{"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
{"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
{"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
{"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
{"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
]
def _iso_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
ssh_cmd = [
"sshpass",
"-p",
SSH_PASSWORD,
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"ConnectTimeout=7",
f"{SSH_USER}@{host}",
command,
]
try:
proc = subprocess.run( # noqa: S603
ssh_cmd,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except FileNotFoundError as exc:
return {"ok": False, "error": f"ssh tooling missing: {exc}"}
except subprocess.TimeoutExpired:
return {"ok": False, "error": "ssh timeout"}
return {
"ok": proc.returncode == 0,
"code": proc.returncode,
"stdout": (proc.stdout or "").strip(),
"stderr": (proc.stderr or "").strip(),
}
def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]:
req = Request(url, headers=headers or {})
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310
payload = resp.read().decode("utf-8")
return json.loads(payload)
def _build_token_header(token_value: str) -> str:
val = token_value.strip()
if val.startswith("PVEAPIToken="):
return val
return f"PVEAPIToken={val}"
def _create_api_token_via_ssh() -> str | None:
token_name = f"ops{int(time.time())}"
cmd = (
f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} "
"--privsep 0 --expire 0 --output-format json"
)
result = _run_ssh(PROXMOX_HOST, cmd, timeout=15)
if not result.get("ok"):
return None
try:
parsed = json.loads(result.get("stdout") or "{}")
except json.JSONDecodeError:
return None
tokenid = parsed.get("full-tokenid")
secret = parsed.get("value")
if tokenid and secret:
return f"PVEAPIToken={tokenid}={secret}"
return None
def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
token = os.getenv("PROXMOX_TOKEN")
tried = []
if token:
tried.append("env-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": _build_token_header(token)},
)
return data.get("data") or [], "api-token-env", None
except Exception as exc: # noqa: BLE001
tried.append(f"env-failed:{exc}")
created = _create_api_token_via_ssh()
if created:
tried.append("ssh-created-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": created},
)
return data.get("data") or [], "api-token-ssh", None
except Exception as exc: # noqa: BLE001
tried.append(f"ssh-token-failed:{exc}")
return [], "none", ", ".join(tried) if tried else "no-token"
def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
if pvesh.get("ok"):
try:
return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
except json.JSONDecodeError:
pass
qm = _run_ssh(PROXMOX_HOST, "qm list")
rows: list[dict[str, Any]] = []
if qm.get("ok") and qm.get("stdout"):
lines = (qm["stdout"] or "").splitlines()
for line in lines[1:]:
parts = line.split()
if not parts:
continue
vmid = parts[0]
rows.append(
{
"node": "pve",
"type": "qemu",
"id": f"qemu/{vmid}",
"vmid": int(vmid) if vmid.isdigit() else vmid,
"status": parts[2] if len(parts) > 2 else "unknown",
}
)
return rows, "ssh-qm-list", None
err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
return [], "none", err
def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
return sock.connect_ex((host, port)) == 0
finally:
sock.close()
def check_docker_services() -> dict[str, Any]:
result = _run_ssh(VM_106_IP, "docker ps --format json")
method = "docker-ps-json"
if not result.get("ok"):
result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
method = "docker-ps-template-json"
if not result.get("ok"):
return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}
containers: list[dict[str, Any]] = []
for line in (result.get("stdout") or "").splitlines():
line = line.strip()
if not line:
continue
try:
parsed = json.loads(line)
containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
except json.JSONDecodeError:
containers.append({"raw": line})
return {"ok": True, "source": method, "containers": containers}
def get_topology() -> dict[str, Any]:
api_nodes, api_source, api_error = _fetch_nodes_via_api()
ssh_nodes: list[dict[str, Any]] = []
ssh_source = "none"
ssh_error: str | None = None
if not api_nodes:
ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()
api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
host_status = api_node.get("status") if api_node else "unknown"
if host_status == "unknown" and ssh_nodes:
host_status = "online"
vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
source_rows = api_nodes or ssh_nodes
for row in source_rows:
vmid = str(row.get("vmid") or "").strip()
if vmid in vm_states:
vm_states[vmid] = str(row.get("status") or "unknown")
docker_state = check_docker_services()
docker_names = {
str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
}
vm105_children: list[dict[str, Any]] = []
vm106_children: list[dict[str, Any]] = []
for svc in SERVICE_LAYOUT:
up = _port_health(str(svc["host"]), int(svc["port"]))
hinted = "unknown"
for name, details in docker_names.items():
if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
hinted = str(details.get("State") or details.get("Status") or "running")
break
item = {
"id": svc["id"],
"label": svc["label"],
"type": "service",
"status": "online" if up else "offline",
"cpu": None,
"mem": None,
"host": svc["host"],
"hint": hinted,
"children": [],
}
if svc["parent"] == "vm105-hermes":
vm105_children.append(item)
else:
vm106_children.append(item)
topology_nodes = [
{
"id": "proxmox-host",
"label": f"proxmox-host ({PROXMOX_HOST})",
"type": "proxmox",
"status": host_status,
"cpu": host_cpu,
"mem": host_mem,
"children": [
{
"id": "vm105-hermes",
"label": f"vm105-hermes ({VM_105_IP})",
"type": "vm",
"status": vm_states["105"],
"cpu": None,
"mem": None,
"children": vm105_children,
},
{
"id": "vm106-command",
"label": f"vm106-command ({VM_106_IP})",
"type": "vm",
"status": vm_states["106"],
"cpu": None,
"mem": None,
"children": vm106_children,
},
],
}
]
return {
"generated_at": _iso_now(),
"nodes": topology_nodes,
"meta": {
"proxmox_host": PROXMOX_HOST,
"api_source": api_source,
"api_error": api_error,
"ssh_source": ssh_source,
"ssh_error": ssh_error,
"docker_source": docker_state.get("source"),
"docker_ok": docker_state.get("ok", False),
},
}
def get_status_summary() -> dict[str, Any]:
topo = get_topology()
flat: list[dict[str, Any]] = []
def _collect(node: dict[str, Any]) -> None:
flat.append(node)
for child in node.get("children") or []:
_collect(child)
for root in topo.get("nodes") or []:
_collect(root)
total = len(flat)
online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"})
degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"})
offline = max(0, total - online - degraded)
return {
"generated_at": topo.get("generated_at"),
"health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"),
"counts": {"total": total, "online": online, "degraded": degraded, "offline": offline},
"sources": topo.get("meta", {}),
"topology": topo,
}
def _table_exists(table_name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = %s
) AS ok
""",
(table_name,),
)
return bool(row and row.get("ok"))
def poll_and_snapshot() -> dict[str, Any]:
status = get_status_summary()
if not _table_exists("infra_snapshots"):
return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status}
cols = fetch_all(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'infra_snapshots'
ORDER BY ordinal_position
"""
)
colset = {c.get("column_name") for c in cols}
payload = {
"source": "proxmox",
"topology": status.get("topology"),
"summary": {k: v for k, v in status.items() if k != "topology"},
"generated_at": status.get("generated_at"),
}
value_map: dict[str, Any] = {}
if "source" in colset:
value_map["source"] = "proxmox"
if "provider" in colset:
value_map["provider"] = "proxmox"
if "snapshot" in colset:
value_map["snapshot"] = json.dumps(payload)
if "payload" in colset:
value_map["payload"] = json.dumps(payload)
if "topology" in colset:
value_map["topology"] = json.dumps(status.get("topology"))
if "summary" in colset:
value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"})
if "created_at" in colset:
value_map["created_at"] = datetime.now(timezone.utc)
if not value_map:
return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}
columns = list(value_map.keys())
placeholders = ", ".join(["%s"] * len(columns))
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
execute(sql, tuple(value_map[c] for c in columns))
return {"ok": True, "saved": True, "columns": columns, "status": status}
tools-api/app/connectors/reclamefolder.py
+211
View File
@@ -0,0 +1,211 @@
"""Fetch live supermarket folders from reclamefolder.nl — all chains."""
from __future__ import annotations
import json
import re
import xml.etree.ElementTree as ET
from datetime import date, datetime, timezone
from typing import Any, Optional
from urllib.parse import unquote
from urllib.request import HTTPCookieProcessor, Request, build_opener
from app.db import execute, fetch_all, fetch_one
BASE = "https://www.reclamefolder.nl"
SUPERMARKT_URL = f"{BASE}/categorieen/supermarkt/"
SITEMAP_RETAILERS = f"{BASE}/sitemap/retailers.xml"
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
SUPERMARKT_KEYWORDS = (
"supermarkt", "markt", "ah", "jumbo", "lidl", "aldi", "plus", "dirk",
"coop", "boni", "vomar", "deka", "ekoplaza", "mitra", "spar", "hoogvliet",
"food", "gall", "poiesz", "nettorama",
)
_opener = None
def _get_opener():
global _opener
if _opener is None:
_opener = build_opener(HTTPCookieProcessor())
return _opener
def _fetch_html(url: str) -> str:
headers = {"User-Agent": USER_AGENT, "Accept-Language": "nl-NL,nl;q=0.9"}
html = _get_opener().open(Request(url, headers=headers), timeout=35).read().decode("utf-8", "ignore")
if "__NEXT_DATA__" not in html:
cb = re.search(r"decodeURIComponent\('([^']+)'\)", html)
if cb:
html = _get_opener().open(Request(unquote(cb.group(1)), headers=headers), timeout=35).read().decode("utf-8", "ignore")
return html
def _fetch_page_props(url: str) -> dict[str, Any]:
html = _fetch_html(url)
match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.+?)</script>', html)
if not match:
return {}
data = json.loads(match.group(1))
return data.get("props", {}).get("pageProps", {}) or {}
def _parse_day(raw: Optional[str]) -> Optional[date]:
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00")).date()
except Exception:
return None
def _folder_item(row: dict[str, Any], source: str = "category") -> Optional[dict[str, Any]]:
retailer = row.get("retailer") or {}
if source == "retailer_page":
chain = retailer.get("name") or row.get("name", "")
edition_id = row.get("id")
valid_label = ""
cover = row.get("cover") or {}
folder_name = row.get("name") or "Folder"
else:
chain = retailer.get("name")
edition_id = row.get("editionId") or row.get("id")
valid_label = row.get("validLabel") or ""
cover = row.get("cover") or {}
folder_name = valid_label or "Folder"
if not edition_id or not chain:
return None
valid_to = _parse_day(row.get("validThru"))
today = datetime.now(timezone.utc).date()
if valid_to and valid_to < today:
return None
permaname = retailer.get("permaname") or ""
url = f"{BASE}/f/folders/{edition_id}/"
title = f"{chain} — {folder_name}" if folder_name != chain else f"{chain} folder ({valid_label})".strip(" ()")
return {
"chain": chain,
"title": title,
"folder_path": url,
"folder_label": valid_label or folder_name or chain,
"description": f"Reclamefolder.nl · {valid_label or folder_name}".strip(" ·"),
"valid_from": _parse_day(row.get("validFrom")),
"valid_to": valid_to,
"image_url": cover.get("imageUrl") if isinstance(cover, dict) else None,
"status": "active",
"promo_type": row.get("name") or "folder",
"source": "reclamefolder.nl",
"metadata": {
"edition_id": str(edition_id),
"version_id": str(row.get("versionId") or row.get("id") or ""),
"retailer_permaname": permaname,
"retailer_url": f"{BASE}/winkels/{permaname}/" if permaname else "",
"folder_type": row.get("name") or "folder",
},
}
def fetch_retailer_slugs() -> list[str]:
try:
req = Request(SITEMAP_RETAILERS, headers={"User-Agent": USER_AGENT})
data = _get_opener().open(req, timeout=45).read()
text = data.decode("utf-8", "ignore")
slugs = set()
for m in re.finditer(r"https://www\.reclamefolder\.nl/winkels/([a-z0-9-]+)/", text):
slug = m.group(1)
if "vestiging" in slug:
continue
if any(k in slug for k in SUPERMARKET_KEYWORDS):
slugs.add(slug)
return sorted(slugs)
except Exception:
return [
"albert-heijn", "jumbo", "lidl", "plus", "dirk", "aldi", "dekamarkt",
"ekoplaza", "vomar", "coop-supermarkten", "boni-supermarkt", "mitra",
]
def fetch_supermarkt_folders(include_all_retailers: bool = True) -> list[dict[str, Any]]:
seen: set[str] = set()
items: list[dict[str, Any]] = []
props = _fetch_page_props(SUPERMARKT_URL)
for row in props.get("foldersFromProps") or []:
item = _folder_item(row, "category")
if item and item["metadata"]["edition_id"] not in seen:
seen.add(item["metadata"]["edition_id"])
items.append(item)
if include_all_retailers:
for slug in fetch_retailer_slugs():
try:
rprops = _fetch_page_props(f"{BASE}/winkels/{slug}/")
retailer = rprops.get("retailer") or {}
for folder in retailer.get("folders") or []:
folder = dict(folder)
folder["retailer"] = retailer
item = _folder_item(folder, "retailer_page")
if item and item["metadata"]["edition_id"] not in seen:
seen.add(item["metadata"]["edition_id"])
items.append(item)
except Exception:
continue
items.sort(key=lambda x: (x.get("chain") or "", x.get("valid_to") or date.max))
return items
def sync_to_db() -> dict[str, Any]:
folders = fetch_supermarkt_folders()
execute(
"UPDATE promo_campaigns SET status = 'expired', updated_at = NOW() WHERE source = 'reclamefolder.nl'"
)
inserted = 0
chains: set[str] = set()
for f in folders:
fetch_one(
"""INSERT INTO promo_campaigns
(chain, title, folder_path, folder_label, description, valid_from, valid_to,
status, promo_type, image_url, source, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id""",
(
f["chain"], f["title"], f["folder_path"], f["folder_label"],
f["description"], f["valid_from"], f["valid_to"],
f["status"], f["promo_type"], f.get("image_url"),
f["source"], json.dumps(f["metadata"]),
),
)
inserted += 1
chains.add(f["chain"])
return {
"ok": True,
"source": "reclamefolder.nl",
"synced": inserted,
"chains": len(chains),
"items": folders,
}
def list_cached(limit: int = 200) -> list[dict[str, Any]]:
rows = fetch_all(
"""SELECT * FROM promo_campaigns
WHERE source = 'reclamefolder.nl' AND status = 'active'
ORDER BY valid_to ASC NULLS LAST, chain ASC
LIMIT %s""",
(limit,),
)
return [dict(r) for r in rows]
def list_chains() -> list[str]:
rows = fetch_all(
"""SELECT DISTINCT chain FROM promo_campaigns
WHERE source = 'reclamefolder.nl' AND status = 'active' AND chain IS NOT NULL
ORDER BY chain"""
)
return [r["chain"] for r in rows if r.get("chain")]
tools-api/app/connectors/retail_360_routes.py
+239
View File
@@ -0,0 +1,239 @@
"""Retail 360 workspace API — notes, media, milestones, RSS, wholesalers."""
from __future__ import annotations
import json
import urllib.request
from datetime import datetime
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from app.db import fetch_all, fetch_one
from app.middleware import log_agent_event
from app import retail_360
from app import wholesaler_scrapers
from app.connectors import market_stocks, rss_feeds
router = APIRouter(prefix="/retail", tags=["retail-360"])
class NoteIn(BaseModel):
body: str = Field(..., min_length=1)
title: Optional[str] = None
note_type: str = "general"
class MilestoneIn(BaseModel):
title: str
milestone_type: str = "custom"
client_id: Optional[int] = None
deal_id: Optional[int] = None
target_date: Optional[str] = None
value_eur: Optional[float] = None
notes: Optional[str] = None
class OwnershipIn(BaseModel):
new_owner: str
previous_owner: Optional[str] = None
change_type: str = "acquisition"
effective_date: Optional[str] = None
source: Optional[str] = None
notes: Optional[str] = None
class CalendarIn(BaseModel):
title: str
starts_at: str
description: Optional[str] = None
ends_at: Optional[str] = None
client_id: Optional[int] = None
deal_id: Optional[int] = None
location: Optional[str] = None
class MediaIn(BaseModel):
filename: str
storage_path: str
content_type: str = "image/jpeg"
caption: Optional[str] = None
def _row(row: dict | None) -> dict[str, Any]:
if not row:
raise HTTPException(404, "Not found")
out: dict[str, Any] = {}
for k, v in row.items():
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
elif v is not None and hasattr(v, "__float__") and type(v).__name__ == "Decimal":
out[k] = float(v)
else:
out[k] = v
return out
def _fetch_weather_forecast(lat: float, lon: float) -> list[dict[str, Any]]:
url = (
f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}"
f"&daily=temperature_2m_max,precipitation_sum,weathercode"
f"&timezone=Europe%2FAmsterdam&forecast_days=7"
)
try:
with urllib.request.urlopen(url, timeout=15) as resp:
data = json.loads(resp.read().decode())
days = data.get("daily", {}).get("time", [])
temps = data.get("daily", {}).get("temperature_2m_max", [])
prec = data.get("daily", {}).get("precipitation_sum", [])
return [
{"date": days[i], "temperature_c": temps[i] if i < len(temps) else None,
"precipitation_mm": prec[i] if i < len(prec) else None, "source": "open-meteo-live"}
for i in range(len(days))
]
except Exception:
return []
@router.get("/360/{store_id}")
def get_360_view(store_id: int) -> dict[str, Any]:
try:
data = retail_360.get_store_360(store_id)
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
store = data["store"]
if store.get("latitude") and store.get("longitude"):
live = _fetch_weather_forecast(float(store["latitude"]), float(store["longitude"]))
if live:
data["weather_forecast"] = live
for key in ("notes", "media", "milestones", "ownership_changes", "calendar", "weather"):
data[key] = [_row(x) for x in data.get(key, [])]
if data.get("area_analysis"):
data["area_analysis"] = _row(data["area_analysis"])
return data
@router.post("/360/{store_id}/notes")
def add_store_note(store_id: int, payload: NoteIn) -> dict[str, Any]:
note = retail_360.add_note("supermarket", store_id, payload.body, payload.title, payload.note_type)
log_agent_event(agent_name="retail_360", event_type="note", title=f"Note on store {store_id}")
return {"note": _row(note)}
@router.post("/360/{store_id}/milestones")
def add_store_milestone(store_id: int, payload: MilestoneIn) -> dict[str, Any]:
ms = retail_360.add_milestone(store_id, payload.title, payload.milestone_type, **payload.model_dump(exclude={"title", "milestone_type"}))
return {"milestone": _row(ms)}
@router.post("/360/{store_id}/ownership")
def add_store_ownership(store_id: int, payload: OwnershipIn) -> dict[str, Any]:
store = fetch_one("SELECT chain FROM supermarkets WHERE id = %s", (store_id,))
row = retail_360.add_ownership(entity_id=store_id, chain=store.get("chain") if store else None, **payload.model_dump())
return {"ownership": _row(row)}
@router.post("/360/{store_id}/calendar")
def add_store_calendar(store_id: int, payload: CalendarIn) -> dict[str, Any]:
ev = retail_360.add_calendar_event(store_id, payload.title, payload.starts_at, **payload.model_dump(exclude={"title", "starts_at"}))
return {"event": _row(ev)}
@router.post("/360/{store_id}/media")
def register_store_media(store_id: int, payload: MediaIn) -> dict[str, Any]:
media = retail_360.register_media("supermarket", store_id, payload.filename, payload.storage_path, payload.content_type, payload.caption)
return {"media": _row(media)}
@router.get("/cities")
def list_cities(limit: int = Query(200, ge=1, le=1000)) -> dict[str, Any]:
rows = fetch_all(
"""SELECT c.*, (SELECT COUNT(*) FROM supermarkets s WHERE s.city ILIKE c.city) AS store_count
FROM city_demographics c ORDER BY c.population DESC NULLS LAST LIMIT %s""",
(limit,),
)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/cities/sync")
def sync_cities(limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:
return retail_360.sync_city_demographics(limit)
@router.get("/wholesalers")
def list_wholesalers(limit: int = Query(500, ge=1, le=2000), q: Optional[str] = None) -> dict[str, Any]:
clauses, params = [], []
if q:
clauses.append("(name ILIKE %s OR city ILIKE %s OR address ILIKE %s)")
like = f"%{q}%"
params.extend([like, like, like])
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
rows = fetch_all(f"SELECT * FROM wholesalers{where} ORDER BY name LIMIT %s", tuple(params + [limit]))
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/wholesalers/import")
def import_wholesalers(background: bool = Query(False)) -> dict[str, Any]:
log_agent_event(agent_name="wholesale_scraper", event_type="import", title="OSM wholesalers import")
if background:
import threading
threading.Thread(target=wholesaler_scrapers.import_wholesalers, daemon=True).start()
return {"status": "started", "message": "Wholesaler import running in background"}
return wholesaler_scrapers.import_wholesalers()
@router.get("/rss/live")
def rss_live(limit: int = Query(30, ge=1, le=100), category: Optional[str] = None) -> dict[str, Any]:
rows = rss_feeds.list_live_feed(limit, category)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/rss/refresh")
def rss_refresh() -> dict[str, Any]:
log_agent_event(agent_name="rss_feeds", event_type="refresh", title="RSS feeds refresh")
return rss_feeds.refresh_all_feeds()
@router.get("/market/stocks")
def retail_market_stocks() -> dict[str, Any]:
quotes = market_stocks.fetch_retail_quotes()
return {
"items": quotes,
"summary": market_stocks.market_summary(quotes),
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/regulations")
def retail_regulations(limit: int = Query(30, ge=1, le=100)) -> dict[str, Any]:
reg = rss_feeds.list_live_feed(limit, "regelgeving")
cbs = rss_feeds.list_live_feed(limit, "cbs")
markt = rss_feeds.list_live_feed(min(limit, 15), "markt")
return {
"regelgeving": [_row(r) for r in reg],
"cbs": [_row(r) for r in cbs],
"markt": [_row(r) for r in markt],
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/live-dashboard")
def live_dashboard() -> dict[str, Any]:
trends = fetch_all(
"SELECT * FROM market_trends ORDER BY updated_at DESC NULLS LAST LIMIT 8"
)
rss = rss_feeds.list_live_feed(12)
opportunities = fetch_all(
"""SELECT s.name, s.chain, s.city, ros.halal_opportunity_score
FROM retail_opportunity_scores ros JOIN supermarkets s ON s.id = ros.supermarket_id
ORDER BY ros.halal_opportunity_score DESC LIMIT 5"""
)
quotes = market_stocks.fetch_retail_quotes()
return {
"trends": [_row(t) for t in trends],
"rss": [_row(r) for r in rss],
"top_opportunities": [_row(o) for o in opportunities],
"market_stocks": quotes,
"market_summary": market_stocks.market_summary(quotes),
"updated_at": datetime.utcnow().isoformat(),
}
tools-api/app/connectors/rss_feeds.py
+152
View File
@@ -0,0 +1,152 @@
"""RSS feed ingestion — filtered for kant-en-klaar & supermarkt only."""
from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Optional
from urllib.request import Request, urlopen
from app.db import execute, execute_returning, fetch_all, fetch_one
USER_AGENT = "Foodlinkk-Intel/1.0"
INCLUDE_KEYWORDS = (
"kant en klaar", "kant-en-klaar", "kant&klaa", "ready meal", "ready-to-eat",
"maaltijd", "maaltijden", "supermarkt", "supermarket", "retail", "jumbo",
"albert heijn", "ah ", " plus ", "lidl", "aldi", "dirk", "halal",
"convenience", "schap", "filiaal", "foodservice", "vers", "meal",
"grocery", "food retail", "kant-en-klaar",
)
EXCLUDE_KEYWORDS = (
"voetbal", "sport", "politiek", "verkiezing", "trump", "bbc", "oorlog",
"crypto", "bitcoin", "aandelenbeurs", "beurs ", "weerbericht",
)
def _parse_date(raw: Optional[str]) -> Optional[datetime]:
if not raw:
return None
try:
return parsedate_to_datetime(raw).astimezone(timezone.utc)
except Exception:
pass
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except Exception:
return None
def _strip_html(text: str) -> str:
return re.sub(r"<[^>]+>", "", text or "").strip()[:2000]
def is_relevant(title: str, summary: Optional[str] = None) -> bool:
blob = f"{title} {summary or ''}".lower()
for bad in EXCLUDE_KEYWORDS:
if bad in blob:
return False
for good in INCLUDE_KEYWORDS:
if good in blob:
return True
return False
def _fetch_xml(url: str) -> ET.Element:
req = Request(url, headers={"User-Agent": USER_AGENT})
with urlopen(req, timeout=25) as resp:
data = resp.read()
return ET.fromstring(data)
def _skip_keyword_filter(category: Optional[str]) -> bool:
return category in ("regelgeving", "cbs", "markt")
def refresh_feed(feed_id: int) -> dict[str, Any]:
feed = fetch_one("SELECT * FROM rss_feeds WHERE id = %s AND is_active = TRUE", (feed_id,))
if not feed:
return {"error": "feed not found"}
skip_filter = _skip_keyword_filter(feed.get("category"))
root = _fetch_xml(feed["url"])
items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
inserted = skipped = 0
for item in items[:50]:
title = (item.findtext("title") or item.findtext("{http://www.w3.org/2005/Atom}title") or "").strip()
link = (item.findtext("link") or "").strip()
if not link:
link_el = item.find("{http://www.w3.org/2005/Atom}link")
if link_el is not None:
link = link_el.get("href") or ""
summary = item.findtext("description") or item.findtext("summary") or item.findtext("{http://www.w3.org/2005/Atom}summary") or ""
pub = item.findtext("pubDate") or item.findtext("published") or item.findtext("{http://www.w3.org/2005/Atom}published")
if not title or not link:
continue
clean_summary = _strip_html(summary)
cat = (feed.get("category") or "").lower()
if cat not in ("regelgeving", "cbs", "markt") and not is_relevant(title, clean_summary):
skipped += 1
continue
try:
execute_returning(
"""INSERT INTO rss_items (feed_id, title, link, summary, published_at)
VALUES (%s, %s, %s, %s, %s) RETURNING id""",
(feed_id, title[:500], link[:1000], clean_summary, _parse_date(pub)),
)
inserted += 1
except Exception:
pass
execute(
"UPDATE rss_feeds SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
(feed_id,),
)
return {"feed": feed["name"], "inserted": inserted, "skipped": skipped}
def refresh_all_feeds() -> dict[str, Any]:
feeds = fetch_all("SELECT id, name FROM rss_feeds WHERE is_active = TRUE")
results = []
for f in feeds:
try:
results.append(refresh_feed(int(f["id"])))
except Exception as exc: # noqa: BLE001
execute("UPDATE rss_feeds SET last_status = %s WHERE id = %s", (str(exc)[:32], f["id"]))
results.append({"feed": f["name"], "error": str(exc)})
return {"feeds": len(feeds), "results": results}
def list_live_feed(limit: int = 40, category: Optional[str] = None) -> list[dict[str, Any]]:
params: list[Any] = []
if category and category.lower() in ("regelgeving", "cbs", "markt"):
base = """
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
FROM rss_items i
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
WHERE f.category = %s
"""
params.append(category.lower())
else:
like_clauses = " OR ".join(
f"(i.title ILIKE %s OR COALESCE(i.summary,'') ILIKE %s)" for _ in INCLUDE_KEYWORDS[:12]
)
for kw in INCLUDE_KEYWORDS[:12]:
p = f"%{kw}%"
params.extend([p, p])
base = f"""
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
FROM rss_items i
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
WHERE ({like_clauses})
"""
if category:
base += " AND f.category = %s"
params.append(category)
base += " ORDER BY i.published_at DESC NULLS LAST, i.fetched_at DESC LIMIT %s"
params.append(limit)
rows = fetch_all(base, tuple(params))
skip_filter = category and category.lower() in ("regelgeving", "cbs", "markt")
if skip_filter:
return [dict(r) for r in rows]
return [dict(r) for r in rows if is_relevant(r.get("title") or "", r.get("summary"))]
tools-api/app/connectors/trends_feed.py
+104
View File
@@ -0,0 +1,104 @@
"""Market trends feed for kant-en-klaar / halal ready meals."""
from __future__ import annotations
from typing import Any
from app.db import execute_returning, fetch_all, json_param
TREND_SEEDS = [
{
"category": "kant-en-klaar",
"trend_name": "Halal ready-meals groei stedelijk",
"description": "Stedelijke gebieden met hoge niet-westerse bevolking tonen vraag naar halal kant-en-klaar zonder voldoende schap-aanbod.",
"source": "CBS + retail intelligence",
"confidence_score": 0.82,
"opportunity_score": 0.88,
"related_products": ["halal maaltijden", "microwave meals", "salades"],
"action_items": ["Target Plus/Jumbo regio's met halal-gap score >60", "Pilot schap bij 3 filialen"],
},
{
"category": "halal",
"trend_name": "Certificering als vertrouwen-driver",
"description": "Filialen met halal-certificering maar beperkt ready-meal assortiment = upsell kans voor Foodlinkk.",
"source": "halal_registry + CRM",
"confidence_score": 0.75,
"opportunity_score": 0.80,
"related_products": ["HQC gecertificeerde maaltijden"],
"action_items": ["Match HQC stores met CRM pipeline", "Cross-sell bestaande klanten"],
},
{
"category": "kant-en-klaar",
"trend_name": "Convenience trend post-COVID",
"description": "Gemiddeld inkomen en eenpersoonshuishoudens correleren met groei kant-en-klaar segment.",
"source": "CBS kerncijfers",
"confidence_score": 0.70,
"opportunity_score": 0.72,
"related_products": ["single-serve", "meal kits"],
"action_items": ["Filter winkels op huishoudens + inkomen >€35k"],
},
]
def refresh_trends_from_social() -> dict[str, Any]:
"""Derive trend signals from social mentions keywords."""
mentions = fetch_all(
"""SELECT platform, text, sentiment_score, created_at FROM social_mentions
WHERE created_at > NOW() - interval '30 days'
ORDER BY created_at DESC LIMIT 100"""
)
keywords = {
"halal": 0, "kant-en-klaar": 0, "ready meal": 0, "meal prep": 0,
"supermarkt": 0, "schap": 0, "afhalen": 0,
}
for m in mentions:
text = (m.get("text") or "").lower()
for kw in keywords:
if kw in text:
keywords[kw] += 1
created = 0
for kw, count in keywords.items():
if count < 1:
continue
execute_returning(
"""INSERT INTO market_trends (category, trend_name, description, source,
confidence_score, opportunity_score, related_products, data_source)
VALUES (%s, %s, %s, 'social_mentions', %s, %s, %s, 'live_feed')
RETURNING id""",
(
"kant-en-klaar" if "meal" in kw or "kant" in kw else "halal",
f"Social buzz: {kw} ({count} mentions)",
f"{count} vermeldingen afgelopen 30 dagen rond '{kw}'.",
min(0.95, 0.5 + count * 0.05),
min(0.95, 0.4 + count * 0.06),
[kw],
),
)
created += 1
for seed in TREND_SEEDS:
exists = fetch_all(
"SELECT id FROM market_trends WHERE trend_name = %s LIMIT 1",
(seed["trend_name"],),
)
if not exists:
execute_returning(
"""INSERT INTO market_trends (category, trend_name, description, source,
confidence_score, opportunity_score, related_products, action_items, data_source)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'seed') RETURNING id""",
(
seed["category"], seed["trend_name"], seed["description"], seed["source"],
seed["confidence_score"], seed["opportunity_score"],
seed["related_products"], seed["action_items"],
),
)
created += 1
return {"trends_created": created, "keyword_hits": keywords}
def list_live_trends(limit: int = 20) -> list[dict[str, Any]]:
return fetch_all(
"""SELECT * FROM market_trends ORDER BY updated_at DESC NULLS LAST, created_at DESC LIMIT %s""",
(limit,),
)
tools-api/app/db.py
+76
View File
@@ -0,0 +1,76 @@
from contextlib import contextmanager
from typing import Any, Optional
import psycopg2
from psycopg2 import pool
from psycopg2.extras import RealDictCursor, Json
from app.config import settings
_connection_pool: Optional[pool.SimpleConnectionPool] = None
def init_pool(minconn: int = 1, maxconn: int = 10) -> None:
global _connection_pool
if _connection_pool is None:
_connection_pool = pool.SimpleConnectionPool(
minconn,
maxconn,
dsn=settings.database_dsn,
)
def close_pool() -> None:
global _connection_pool
if _connection_pool is not None:
_connection_pool.closeall()
_connection_pool = None
@contextmanager
def get_connection():
if _connection_pool is None:
init_pool()
conn = _connection_pool.getconn()
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
_connection_pool.putconn(conn)
def fetch_all(query: str, params: Optional[tuple] = None) -> list[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
return [dict(row) for row in cur.fetchall()]
def fetch_one(query: str, params: Optional[tuple] = None) -> Optional[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
row = cur.fetchone()
return dict(row) if row else None
def execute_returning(query: str, params: Optional[tuple] = None) -> Optional[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
row = cur.fetchone()
return dict(row) if row else None
def execute(query: str, params: Optional[tuple] = None) -> int:
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(query, params)
return cur.rowcount
def json_param(value: Any) -> Json:
return Json(value or {})
tools-api/app/email_config.py
+46
View File
@@ -0,0 +1,46 @@
"""Load active email account from PostgreSQL for tools-api."""
from __future__ import annotations
import os
from typing import Any
from app.db import fetch_one
def get_active_email_config() -> dict[str, Any]:
"""Return SMTP config: DB active account first, then env fallback."""
try:
row = fetch_one(
"""
SELECT id, label, email_address, provider, smtp_host, smtp_port,
smtp_user, smtp_password, imap_host, imap_port, imap_user, imap_password
FROM email_accounts WHERE is_active = TRUE
ORDER BY updated_at DESC LIMIT 1
"""
)
if row and row.get("smtp_host"):
return {
"source": "database",
"account_id": row.get("id"),
"label": row.get("label"),
"smtp_host": (row.get("smtp_host") or "").strip(),
"smtp_port": int(row.get("smtp_port") or 587),
"smtp_user": (row.get("smtp_user") or row.get("email_address") or "").strip(),
"smtp_pass": row.get("smtp_password") or "",
"smtp_from": (row.get("email_address") or row.get("smtp_user") or "").strip(),
}
except Exception:
pass
smtp_user = os.getenv("SMTP_USER", "").strip()
return {
"source": "env",
"account_id": None,
"label": "Environment",
"smtp_host": os.getenv("SMTP_HOST", "").strip(),
"smtp_port": int(os.getenv("SMTP_PORT", "587")),
"smtp_user": smtp_user,
"smtp_pass": os.getenv("SMTP_PASS", "").strip(),
"smtp_from": os.getenv("SMTP_FROM", smtp_user).strip(),
}
tools-api/app/logging_middleware.py
+26
View File
@@ -0,0 +1,26 @@
"""Log Tools API mutations to agent_events."""
from __future__ import annotations
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from app.middleware import log_agent_event
class AgentLoggingMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if request.method in ("POST", "PUT", "PATCH", "DELETE") and response.status_code < 400:
path = request.url.path
if path.startswith(("/research", "/recommendations", "/retail", "/events")):
try:
log_agent_event(
agent_name="tools_api",
event_type="api_call",
title=f"{request.method} {path}",
channel="api",
metadata={"status": response.status_code},
)
except Exception:
pass
return response
tools-api/app/main.py
+750
View File
@@ -0,0 +1,750 @@
from datetime import date, datetime
from decimal import Decimal
from typing import Any, Optional
from fastapi import FastAPI, HTTPException, Query
from pydantic import BaseModel, Field
from app.db import close_pool, execute_returning, fetch_all, fetch_one, init_pool
from app.middleware import log_agent_event
from app.packaging_routes import router as packaging_router
from app.retail import router as retail_router
from app.retail_360_routes import router as retail_360_router
from app.research import router as research_router
from app.recommendations import router as recommendations_router
from app.ops_routes import router as ops_router
from app.logging_middleware import AgentLoggingMiddleware
from app import brain as brain_svc
class BrainMessageIn(BaseModel):
chat_id: int
direction: str = Field(..., pattern="^(in|out)$")
content_text: Optional[str] = None
content_type: str = "text"
role: str = "user"
telegram_message_id: Optional[int] = None
reply_to_db_id: Optional[int] = None
agent_name: Optional[str] = None
content_json: dict[str, Any] = Field(default_factory=dict)
user_name: Optional[str] = None
user_role: Optional[str] = None
chat_type: str = "private"
embed: bool = True
class BrainEdgeIn(BaseModel):
source_message_id: int
edge_type: str
target_message_id: Optional[int] = None
target_entity_type: Optional[str] = None
target_entity_id: Optional[int] = None
weight: float = 1.0
metadata: dict[str, Any] = Field(default_factory=dict)
class BrainSearchIn(BaseModel):
query: str = Field(..., min_length=2)
chat_id: Optional[int] = None
limit: int = Field(default=8, ge=1, le=30)
from app.email_config import get_active_email_config
from app.comfyui import fetch_image_bytes, generate_image, get_job, start_generation, QUALITY_PRESETS
import json
import os
DOC_INGEST_URL = os.getenv("DOC_INGEST_URL", "http://10.4.7.19:8750")
app = FastAPI(title="Foodlinkk Tools API", version="1.1.0")
app.add_middleware(AgentLoggingMiddleware)
app.include_router(retail_router)
app.include_router(retail_360_router)
app.include_router(research_router)
app.include_router(recommendations_router)
app.include_router(packaging_router)
app.include_router(ops_router)
class AgentEventCreate(BaseModel):
agent_name: str = Field(..., max_length=64)
event_type: str = Field(..., max_length=64)
title: str = Field(..., max_length=255)
body: Optional[str] = None
agent_type: str = Field(default="openswarm", max_length=32)
metadata: dict[str, Any] = Field(default_factory=dict)
status: str = Field(default="completed", max_length=32)
related_table: Optional[str] = Field(default=None, max_length=64)
related_id: Optional[int] = None
channel: str = Field(default="dashboard", max_length=32)
def _serialize_row(row: dict[str, Any]) -> dict[str, Any]:
out: dict[str, Any] = {}
for key, value in row.items():
if isinstance(value, (datetime, date)):
out[key] = value.isoformat()
elif isinstance(value, Decimal):
out[key] = float(value)
else:
out[key] = value
return out
@app.on_event("startup")
def on_startup() -> None:
init_pool()
@app.on_event("shutdown")
def on_shutdown() -> None:
close_pool()
@app.get("/health")
def health() -> dict[str, str]:
try:
fetch_one("SELECT 1 AS ok")
return {"status": "ok", "database": "connected"}
except Exception as exc:
return {"status": "degraded", "database": str(exc)}
@app.post("/events")
def create_event(payload: AgentEventCreate) -> dict[str, Any]:
try:
row = log_agent_event(**payload.model_dump())
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return _serialize_row(row)
@app.get("/events")
def list_events(
limit: int = Query(default=50, ge=1, le=200),
status: Optional[str] = Query(default=None),
agent_name: Optional[str] = Query(default=None),
) -> dict[str, Any]:
clauses: list[str] = []
params: list[Any] = []
if status:
clauses.append("status = %s")
params.append(status)
if agent_name:
clauses.append("agent_name = %s")
params.append(agent_name)
where = f"WHERE { AND .join(clauses)}" if clauses else ""
params.append(limit)
try:
rows = fetch_all(
f"""
SELECT *
FROM agent_events
{where}
ORDER BY created_at DESC
LIMIT %s
""",
tuple(params),
)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {"events": [_serialize_row(r) for r in rows]}
@app.post("/events/{event_id}/approve")
def approve_event(event_id: int) -> dict[str, Any]:
row = execute_returning(
"""
UPDATE agent_events
SET status = %s, completed_at = NOW()
WHERE id = %s
RETURNING *
""",
("approved", event_id),
)
if row is None:
raise HTTPException(status_code=404, detail="Event not found")
return _serialize_row(row)
@app.post("/events/{event_id}/reject")
def reject_event(event_id: int) -> dict[str, Any]:
row = execute_returning(
"""
UPDATE agent_events
SET status = %s, completed_at = NOW()
WHERE id = %s
RETURNING *
""",
("rejected", event_id),
)
if row is None:
raise HTTPException(status_code=404, detail="Event not found")
return _serialize_row(row)
class BrowserTask(BaseModel):
url: str
task: str = Field(default="open")
metadata: dict[str, Any] = Field(default_factory=dict)
class HermanDelegate(BaseModel):
message: str
target_agent: Optional[str] = Field(default=None, max_length=64)
@app.post("/browser/task")
async def browser_task(payload: BrowserTask) -> dict[str, Any]:
import httpx
import os
browser_url = os.getenv("BROWSER_USE_URL", "http://browser-agent:7790")
result = {"ok": True, "browser_url": browser_url, "url": payload.url}
try:
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.post(f"{browser_url.rstrip('/')}/task", json=payload.model_dump())
result["upstream_status"] = resp.status_code
if resp.status_code < 500:
try:
result["upstream"] = resp.json()
except Exception:
result["upstream"] = resp.text[:500]
except Exception as exc:
result["upstream_error"] = str(exc)
try:
log_agent_event(
agent_name="browser",
event_type="browser_task",
title=f"Browser task: {payload.url[:120]}",
body=payload.task,
metadata={"url": payload.url, **payload.metadata, **result},
channel="tools-api",
)
except Exception:
pass
return result
@app.post("/herman/delegate")
def herman_delegate(payload: HermanDelegate) -> dict[str, Any]:
agent = payload.target_agent or "herman"
try:
row = log_agent_event(
agent_name=agent,
agent_type="herman_delegate",
event_type="delegate",
title="Herman delegation",
body=payload.message,
metadata={"target_agent": agent},
channel="herman",
)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return _serialize_row(row)
@app.get("/knowledge/search")
async def knowledge_search(q: str = Query(..., min_length=1), limit: int = Query(default=10, ge=1, le=50)) -> dict[str, Any]:
import httpx
try:
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.get(f"{DOC_INGEST_URL.rstrip('/')}/search", params={"q": q, "limit": limit})
if r.status_code == 200:
data = r.json()
return {"query": q, "results": data.get("results", []), "source": "chroma"}
except Exception:
pass
try:
rows = fetch_all(
"""
SELECT id, title, source, doc_type, metadata, created_at
FROM knowledge_documents
WHERE title ILIKE %s OR source ILIKE %s
ORDER BY updated_at DESC NULLS LAST
LIMIT %s
""",
(f"%{q}%", f"%{q}%", limit),
)
except Exception:
rows = []
return {"query": q, "results": [_serialize_row(r) for r in rows], "source": "postgres"}
@app.post("/knowledge/ingest")
async def knowledge_ingest(force: bool = Query(default=False)) -> dict[str, Any]:
import httpx
try:
async with httpx.AsyncClient(timeout=600.0) as client:
r = await client.post(f"{DOC_INGEST_URL.rstrip('/')}/ingest/scan", params={"force": force})
r.raise_for_status()
return r.json()
except Exception as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
@app.get("/knowledge/nas")
async def knowledge_nas(limit: int = Query(default=50, ge=1, le=500)) -> dict[str, Any]:
import httpx
try:
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.get(f"{DOC_INGEST_URL.rstrip('/')}/nas/list", params={"limit": limit})
r.raise_for_status()
return r.json()
except Exception as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
# --- CRM / Email / LLM memory (2nd brain) ---
class EmailSend(BaseModel):
to: list[str] = Field(..., min_length=1)
subject: str = Field(..., max_length=500)
body: str
client_id: Optional[int] = None
deal_id: Optional[int] = None
cc: list[str] = Field(default_factory=list)
class LlmMemoryCreate(BaseModel):
category: str = Field(default="fact", max_length=64)
subject: Optional[str] = Field(default=None, max_length=255)
content: str
client_id: Optional[int] = None
deal_id: Optional[int] = None
source: str = Field(default="herman", max_length=64)
metadata: dict[str, Any] = Field(default_factory=dict)
def _crm_safe_all(sql: str, params: tuple = ()) -> list[dict[str, Any]]:
try:
return fetch_all(sql, params or None)
except Exception:
return []
@app.get("/crm/context")
def crm_context(days: int = Query(default=7, ge=1, le=30)) -> dict[str, Any]:
"""Live PostgreSQL bundle for Herman / morning briefing."""
pipeline = fetch_one(
"SELECT COALESCE(SUM(value), 0) AS total, COUNT(*) AS cnt FROM deals WHERE stage NOT IN ('won', 'lost')"
)
clients = _crm_safe_all(
"""SELECT id, name, contact, email, stage, sector, notes, mrr_estimate
FROM clients ORDER BY updated_at DESC NULLS LAST, created_at DESC LIMIT 25"""
)
deals = _crm_safe_all(
"""
SELECT d.id, d.title, d.value, d.stage, d.next_action, d.deadline, d.agent_owner,
c.name AS client_name, c.email AS client_email
FROM deals d
LEFT JOIN clients c ON c.id = d.client_id
WHERE d.stage NOT IN ('won', 'lost')
ORDER BY d.deadline ASC NULLS LAST, d.updated_at DESC NULLS LAST
LIMIT 20
"""
)
upcoming = _crm_safe_all(
"""
SELECT d.id, d.title, d.value, d.stage, d.deadline, d.next_action, c.name AS client_name
FROM deals d
LEFT JOIN clients c ON c.id = d.client_id
WHERE d.deadline IS NOT NULL
AND d.deadline <= CURRENT_DATE + make_interval(days => %s)
AND d.stage NOT IN ('won', 'lost')
ORDER BY d.deadline ASC
LIMIT 15
""",
(days,),
)
calendar = _crm_safe_all(
"""
SELECT ce.id, ce.title, ce.starts_at, ce.ends_at, ce.location, ce.source,
c.name AS client_name, d.title AS deal_title
FROM calendar_events ce
LEFT JOIN clients c ON c.id = ce.client_id
LEFT JOIN deals d ON d.id = ce.deal_id
WHERE ce.starts_at >= NOW() - INTERVAL '1 day'
AND ce.starts_at <= NOW() + make_interval(days => %s)
ORDER BY ce.starts_at ASC
LIMIT 25
""",
(days,),
)
pending = _crm_safe_all(
"""
SELECT id, agent_name, title, event_type, created_at
FROM agent_events WHERE status = 'needs_approval'
ORDER BY created_at ASC LIMIT 10
"""
)
emails_recent = _crm_safe_all(
"""
SELECT id, from_addr, subject, received_at, sent_at, direction, is_read, client_id
FROM emails ORDER BY COALESCE(received_at, sent_at) DESC NULLS LAST LIMIT 15
"""
)
memories = _crm_safe_all(
"""
SELECT id, category, subject, content, client_id, deal_id, source, updated_at
FROM llm_memory ORDER BY updated_at DESC LIMIT 20
"""
)
return {
"generated_at": datetime.utcnow().isoformat() + "Z",
"pipeline_eur": float(pipeline["total"]) if pipeline else 0.0,
"active_deals": int(pipeline["cnt"]) if pipeline else 0,
"clients_count": len(clients),
"clients": [_serialize_row(c) for c in clients],
"deals": [_serialize_row(d) for d in deals],
"upcoming_deadlines": [_serialize_row(u) for u in upcoming],
"calendar": [_serialize_row(c) for c in calendar],
"pending_approvals": [_serialize_row(p) for p in pending],
"recent_emails": [_serialize_row(e) for e in emails_recent],
"llm_memory": [_serialize_row(m) for m in memories],
"links": {
"dashboard": "http://10.4.7.18:8600",
"clients": "http://10.4.7.18:8600/clients",
"deals": "http://10.4.7.18:8600/deals",
"reports": "http://10.4.7.18:8600/reports",
},
}
@app.get("/crm/clients")
def crm_clients(limit: int = Query(default=25, ge=1, le=100)) -> dict[str, Any]:
rows = _crm_safe_all(
"SELECT id, name, contact, email, stage, sector, notes FROM clients ORDER BY name LIMIT %s",
(limit,),
)
return {"clients": [_serialize_row(r) for r in rows]}
@app.get("/crm/deals/upcoming")
def crm_deals_upcoming(days: int = Query(default=7, ge=1, le=60)) -> dict[str, Any]:
rows = _crm_safe_all(
"""
SELECT d.id, d.title, d.value, d.stage, d.deadline, d.next_action, c.name AS client_name
FROM deals d LEFT JOIN clients c ON c.id = d.client_id
WHERE d.deadline IS NOT NULL AND d.deadline <= CURRENT_DATE + make_interval(days => %s)
AND d.stage NOT IN ('won', 'lost')
ORDER BY d.deadline ASC LIMIT 30
""",
(days,),
)
return {"days": days, "deals": [_serialize_row(r) for r in rows]}
@app.get("/emails/recent")
def emails_recent(limit: int = Query(default=20, ge=1, le=100)) -> dict[str, Any]:
rows = _crm_safe_all(
"""
SELECT id, message_id, from_addr, to_addrs, subject, direction,
received_at, sent_at, is_read, client_id
FROM emails ORDER BY COALESCE(received_at, sent_at) DESC NULLS LAST LIMIT %s
""",
(limit,),
)
return {"emails": [_serialize_row(r) for r in rows]}
@app.post("/emails/send")
def emails_send(payload: EmailSend) -> dict[str, Any]:
"""Verstuur e-mail via SMTP (configureer SMTP_* env vars). Log in PostgreSQL."""
import smtplib
from email.mime.text import MIMEText
from email.utils import formatdate, make_msgid
import uuid
cfg = get_active_email_config()
smtp_host = cfg["smtp_host"]
smtp_port = cfg["smtp_port"]
smtp_user = cfg["smtp_user"]
smtp_pass = cfg["smtp_pass"]
smtp_from = cfg["smtp_from"]
if not smtp_host or not smtp_from:
raise HTTPException(
status_code=503,
detail="Geen email account geconfigureerd. Ga naar Settings → Email in het dashboard.",
)
msg = MIMEText(payload.body, "plain", "utf-8")
msg["Subject"] = payload.subject
msg["From"] = smtp_from
msg["To"] = ", ".join(payload.to)
if payload.cc:
msg["Cc"] = ", ".join(payload.cc)
msg["Date"] = formatdate(localtime=True)
message_id = make_msgid()
msg["Message-ID"] = message_id
recipients = list(payload.to) + list(payload.cc)
try:
with smtplib.SMTP(smtp_host, smtp_port, timeout=30) as server:
server.ehlo()
if smtp_port == 587:
server.starttls()
if smtp_user and smtp_pass:
server.login(smtp_user, smtp_pass)
server.sendmail(smtp_from, recipients, msg.as_string())
except Exception as exc:
raise HTTPException(status_code=502, detail=f"SMTP send failed: {exc}") from exc
mid = message_id.strip("<>")
try:
row = execute_returning(
"""
INSERT INTO emails (
message_id, client_id, deal_id, direction, from_addr, to_addrs,
subject, body_text, sent_at, is_read
) VALUES (%s, %s, %s, 'out', %s, %s, %s, %s, NOW(), TRUE)
RETURNING id, message_id, subject, sent_at
""",
(mid, payload.client_id, payload.deal_id, smtp_from, payload.to, payload.subject, payload.body),
)
except Exception:
row = {"message_id": mid, "subject": payload.subject}
try:
log_agent_event(
agent_name="herman",
event_type="email_sent",
title=f"Email: {payload.subject[:120]}",
body=payload.body[:2000],
metadata={"to": payload.to, "client_id": payload.client_id},
channel="email",
)
except Exception:
pass
return {"ok": True, "message_id": mid, "email": _serialize_row(row) if isinstance(row, dict) else row}
@app.post("/llm/memory")
def llm_memory_create(payload: LlmMemoryCreate) -> dict[str, Any]:
row = execute_returning(
"""
INSERT INTO llm_memory (category, subject, content, client_id, deal_id, source, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id, category, subject, content, created_at
""",
(
payload.category,
payload.subject,
payload.content,
payload.client_id,
payload.deal_id,
payload.source,
json.dumps(payload.metadata),
),
)
return {"ok": True, "memory": _serialize_row(row)}
@app.get("/llm/memory/search")
def llm_memory_search(q: str = Query(..., min_length=1), limit: int = Query(default=15, ge=1, le=50)) -> dict[str, Any]:
rows = _crm_safe_all(
"""
SELECT id, category, subject, content, client_id, deal_id, source, updated_at
FROM llm_memory
WHERE content ILIKE %s OR subject ILIKE %s OR category ILIKE %s
ORDER BY updated_at DESC LIMIT %s
""",
(f"%{q}%", f"%{q}%", f"%{q}%", limit),
)
return {"query": q, "results": [_serialize_row(r) for r in rows]}
@app.get("/settings/email/active")
def settings_email_active() -> dict[str, Any]:
cfg = get_active_email_config()
return {
"configured": bool(cfg.get("smtp_host") and cfg.get("smtp_from")),
"source": cfg.get("source"),
"label": cfg.get("label"),
"from": cfg.get("smtp_from"),
"account_id": cfg.get("account_id"),
}
class ImageGenerateBody(BaseModel):
prompt: str = Field(..., min_length=3, max_length=2000)
width: int = Field(default=1024, ge=256, le=1024)
height: int = Field(default=1024, ge=256, le=1024)
steps: int = Field(default=28, ge=5, le=40)
seed: Optional[int] = None
quality: str = Field(default="hd", pattern="^(fast|hd|ultra|custom)$")
class ImageStartBody(BaseModel):
prompt: str = Field(..., min_length=3, max_length=2000)
negative_prompt: str = Field(default="blurry, low quality, watermark, text, ugly, deformed", max_length=2000)
quality: str = Field(default="hd", pattern="^(fast|hd|ultra)$")
seed: Optional[int] = None
@app.post("/images/generate/start")
async def images_generate_start(body: ImageStartBody) -> dict[str, Any]:
try:
return await start_generation(
body.prompt.strip(),
negative=body.negative_prompt.strip(),
quality=body.quality,
seed=body.seed,
)
except Exception as exc:
raise HTTPException(status_code=502, detail=f"ComfyUI: {exc}") from exc
@app.get("/images/progress/{prompt_id}")
async def images_progress(prompt_id: str) -> dict[str, Any]:
job = get_job(prompt_id)
if not job:
raise HTTPException(status_code=404, detail="Onbekende job")
return job
@app.get("/images/presets")
def images_presets() -> dict[str, Any]:
return {"presets": QUALITY_PRESETS}
@app.post("/images/generate")
async def images_generate(body: ImageGenerateBody) -> dict[str, Any]:
try:
result = await generate_image(
body.prompt.strip(),
width=body.width,
height=body.height,
steps=body.steps,
seed=body.seed,
quality=body.quality,
)
except TimeoutError as exc:
raise HTTPException(status_code=504, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=502, detail=f"ComfyUI: {exc}") from exc
try:
log_agent_event(
agent_name="design",
event_type="image_generated",
title="ComfyUI foto gegenereerd",
body=body.prompt[:500],
metadata={"filename": result["filename"], "prompt_id": result["prompt_id"]},
channel="cockpit",
)
except Exception:
pass
return {"ok": True, **result}
@app.get("/images/view")
async def images_view(
filename: str = Query(..., min_length=1),
subfolder: str = Query(default=""),
type: str = Query(default="output"),
):
from fastapi.responses import Response
try:
data = await fetch_image_bytes(filename, subfolder, type)
except Exception as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
media = "image/png" if filename.lower().endswith(".png") else "image/jpeg"
return Response(content=data, media_type=media)
@app.post("/brain/messages")
async def brain_store_message(body: BrainMessageIn) -> dict[str, Any]:
try:
if body.embed and (body.content_text or "").strip():
row = await brain_svc.store_message_with_embedding(
body.chat_id,
direction=body.direction,
content_text=body.content_text,
content_type=body.content_type,
role=body.role,
telegram_message_id=body.telegram_message_id,
reply_to_db_id=body.reply_to_db_id,
agent_name=body.agent_name,
content_json=body.content_json,
user_name=body.user_name,
user_role=body.user_role,
chat_type=body.chat_type,
)
else:
row = brain_svc.store_message(
body.chat_id,
direction=body.direction,
content_text=body.content_text,
content_type=body.content_type,
role=body.role,
telegram_message_id=body.telegram_message_id,
reply_to_db_id=body.reply_to_db_id,
agent_name=body.agent_name,
content_json=body.content_json,
user_name=body.user_name,
user_role=body.user_role,
chat_type=body.chat_type,
)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return _serialize_row(row)
@app.post("/brain/edges")
def brain_store_edge(body: BrainEdgeIn) -> dict[str, Any]:
try:
row = brain_svc.store_edge(
body.source_message_id,
body.edge_type,
target_message_id=body.target_message_id,
target_entity_type=body.target_entity_type,
target_entity_id=body.target_entity_id,
weight=body.weight,
metadata=body.metadata,
)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return _serialize_row(row)
@app.get("/brain/graph/{chat_id}")
def brain_graph(chat_id: int, limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
return brain_svc.get_graph(chat_id, limit=limit)
@app.post("/brain/search")
async def brain_search(body: BrainSearchIn) -> dict[str, Any]:
try:
results = await brain_svc.search_memory(body.query, chat_id=body.chat_id, limit=body.limit)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {"results": [_serialize_row(r) for r in results]}
# Append to tools-api main.py before end
@app.get("/brain/conversations")
def brain_list_conversations(limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
rows = brain_svc.list_conversations(limit=limit)
return {"items": [_serialize_row(r) for r in rows]}
@app.get("/brain/feed")
def brain_feed(
chat_id: Optional[int] = Query(default=None),
limit: int = Query(default=80, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
rows = brain_svc.list_feed(chat_id=chat_id, limit=limit, offset=offset)
return {"items": [_serialize_row(r) for r in rows]}
@app.get("/brain/stats")
def brain_stats() -> dict[str, Any]:
data = brain_svc.get_dashboard_stats()
return {
"stats": _serialize_row(data.get("stats") or {}),
"recent_messages": [_serialize_row(r) for r in data.get("recent_messages") or []],
"agent_events": [_serialize_row(r) for r in data.get("agent_events") or []],
}
@app.get("/brain/graph")
def brain_global_graph(limit: int = Query(default=100, ge=1, le=300)) -> dict[str, Any]:
return brain_svc.get_global_graph(limit=limit)
tools-api/app/middleware.py
+45
View File
@@ -0,0 +1,45 @@
"""Agent event logging helpers used by the Tools API."""
from typing import Any, Optional
from app.db import execute_returning, json_param
def log_agent_event(
*,
agent_name: str,
event_type: str,
title: str,
body: Optional[str] = None,
agent_type: str = "openswarm",
metadata: Optional[dict[str, Any]] = None,
status: str = "completed",
related_table: Optional[str] = None,
related_id: Optional[int] = None,
channel: str = "dashboard",
) -> dict[str, Any]:
row = execute_returning(
"""
INSERT INTO agent_events (
agent_name, agent_type, event_type, title, body, metadata,
status, related_table, related_id, channel
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING *
""",
(
agent_name,
agent_type,
event_type,
title,
body,
json_param(metadata),
status,
related_table,
related_id,
channel,
),
)
if row is None:
raise RuntimeError("Failed to insert agent event")
return row
tools-api/app/ops_routes.py
+31
View File
@@ -0,0 +1,31 @@
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from app.connectors.proxmox import get_status_summary, get_topology, poll_and_snapshot
router = APIRouter(prefix="/ops", tags=["ops"])
@router.get("/status")
def ops_status() -> dict:
try:
return get_status_summary()
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=f"ops status failed: {exc}") from exc
@router.get("/topology")
def ops_topology() -> dict:
try:
return get_topology()
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=f"ops topology failed: {exc}") from exc
@router.post("/refresh")
def ops_refresh() -> dict:
try:
return poll_and_snapshot()
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=500, detail=f"ops refresh failed: {exc}") from exc
tools-api/app/packaging/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Packaging helpers for the tools API."""
tools-api/app/packaging/export.py
+52
View File
@@ -0,0 +1,52 @@
"""Export helpers for packaging assets."""
from __future__ import annotations
from io import BytesIO
from PIL import Image, ImageDraw
from reportlab.lib.pagesizes import A4
from reportlab.lib.utils import ImageReader
from reportlab.pdfgen import canvas
try:
import cairosvg # type: ignore
except Exception: # pragma: no cover
cairosvg = None
def svg_to_png_bytes(svg_content: str, width: int = 1400, height: int = 1000) -> bytes:
"""Convert SVG content to PNG bytes with a Pillow fallback."""
if cairosvg is not None:
return cairosvg.svg2png(bytestring=svg_content.encode("utf-8"))
# Fallback when cairosvg is unavailable: branded placeholder raster.
img = Image.new("RGB", (width, height), "#0b1220")
draw = ImageDraw.Draw(img)
draw.rectangle((24, 24, width - 24, height - 24), outline="#00e5ff", width=3)
draw.text((48, 56), "Foodlinkk Packaging Preview", fill="#e2e8f0")
draw.text((48, 92), "Install cairosvg for full SVG rendering.", fill="#94a3b8")
stream = BytesIO()
img.save(stream, format="PNG")
return stream.getvalue()
def svg_to_pdf_bytes(svg_content: str) -> bytes:
"""Render SVG in a PDF by first rasterizing to PNG."""
png_data = svg_to_png_bytes(svg_content, width=1800, height=1300)
png_image = Image.open(BytesIO(png_data)).convert("RGB")
output = BytesIO()
pdf = canvas.Canvas(output, pagesize=A4)
page_w, page_h = A4
img_w, img_h = png_image.size
scale = min((page_w - 64) / img_w, (page_h - 64) / img_h)
draw_w = img_w * scale
draw_h = img_h * scale
x = (page_w - draw_w) / 2
y = (page_h - draw_h) / 2
pdf.drawImage(ImageReader(png_image), x, y, width=draw_w, height=draw_h, preserveAspectRatio=True, mask="auto")
pdf.showPage()
pdf.save()
return output.getvalue()
tools-api/app/packaging/generator.py
+233
View File
@@ -0,0 +1,233 @@
"""SVG packaging generator for Foodlinkk."""
from __future__ import annotations
import base64
from io import BytesIO
from typing import Any
import svgwrite
from barcode import Code128
from barcode.writer import SVGWriter
MM_TO_PX = 3.7795275591 # 96 DPI conversion
DEFAULT_BARCODE_VALUE = "8710000000012"
FOODLINKK_BRAND = {
"bg": "#0b1220",
"panel": "#101a2d",
"primary": "#00e5ff",
"secondary": "#ffd700",
"text": "#e2e8f0",
"cut_line": "#ef4444",
"fold_line": "#60a5fa",
}
def _mm(mm: float) -> float:
return round(float(mm) * MM_TO_PX, 2)
def _elements_enabled(elements: Any, name: str) -> bool:
if isinstance(elements, dict):
return bool(elements.get(name))
if isinstance(elements, list):
return name in elements
return False
def _barcode_data_uri(value: str) -> str:
barcode = Code128(value, writer=SVGWriter())
stream = BytesIO()
barcode.write(stream)
encoded = base64.b64encode(stream.getvalue()).decode("ascii")
return f"data:image/svg+xml;base64,{encoded}"
def generate_packaging(spec: dict[str, Any]) -> str:
"""Create an SVG packaging design based on a simple spec."""
ptype = (spec.get("type") or "folding_box").strip().lower()
width_mm = float(spec.get("width_mm", 120))
height_mm = float(spec.get("height_mm", 80))
depth_mm = float(spec.get("depth_mm", 40))
elements = spec.get("elements", {})
brand = {**FOODLINKK_BRAND, **(spec.get("brand") or {})}
if ptype == "folding_box":
canvas_w = _mm((width_mm * 2) + (depth_mm * 2) + 20)
canvas_h = _mm(height_mm + depth_mm + 20)
elif ptype == "wrap":
canvas_w = _mm(width_mm + 20)
canvas_h = _mm(height_mm + 20)
elif ptype == "round_label":
diameter = max(min(width_mm, height_mm), 20)
canvas_w = _mm(diameter + 20)
canvas_h = _mm(diameter + 20)
else:
raise ValueError(f"Unsupported packaging type: {ptype}")
dwg = svgwrite.Drawing(size=(canvas_w, canvas_h))
dwg.viewbox(0, 0, canvas_w, canvas_h)
# Background and frame
dwg.add(dwg.rect(insert=(0, 0), size=(canvas_w, canvas_h), fill=brand["bg"]))
dwg.add(
dwg.rect(
insert=(4, 4),
size=(canvas_w - 8, canvas_h - 8),
fill=brand["panel"],
rx=10,
ry=10,
stroke=brand["primary"],
stroke_opacity=0.25,
stroke_width=2,
)
)
margin = 24
if ptype == "folding_box":
body_w = _mm(width_mm)
body_h = _mm(height_mm)
depth_w = _mm(depth_mm)
x0 = margin
y0 = margin
panels = [depth_w, body_w, depth_w, body_w]
x = x0
for idx, panel_w in enumerate(panels):
dwg.add(
dwg.rect(
insert=(x, y0),
size=(panel_w, body_h),
fill="none",
stroke=brand["primary"] if idx % 2 else brand["secondary"],
stroke_opacity=0.45,
stroke_width=1.6,
)
)
x += panel_w
if _elements_enabled(elements, "fold_lines"):
x = x0 + panels[0]
for panel_w in panels[1:]:
dwg.add(
dwg.line(
start=(x, y0),
end=(x, y0 + body_h),
stroke=brand["fold_line"],
stroke_dasharray="8,6",
stroke_width=1.2,
)
)
x += panel_w
if _elements_enabled(elements, "cut_lines"):
dwg.add(
dwg.rect(
insert=(x0, y0),
size=(sum(panels), body_h),
fill="none",
stroke=brand["cut_line"],
stroke_dasharray="5,4",
stroke_width=1.1,
)
)
logo_x = x0 + panels[0] + (_mm(width_mm) * 0.12)
logo_y = y0 + (_mm(height_mm) * 0.16)
logo_w = _mm(width_mm) * 0.76
logo_h = _mm(height_mm) * 0.42
elif ptype == "wrap":
body_w = _mm(width_mm)
body_h = _mm(height_mm)
x0 = margin
y0 = margin
dwg.add(
dwg.rect(
insert=(x0, y0),
size=(body_w, body_h),
fill="none",
stroke=brand["primary"],
stroke_width=2.2,
)
)
if _elements_enabled(elements, "cut_lines"):
dwg.add(
dwg.rect(
insert=(x0, y0),
size=(body_w, body_h),
fill="none",
stroke=brand["cut_line"],
stroke_dasharray="6,4",
stroke_width=1.1,
)
)
logo_x = x0 + (body_w * 0.14)
logo_y = y0 + (body_h * 0.14)
logo_w = body_w * 0.72
logo_h = body_h * 0.36
else:
diameter = min(canvas_w, canvas_h) - (margin * 2)
cx = canvas_w / 2
cy = canvas_h / 2
dwg.add(
dwg.circle(
center=(cx, cy),
r=diameter / 2,
fill="none",
stroke=brand["primary"],
stroke_width=2.4,
)
)
if _elements_enabled(elements, "cut_lines"):
dwg.add(
dwg.circle(
center=(cx, cy),
r=(diameter / 2) - 4,
fill="none",
stroke=brand["cut_line"],
stroke_dasharray="4,4",
stroke_width=1.0,
)
)
logo_w = diameter * 0.64
logo_h = diameter * 0.22
logo_x = cx - (logo_w / 2)
logo_y = cy - (logo_h / 2) - 8
body_w = diameter
body_h = diameter
x0 = cx - (diameter / 2)
y0 = cy - (diameter / 2)
if _elements_enabled(elements, "logo_area"):
dwg.add(
dwg.rect(
insert=(logo_x, logo_y),
size=(logo_w, logo_h),
rx=8,
ry=8,
fill="none",
stroke=brand["secondary"],
stroke_width=2,
)
)
dwg.add(
dwg.text(
"FOODLINKK",
insert=(logo_x + 12, logo_y + (logo_h / 2) + 5),
fill=brand["text"],
font_size=18,
font_family="Arial, sans-serif",
font_weight="bold",
)
)
if _elements_enabled(elements, "barcode"):
barcode_uri = _barcode_data_uri(str(spec.get("barcode_value") or DEFAULT_BARCODE_VALUE))
bar_w = max(140, body_w * 0.35)
bar_h = max(50, body_h * 0.16)
bar_x = x0 + body_w - bar_w - 14
bar_y = y0 + body_h - bar_h - 14
dwg.add(dwg.rect(insert=(bar_x - 4, bar_y - 4), size=(bar_w + 8, bar_h + 8), fill="#ffffff"))
dwg.add(dwg.image(href=barcode_uri, insert=(bar_x, bar_y), size=(bar_w, bar_h)))
return dwg.tostring()
tools-api/app/packaging_routes.py
+80
View File
@@ -0,0 +1,80 @@
"""Packaging generation API routes."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import Response
from pydantic import BaseModel, Field
from app.packaging.export import svg_to_pdf_bytes, svg_to_png_bytes
from app.packaging.generator import FOODLINKK_BRAND, generate_packaging
router = APIRouter(prefix="/packaging", tags=["packaging"])
_PROJECTS: dict[str, dict[str, Any]] = {}
class PackagingSpec(BaseModel):
type: Literal["folding_box", "wrap", "round_label"]
width_mm: float = Field(default=120, gt=0, le=4000)
height_mm: float = Field(default=80, gt=0, le=4000)
depth_mm: float = Field(default=40, ge=0, le=4000)
elements: dict[str, bool] = Field(default_factory=dict)
brand: dict[str, str] = Field(default_factory=dict)
barcode_value: str | None = Field(default=None, max_length=64)
@router.post("/generate")
def packaging_generate(spec: PackagingSpec) -> dict[str, Any]:
spec_data = spec.model_dump()
if not spec_data["brand"]:
spec_data["brand"] = dict(FOODLINKK_BRAND)
svg = generate_packaging(spec_data)
project_id = uuid4().hex
now = datetime.utcnow().isoformat() + "Z"
_PROJECTS[project_id] = {
"id": project_id,
"created_at": now,
"spec": spec_data,
"svg": svg,
}
return {"id": project_id, "created_at": now, "svg": svg, "spec": spec_data}
@router.get("/projects")
def packaging_projects(limit: int = Query(default=30, ge=1, le=200)) -> dict[str, Any]:
items = sorted(_PROJECTS.values(), key=lambda x: x["created_at"], reverse=True)[:limit]
return {
"items": [{"id": p["id"], "created_at": p["created_at"], "spec": p["spec"]} for p in items],
"count": len(items),
}
@router.get("/download/{project_id}")
def packaging_download(project_id: str, format: str = Query(default="svg", pattern="^(svg|png|pdf)$")):
project = _PROJECTS.get(project_id)
if not project:
raise HTTPException(status_code=404, detail="Packaging project not found")
svg = project["svg"]
filename = f"foodlinkk-packaging-{project_id[:8]}.{format}"
if format == "svg":
return Response(
content=svg.encode("utf-8"),
media_type="image/svg+xml",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
if format == "png":
return Response(
content=svg_to_png_bytes(svg),
media_type="image/png",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
return Response(
content=svg_to_pdf_bytes(svg),
media_type="application/pdf",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
tools-api/app/recommendations.py
+165
View File
@@ -0,0 +1,165 @@
"""AI recommendations — Herman filter output."""
from __future__ import annotations
import json
import os
import urllib.request
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
from app.middleware import log_agent_event
router = APIRouter(prefix="/recommendations", tags=["recommendations"])
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.4.7.19:11434")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3:8b")
HERMAN_URL = os.getenv("HERMAN_ORCHESTRATOR_URL", "http://10.4.7.19:8090")
def _serialize(row: dict) -> dict[str, Any]:
out = {}
for k, v in row.items():
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
elif type(v).__name__ == "Decimal":
out[k] = float(v)
else:
out[k] = v
return out
@router.get("/pending")
def pending_recommendations(limit: int = Query(10, ge=1, le=50)) -> dict[str, Any]:
rows = fetch_all(
"""SELECT * FROM ai_recommendations WHERE status = 'pending'
ORDER BY impact_score DESC NULLS LAST, created_at DESC LIMIT %s""",
(limit,),
)
return {"items": [_serialize(r) for r in rows]}
@router.get("/strategies")
def list_strategies(active_only: bool = True) -> dict[str, Any]:
q = "SELECT * FROM marketing_strategies"
if active_only:
q += " WHERE is_active = true"
q += " ORDER BY updated_at DESC LIMIT 20"
return {"items": [_serialize(r) for r in fetch_all(q)]}
@router.post("/generate")
def generate_recommendations() -> dict[str, Any]:
briefs = fetch_all("SELECT domain, title, summary FROM research_briefs ORDER BY generated_at DESC LIMIT 5")
deals = fetch_all(
"SELECT d.id, d.title, d.value, d.stage, d.next_action, c.name AS client_name FROM deals d LEFT JOIN clients c ON c.id = d.client_id WHERE d.stage NOT IN ('won','lost')"
)
pending_approvals = fetch_all(
"SELECT id, agent_name, title FROM agent_events WHERE status = 'needs_approval' LIMIT 5"
)
context = {
"briefs": [dict(b) for b in briefs],
"deals": [dict(d) for d in deals],
"approvals": [dict(a) for a in pending_approvals],
}
created = []
for deal in deals:
val = float(deal.get("value") or 0)
title = f"{deal.get('client_name') or 'Deal'} — {deal.get('next_action') or 'follow-up'}"
existing = fetch_one(
"SELECT id FROM ai_recommendations WHERE title = %s AND status = 'pending'",
(title[:255],),
)
if existing:
continue
row = execute_returning(
"""INSERT INTO ai_recommendations
(recommendation_type, title, description, priority, impact_score, confidence_score,
data_sources, action_items, generated_by, related_entity_type, related_entity_id, status, expires_at)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'pending',%s) RETURNING id""",
(
"deal_action",
title[:255],
f"Deal {deal.get('title')} — stage {deal.get('stage')} — €{val:,.0f}",
"high" if val >= 50000 else "medium",
min(0.95, 0.5 + val / 200000),
0.85,
json_param({"source": "crm", "deal_id": deal.get("id")}),
[deal.get("next_action") or "Follow-up"],
"herman",
"deal",
deal.get("id"),
datetime.now(timezone.utc) + timedelta(days=7),
),
)
created.append(row["id"])
for appr in pending_approvals:
title = f"Goedkeuren: {appr.get('title')}"
existing = fetch_one(
"SELECT id FROM ai_recommendations WHERE title = %s AND status = 'pending'",
(title[:255],),
)
if not existing:
row = execute_returning(
"""INSERT INTO ai_recommendations
(recommendation_type, title, description, priority, impact_score, confidence_score,
generated_by, status, expires_at)
VALUES ('operational', %s, %s, 'high', 0.80, 0.90, 'herman', 'pending', %s)
RETURNING id""",
(
title[:255],
f"Agent {appr.get('agent_name')} wacht op CEO",
datetime.now(timezone.utc) + timedelta(days=3),
),
)
created.append(row["id"])
if not created and not fetch_one("SELECT id FROM ai_recommendations WHERE status='pending' LIMIT 1"):
row = execute_returning(
"""INSERT INTO ai_recommendations
(recommendation_type, title, description, priority, impact_score, confidence_score,
generated_by, status)
VALUES ('retail_target', 'Retail intelligence uitbreiden', 'Run AH/Jumbo scrapers voor volledige kaart', 'medium', 0.70, 0.75, 'herman', 'pending')
RETURNING id"""
)
created.append(row["id"])
log_agent_event(
agent_name="herman",
event_type="recommendations_generated",
title=f"Generated {len(created)} recommendations",
metadata={"ids": created, "context_keys": list(context.keys())},
)
return {"created": created, "pending_count": len(fetch_all("SELECT id FROM ai_recommendations WHERE status='pending'"))}
@router.post("/{rec_id}/approve")
def approve_recommendation(rec_id: int) -> dict[str, Any]:
row = execute_returning(
"UPDATE ai_recommendations SET status = 'approved', updated_at = NOW() WHERE id = %s RETURNING *",
(rec_id,),
)
if not row:
raise HTTPException(404, "Recommendation not found")
execute_returning(
"""INSERT INTO herman_actions (action_type, title, description, status, completed_at)
VALUES ('recommendation_approved', %s, %s, 'completed', NOW()) RETURNING id""",
(row["title"], row.get("description")),
)
log_agent_event(agent_name="herman", event_type="approval", title=f"Approved recommendation {rec_id}", status="completed")
return _serialize(row)
@router.post("/{rec_id}/dismiss")
def dismiss_recommendation(rec_id: int) -> dict[str, Any]:
row = execute_returning(
"UPDATE ai_recommendations SET status = 'dismissed', updated_at = NOW() WHERE id = %s RETURNING *",
(rec_id,),
)
if not row:
raise HTTPException(404, "Recommendation not found")
return _serialize(row)
tools-api/app/research.py
+183
View File
@@ -0,0 +1,183 @@
"""Research pipeline: data providers, snapshots, briefs."""
from __future__ import annotations
import json
import urllib.request
from datetime import date, datetime, timedelta, timezone
from typing import Any
from fastapi import APIRouter, HTTPException
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
from app.middleware import log_agent_event
router = APIRouter(prefix="/research", tags=["research"])
def _json_safe(obj: Any) -> Any:
if isinstance(obj, dict):
return {k: _json_safe(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_json_safe(x) for x in obj]
if hasattr(obj, "isoformat"):
return obj.isoformat()
if type(obj).__name__ == "Decimal":
return float(obj)
return obj
def _http_get(url: str, timeout: int = 30) -> dict:
with urllib.request.urlopen(url, timeout=timeout) as resp:
return json.loads(resp.read().decode())
@router.get("/providers")
def list_providers() -> dict[str, Any]:
rows = fetch_all("SELECT * FROM data_providers ORDER BY name")
return {"items": [dict(r) for r in rows]}
@router.post("/providers/{provider_id}/refresh")
def refresh_provider(provider_id: int) -> dict[str, Any]:
prov = fetch_one("SELECT * FROM data_providers WHERE id = %s", (provider_id,))
if not prov:
raise HTTPException(404, "Provider not found")
name = prov["name"]
payload: dict[str, Any] = {}
count = 0
if name == "crm":
clients = fetch_all("SELECT id, name, stage FROM clients ORDER BY updated_at DESC LIMIT 20")
deals = fetch_all(
"SELECT id, title, value, stage, next_action, deadline FROM deals ORDER BY updated_at DESC LIMIT 20"
)
payload = {
"clients": _json_safe([dict(c) for c in clients]),
"deals": _json_safe([dict(d) for d in deals]),
}
count = len(clients) + len(deals)
elif name == "weather":
url = (
"https://api.open-meteo.com/v1/forecast?"
"latitude=52.37&longitude=4.89&daily=temperature_2m_max,precipitation_sum&timezone=Europe%2FAmsterdam&forecast_days=7"
)
payload = _http_get(url)
count = len(payload.get("daily", {}).get("time", []))
for i, day in enumerate(payload.get("daily", {}).get("time", [])[:7]):
temps = payload["daily"].get("temperature_2m_max", [])
prec = payload["daily"].get("precipitation_sum", [])
execute(
"""INSERT INTO weather_data (region, city, date, temperature_c, precipitation_mm, weather_condition, data_source)
VALUES ('Noord-Holland', 'Amsterdam', %s, %s, %s, 'forecast', 'open-meteo')""",
(day, temps[i] if i < len(temps) else None, prec[i] if i < len(prec) else None),
)
elif name == "social":
rows = fetch_all(
"SELECT platform, text, sentiment_score, created_at FROM social_mentions ORDER BY created_at DESC LIMIT 30"
)
payload = _json_safe({"mentions": [dict(r) for r in rows]})
count = len(rows)
elif name == "retail_manual":
rows = fetch_all("SELECT id, name, chain, city, partnership_status FROM supermarkets ORDER BY id")
payload = _json_safe({"stores": [dict(r) for r in rows]})
count = len(rows)
elif name == "cbs":
from app import retail_enrichment
status = retail_enrichment.enrichment_status()
batch = retail_enrichment.enrich_batch(limit=30, offset=0)
payload = _json_safe({"status": status, "batch": batch})
count = batch.get("ok", 0)
elif name == "pdok":
from app.connectors import pdok as pdok_conn
sample = fetch_all(
"SELECT DISTINCT postcode FROM supermarkets WHERE postcode <> '0000AA' LIMIT 5"
)
lookups = [pdok_conn.lookup_postcode(r["postcode"]) for r in sample]
payload = _json_safe({"lookups": [x for x in lookups if x]})
count = len(payload.get("lookups", []))
else:
payload = {"status": "noop"}
snap = execute_returning(
"""INSERT INTO data_snapshots (provider_id, payload, record_count)
VALUES (%s, %s, %s) RETURNING id, fetched_at""",
(provider_id, json_param(payload), count),
)
execute(
"UPDATE data_providers SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
(provider_id,),
)
log_agent_event(
agent_name="research",
event_type="data_refresh",
title=f"Provider {name} refreshed",
metadata={"provider_id": provider_id, "records": count},
)
return {"provider": name, "snapshot_id": snap["id"], "record_count": count}
@router.get("/briefs")
def list_briefs(limit: int = 20) -> dict[str, Any]:
rows = fetch_all(
"SELECT * FROM research_briefs ORDER BY generated_at DESC LIMIT %s",
(limit,),
)
return {"items": [dict(r) for r in rows]}
def _build_brief(domain: str, title: str, summary: str, findings: list[dict]) -> dict:
row = execute_returning(
"""INSERT INTO research_briefs (domain, title, summary, key_findings, expires_at)
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
(
domain,
title,
summary,
json_param(findings),
datetime.now(timezone.utc) + timedelta(days=1),
),
)
return dict(row)
@router.post("/run")
def run_research() -> dict[str, Any]:
providers = fetch_all("SELECT id, name FROM data_providers WHERE is_active = true")
snapshot_ids = []
for p in providers:
result = refresh_provider(int(p["id"]))
snapshot_ids.append(result.get("snapshot_id"))
clients_n = fetch_one("SELECT COUNT(*) AS c FROM clients")["c"]
deals = fetch_all("SELECT title, value, stage FROM deals WHERE stage NOT IN ('won','lost')")
pipeline = sum(float(d.get("value") or 0) for d in deals)
stores = fetch_one("SELECT COUNT(*) AS c FROM supermarkets")["c"]
mentions = fetch_one("SELECT COUNT(*) AS c FROM social_mentions WHERE created_at > NOW() - interval '7 days'")["c"]
crm_brief = _build_brief(
"crm",
f"CRM snapshot {date.today()}",
f"{clients_n} klanten, pipeline €{pipeline:,.0f}, {len(deals)} actieve deals.",
[{"finding": f"Pipeline €{pipeline:,.0f}", "relevance": "high"}],
)
retail_brief = _build_brief(
"retail",
f"Retail NL {date.today()}",
f"{stores} supermarkten in database, partnerships actief/proposal gemapt.",
[{"finding": f"{stores} locaties geladen", "relevance": "medium"}],
)
social_brief = _build_brief(
"social",
f"Social week {date.today()}",
f"{mentions} mentions afgelopen 7 dagen.",
[{"finding": f"{mentions} mentions", "relevance": "medium"}],
)
log_agent_event(
agent_name="research",
event_type="research_run",
title="Full research cycle completed",
metadata={"briefs": 3, "snapshots": len(snapshot_ids)},
)
return {
"snapshots": snapshot_ids,
"briefs": [crm_brief["id"], retail_brief["id"], social_brief["id"]],
}
tools-api/app/retail.html
+411
View File
@@ -0,0 +1,411 @@
{% extends "base.html" %}
{% block content %}
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" />
<link rel="stylesheet" href="https://unpkg.com/leaflet.markercluster@1.5.3/dist/MarkerCluster.css" />
<link rel="stylesheet" href="https://unpkg.com/leaflet.markercluster@1.5.3/dist/MarkerCluster.Default.css" />
<link rel="stylesheet" href="/static/retail.css" />
<div x-data="retailIntel()" x-init="init()">
<header class="page-header">
<div>
<h1>Retail Intelligence <span class="live-badge">Live</span></h1>
<p class="subtitle">360° filiaal view · CBS · halal · CRM · groothandels · trends</p>
</div>
<div class="retail-actions">
<button class="btn btn-sm btn-pulse" @click="refreshRss()" :disabled="busy">↻ RSS feeds</button>
<button class="btn btn-sm btn-pulse-green" @click="importWholesale()" :disabled="busy">Groothandels import</button>
<button class="btn btn-sm btn-pulse-purple" @click="syncCities()" :disabled="busy">Stad demografie</button>
<a class="btn btn-sm" :href="'/api/retail/export?' + params()" target="_blank">Export CSV</a>
</div>
</header>
<div class="live-feed-bar" x-show="rssItems.length">
<div class="ticker-track">
<template x-for="r in rssItems.slice(0,10)" :key="'r'+r.id">
<a class="ticker-link" :href="r.link" target="_blank" rel="noopener" x-text="'📰 '+r.feed_name+': '+r.title"></a>
</template>
</div>
</div>
<div class="kpi-row">
<div class="kpi-card kpi-pulse"><span>Totaal filialen</span><strong x-text="stats.total||0"></strong></div>
<div class="kpi-card kpi-green"><span>CRM actief</span><strong x-text="stats.active_partnerships||0"></strong></div>
<div class="kpi-card"><span>Halal cert.</span><strong x-text="stats.halal_certified_count||0"></strong></div>
<div class="kpi-card"><span>CBS data</span><strong x-text="stats.with_area_data||0"></strong></div>
<div class="kpi-card"><span>Groothandels</span><strong x-text="wholesaleCount"></strong></div>
<div class="kpi-card"><span>Resultaat</span><strong x-text="resultCount"></strong></div>
</div>
<div class="retail-workspace">
<aside class="retail-sidebar panel">
<nav class="side-nav">
<button class="side-nav-btn" :class="mode==='stores'?'active':''" @click="mode='stores'; apply()">🏪 Supermarkten</button>
<button class="side-nav-btn" :class="mode==='wholesale'?'active':''" @click="mode='wholesale'; loadWholesale()">📦 Groothandels</button>
<button class="side-nav-btn" :class="mode==='opportunities'?'active':''" @click="mode='opportunities'; loadOpportunities()">🎯 Kansen</button>
<button class="side-nav-btn" :class="mode==='cities'?'active':''" @click="mode='cities'; loadCities()">🏙️ Steden</button>
<a class="side-nav-btn" href="/clients">👥 CRM Clients</a>
<a class="side-nav-btn" href="/deals">💼 CRM Deals</a>
<a class="side-nav-btn" href="/marketing">📣 Marketing Hub</a>
</nav>
</aside>
<aside class="retail-filters panel" x-show="mode==='stores'">
<h3>Filters</h3>
<label>Keten<select x-model="f.chain" @change="apply()"><option value="">Alle</option>
<template x-for="c in meta.chains" :key="c.chain"><option :value="c.chain" x-text="c.chain+' ('+c.n+')'"></option></template>
</select></label>
<label>Provincie<select x-model="f.province" @change="apply()"><option value="">Alle</option>
<template x-for="p in meta.provinces" :key="p.province"><option :value="p.province" x-text="p.province+' ('+p.n+')'"></option></template>
</select></label>
<label>Stad<input x-model="f.city" @change="apply()" placeholder="Amsterdam"></label>
<label>Zoek<input x-model="f.q" @keyup.enter="apply()" placeholder="Naam, adres"></label>
<label class="checkbox"><input type="checkbox" x-model="f.halal_gap_only" @change="apply()"> Halal-gap kansen</label>
<label class="checkbox"><input type="checkbox" x-model="f.linked_to_crm" @change="apply()"> Gekoppeld CRM</label>
<label class="checkbox"><input type="checkbox" x-model="f.has_area_data" @change="apply()"> Met CBS data</label>
<h4>Data sync</h4>
<button class="btn btn-sm btn-block btn-pulse" @click="sync('contacts')" :disabled="busy">OSM contacten</button>
<button class="btn btn-sm btn-block" @click="sync('halal')" :disabled="busy">Halal registry</button>
<button class="btn btn-sm btn-block" @click="sync('opportunities')" :disabled="busy">Kans-scores</button>
<button class="btn btn-sm btn-block btn-primary" @click="runEnrich()" :disabled="busy">CBS/PDOK batch</button>
</aside>
<div class="retail-main">
<div x-show="mode==='stores'" class="retail-map-wrap">
<div id="map" class="retail-map"></div>
<div class="map-loading" x-show="loading">Laden…</div>
</div>
<div x-show="mode==='wholesale'" class="panel wholesale-list">
<h3>Groothandels <span class="live-badge">OSM</span></h3>
<input class="form-input" x-model="wholesaleQ" @keyup.enter="loadWholesale()" placeholder="Zoek groothandel…" style="margin-bottom:0.5rem">
<template x-for="w in wholesaleRows" :key="w.id">
<div class="wholesale-item" :class="selectedWholesale?.id===w.id?'active':''" @click="selectWholesale(w)">
<strong x-text="w.name"></strong><br>
<small x-text="w.address+', '+w.city"></small>
</div>
</template>
<p x-show="!wholesaleRows.length" class="muted">Geen groothandels — klik Groothandels import.</p>
</div>
<div x-show="mode==='opportunities'" class="panel retail-table-wrap">
<h3>Top halal-markt kansen</h3>
<table class="retail-table">
<thead><tr><th>Score</th><th>Filiaal</th><th>Stad</th><th>Halal%</th><th>CRM</th></tr></thead>
<tbody>
<template x-for="o in oppRows" :key="o.id">
<tr @click="selectStore(o.id); mode='stores'" class="clickable-row">
<td><strong x-text="Math.round(o.halal_opportunity_score||0)"></strong></td>
<td x-text="o.chain+' · '+o.name"></td>
<td x-text="o.city"></td>
<td x-text="formatPct(o.muslim_proxy_pct)"></td>
<td x-text="o.partnership_status||'—'"></td>
</tr>
</template>
</tbody>
</table>
</div>
<div x-show="mode==='cities'" class="panel retail-table-wrap">
<h3>Stad demografie <span class="live-badge">CBS</span></h3>
<table class="retail-table">
<thead><tr><th>Stad</th><th>Inwoners</th><th>Huishoudens</th><th>Halal-markt%</th><th>Filialen</th></tr></thead>
<tbody>
<template x-for="c in cityRows" :key="c.id">
<tr><td x-text="c.city"></td><td x-text="formatNum(c.population)"></td>
<td x-text="formatNum(c.households)"></td><td x-text="formatPct(c.muslim_proxy_pct)"></td>
<td x-text="c.store_count"></td></tr>
</template>
</tbody>
</table>
</div>
</div>
<aside class="retail-panel panel">
<template x-if="selected && mode==='stores'">
<div class="store-detail">
<h3 x-text="view360.store?.name||detail.name"></h3>
<p class="muted"><span x-text="view360.store?.chain||detail.chain"></span> · <span x-text="view360.store?.city||detail.city"></span></p>
<p><strong x-text="view360.store?.address||detail.address"></strong></p>
<p class="muted" x-text="(view360.store?.postcode||'')+' '+(view360.store?.province||'')"></p>
<div class="section-tabs">
<button class="section-tab" :class="section==='overview'?'active':''" @click="section='overview'">Overzicht</button>
<button class="section-tab" :class="section==='crm'?'active':''" @click="section='crm'">CRM</button>
<button class="section-tab" :class="section==='notes'?'active':''" @click="section='notes'">Notes</button>
<button class="section-tab" :class="section==='media'?'active':''" @click="section='media'">Media</button>
<button class="section-tab" :class="section==='milestones'?'active':''" @click="section='milestones'">Sales</button>
<button class="section-tab" :class="section==='ownership'?'active':''" @click="section='ownership'">Overnames</button>
<button class="section-tab" :class="section==='weather'?'active':''" @click="section='weather'">Weer</button>
<button class="section-tab" :class="section==='agenda'?'active':''" @click="section='agenda'">Agenda</button>
</div>
<div x-show="section==='overview'">
<dl class="detail-dl">
<dt>Telefoon</dt><dd x-text="detail.phone||'—'"></dd>
<dt>Leidinggevende</dt><dd x-text="detail.manager_name||'—'"></dd>
<dt>Halal kans</dt><dd x-text="Math.round(detail.opportunity?.halal_opportunity_score||0)+'/100'"></dd>
<dt>Stad inwoners</dt><dd x-text="formatNum(view360.catchment?.city_population)"></dd>
<dt>Filialen in stad</dt><dd x-text="view360.catchment?.stores_in_city||0"></dd>
<dt>Postcode bevolking</dt><dd x-text="formatNum(view360.catchment?.postcode_population_proxy)"></dd>
</dl>
<div class="detail-section" x-show="view360.area_analysis">
<h4>CBS Demografie</h4>
<dl class="detail-dl">
<dt>Bevolking</dt><dd x-text="formatNum(view360.area_analysis.population)"></dd>
<dt>Gem. inkomen</dt><dd x-text="formatEuro(view360.area_analysis.avg_income)"></dd>
<dt>Halal-markt proxy</dt><dd x-text="formatPct(view360.area_analysis.religious_composition?.muslim_proxy_pct)"></dd>
</dl>
</div>
</div>
<div x-show="section==='crm'">
<template x-for="link in (detail.crm?.links||[])" :key="link.id">
<p><strong x-text="link.client_name"></strong> · <span x-text="link.relationship_type"></span></p>
</template>
<label>Klant<select x-model.number="linkForm.client_id">
<option value="">— kies klant —</option>
<template x-for="c in crmOptions.clients" :key="c.id">
<option :value="c.id" x-text="c.name"></option>
</template>
</select></label>
<label>Deal<select x-model.number="linkForm.deal_id">
<option value="">— optioneel —</option>
<template x-for="d in crmOptions.deals" :key="d.id">
<option :value="d.id" x-text="d.title"></option>
</template>
</select></label>
<button class="btn btn-sm btn-primary btn-block btn-pulse" @click="linkCrm()" :disabled="!linkForm.client_id">Koppel aan CRM</button>
<a class="btn btn-sm btn-block" href="/clients">Open CRM →</a>
</div>
<div x-show="section==='notes'">
<template x-for="n in (view360.notes||[])" :key="n.id">
<div class="note-bubble">
<strong x-text="n.title||'Note'"></strong>
<p x-text="n.body"></p>
<small x-text="n.created_at"></small>
</div>
</template>
<textarea x-model="noteForm.body" class="form-input" rows="3" placeholder="Nieuwe note…"></textarea>
<input x-model="noteForm.title" class="form-input" placeholder="Titel (optioneel)">
<button class="btn btn-sm btn-pulse btn-block" @click="addNote()" :disabled="!noteForm.body">Note opslaan</button>
</div>
<div x-show="section==='media'">
<div class="media-grid">
<template x-for="m in (view360.media||[])" :key="m.id">
<img class="media-thumb" :src="m.storage_path" :alt="m.caption||m.filename">
</template>
</div>
<input x-model="mediaForm.url" class="form-input" placeholder="Foto URL">
<input x-model="mediaForm.caption" class="form-input" placeholder="Bijschrift">
<button class="btn btn-sm btn-pulse-purple btn-block" @click="addMedia()" :disabled="!mediaForm.url">Foto toevoegen</button>
</div>
<div x-show="section==='milestones'">
<template x-for="m in (view360.milestones||[])" :key="m.id">
<div class="milestone-item">
<span class="milestone-dot" :class="m.status==='completed'?'done':''"></span>
<div><strong x-text="m.title"></strong><br>
<small x-text="m.milestone_type+' · '+m.status+(m.value_eur?' · €'+m.value_eur:'')"></small></div>
</div>
</template>
<input x-model="milestoneForm.title" class="form-input" placeholder="Milestone titel">
<select x-model="milestoneForm.milestone_type" class="form-input">
<option value="first_order">Eerste order</option>
<option value="listing">Listing</option>
<option value="promo">Promotie</option>
<option value="renewal">Verlenging</option>
<option value="custom">Custom</option>
</select>
<input type="number" x-model.number="milestoneForm.value_eur" class="form-input" placeholder="Waarde €">
<button class="btn btn-sm btn-pulse-green btn-block" @click="addMilestone()" :disabled="!milestoneForm.title">Milestone toevoegen</button>
</div>
<div x-show="section==='ownership'">
<template x-for="o in (view360.ownership_changes||[])" :key="o.id">
<div class="note-bubble">
<strong x-text="o.new_owner"></strong>
<p x-text="'Was: '+(o.previous_owner||'?')+' · '+o.change_type"></p>
<small x-text="o.effective_date||o.created_at"></small>
</div>
</template>
<input x-model="ownershipForm.new_owner" class="form-input" placeholder="Nieuwe eigenaar">
<input x-model="ownershipForm.previous_owner" class="form-input" placeholder="Vorige eigenaar">
<select x-model="ownershipForm.change_type" class="form-input">
<option value="acquisition">Overname</option>
<option value="merger">Fusie</option>
<option value="rebrand">Rebrand</option>
</select>
<button class="btn btn-sm btn-block" @click="addOwnership()" :disabled="!ownershipForm.new_owner">Overname registreren</button>
</div>
<div x-show="section==='weather'">
<p class="live-badge">7-daagse voorspelling · Open-Meteo</p>
<div class="weather-row">
<template x-for="d in (view360.weather_forecast||[])" :key="d.date">
<div class="weather-day">
<span x-text="d.date"></span>
<strong x-text="Math.round(d.temperature_c||0)+'°'"></strong>
<span x-text="(d.precipitation_mm||0)+'mm'"></span>
</div>
</template>
</div>
</div>
<div x-show="section==='agenda'">
<template x-for="ev in (view360.calendar||[])" :key="ev.id">
<div class="note-bubble">
<strong x-text="ev.title"></strong>
<p x-text="ev.starts_at"></p>
</div>
</template>
<input x-model="calendarForm.title" class="form-input" placeholder="Afspraak titel">
<input type="datetime-local" x-model="calendarForm.starts_at" class="form-input">
<button class="btn btn-sm btn-pulse btn-block" @click="addCalendar()" :disabled="!calendarForm.title">In agenda zetten</button>
</div>
</div>
</template>
<template x-if="selectedWholesale && mode==='wholesale'">
<div>
<h3 x-text="selectedWholesale.name"></h3>
<p x-text="selectedWholesale.address"></p>
<p x-text="selectedWholesale.city+' · '+selectedWholesale.phone"></p>
<a class="btn btn-sm" :href="selectedWholesale.website" target="_blank" x-show="selectedWholesale.website">Website →</a>
</div>
</template>
<template x-if="!selected && mode==='stores'">
<p class="muted">Selecteer een filiaal op de kaart voor de volledige 360° view.</p>
</template>
</aside>
</div>
</div>
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
<script src="https://unpkg.com/leaflet.markercluster@1.5.3/dist/leaflet.markercluster.js"></script>
<script>
function retailIntel(){
return {
mode:'stores', section:'overview', map:null, cluster:null, loading:false, busy:false,
stats: {{ stats | tojson }}, meta: {{ filters | tojson }}, trends: {{ trends | tojson }},
crmOptions: {{ crm_options | tojson }},
f:{ chain:'',province:'',city:'',q:'',halal_gap_only:false,linked_to_crm:false,has_area_data:false },
selected:null, detail:{}, view360:{}, tableRows:[], oppRows:[], cityRows:[],
wholesaleRows:[], wholesaleCount:0, wholesaleQ:'', selectedWholesale:null,
rssItems:[], resultCount:0,
linkForm:{ client_id:'', deal_id:'', partnership_status:'proposal', relationship_type:'prospect' },
noteForm:{ title:'', body:'' }, mediaForm:{ url:'', caption:'' },
milestoneForm:{ title:'', milestone_type:'custom', value_eur:null },
ownershipForm:{ new_owner:'', previous_owner:'', change_type:'acquisition' },
calendarForm:{ title:'', starts_at:'' },
async init(){
this.map = L.map('map').setView([52.15,5.3],8);
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{maxZoom:18}).addTo(this.map);
this.cluster = L.markerClusterGroup(); this.map.addLayer(this.cluster);
this.loadRss(); this.loadWholesale(); await this.apply();
},
params(){
const p = new URLSearchParams();
Object.entries(this.f).forEach(([k,v])=>{
if(v===null||v===''||v===false) return;
if(typeof v==='boolean') p.set(k, v?'true':'false'); else p.set(k,v);
});
return p.toString();
},
async apply(){
this.loading=true;
try{
const [stats,mapData]=await Promise.all([
fetch('/api/retail/stats?'+this.params()).then(r=>r.json()),
fetch('/api/retail/map?'+this.params()).then(r=>r.json()),
]);
this.stats=stats; this.resultCount=mapData.count||0;
this.renderMarkers(mapData.items||[]);
}finally{ this.loading=false; }
},
renderMarkers(stores){
this.cluster.clearLayers();
const colors={'Albert Heijn':'#0066cc','Jumbo':'#ffcc00','Plus':'#008040','Lidl':'#0050aa','ALDI':'#0066b3','Dirk':'#e30613'};
stores.forEach(s=>{
if(!s.latitude||!s.longitude) return;
const active=s.partnership_status==='active';
const score=s.halal_opportunity_score||0;
const color=active?'#22c55e':(s.halal_certified?'#a855f7':(colors[s.chain]||'#64748b'));
const m=L.circleMarker([+s.latitude,+s.longitude],{radius:active?10:7,color,weight:2,fillColor:color,fillOpacity:0.85});
m.on('click',()=>this.selectStore(s.id));
m.bindTooltip(`${s.chain}: ${s.name}${score?' · kans '+Math.round(score):''}`,{direction:'top'});
this.cluster.addLayer(m);
});
},
async selectStore(id){
const [detail, v360] = await Promise.all([
fetch('/api/retail/supermarkets/'+id).then(r=>r.json()),
fetch('/api/retail/360/'+id).then(r=>r.json()),
]);
this.detail=detail; this.view360=v360; this.selected=id; this.section='overview';
const lat=v360.store?.latitude||detail.latitude;
const lon=v360.store?.longitude||detail.longitude;
if(lat) this.map.setView([+lat,+lon],14);
},
async loadOpportunities(){
const d=await fetch('/api/retail/opportunities?min_score=35&limit=100').then(r=>r.json());
this.oppRows=d.items||[];
},
async loadCities(){
const d=await fetch('/api/retail/cities?limit=200').then(r=>r.json());
this.cityRows=d.items||[];
},
async loadWholesale(){
const q=this.wholesaleQ?'&q='+encodeURIComponent(this.wholesaleQ):'';
const d=await fetch('/api/retail/wholesalers?limit=500'+q).then(r=>r.json());
this.wholesaleRows=d.items||[]; this.wholesaleCount=d.count||0;
},
selectWholesale(w){ this.selectedWholesale=w; if(w.latitude) this.map.setView([+w.latitude,+w.longitude],13); },
async loadRss(){
try{ const d=await fetch('/api/retail/rss/live?limit=20&category=kant-en-klaar').then(r=>r.json());
if(!(d.items||[]).length) {
const d2=await fetch('/api/retail/rss/live?limit=20').then(r=>r.json());
this.rssItems=d2.items||[];
} else { this.rssItems=d.items||[]; }
}catch(e){}
},
async refreshRss(){ this.busy=true; try{ await fetch('/api/retail/rss/refresh',{method:'POST'}); await this.loadRss(); }finally{ this.busy=false; } },
async importWholesale(){ this.busy=true; try{ await fetch('/api/retail/wholesalers/import',{method:'POST'}); await this.loadWholesale(); }finally{ this.busy=false; } },
async syncCities(){ this.busy=true; try{ await fetch('/api/retail/cities/sync?limit=50',{method:'POST'}); await this.loadCities(); }finally{ this.busy=false; } },
async linkCrm(){
if(!this.selected||!this.linkForm.client_id) return;
await fetch('/api/retail/supermarkets/'+this.selected+'/link',{
method:'POST', headers:{'Content-Type':'application/json'},
body:JSON.stringify({...this.linkForm, deal_id:this.linkForm.deal_id||null})
});
await this.selectStore(this.selected); await this.apply();
},
async addNote(){
await fetch('/api/retail/360/'+this.selected+'/notes',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(this.noteForm)});
this.noteForm={title:'',body:''}; await this.selectStore(this.selected);
},
async addMedia(){
const fn=this.mediaForm.url.split('/').pop()||'photo.jpg';
await fetch('/api/retail/360/'+this.selected+'/media',{method:'POST',headers:{'Content-Type':'application/json'},
body:JSON.stringify({filename:fn,storage_path:this.mediaForm.url,caption:this.mediaForm.caption})});
this.mediaForm={url:'',caption:''}; await this.selectStore(this.selected);
},
async addMilestone(){
await fetch('/api/retail/360/'+this.selected+'/milestones',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(this.milestoneForm)});
this.milestoneForm={title:'',milestone_type:'custom',value_eur:null}; await this.selectStore(this.selected);
},
async addOwnership(){
await fetch('/api/retail/360/'+this.selected+'/ownership',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(this.ownershipForm)});
this.ownershipForm={new_owner:'',previous_owner:'',change_type:'acquisition'}; await this.selectStore(this.selected);
},
async addCalendar(){
await fetch('/api/retail/360/'+this.selected+'/calendar',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(this.calendarForm)});
this.calendarForm={title:'',starts_at:''}; await this.selectStore(this.selected);
},
async sync(action){ this.busy=true; try{ await fetch('/api/retail/sync/'+action,{method:'POST'}); await this.apply(); }finally{ this.busy=false; } },
async runEnrich(){ this.busy=true; try{ await fetch('/api/retail/enrich?limit=100',{method:'POST'}); await this.apply(); }finally{ this.busy=false; } },
formatEuro(v){ return v?'€ '+Math.round(v).toLocaleString('nl-NL'):'—'; },
formatNum(v){ return v?Number(v).toLocaleString('nl-NL'):'—'; },
formatPct(v){ return (v!=null&&v!=='')?Number(v).toFixed(1)+'%':'—'; },
};
}
</script>
{% endblock %}
tools-api/app/retail.py
+557
View File
@@ -0,0 +1,557 @@
"""Retail intelligence API routes."""
from __future__ import annotations
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from app.db import execute_returning, fetch_all, fetch_one
from app.middleware import log_agent_event
from app import retail_scrapers
from app import retail_enrichment
from app import retail_crm
from app import retail_opportunities
from app.connectors import halal_registry, trends_feed
router = APIRouter(prefix="/retail", tags=["retail"])
FIELD_SCHEMA = {
"locatie": ["id", "name", "chain", "address", "postcode", "city", "province", "store_type", "latitude", "longitude"],
"contact": ["phone", "email", "website", "manager_name", "employee_count"],
"halal": ["halal_certified", "halal_certifier", "has_halal_section", "halal_certificate_number", "halal_expiry_date"],
"crm": ["partnership_status", "client_id", "deal_id", "halal_opportunity_score"],
"cbs": ["area_population", "area_avg_income", "area_households", "muslim_proxy_pct", "area_data_source"],
"meta": ["data_source", "external_id", "last_updated", "enrichment_score"],
}
class CrmLinkIn(BaseModel):
client_id: int
deal_id: Optional[int] = None
relationship_type: str = "prospect"
partnership_status: Optional[str] = None
notes: Optional[str] = None
STORE_SELECT = """
SELECT s.*,
a.population AS area_population,
a.avg_income AS area_avg_income,
a.households AS area_households,
a.religious_composition AS area_religious,
a.ethnic_composition AS area_ethnic,
a.data_source AS area_data_source,
ros.halal_opportunity_score AS opp_halal_score,
ros.market_potential_score AS opp_market_score,
sp.manager_name AS profile_manager,
sp.manager_phone AS profile_manager_phone,
sp.manager_email AS profile_manager_email,
sp.staff_count_estimate,
sp.data_completeness AS profile_completeness
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id
LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id
"""
def _row(row: dict | None) -> dict[str, Any]:
if not row:
raise HTTPException(404, "Not found")
out: dict[str, Any] = {}
for k, v in row.items():
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
elif v is not None and hasattr(v, "__float__") and type(v).__name__ == "Decimal":
out[k] = float(v)
else:
out[k] = v
if out.get("area_religious") and isinstance(out["area_religious"], dict):
out["muslim_proxy_pct"] = out["area_religious"].get("muslim_proxy_pct")
return out
def _build_filters(
chain: Optional[str] = None,
province: Optional[str] = None,
city: Optional[str] = None,
partnership: Optional[str] = None,
halal_certified: Optional[bool] = None,
has_halal_section: Optional[bool] = None,
store_type: Optional[str] = None,
postcode_prefix: Optional[str] = None,
q: Optional[str] = None,
min_population: Optional[int] = None,
max_population: Optional[int] = None,
min_avg_income: Optional[float] = None,
max_avg_income: Optional[float] = None,
min_muslim_pct: Optional[float] = None,
has_area_data: Optional[bool] = None,
min_halal_opportunity: Optional[float] = None,
max_halal_opportunity: Optional[float] = None,
has_phone: Optional[bool] = None,
has_email: Optional[bool] = None,
has_manager: Optional[bool] = None,
halal_gap_only: Optional[bool] = None,
linked_to_crm: Optional[bool] = None,
has_halal_cert_registry: Optional[bool] = None,
) -> tuple[list[str], list[Any]]:
clauses: list[str] = ["s.postcode <> '0000AA'"]
params: list[Any] = []
if chain:
clauses.append("s.chain ILIKE %s")
params.append(f"%{chain}%")
if province:
clauses.append("s.province ILIKE %s")
params.append(f"%{province}%")
if city:
clauses.append("s.city ILIKE %s")
params.append(f"%{city}%")
if partnership:
clauses.append("s.partnership_status = %s")
params.append(partnership)
if halal_certified is not None:
clauses.append("s.halal_certified = %s")
params.append(halal_certified)
if has_halal_section is not None:
clauses.append("s.has_halal_section = %s")
params.append(has_halal_section)
if store_type:
clauses.append("s.store_type ILIKE %s")
params.append(f"%{store_type}%")
if postcode_prefix:
clauses.append("s.postcode LIKE %s")
params.append(f"{postcode_prefix.upper()}%")
if q:
clauses.append("(s.name ILIKE %s OR s.address ILIKE %s OR s.city ILIKE %s)")
like = f"%{q}%"
params.extend([like, like, like])
if min_population is not None:
clauses.append("a.population >= %s")
params.append(min_population)
if max_population is not None:
clauses.append("a.population <= %s")
params.append(max_population)
if min_avg_income is not None:
clauses.append("a.avg_income >= %s")
params.append(min_avg_income)
if max_avg_income is not None:
clauses.append("a.avg_income <= %s")
params.append(max_avg_income)
if min_muslim_pct is not None:
clauses.append("(a.religious_composition->>'muslim_proxy_pct')::float >= %s")
params.append(min_muslim_pct)
if has_area_data is True:
clauses.append("a.id IS NOT NULL")
elif has_area_data is False:
clauses.append("a.id IS NULL")
if min_halal_opportunity is not None:
clauses.append("COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score, 0) >= %s")
params.append(min_halal_opportunity)
if max_halal_opportunity is not None:
clauses.append("COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score, 0) <= %s")
params.append(max_halal_opportunity)
if has_phone is True:
clauses.append("(s.phone IS NOT NULL AND s.phone <> '')")
elif has_phone is False:
clauses.append("(s.phone IS NULL OR s.phone = '')")
if has_email is True:
clauses.append("(s.email IS NOT NULL AND s.email <> '')")
elif has_email is False:
clauses.append("(s.email IS NULL OR s.email = '')")
if has_manager is True:
clauses.append("(s.manager_name IS NOT NULL OR sp.manager_name IS NOT NULL)")
elif has_manager is False:
clauses.append("(s.manager_name IS NULL AND sp.manager_name IS NULL)")
if halal_gap_only:
clauses.append("s.halal_certified = FALSE AND s.has_halal_section = FALSE")
clauses.append("(a.religious_composition->>'muslim_proxy_pct')::float >= 5")
if linked_to_crm is True:
clauses.append("s.client_id IS NOT NULL")
elif linked_to_crm is False:
clauses.append("s.client_id IS NULL")
if has_halal_cert_registry:
clauses.append(
"EXISTS (SELECT 1 FROM halal_certifications h WHERE h.supermarket_id = s.id AND h.status = 'active')"
)
return clauses, params
@router.get("/filters")
def retail_filters() -> dict[str, Any]:
chains = fetch_all(
"SELECT chain, COUNT(*) AS n FROM supermarkets GROUP BY chain ORDER BY n DESC"
)
provinces = fetch_all(
"""SELECT COALESCE(province, 'Onbekend') AS province, COUNT(*) AS n
FROM supermarkets GROUP BY province ORDER BY n DESC"""
)
partnerships = fetch_all(
"SELECT partnership_status, COUNT(*) AS n FROM supermarkets GROUP BY partnership_status"
)
status = retail_enrichment.enrichment_status()
income = fetch_one(
"""SELECT MIN(avg_income) AS min_income, MAX(avg_income) AS max_income,
MIN(population) AS min_pop, MAX(population) AS max_pop
FROM area_analysis WHERE avg_income IS NOT NULL"""
)
return {
"chains": [dict(r) for r in chains],
"provinces": [dict(r) for r in provinces],
"partnerships": [dict(r) for r in partnerships],
"enrichment": status,
"ranges": {
"min_income": float(income["min_income"]) if income and income.get("min_income") else None,
"max_income": float(income["max_income"]) if income and income.get("max_income") else None,
"min_population": int(income["min_pop"]) if income and income.get("min_pop") else None,
"max_population": int(income["max_pop"]) if income and income.get("max_pop") else None,
},
}
@router.get("/supermarkets")
def list_supermarkets(
chain: Optional[str] = None,
province: Optional[str] = None,
city: Optional[str] = None,
partnership: Optional[str] = None,
halal_certified: Optional[bool] = None,
has_halal_section: Optional[bool] = None,
store_type: Optional[str] = None,
postcode_prefix: Optional[str] = None,
q: Optional[str] = None,
min_population: Optional[int] = Query(None, ge=0),
max_population: Optional[int] = Query(None, ge=0),
min_avg_income: Optional[float] = Query(None, ge=0),
max_avg_income: Optional[float] = Query(None, ge=0),
min_muslim_pct: Optional[float] = Query(None, ge=0, le=100),
has_area_data: Optional[bool] = None,
min_halal_opportunity: Optional[float] = Query(None, ge=0, le=100),
max_halal_opportunity: Optional[float] = Query(None, ge=0, le=100),
has_phone: Optional[bool] = None,
has_email: Optional[bool] = None,
has_manager: Optional[bool] = None,
halal_gap_only: Optional[bool] = None,
linked_to_crm: Optional[bool] = None,
has_halal_cert_registry: Optional[bool] = None,
sort: Optional[str] = Query("name", pattern="^(name|halal_opportunity|population|chain)$"),
limit: int = Query(2000, ge=1, le=5000),
offset: int = Query(0, ge=0),
) -> dict[str, Any]:
clauses, params = _build_filters(
chain, province, city, partnership, halal_certified, has_halal_section,
store_type, postcode_prefix, q, min_population, max_population,
min_avg_income, max_avg_income, min_muslim_pct, has_area_data,
min_halal_opportunity, max_halal_opportunity, has_phone, has_email,
has_manager, halal_gap_only, linked_to_crm, has_halal_cert_registry,
)
where = " WHERE " + " AND ".join(clauses)
order = {
"halal_opportunity": "COALESCE(ros.halal_opportunity_score,0) DESC, s.name",
"population": "COALESCE(a.population,0) DESC, s.name",
"chain": "s.chain, s.name",
"name": "s.chain, s.name",
}.get(sort or "name", "s.chain, s.name")
rows = fetch_all(
f"{STORE_SELECT}{where} ORDER BY {order} LIMIT %s OFFSET %s",
tuple(params + [limit, offset]),
)
total = fetch_one(
f"""SELECT COUNT(*) AS n FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id
LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id
{where}""",
tuple(params),
)
return {
"items": [_row(r) for r in rows],
"count": len(rows),
"total": int((total or {}).get("n") or 0),
}
@router.get("/map")
def map_points(
chain: Optional[str] = None,
province: Optional[str] = None,
partnership: Optional[str] = None,
halal_certified: Optional[bool] = None,
has_halal_section: Optional[bool] = None,
min_muslim_pct: Optional[float] = None,
min_population: Optional[int] = None,
min_halal_opportunity: Optional[float] = None,
halal_gap_only: Optional[bool] = None,
linked_to_crm: Optional[bool] = None,
q: Optional[str] = None,
limit: int = Query(5000, ge=1, le=5000),
) -> dict[str, Any]:
clauses, params = _build_filters(
chain, province, None, partnership, halal_certified, has_halal_section,
None, None, q, min_population, None, None, None, min_muslim_pct, None,
min_halal_opportunity, None, None, None, None, halal_gap_only, linked_to_crm, None,
)
where = " WHERE " + " AND ".join(clauses) + " AND s.latitude IS NOT NULL AND s.longitude IS NOT NULL"
rows = fetch_all(
f"""SELECT s.id, s.name, s.chain, s.city, s.province, s.postcode,
s.latitude, s.longitude, s.partnership_status, s.halal_certified,
s.has_halal_section, s.phone, s.manager_name,
a.population AS area_population,
(a.religious_composition->>'muslim_proxy_pct')::float AS muslim_proxy_pct,
COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score) AS halal_opportunity_score
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id
LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id
{where} LIMIT %s""",
tuple(params + [limit]),
)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.get("/supermarkets/{store_id}")
def get_supermarket(store_id: int) -> dict[str, Any]:
row = fetch_one(f"{STORE_SELECT} WHERE s.id = %s", (store_id,))
data = _row(row)
area = fetch_one("SELECT * FROM area_analysis WHERE postcode = %s", (data.get("postcode"),))
if area:
data["area_analysis"] = _row(area)
weather = fetch_all(
"SELECT * FROM weather_data WHERE city ILIKE %s ORDER BY date DESC LIMIT 3",
(f"%{data.get('city', '')}%",),
)
data["weather"] = [_row(w) for w in weather]
recs = fetch_all(
"""SELECT * FROM ai_recommendations
WHERE related_entity_type = 'supermarket' AND related_entity_id = %s
ORDER BY created_at DESC LIMIT 3""",
(store_id,),
)
data["recommendations"] = [_row(r) for r in recs]
nearby = fetch_all(
"""
SELECT id, name, chain, partnership_status, distance_km FROM (
SELECT id, name, chain, partnership_status,
(6371 * acos(
LEAST(1.0, cos(radians(%s)) * cos(radians(latitude))
* cos(radians(longitude) - radians(%s))
+ sin(radians(%s)) * sin(radians(latitude)))
)) AS distance_km
FROM supermarkets
WHERE id <> %s AND latitude IS NOT NULL AND longitude IS NOT NULL
) nearby_q
WHERE distance_km < 3
ORDER BY distance_km LIMIT 8
""",
(
data.get("latitude"), data.get("longitude"), data.get("latitude"),
store_id,
),
)
data["nearby_stores"] = [_row(n) for n in nearby]
data["crm"] = retail_crm.get_store_crm_context(store_id)
opp = fetch_one("SELECT * FROM retail_opportunity_scores WHERE supermarket_id = %s", (store_id,))
if opp:
data["opportunity"] = _row(opp)
return data
@router.get("/scrape/chains")
def list_scrape_chains() -> dict[str, Any]:
return {"chains": retail_scrapers.list_chains()}
@router.post("/scrape/all")
def scrape_all_chains() -> dict[str, Any]:
log_agent_event(agent_name="retail_scraper", event_type="scrape", title="OSM import all chains")
return retail_scrapers.import_all_chains()
@router.post("/scrape/{chain_key}")
def scrape_chain(chain_key: str) -> dict[str, Any]:
log_agent_event(
agent_name="retail_scraper",
event_type="scrape",
title=f"OSM import {chain_key}",
)
try:
return retail_scrapers.import_chain(chain_key.lower())
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(502, str(exc)) from exc
@router.post("/enrich")
def enrich_areas(
limit: int = Query(50, ge=1, le=200),
offset: int = Query(0, ge=0),
) -> dict[str, Any]:
log_agent_event(
agent_name="retail_enrichment",
event_type="enrich",
title=f"CBS/PDOK enrichment batch limit={limit}",
)
return retail_enrichment.enrich_batch(limit=limit, offset=offset)
@router.get("/enrich/status")
def enrich_status() -> dict[str, Any]:
return retail_enrichment.enrichment_status()
@router.get("/stats")
def retail_stats(
chain: Optional[str] = None,
province: Optional[str] = None,
partnership: Optional[str] = None,
min_muslim_pct: Optional[float] = None,
) -> dict[str, Any]:
clauses, params = _build_filters(
chain, province, None, partnership, None, None, None, None, None,
None, None, None, None, min_muslim_pct, None,
)
where = " WHERE " + " AND ".join(clauses)
row = fetch_one(
f"""
SELECT COUNT(*) AS total,
COUNT(*) FILTER (WHERE s.partnership_status = 'active') AS active_partnerships,
COUNT(*) FILTER (WHERE s.halal_certified) AS halal_certified,
COUNT(*) FILTER (WHERE a.id IS NOT NULL) AS with_area_data,
ROUND(AVG(a.avg_income)::numeric, 0) AS avg_area_income,
ROUND(AVG((a.religious_composition->>'muslim_proxy_pct')::float)::numeric, 1) AS avg_muslim_proxy_pct
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
{where}
""",
tuple(params),
)
out = {k: int(v or 0) if k in ("total", "active_partnerships", "halal_certified", "with_area_data") else v
for k, v in (row or {}).items()}
if out.get("avg_area_income") is not None:
out["avg_area_income"] = float(out["avg_area_income"])
if out.get("avg_muslim_proxy_pct") is not None:
out["avg_muslim_proxy_pct"] = float(out["avg_muslim_proxy_pct"])
halal_n = fetch_one("SELECT COUNT(*) AS n FROM supermarkets WHERE halal_certified = TRUE")
out["halal_certified_count"] = int((halal_n or {}).get("n") or 0)
return out
@router.get("/schema")
def retail_schema() -> dict[str, Any]:
return {"groups": FIELD_SCHEMA, "all_fields": [f for fields in FIELD_SCHEMA.values() for f in fields]}
@router.get("/crm/options")
def crm_options() -> dict[str, Any]:
return retail_crm.list_crm_options()
@router.post("/supermarkets/{store_id}/link")
def link_store_crm(store_id: int, payload: CrmLinkIn) -> dict[str, Any]:
try:
row = retail_crm.link_client_to_store(
store_id, payload.client_id, payload.deal_id,
payload.relationship_type, payload.partnership_status, payload.notes,
)
log_agent_event(agent_name="retail_crm", event_type="link", title=f"Linked store {store_id} to client {payload.client_id}")
return {"link": row}
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
@router.delete("/supermarkets/{store_id}/link/{client_id}")
def unlink_store_crm(store_id: int, client_id: int) -> dict[str, Any]:
ok = retail_crm.unlink_client_from_store(store_id, client_id)
return {"unlinked": ok}
@router.get("/halal")
def list_halal_stores(limit: int = Query(500, ge=1, le=2000)) -> dict[str, Any]:
rows = halal_registry.list_halal_certified(limit)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.get("/opportunities")
def list_opportunities(
limit: int = Query(50, ge=1, le=500),
min_score: float = Query(30, ge=0, le=100),
chain: Optional[str] = None,
province: Optional[str] = None,
) -> dict[str, Any]:
rows = retail_opportunities.top_opportunities(limit, min_score, chain, province)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.get("/trends")
def list_trends(limit: int = Query(20, ge=1, le=100)) -> dict[str, Any]:
rows = trends_feed.list_live_trends(limit)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/sync/halal")
def sync_halal() -> dict[str, Any]:
log_agent_event(agent_name="halal_registry", event_type="sync", title="Halal OSM sync")
return halal_registry.sync_osm_halal_tags()
@router.post("/sync/contacts")
def sync_contacts(limit: int = Query(100, ge=1, le=300)) -> dict[str, Any]:
log_agent_event(agent_name="branch_scraper", event_type="sync", title=f"OSM contacts limit={limit}")
return halal_registry.sync_osm_contact_tags(limit)
@router.post("/sync/trends")
def sync_trends() -> dict[str, Any]:
return trends_feed.refresh_trends_from_social()
@router.post("/compute-opportunities")
def compute_opportunities(limit: int = Query(5000, ge=100, le=10000)) -> dict[str, Any]:
log_agent_event(agent_name="retail_intel", event_type="score", title="Compute halal opportunity scores")
return retail_opportunities.compute_all_scores(limit)
@router.get("/export")
def export_csv(
chain: Optional[str] = None,
province: Optional[str] = None,
halal_certified: Optional[bool] = None,
min_halal_opportunity: Optional[float] = None,
limit: int = Query(5000, ge=1, le=5000),
):
clauses, params = _build_filters(
chain, province, None, None, halal_certified, None, None, None, None,
None, None, None, None, None, None, min_halal_opportunity, None,
None, None, None, None, None, None,
)
where = " WHERE " + " AND ".join(clauses)
rows = fetch_all(
f"{STORE_SELECT}{where} ORDER BY s.chain, s.name LIMIT %s",
tuple(params + [limit]),
)
def generate():
headers = ["id", "name", "chain", "city", "province", "postcode", "phone", "email",
"manager_name", "halal_certified", "has_halal_section", "partnership_status",
"muslim_proxy_pct", "area_population", "area_avg_income", "halal_opportunity_score"]
yield ",".join(headers) + "\n"
for r in rows:
rel = r.get("area_religious") or {}
if isinstance(rel, str):
rel = {}
vals = [
r.get("id"), r.get("name"), r.get("chain"), r.get("city"), r.get("province"),
r.get("postcode"), r.get("phone"), r.get("email"), r.get("manager_name"),
r.get("halal_certified"), r.get("has_halal_section"), r.get("partnership_status"),
rel.get("muslim_proxy_pct") if isinstance(rel, dict) else None,
r.get("area_population"), r.get("area_avg_income"),
r.get("opp_halal_score") or r.get("halal_opportunity_score"),
]
yield ",".join('"' + str(v or "").replace('"', '""') + '"' for v in vals) + "\n"
return StreamingResponse(generate(), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=retail_export.csv"})
tools-api/app/retail_360.py
+182
View File
@@ -0,0 +1,182 @@
"""City demographics and 360 entity workspace."""
from __future__ import annotations
from typing import Any, Optional
from app.connectors import cbs, pdok
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
def sync_city_demographics(limit: int = 100) -> dict[str, Any]:
cities = fetch_all(
"""SELECT DISTINCT city, province FROM supermarkets
WHERE city IS NOT NULL AND city <> 'Onbekend'
AND city NOT IN (SELECT city FROM city_demographics)
LIMIT %s""",
(limit,),
)
synced = 0
for row in cities:
pc_rows = fetch_all(
"SELECT postcode FROM supermarkets WHERE city = %s AND postcode <> '0000AA' LIMIT 1",
(row["city"],),
)
if not pc_rows:
continue
pd = pdok.lookup_postcode(pc_rows[0]["postcode"])
if not pd or not pd.get("municipality_code"):
continue
stats = cbs.fetch_gemeente_stats(pd["municipality_code"])
if not stats:
continue
rel = stats.get("religious_composition") or {}
execute_returning(
"""INSERT INTO city_demographics (city, province, gemeente_code, population, households,
avg_income, muslim_proxy_pct, data_source, last_updated)
VALUES (%s,%s,%s,%s,%s,%s,%s,'cbs+pdok',NOW())
ON CONFLICT (city, province) DO UPDATE SET
population=EXCLUDED.population, households=EXCLUDED.households,
avg_income=EXCLUDED.avg_income, muslim_proxy_pct=EXCLUDED.muslim_proxy_pct,
last_updated=NOW() RETURNING id""",
(
row["city"], row.get("province") or pd.get("province"),
pd.get("municipality_code"), stats.get("population"), stats.get("households"),
stats.get("avg_income"), rel.get("muslim_proxy_pct"),
),
)
synced += 1
return {"synced": synced}
def get_city_context(city: str, province: Optional[str] = None) -> Optional[dict[str, Any]]:
if province:
row = fetch_one(
"SELECT * FROM city_demographics WHERE city ILIKE %s AND province ILIKE %s",
(city, province),
)
else:
row = fetch_one("SELECT * FROM city_demographics WHERE city ILIKE %s LIMIT 1", (city,))
if not row:
return None
stores = fetch_one(
"SELECT COUNT(*) AS n FROM supermarkets WHERE city ILIKE %s", (city,)
)
out = dict(row)
out["stores_in_city"] = int((stores or {}).get("n") or 0)
return out
def get_store_360(supermarket_id: int) -> dict[str, Any]:
store = fetch_one("SELECT * FROM supermarkets WHERE id = %s", (supermarket_id,))
if not store:
raise ValueError("Store not found")
city_ctx = get_city_context(store["city"], store.get("province"))
notes = fetch_all(
"SELECT * FROM entity_notes WHERE entity_type='supermarket' AND entity_id=%s ORDER BY pinned DESC, created_at DESC",
(supermarket_id,),
)
media = fetch_all(
"SELECT * FROM entity_media WHERE entity_type='supermarket' AND entity_id=%s ORDER BY created_at DESC",
(supermarket_id,),
)
milestones = fetch_all(
"SELECT * FROM sales_milestones WHERE supermarket_id=%s ORDER BY sort_order, target_date NULLS LAST",
(supermarket_id,),
)
ownership = fetch_all(
"""SELECT * FROM ownership_changes
WHERE (entity_id=%s AND entity_type='supermarket') OR chain ILIKE %s
ORDER BY effective_date DESC NULLS LAST LIMIT 10""",
(supermarket_id, f"%{store['chain']}%"),
)
calendar = fetch_all(
"""SELECT * FROM calendar_events WHERE supermarket_id=%s OR (client_id=%s AND client_id IS NOT NULL)
ORDER BY starts_at DESC LIMIT 10""",
(supermarket_id, store.get("client_id")),
)
nearby_count = fetch_one(
"SELECT COUNT(*) AS n FROM supermarkets WHERE city ILIKE %s AND id <> %s",
(store["city"], supermarket_id),
)
area = fetch_one("SELECT * FROM area_analysis WHERE postcode = %s", (store.get("postcode"),))
weather = fetch_all(
"SELECT * FROM weather_data WHERE city ILIKE %s ORDER BY date DESC LIMIT 7",
(f"%{store.get('city', '')}%",),
)
return {
"store": dict(store),
"city": city_ctx,
"catchment": {
"city_population": (city_ctx or {}).get("population"),
"stores_in_city": int((nearby_count or {}).get("n") or 0),
"gemeente_population": (area or {}).get("population"),
"postcode_population_proxy": (area or {}).get("population"),
},
"notes": [dict(n) for n in notes],
"media": [dict(m) for m in media],
"milestones": [dict(m) for m in milestones],
"ownership_changes": [dict(o) for o in ownership],
"calendar": [dict(c) for c in calendar],
"area_analysis": dict(area) if area else None,
"weather": [dict(w) for w in weather],
}
def add_note(entity_type: str, entity_id: int, body: str, title: Optional[str] = None, note_type: str = "general") -> dict[str, Any]:
row = execute_returning(
"""INSERT INTO entity_notes (entity_type, entity_id, title, body, note_type)
VALUES (%s,%s,%s,%s,%s) RETURNING *""",
(entity_type, entity_id, title, body, note_type),
)
return dict(row or {})
def add_milestone(supermarket_id: int, title: str, milestone_type: str = "custom", **kwargs: Any) -> dict[str, Any]:
row = execute_returning(
"""INSERT INTO sales_milestones (supermarket_id, client_id, deal_id, milestone_type, title,
status, target_date, value_eur, notes, sort_order)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *""",
(
supermarket_id, kwargs.get("client_id"), kwargs.get("deal_id"), milestone_type, title,
kwargs.get("status", "pending"), kwargs.get("target_date"), kwargs.get("value_eur"),
kwargs.get("notes"), kwargs.get("sort_order", 0),
),
)
return dict(row or {})
def add_ownership(**kwargs: Any) -> dict[str, Any]:
row = execute_returning(
"""INSERT INTO ownership_changes (entity_type, entity_id, chain, previous_owner, new_owner,
change_type, effective_date, source, notes)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *""",
(
kwargs.get("entity_type", "supermarket"), kwargs.get("entity_id"), kwargs.get("chain"),
kwargs.get("previous_owner"), kwargs["new_owner"], kwargs.get("change_type", "acquisition"),
kwargs.get("effective_date"), kwargs.get("source"), kwargs.get("notes"),
),
)
return dict(row or {})
def add_calendar_event(supermarket_id: int, title: str, starts_at: str, **kwargs: Any) -> dict[str, Any]:
row = execute_returning(
"""INSERT INTO calendar_events (title, description, starts_at, ends_at, client_id, deal_id,
supermarket_id, location, source)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'retail_360') RETURNING *""",
(
title, kwargs.get("description"), starts_at, kwargs.get("ends_at"),
kwargs.get("client_id"), kwargs.get("deal_id"), supermarket_id,
kwargs.get("location"),
),
)
return dict(row or {})
def register_media(entity_type: str, entity_id: int, filename: str, storage_path: str, content_type: str, caption: Optional[str] = None) -> dict[str, Any]:
row = execute_returning(
"""INSERT INTO entity_media (entity_type, entity_id, filename, storage_path, content_type, caption)
VALUES (%s,%s,%s,%s,%s,%s) RETURNING *""",
(entity_type, entity_id, filename, storage_path, content_type, caption),
)
return dict(row or {})
tools-api/app/retail_360_routes.py
+511
View File
@@ -0,0 +1,511 @@
"""Retail 360 workspace API — notes, media, milestones, RSS, wholesalers."""
from __future__ import annotations
import json
import urllib.request
from datetime import datetime
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from app.db import fetch_all, fetch_one
from app.middleware import log_agent_event
from app import retail_360
from app import wholesaler_scrapers
from app.connectors import market_stocks, rss_feeds
from app.connectors import food_trends
router = APIRouter(prefix="/retail", tags=["retail-360"])
class NoteIn(BaseModel):
body: str = Field(..., min_length=1)
title: Optional[str] = None
note_type: str = "general"
class MilestoneIn(BaseModel):
title: str
milestone_type: str = "custom"
client_id: Optional[int] = None
deal_id: Optional[int] = None
target_date: Optional[str] = None
value_eur: Optional[float] = None
notes: Optional[str] = None
class OwnershipIn(BaseModel):
new_owner: str
previous_owner: Optional[str] = None
change_type: str = "acquisition"
effective_date: Optional[str] = None
source: Optional[str] = None
notes: Optional[str] = None
class CalendarIn(BaseModel):
title: str
starts_at: str
description: Optional[str] = None
ends_at: Optional[str] = None
client_id: Optional[int] = None
deal_id: Optional[int] = None
location: Optional[str] = None
class MediaIn(BaseModel):
filename: str
storage_path: str
content_type: str = "image/jpeg"
caption: Optional[str] = None
def _row(row: dict | None) -> dict[str, Any]:
if not row:
raise HTTPException(404, "Not found")
out: dict[str, Any] = {}
for k, v in row.items():
if hasattr(v, "isoformat"):
out[k] = v.isoformat()
elif v is not None and hasattr(v, "__float__") and type(v).__name__ == "Decimal":
out[k] = float(v)
else:
out[k] = v
return out
def _fetch_weather_forecast(lat: float, lon: float) -> list[dict[str, Any]]:
url = (
f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}"
f"&daily=temperature_2m_max,precipitation_sum,weathercode"
f"&timezone=Europe%2FAmsterdam&forecast_days=7"
)
try:
with urllib.request.urlopen(url, timeout=15) as resp:
data = json.loads(resp.read().decode())
days = data.get("daily", {}).get("time", [])
temps = data.get("daily", {}).get("temperature_2m_max", [])
prec = data.get("daily", {}).get("precipitation_sum", [])
return [
{"date": days[i], "temperature_c": temps[i] if i < len(temps) else None,
"precipitation_mm": prec[i] if i < len(prec) else None, "source": "open-meteo-live"}
for i in range(len(days))
]
except Exception:
return []
@router.get("/360/{store_id}")
def get_360_view(store_id: int) -> dict[str, Any]:
try:
data = retail_360.get_store_360(store_id)
except ValueError as exc:
raise HTTPException(404, str(exc)) from exc
store = data["store"]
if store.get("latitude") and store.get("longitude"):
live = _fetch_weather_forecast(float(store["latitude"]), float(store["longitude"]))
if live:
data["weather_forecast"] = live
for key in ("notes", "media", "milestones", "ownership_changes", "calendar", "weather"):
data[key] = [_row(x) for x in data.get(key, [])]
if data.get("area_analysis"):
data["area_analysis"] = _row(data["area_analysis"])
return data
@router.post("/360/{store_id}/notes")
def add_store_note(store_id: int, payload: NoteIn) -> dict[str, Any]:
note = retail_360.add_note("supermarket", store_id, payload.body, payload.title, payload.note_type)
log_agent_event(agent_name="retail_360", event_type="note", title=f"Note on store {store_id}")
return {"note": _row(note)}
@router.post("/360/{store_id}/milestones")
def add_store_milestone(store_id: int, payload: MilestoneIn) -> dict[str, Any]:
ms = retail_360.add_milestone(store_id, payload.title, payload.milestone_type, **payload.model_dump(exclude={"title", "milestone_type"}))
return {"milestone": _row(ms)}
@router.post("/360/{store_id}/ownership")
def add_store_ownership(store_id: int, payload: OwnershipIn) -> dict[str, Any]:
store = fetch_one("SELECT chain FROM supermarkets WHERE id = %s", (store_id,))
row = retail_360.add_ownership(entity_id=store_id, chain=store.get("chain") if store else None, **payload.model_dump())
return {"ownership": _row(row)}
@router.post("/360/{store_id}/calendar")
def add_store_calendar(store_id: int, payload: CalendarIn) -> dict[str, Any]:
ev = retail_360.add_calendar_event(store_id, payload.title, payload.starts_at, **payload.model_dump(exclude={"title", "starts_at"}))
return {"event": _row(ev)}
@router.post("/360/{store_id}/media")
def register_store_media(store_id: int, payload: MediaIn) -> dict[str, Any]:
media = retail_360.register_media("supermarket", store_id, payload.filename, payload.storage_path, payload.content_type, payload.caption)
return {"media": _row(media)}
@router.get("/cities")
def list_cities(
limit: int = Query(200, ge=1, le=1000),
q: Optional[str] = None,
min_population: Optional[int] = None,
min_muslim_pct: Optional[float] = None,
sort: str = Query("population", pattern="^(population|muslim|stores|city)$"),
) -> dict[str, Any]:
clauses, params = [], []
if q:
clauses.append("c.city ILIKE %s")
params.append(f"%{q}%")
if min_population:
clauses.append("c.population >= %s")
params.append(min_population)
if min_muslim_pct:
clauses.append("c.muslim_proxy_pct >= %s")
params.append(min_muslim_pct)
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
order_map = {
"population": "c.population DESC NULLS LAST",
"muslim": "c.muslim_proxy_pct DESC NULLS LAST",
"stores": "store_count DESC",
"city": "c.city ASC",
}
order = order_map.get(sort, order_map["population"])
rows = fetch_all(
f"""SELECT c.*, (SELECT COUNT(*) FROM supermarkets s WHERE s.city ILIKE c.city) AS store_count
FROM city_demographics c{where} ORDER BY {order} LIMIT %s""",
tuple(params + [limit]),
)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/cities/sync")
def sync_cities(limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:
return retail_360.sync_city_demographics(limit)
@router.get("/wholesalers")
def list_wholesalers(
limit: int = Query(500, ge=1, le=2000),
q: Optional[str] = None,
province: Optional[str] = None,
city: Optional[str] = None,
halal_certified: Optional[bool] = None,
has_phone: Optional[bool] = None,
has_email: Optional[bool] = None,
sort: str = Query("name", pattern="^(name|city|province)$"),
) -> dict[str, Any]:
clauses, params = [], []
if q:
clauses.append("(name ILIKE %s OR city ILIKE %s OR address ILIKE %s OR email ILIKE %s)")
like = f"%{q}%"
params.extend([like, like, like, like])
if province:
clauses.append("province ILIKE %s")
params.append(province)
if city:
clauses.append("city ILIKE %s")
params.append(f"%{city}%")
if halal_certified is True:
clauses.append("halal_certified = TRUE")
if has_phone is True:
clauses.append("phone IS NOT NULL AND phone <> ''")
if has_email is True:
clauses.append("email IS NOT NULL AND email <> ''")
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
order = {"name": "name", "city": "city", "province": "province"}.get(sort, "name")
rows = fetch_all(f"SELECT * FROM wholesalers{where} ORDER BY {order} LIMIT %s", tuple(params + [limit]))
total = fetch_one(f"SELECT COUNT(*) AS n FROM wholesalers{where}", tuple(params) if params else None)
return {"items": [_row(r) for r in rows], "count": len(rows), "total": int(total["n"]) if total else len(rows)}
@router.get("/wholesalers/meta")
def wholesalers_meta() -> dict[str, Any]:
provinces = fetch_all(
"SELECT province, COUNT(*) AS n FROM wholesalers WHERE province IS NOT NULL GROUP BY province ORDER BY n DESC"
)
return {
"provinces": [_row(p) for p in provinces],
"total": _safe_count_wh("wholesalers"),
}
def _safe_count_wh(table: str) -> int:
row = fetch_one(f"SELECT COUNT(*) AS n FROM {table}")
return int(row["n"]) if row else 0
@router.get("/wholesalers/{wh_id}/contacts")
def wholesaler_contacts(wh_id: int) -> dict[str, Any]:
rows = fetch_all(
"SELECT * FROM wholesaler_contacts WHERE wholesaler_id = %s ORDER BY confidence DESC, full_name",
(wh_id,),
)
wh = fetch_one("SELECT id, name, phone, email, address, city, province, website, linkedin_url FROM wholesalers WHERE id = %s", (wh_id,))
if not wh:
raise HTTPException(404, "Wholesaler not found")
return {"wholesaler": _row(wh), "contacts": [_row(r) for r in rows]}
class WholesalerContactIn(BaseModel):
full_name: str
role: str = "contact"
phone: Optional[str] = None
email: Optional[str] = None
linkedin_url: Optional[str] = None
@router.post("/wholesalers/{wh_id}/contacts")
def add_wholesaler_contact(wh_id: int, payload: WholesalerContactIn) -> dict[str, Any]:
row = fetch_one(
"""INSERT INTO wholesaler_contacts (wholesaler_id, full_name, role, phone, email, linkedin_url, source)
VALUES (%s, %s, %s, %s, %s, %s, 'manual') RETURNING *""",
(wh_id, payload.full_name, payload.role, payload.phone, payload.email, payload.linkedin_url),
)
return {"contact": _row(row)}
class RssBookmarkIn(BaseModel):
rss_item_id: int
title: Optional[str] = None
link: Optional[str] = None
feed_name: Optional[str] = None
notes: Optional[str] = None
@router.get("/rss/bookmarks")
def list_rss_bookmarks(limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:
rows = fetch_all(
"""SELECT b.*, i.title AS item_title, i.link AS item_link, f.name AS feed_name
FROM rss_bookmarks b
LEFT JOIN rss_items i ON i.id = b.rss_item_id
LEFT JOIN rss_feeds f ON f.id = i.feed_id
ORDER BY b.created_at DESC LIMIT %s""",
(limit,),
)
out = []
for r in rows:
row = _row(r)
row["title"] = row.get("title") or row.get("item_title")
row["link"] = row.get("link") or row.get("item_link")
out.append(row)
return {"items": out, "count": len(out)}
@router.post("/rss/bookmarks")
def add_rss_bookmark(payload: RssBookmarkIn) -> dict[str, Any]:
from app.db import execute
item = fetch_one("SELECT id, title, link FROM rss_items WHERE id = %s", (payload.rss_item_id,))
if not item:
raise HTTPException(404, "RSS item not found")
execute(
"""INSERT INTO rss_bookmarks (rss_item_id, title, link, feed_name, notes)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (rss_item_id) DO UPDATE SET title=EXCLUDED.title, link=EXCLUDED.link, feed_name=EXCLUDED.feed_name, notes=EXCLUDED.notes""",
(
payload.rss_item_id,
payload.title or item.get("title"),
payload.link or item.get("link"),
payload.feed_name,
payload.notes,
),
)
row = fetch_one("SELECT * FROM rss_bookmarks WHERE rss_item_id = %s", (payload.rss_item_id,))
return {"bookmark": _row(row)}
@router.delete("/rss/bookmarks/{rss_item_id}")
def delete_rss_bookmark(rss_item_id: int) -> dict[str, Any]:
from app.db import execute
execute("DELETE FROM rss_bookmarks WHERE rss_item_id = %s", (rss_item_id,))
return {"ok": True}
@router.get("/promo-campaigns")
def list_promo_campaigns(
chain: Optional[str] = None,
status: str = Query("active"),
q: Optional[str] = None,
folder_type: Optional[str] = None,
valid_days: Optional[int] = Query(None, ge=1, le=365),
limit: int = Query(100, ge=1, le=500),
) -> dict[str, Any]:
clauses, params = ["p.status = %s"], [status]
if chain:
clauses.append("p.chain ILIKE %s")
params.append(f"%{chain}%")
if q:
clauses.append("(p.title ILIKE %s OR p.chain ILIKE %s OR p.description ILIKE %s)")
params.extend([f"%{q}%"] * 3)
if folder_type:
clauses.append("(p.promo_type ILIKE %s OR p.metadata->>'folder_type' ILIKE %s)")
params.extend([f"%{folder_type}%", f"%{folder_type}%"])
if valid_days:
clauses.append("p.valid_to IS NOT NULL AND p.valid_to <= CURRENT_DATE + %s * INTERVAL '1 day'")
params.append(valid_days)
where = " WHERE " + " AND ".join(clauses)
rows = fetch_all(
f"""SELECT p.*, s.name AS store_name FROM promo_campaigns p
LEFT JOIN supermarkets s ON s.id = p.supermarket_id
{where} ORDER BY p.valid_to ASC NULLS LAST, p.chain ASC, p.created_at DESC LIMIT %s""",
tuple(params + [limit]),
)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.get("/reclamefolder/chains")
def reclamefolder_chains() -> dict[str, Any]:
from app.connectors import reclamefolder
chains = reclamefolder.list_chains()
return {"chains": chains, "count": len(chains)}
class PromoCampaignIn(BaseModel):
chain: Optional[str] = None
title: str
folder_path: Optional[str] = None
folder_label: Optional[str] = None
description: Optional[str] = None
valid_from: Optional[str] = None
valid_to: Optional[str] = None
status: str = "active"
promo_type: str = "folder"
@router.post("/promo-campaigns")
def add_promo_campaign(payload: PromoCampaignIn) -> dict[str, Any]:
row = fetch_one(
"""INSERT INTO promo_campaigns (chain, title, folder_path, folder_label, description, valid_from, valid_to, status, promo_type)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING *""",
(
payload.chain, payload.title, payload.folder_path, payload.folder_label,
payload.description, payload.valid_from, payload.valid_to, payload.status, payload.promo_type,
),
)
return {"campaign": _row(row)}
@router.post("/reclamefolder/refresh")
def refresh_reclamefolders() -> dict[str, Any]:
from app.connectors import reclamefolder
from app.middleware import log_agent_event
log_agent_event(
agent_name="reclamefolder",
event_type="refresh",
title="Reclamefolder.nl sync",
)
return reclamefolder.sync_to_db()
@router.get("/reclamefolder/live")
def live_reclamefolders(limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:
from app.connectors import reclamefolder
try:
result = reclamefolder.sync_to_db()
items = result.get("items") or []
except Exception as exc:
items = reclamefolder.list_cached(limit)
return {"items": items, "count": len(items), "cached": True, "error": str(exc)}
return {"items": items[:limit], "count": len(items), "synced": True, "source": "reclamefolder.nl"}
@router.post("/wholesalers/import")
def import_wholesalers(background: bool = Query(False)) -> dict[str, Any]:
log_agent_event(agent_name="wholesale_scraper", event_type="import", title="OSM wholesalers import")
if background:
import threading
threading.Thread(target=wholesaler_scrapers.import_wholesalers, daemon=True).start()
return {"status": "started", "message": "Wholesaler import running in background"}
return wholesaler_scrapers.import_wholesalers()
@router.get("/rss/live")
def rss_live(limit: int = Query(30, ge=1, le=100), category: Optional[str] = None) -> dict[str, Any]:
rows = rss_feeds.list_live_feed(limit, category)
return {"items": [_row(r) for r in rows], "count": len(rows)}
@router.post("/rss/refresh")
def rss_refresh() -> dict[str, Any]:
log_agent_event(agent_name="rss_feeds", event_type="refresh", title="RSS feeds refresh")
return rss_feeds.refresh_all_feeds()
@router.get("/market/stocks")
def retail_market_stocks() -> dict[str, Any]:
quotes = market_stocks.fetch_retail_quotes()
return {
"items": quotes,
"summary": market_stocks.market_summary(quotes),
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/market/supermarkets")
def supermarket_market_board() -> dict[str, Any]:
quotes = market_stocks.fetch_supermarket_quotes()
listed = [q for q in quotes if q.get("listed")]
return {
"items": [_row(q) for q in quotes],
"listed": [_row(q) for q in listed],
"unlisted_nl": [_row(q) for q in quotes if not q.get("listed")],
"summary": market_stocks.market_summary(listed),
"data_source": market_stocks.DATA_SOURCE,
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/market/food-trends")
def market_food_trends() -> dict[str, Any]:
return food_trends.food_trends_dashboard()
@router.get("/market/concepts")
def market_concepts(limit: int = Query(6, ge=1, le=12)) -> dict[str, Any]:
listed = market_stocks.fetch_retail_quotes()
summary = market_stocks.market_summary(listed)
best = summary.get("best_performer")
concepts = food_trends.generate_concepts(market_best=best, limit=limit)
return {
"concepts": concepts,
"summary": summary,
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/regulations")
def retail_regulations(limit: int = Query(30, ge=1, le=100)) -> dict[str, Any]:
reg = rss_feeds.list_live_feed(limit, "regelgeving")
cbs = rss_feeds.list_live_feed(limit, "cbs")
markt = rss_feeds.list_live_feed(min(limit, 15), "markt")
return {
"regelgeving": [_row(r) for r in reg],
"cbs": [_row(r) for r in cbs],
"markt": [_row(r) for r in markt],
"updated_at": datetime.utcnow().isoformat(),
}
@router.get("/live-dashboard")
def live_dashboard() -> dict[str, Any]:
trends = fetch_all(
"SELECT * FROM market_trends ORDER BY updated_at DESC NULLS LAST LIMIT 8"
)
rss = rss_feeds.list_live_feed(12)
opportunities = fetch_all(
"""SELECT s.name, s.chain, s.city, ros.halal_opportunity_score
FROM retail_opportunity_scores ros JOIN supermarkets s ON s.id = ros.supermarket_id
ORDER BY ros.halal_opportunity_score DESC LIMIT 5"""
)
quotes = market_stocks.fetch_retail_quotes()
return {
"trends": [_row(t) for t in trends],
"rss": [_row(r) for r in rss],
"top_opportunities": [_row(o) for o in opportunities],
"market_stocks": quotes,
"market_summary": market_stocks.market_summary(quotes),
"updated_at": datetime.utcnow().isoformat(),
}
tools-api/app/retail_crm.py
+96
View File
@@ -0,0 +1,96 @@
"""CRM linking for retail locations."""
from __future__ import annotations
from typing import Any, Optional
from app.db import execute, execute_returning, fetch_all, fetch_one
def link_client_to_store(
supermarket_id: int,
client_id: int,
deal_id: Optional[int] = None,
relationship_type: str = "prospect",
partnership_status: Optional[str] = None,
notes: Optional[str] = None,
) -> dict[str, Any]:
store = fetch_one("SELECT id FROM supermarkets WHERE id = %s", (supermarket_id,))
client = fetch_one("SELECT id, name FROM clients WHERE id = %s", (client_id,))
if not store or not client:
raise ValueError("Store or client not found")
row = execute_returning(
"""
INSERT INTO client_supermarket_links (client_id, supermarket_id, deal_id, relationship_type, notes)
VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (client_id, supermarket_id) DO UPDATE SET
deal_id = COALESCE(EXCLUDED.deal_id, client_supermarket_links.deal_id),
relationship_type = EXCLUDED.relationship_type,
notes = COALESCE(EXCLUDED.notes, client_supermarket_links.notes)
RETURNING *
""",
(client_id, supermarket_id, deal_id, relationship_type, notes),
)
if partnership_status:
execute(
"""UPDATE supermarkets SET client_id = %s, deal_id = %s,
partnership_status = %s, last_updated = NOW() WHERE id = %s""",
(client_id, deal_id, partnership_status, supermarket_id),
)
return dict(row or {})
def unlink_client_from_store(supermarket_id: int, client_id: int) -> bool:
n = execute(
"DELETE FROM client_supermarket_links WHERE supermarket_id = %s AND client_id = %s",
(supermarket_id, client_id),
)
execute(
"""UPDATE supermarkets SET client_id = NULL, deal_id = NULL,
partnership_status = 'none', last_updated = NOW()
WHERE id = %s AND client_id = %s""",
(supermarket_id, client_id),
)
return n > 0
def get_store_crm_context(supermarket_id: int) -> dict[str, Any]:
links = fetch_all(
"""
SELECT l.*, c.name AS client_name, c.email AS client_email, c.contact AS client_contact,
c.stage AS client_stage, d.title AS deal_title, d.value AS deal_value, d.stage AS deal_stage
FROM client_supermarket_links l
JOIN clients c ON c.id = l.client_id
LEFT JOIN deals d ON d.id = l.deal_id
WHERE l.supermarket_id = %s
ORDER BY l.created_at DESC
""",
(supermarket_id,),
)
contacts = fetch_all(
"SELECT * FROM supermarket_contacts WHERE supermarket_id = %s ORDER BY confidence DESC",
(supermarket_id,),
)
profile = fetch_one("SELECT * FROM supermarket_profiles WHERE supermarket_id = %s", (supermarket_id,))
halal = fetch_all(
"SELECT * FROM halal_certifications WHERE supermarket_id = %s ORDER BY matched_confidence DESC",
(supermarket_id,),
)
return {
"links": [dict(x) for x in links],
"contacts": [dict(x) for x in contacts],
"profile": dict(profile) if profile else None,
"halal_certifications": [dict(x) for x in halal],
}
def list_crm_options() -> dict[str, Any]:
clients = fetch_all(
"SELECT id, name, contact, email, stage, sector FROM clients ORDER BY name LIMIT 500"
)
deals = fetch_all(
"""SELECT d.id, d.title, d.value, d.stage, d.client_id, c.name AS client_name
FROM deals d LEFT JOIN clients c ON c.id = d.client_id
WHERE d.stage NOT IN ('won','lost') ORDER BY d.updated_at DESC LIMIT 200"""
)
return {"clients": [dict(c) for c in clients], "deals": [dict(d) for d in deals]}
tools-api/app/retail_enrichment.py
+158
View File
@@ -0,0 +1,158 @@
"""Enrich supermarket postcodes with PDOK geocoding + CBS demografie."""
from __future__ import annotations
import time
from typing import Any, Optional
from app.connectors import cbs, pdok
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
def _postcodes_to_enrich(limit: int = 100, offset: int = 0) -> list[str]:
rows = fetch_all(
"""
SELECT DISTINCT s.postcode
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
WHERE s.postcode IS NOT NULL
AND s.postcode <> '0000AA'
AND a.id IS NULL
ORDER BY s.postcode
LIMIT %s OFFSET %s
""",
(limit, offset),
)
return [r["postcode"] for r in rows]
def enrich_postcode(postcode: str) -> dict[str, Any]:
pc = pdok.normalize_postcode(postcode)
pdok_data = pdok.lookup_postcode(pc)
if not pdok_data:
return {"postcode": pc, "status": "pdok_not_found"}
gm_code = pdok_data.get("municipality_code")
cbs_data = cbs.fetch_gemeente_stats(gm_code) if gm_code else None
if not cbs_data:
return {"postcode": pc, "status": "cbs_not_found", "pdok": pdok_data}
religious = cbs_data.get("religious_composition") or {}
row = execute_returning(
"""
INSERT INTO area_analysis (
postcode, city, population, households, avg_household_size,
avg_income, median_income, education_level, ethnic_composition,
religious_composition, unemployment_rate, housing_type, car_ownership,
data_source, last_updated
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,NOW())
ON CONFLICT (postcode) DO UPDATE SET
city = EXCLUDED.city,
population = EXCLUDED.population,
households = EXCLUDED.households,
avg_household_size = EXCLUDED.avg_household_size,
avg_income = EXCLUDED.avg_income,
median_income = EXCLUDED.median_income,
education_level = EXCLUDED.education_level,
ethnic_composition = EXCLUDED.ethnic_composition,
religious_composition = EXCLUDED.religious_composition,
housing_type = EXCLUDED.housing_type,
car_ownership = EXCLUDED.car_ownership,
data_source = EXCLUDED.data_source,
last_updated = NOW()
RETURNING id, postcode
""",
(
pc,
pdok_data.get("city") or cbs_data.get("city"),
cbs_data.get("population"),
cbs_data.get("households"),
cbs_data.get("avg_household_size"),
cbs_data.get("avg_income"),
cbs_data.get("median_income"),
json_param(cbs_data.get("education_level")),
json_param(cbs_data.get("ethnic_composition")),
json_param(religious),
cbs_data.get("unemployment_rate"),
json_param(cbs_data.get("housing_type")),
cbs_data.get("car_ownership"),
f"cbs+pdok ({cbs_data.get('data_granularity', 'gemeente')})",
),
)
execute(
"""
UPDATE supermarkets SET
city = COALESCE(NULLIF(city, 'Onbekend'), %s),
province = COALESCE(province, %s),
latitude = COALESCE(latitude, %s),
longitude = COALESCE(longitude, %s),
last_updated = NOW()
WHERE postcode = %s AND (
city = 'Onbekend' OR province IS NULL OR latitude IS NULL OR longitude IS NULL
)
""",
(
pdok_data.get("city"),
pdok_data.get("province"),
pdok_data.get("latitude"),
pdok_data.get("longitude"),
pc,
),
)
return {
"postcode": pc,
"status": "ok",
"area_id": row["id"] if row else None,
"municipality": pdok_data.get("municipality"),
"population": cbs_data.get("population"),
"muslim_proxy_pct": religious.get("muslim_proxy_pct"),
}
def enrich_batch(limit: int = 50, offset: int = 0, delay_sec: float = 0.15) -> dict[str, Any]:
postcodes = _postcodes_to_enrich(limit, offset)
results: list[dict[str, Any]] = []
ok = failed = 0
for pc in postcodes:
try:
result = enrich_postcode(pc)
results.append(result)
if result.get("status") == "ok":
ok += 1
else:
failed += 1
except Exception as exc: # noqa: BLE001
results.append({"postcode": pc, "status": "error", "error": str(exc)})
failed += 1
time.sleep(delay_sec)
remaining = fetch_one(
"""
SELECT COUNT(DISTINCT s.postcode) AS n
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
WHERE s.postcode <> '0000AA' AND a.id IS NULL
"""
)
return {
"processed": len(postcodes),
"ok": ok,
"failed": failed,
"remaining": int((remaining or {}).get("n") or 0),
"results": results[:20],
}
def enrichment_status() -> dict[str, Any]:
total_pc = fetch_one(
"SELECT COUNT(DISTINCT postcode) AS n FROM supermarkets WHERE postcode <> '0000AA'"
)
enriched = fetch_one("SELECT COUNT(*) AS n FROM area_analysis")
providers = fetch_all(
"SELECT name, provider_type, last_fetch_at, last_status, is_active FROM data_providers ORDER BY name"
)
return {
"unique_postcodes": int((total_pc or {}).get("n") or 0),
"enriched_postcodes": int((enriched or {}).get("n") or 0),
"data_providers": [dict(p) for p in providers],
}
tools-api/app/retail_opportunities.py
+108
View File
@@ -0,0 +1,108 @@
"""Halal market opportunity scoring for retail locations."""
from __future__ import annotations
from typing import Any
from app.db import execute, fetch_all, fetch_one, json_param
def compute_all_scores(limit: int = 5000) -> dict[str, Any]:
rows = fetch_all(
"""
SELECT s.id, s.chain, s.city, s.postcode, s.halal_certified, s.has_halal_section,
s.partnership_status, a.population, a.avg_income,
(a.religious_composition->>'muslim_proxy_pct')::float AS muslim_pct,
(a.ethnic_composition->>'niet_westers_pct')::float AS niet_westers_pct
FROM supermarkets s
LEFT JOIN area_analysis a ON a.postcode = s.postcode
WHERE s.postcode <> '0000AA'
LIMIT %s
""",
(limit,),
)
computed = 0
for r in rows:
muslim = float(r.get("muslim_pct") or 0)
niet_w = float(r.get("niet_westers_pct") or 0)
pop = int(r.get("population") or 0)
income = float(r.get("avg_income") or 0)
halal_gap = 0.0
if not r.get("halal_certified") and not r.get("has_halal_section"):
halal_gap = min(100, muslim * 1.5 + niet_w * 0.5)
elif r.get("has_halal_section") and not r.get("halal_certified"):
halal_gap = min(80, muslim * 0.8)
market_potential = 0.0
if pop > 0:
market_potential += min(40, pop / 25000)
if income > 0:
market_potential += min(30, income / 1500)
market_potential += min(30, muslim * 0.4)
partnership_bonus = 15 if r.get("partnership_status") == "active" else 0
halal_opp = round(min(100, halal_gap + market_potential * 0.3), 1)
market_score = round(min(100, market_potential + partnership_bonus), 1)
factors = {
"muslim_proxy_pct": muslim,
"niet_westers_pct": niet_w,
"population": pop,
"avg_income": income,
"halal_gap": round(halal_gap, 1),
"has_halal_section": bool(r.get("has_halal_section")),
"halal_certified": bool(r.get("halal_certified")),
"partnership_status": r.get("partnership_status"),
}
execute(
"""
INSERT INTO retail_opportunity_scores (supermarket_id, halal_opportunity_score,
market_potential_score, factors, computed_at)
VALUES (%s, %s, %s, %s, NOW())
ON CONFLICT (supermarket_id) DO UPDATE SET
halal_opportunity_score = EXCLUDED.halal_opportunity_score,
market_potential_score = EXCLUDED.market_potential_score,
factors = EXCLUDED.factors,
computed_at = NOW()
""",
(r["id"], halal_opp, market_score, json_param(factors)),
)
execute(
"UPDATE supermarkets SET halal_opportunity_score = %s WHERE id = %s",
(halal_opp, r["id"]),
)
computed += 1
return {"computed": computed}
def top_opportunities(
limit: int = 50,
min_score: float = 30,
chain: str | None = None,
province: str | None = None,
) -> list[dict[str, Any]]:
clauses = ["ros.halal_opportunity_score >= %s"]
params: list[Any] = [min_score]
if chain:
clauses.append("s.chain ILIKE %s")
params.append(f"%{chain}%")
if province:
clauses.append("s.province ILIKE %s")
params.append(f"%{province}%")
where = " AND ".join(clauses)
return fetch_all(
f"""
SELECT s.id, s.name, s.chain, s.city, s.province, s.postcode,
s.partnership_status, s.halal_certified, s.has_halal_section,
ros.halal_opportunity_score, ros.market_potential_score, ros.factors,
a.population, a.avg_income,
(a.religious_composition->>'muslim_proxy_pct')::float AS muslim_proxy_pct
FROM retail_opportunity_scores ros
JOIN supermarkets s ON s.id = ros.supermarket_id
LEFT JOIN area_analysis a ON a.postcode = s.postcode
WHERE {where}
ORDER BY ros.halal_opportunity_score DESC
LIMIT %s
""",
tuple(params + [limit]),
)
tools-api/app/retail_scrapers.py
+253
View File
@@ -0,0 +1,253 @@
"""Import supermarket locations from OpenStreetMap via Overpass API."""
from __future__ import annotations
import re
import time
from typing import Any, Optional
import httpx
from app.db import execute, fetch_one, json_param
OVERPASS_URLS = [
"https://overpass-api.de/api/interpreter",
"https://overpass.kumi.systems/api/interpreter",
]
CHAIN_CONFIG: dict[str, dict[str, Any]] = {
"ah": {
"label": "Albert Heijn",
"brands": ["Albert Heijn", "Albert Heijn XL", "AH"],
"db_chain": "Albert Heijn",
},
"jumbo": {
"label": "Jumbo",
"brands": ["Jumbo"],
"db_chain": "Jumbo",
},
"plus": {
"label": "Plus",
"brands": ["Plus", "PLUS"],
"brand_regex": "Plus",
"operators": ["Plus", "Plus Retail", "Plus Supermarkt"],
"db_chain": "Plus",
},
"lidl": {
"label": "Lidl",
"brands": ["Lidl"],
"db_chain": "Lidl",
},
"aldi": {
"label": "ALDI",
"brands": ["ALDI", "Aldi"],
"db_chain": "ALDI",
},
"dirk": {
"label": "Dirk",
"brands": ["Dirk", "Dirk van den Broek"],
"db_chain": "Dirk",
},
}
POSTCODE_RE = re.compile(r"^\d{4}\s?[A-Za-z]{2}$")
def list_chains() -> list[dict[str, str]]:
return [{"key": k, "label": v["label"], "db_chain": v["db_chain"]} for k, v in CHAIN_CONFIG.items()]
def _build_overpass_query(cfg: dict[str, Any]) -> str:
brands: list[str] = cfg.get("brands", [])
brand_regex: Optional[str] = cfg.get("brand_regex")
operators: list[str] = cfg.get("operators", [])
parts: list[str] = []
for b in brands:
parts.append(f'node["shop"="supermarket"]["brand"="{b}"](area.nl);')
parts.append(f'way["shop"="supermarket"]["brand"="{b}"](area.nl);')
if brand_regex:
parts.append(f'node["shop"="supermarket"]["brand"~"{brand_regex}",i](area.nl);')
parts.append(f'way["shop"="supermarket"]["brand"~"{brand_regex}",i](area.nl);')
for op in operators:
parts.append(f'node["shop"="supermarket"]["operator"="{op}"](area.nl);')
parts.append(f'way["shop"="supermarket"]["operator"="{op}"](area.nl);')
return f'[out:json][timeout:180];area["ISO3166-1"="NL"]->.nl;({" ".join(parts)});out center tags;'
def _build_overpass_query_legacy(brands: list[str]) -> str:
return _build_overpass_query({"brands": brands})
def _fetch_overpass(query: str) -> list[dict[str, Any]]:
last_error: Optional[str] = None
for url in OVERPASS_URLS:
for attempt in range(3):
try:
with httpx.Client(timeout=200.0) as client:
resp = client.post(url, data={"data": query})
if resp.status_code == 429:
time.sleep(15 * (attempt + 1))
continue
resp.raise_for_status()
data = resp.json()
return data.get("elements", [])
except Exception as exc: # noqa: BLE001
last_error = str(exc)
time.sleep(5 * (attempt + 1))
raise RuntimeError(f"Overpass query failed: {last_error}")
def _coords(el: dict[str, Any]) -> tuple[Optional[float], Optional[float]]:
if el.get("type") == "node":
return el.get("lat"), el.get("lon")
center = el.get("center") or {}
return center.get("lat"), center.get("lon")
def _normalize_postcode(raw: Optional[str]) -> str:
if not raw:
return "0000AA"
cleaned = raw.strip().upper().replace(" ", "")
if len(cleaned) == 6 and cleaned[:4].isdigit() and cleaned[4:].isalpha():
return cleaned
return "0000AA"
def _parse_store(el: dict[str, Any], db_chain: str) -> Optional[dict[str, Any]]:
tags = el.get("tags") or {}
lat, lon = _coords(el)
if lat is None or lon is None:
return None
street = tags.get("addr:street") or tags.get("addr:place") or ""
housenumber = tags.get("addr:housenumber") or ""
address = " ".join(p for p in [street, housenumber] if p).strip()
if not address:
address = tags.get("name") or f"{db_chain} ({lat:.4f}, {lon:.4f})"
city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village") or "Onbekend"
province = tags.get("addr:province") or tags.get("is_in:state")
name = tags.get("name") or tags.get("brand") or db_chain
brand = tags.get("brand") or db_chain
store_type = None
if "XL" in brand or tags.get("shop") == "supermarket" and "xl" in name.lower():
store_type = "XL"
elif brand == "AH" or "to go" in name.lower():
store_type = "To Go"
external_id = f"osm:{el.get('type')}:{el.get('id')}"
opening_hours = tags.get("opening_hours")
return {
"external_id": external_id,
"name": name[:255],
"chain": db_chain,
"address": address,
"postcode": _normalize_postcode(tags.get("addr:postcode")),
"city": city[:100],
"province": (province or "")[:50] or None,
"latitude": lat,
"longitude": lon,
"store_type": store_type,
"phone": (tags.get("phone") or tags.get("contact:phone") or "")[:20] or None,
"website": (tags.get("website") or tags.get("contact:website") or "")[:255] or None,
"opening_hours": {"raw": opening_hours} if opening_hours else None,
"data_source": "openstreetmap",
}
def _ensure_schema() -> None:
col = fetch_one(
"""
SELECT column_name FROM information_schema.columns
WHERE table_name = 'supermarkets' AND column_name = 'external_id'
"""
)
if not col:
execute("ALTER TABLE supermarkets ADD COLUMN IF NOT EXISTS external_id VARCHAR(64)")
execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS idx_supermarkets_external_id
ON supermarkets (external_id) WHERE external_id IS NOT NULL
"""
)
def import_chain(chain_key: str) -> dict[str, Any]:
cfg = CHAIN_CONFIG.get(chain_key)
if not cfg:
raise ValueError(f"Unknown chain: {chain_key}")
_ensure_schema()
query = _build_overpass_query(cfg)
elements = _fetch_overpass(query)
parsed: list[dict[str, Any]] = []
for el in elements:
store = _parse_store(el, cfg["db_chain"])
if store:
parsed.append(store)
inserted = updated = skipped = 0
for store in parsed:
existing = fetch_one(
"SELECT id FROM supermarkets WHERE external_id = %s",
(store["external_id"],),
)
if existing:
execute(
"""
UPDATE supermarkets SET
name = %s, chain = %s, address = %s, postcode = %s, city = %s,
province = %s, latitude = %s, longitude = %s, store_type = %s,
phone = %s, website = %s, opening_hours = %s,
last_updated = NOW(), data_source = %s
WHERE external_id = %s
""",
(
store["name"], store["chain"], store["address"], store["postcode"],
store["city"], store["province"], store["latitude"], store["longitude"],
store["store_type"], store["phone"], store["website"],
json_param(store["opening_hours"]), store["data_source"], store["external_id"],
),
)
updated += 1
else:
execute(
"""
INSERT INTO supermarkets (
external_id, name, chain, address, postcode, city, province,
latitude, longitude, store_type, phone, website, opening_hours,
data_source, partnership_status
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'none')
""",
(
store["external_id"], store["name"], store["chain"], store["address"],
store["postcode"], store["city"], store["province"], store["latitude"],
store["longitude"], store["store_type"], store["phone"], store["website"],
json_param(store["opening_hours"]), store["data_source"],
),
)
inserted += 1
return {
"chain": chain_key,
"label": cfg["label"],
"fetched": len(elements),
"parsed": len(parsed),
"inserted": inserted,
"updated": updated,
"skipped": skipped,
}
def import_all_chains() -> dict[str, Any]:
results = []
for key in CHAIN_CONFIG:
try:
results.append(import_chain(key))
time.sleep(8)
except Exception as exc: # noqa: BLE001
results.append({"chain": key, "error": str(exc)})
total = fetch_one("SELECT COUNT(*) AS n FROM supermarkets WHERE data_source = 'openstreetmap'")
return {"chains": results, "total_osm_stores": int((total or {}).get("n") or 0)}
tools-api/app/rss_feeds.py
+152
View File
@@ -0,0 +1,152 @@
"""RSS feed ingestion — filtered for kant-en-klaar & supermarkt only."""
from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Optional
from urllib.request import Request, urlopen
from app.db import execute, execute_returning, fetch_all, fetch_one
USER_AGENT = "Foodlinkk-Intel/1.0"
INCLUDE_KEYWORDS = (
"kant en klaar", "kant-en-klaar", "kant&klaa", "ready meal", "ready-to-eat",
"maaltijd", "maaltijden", "supermarkt", "supermarket", "retail", "jumbo",
"albert heijn", "ah ", " plus ", "lidl", "aldi", "dirk", "halal",
"convenience", "schap", "filiaal", "foodservice", "vers", "meal",
"grocery", "food retail", "kant-en-klaar",
)
EXCLUDE_KEYWORDS = (
"voetbal", "sport", "politiek", "verkiezing", "trump", "bbc", "oorlog",
"crypto", "bitcoin", "aandelenbeurs", "beurs ", "weerbericht",
)
def _parse_date(raw: Optional[str]) -> Optional[datetime]:
if not raw:
return None
try:
return parsedate_to_datetime(raw).astimezone(timezone.utc)
except Exception:
pass
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except Exception:
return None
def _strip_html(text: str) -> str:
return re.sub(r"<[^>]+>", "", text or "").strip()[:2000]
def is_relevant(title: str, summary: Optional[str] = None) -> bool:
blob = f"{title} {summary or ''}".lower()
for bad in EXCLUDE_KEYWORDS:
if bad in blob:
return False
for good in INCLUDE_KEYWORDS:
if good in blob:
return True
return False
def _fetch_xml(url: str) -> ET.Element:
req = Request(url, headers={"User-Agent": USER_AGENT})
with urlopen(req, timeout=25) as resp:
data = resp.read()
return ET.fromstring(data)
def _skip_keyword_filter(category: Optional[str]) -> bool:
return category in ("regelgeving", "cbs", "markt")
def refresh_feed(feed_id: int) -> dict[str, Any]:
feed = fetch_one("SELECT * FROM rss_feeds WHERE id = %s AND is_active = TRUE", (feed_id,))
if not feed:
return {"error": "feed not found"}
skip_filter = _skip_keyword_filter(feed.get("category"))
root = _fetch_xml(feed["url"])
items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
inserted = skipped = 0
for item in items[:50]:
title = (item.findtext("title") or item.findtext("{http://www.w3.org/2005/Atom}title") or "").strip()
link = (item.findtext("link") or "").strip()
if not link:
link_el = item.find("{http://www.w3.org/2005/Atom}link")
if link_el is not None:
link = link_el.get("href") or ""
summary = item.findtext("description") or item.findtext("summary") or item.findtext("{http://www.w3.org/2005/Atom}summary") or ""
pub = item.findtext("pubDate") or item.findtext("published") or item.findtext("{http://www.w3.org/2005/Atom}published")
if not title or not link:
continue
clean_summary = _strip_html(summary)
cat = (feed.get("category") or "").lower()
if cat not in ("regelgeving", "cbs", "markt") and not is_relevant(title, clean_summary):
skipped += 1
continue
try:
execute_returning(
"""INSERT INTO rss_items (feed_id, title, link, summary, published_at)
VALUES (%s, %s, %s, %s, %s) RETURNING id""",
(feed_id, title[:500], link[:1000], clean_summary, _parse_date(pub)),
)
inserted += 1
except Exception:
pass
execute(
"UPDATE rss_feeds SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
(feed_id,),
)
return {"feed": feed["name"], "inserted": inserted, "skipped": skipped}
def refresh_all_feeds() -> dict[str, Any]:
feeds = fetch_all("SELECT id, name FROM rss_feeds WHERE is_active = TRUE")
results = []
for f in feeds:
try:
results.append(refresh_feed(int(f["id"])))
except Exception as exc: # noqa: BLE001
execute("UPDATE rss_feeds SET last_status = %s WHERE id = %s", (str(exc)[:32], f["id"]))
results.append({"feed": f["name"], "error": str(exc)})
return {"feeds": len(feeds), "results": results}
def list_live_feed(limit: int = 40, category: Optional[str] = None) -> list[dict[str, Any]]:
params: list[Any] = []
if category and category.lower() in ("regelgeving", "cbs", "markt"):
base = """
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
FROM rss_items i
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
WHERE f.category = %s
"""
params.append(category.lower())
else:
like_clauses = " OR ".join(
f"(i.title ILIKE %s OR COALESCE(i.summary,'') ILIKE %s)" for _ in INCLUDE_KEYWORDS[:12]
)
for kw in INCLUDE_KEYWORDS[:12]:
p = f"%{kw}%"
params.extend([p, p])
base = f"""
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
FROM rss_items i
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
WHERE ({like_clauses})
"""
if category:
base += " AND f.category = %s"
params.append(category)
base += " ORDER BY i.published_at DESC NULLS LAST, i.fetched_at DESC LIMIT %s"
params.append(limit)
rows = fetch_all(base, tuple(params))
skip_filter = category and category.lower() in ("regelgeving", "cbs", "markt")
if skip_filter:
return [dict(r) for r in rows]
return [dict(r) for r in rows if is_relevant(r.get("title") or "", r.get("summary"))]
tools-api/app/wholesaler_scrapers.py
+111
View File
@@ -0,0 +1,111 @@
"""Import wholesalers from OpenStreetMap."""
from __future__ import annotations
import json
import time
import urllib.parse
import urllib.request
from typing import Any, Optional
from app.db import execute, fetch_one
OVERPASS_URL = "https://overpass.kumi.systems/api/interpreter"
WHOLESALE_BRANDS = [
"Sligro", "Hanos", "Makro", "Bidfood", "Metro", "Van Gelder",
"De Klok", "Hoogvliet Groothandel",
]
def _fetch(query: str) -> list[dict[str, Any]]:
data = urllib.parse.urlencode({"data": query}).encode()
req = urllib.request.Request(OVERPASS_URL, data=data, method="POST")
with urllib.request.urlopen(req, timeout=300) as resp:
payload = json.loads(resp.read().decode())
return payload.get("elements", [])
def _coords(el: dict[str, Any]) -> tuple[Optional[float], Optional[float]]:
if el.get("type") == "node":
return el.get("lat"), el.get("lon")
c = el.get("center") or {}
return c.get("lat"), c.get("lon")
def _normalize_pc(raw: Optional[str]) -> str:
if not raw:
return "0000AA"
c = raw.strip().upper().replace(" ", "")
return c if len(c) >= 6 else "0000AA"
def import_wholesalers() -> dict[str, Any]:
query = (
'[out:json][timeout:120];area["ISO3166-1"="NL"]->.nl;('
'node["shop"="wholesale"](area.nl);way["shop"="wholesale"](area.nl);'
'node["shop"="cash_and_carry"](area.nl);way["shop"="cash_and_carry"](area.nl);'
'node["wholesale"](area.nl);way["wholesale"](area.nl);'
');out center tags;'
)
elements: list[dict[str, Any]] = []
try:
elements = _fetch(query)
except Exception:
# Fallback: smaller per-brand queries
for brand in WHOLESALE_BRANDS[:4]:
q = (
f'[out:json][timeout:60];area["ISO3166-1"="NL"]->.nl;('
f'node["name"~"{brand}",i](area.nl);way["name"~"{brand}",i](area.nl);'
f');out center tags;'
)
try:
elements.extend(_fetch(q))
time.sleep(2)
except Exception:
continue
inserted = updated = 0
seen: set[str] = set()
for el in elements:
tags = el.get("tags") or {}
lat, lon = _coords(el)
if lat is None:
continue
external_id = f"osm:{el.get('type')}:{el.get('id')}"
if external_id in seen:
continue
seen.add(external_id)
name = tags.get("name") or tags.get("brand") or "Groothandel"
brand = tags.get("brand") or name.split()[0]
street = tags.get("addr:street") or ""
hn = tags.get("addr:housenumber") or ""
address = f"{street} {hn}".strip() or name
city = tags.get("addr:city") or tags.get("addr:town") or "Onbekend"
existing = fetch_one("SELECT id FROM wholesalers WHERE external_id = %s", (external_id,))
if existing:
execute(
"""UPDATE wholesalers SET name=%s, address=%s, postcode=%s, city=%s, province=%s,
latitude=%s, longitude=%s, phone=%s, email=%s, website=%s, last_updated=NOW()
WHERE external_id=%s""",
(
name[:255], address, _normalize_pc(tags.get("addr:postcode")), city[:100],
(tags.get("addr:province") or "")[:50], lat, lon,
(tags.get("phone") or "")[:20], (tags.get("email") or "")[:255],
(tags.get("website") or "")[:255], external_id,
),
)
updated += 1
else:
execute(
"""INSERT INTO wholesalers (external_id, name, address, postcode, city, province,
latitude, longitude, phone, email, website, data_source, product_categories)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'openstreetmap',%s)""",
(
external_id, name[:255], address, _normalize_pc(tags.get("addr:postcode")),
city[:100], (tags.get("addr:province") or "")[:50], lat, lon,
(tags.get("phone") or "")[:20], (tags.get("email") or "")[:255],
(tags.get("website") or "")[:255], [brand],
),
)
inserted += 1
total = fetch_one("SELECT COUNT(*) AS n FROM wholesalers")
return {"fetched": len(elements), "inserted": inserted, "updated": updated, "total": int((total or {}).get("n") or 0)}
tools-api/db.py
+76
View File
@@ -0,0 +1,76 @@
from contextlib import contextmanager
from typing import Any, Optional
import psycopg2
from psycopg2 import pool
from psycopg2.extras import RealDictCursor, Json
from app.config import settings
_connection_pool: Optional[pool.SimpleConnectionPool] = None
def init_pool(minconn: int = 1, maxconn: int = 10) -> None:
global _connection_pool
if _connection_pool is None:
_connection_pool = pool.SimpleConnectionPool(
minconn,
maxconn,
dsn=settings.database_dsn,
)
def close_pool() -> None:
global _connection_pool
if _connection_pool is not None:
_connection_pool.closeall()
_connection_pool = None
@contextmanager
def get_connection():
if _connection_pool is None:
init_pool()
conn = _connection_pool.getconn()
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
finally:
_connection_pool.putconn(conn)
def fetch_all(query: str, params: Optional[tuple] = None) -> list[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
return [dict(row) for row in cur.fetchall()]
def fetch_one(query: str, params: Optional[tuple] = None) -> Optional[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
row = cur.fetchone()
return dict(row) if row else None
def execute_returning(query: str, params: Optional[tuple] = None) -> Optional[dict[str, Any]]:
with get_connection() as conn:
with conn.cursor(cursor_factory=RealDictCursor) as cur:
cur.execute(query, params)
row = cur.fetchone()
return dict(row) if row else None
def execute(query: str, params: Optional[tuple] = None) -> int:
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(query, params)
return cur.rowcount
def json_param(value: Any) -> Json:
return Json(value or {})
tools-api/patch_main.py
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""Patch tools-api main.py to register new routers."""
from pathlib import Path
p = Path("/home/aissa/foodlinkk-command-center/tools-api/app/main.py")
text = p.read_text()
if "from app.retail import router as retail_router" not in text:
text = text.replace(
"from app.middleware import log_agent_event",
"from app.middleware import log_agent_event\nfrom app.retail import router as retail_router\nfrom app.research import router as research_router\nfrom app.recommendations import router as recommendations_router\nfrom app.logging_middleware import AgentLoggingMiddleware",
)
if "app.include_router(retail_router)" not in text:
text = text.replace(
'app = FastAPI(title="Foodlinkk Tools API", version="1.0.0")',
'app = FastAPI(title="Foodlinkk Tools API", version="1.1.0")\napp.add_middleware(AgentLoggingMiddleware)\napp.include_router(retail_router)\napp.include_router(research_router)\napp.include_router(recommendations_router)',
)
p.write_text(text)
print("patched tools-api main.py")
tools-api/requirements.txt
+10
View File
@@ -0,0 +1,10 @@
fastapi==0.115.6
uvicorn[standard]==0.32.1
psycopg2-binary==2.9.9
pydantic==2.10.3
httpx==0.27.2
websockets==14.1
svgwrite
Pillow
python-barcode
reportlab