Files
foodlinkk-command-center/cockpit/app/services/analytics_data.py
T
Aissa 5e43b6b246 Ship Herman chat, Hermes KPI explorer, and intelligence analytics.
Adds floating Herman assistant with voice, Hermes technical KPI dashboard with file explorer, and a wins/kansen analytics pulse with Hermes findings hooks. Includes optional local HTTPS via Caddy for mic permissions (public certs only).
2026-08-01 17:50:45 +00:00

589 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Intelligence analytics: wins, kansen, pulse score + Hermes findings hooks."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from app.db import execute, fetch_all, fetch_one
def _safe_count(table: str, where: str = "", params: tuple = ()) -> int:
try:
clause = f" WHERE {where}" if where else ""
row = fetch_one(f"SELECT COUNT(*) AS c FROM {table}{clause}", params or None)
return int(row["c"]) if row else 0
except Exception:
return 0
def _iso_rows(rows: list) -> list:
for row in rows:
for key, val in list(row.items()):
if hasattr(val, "isoformat"):
row[key] = val.isoformat()
elif val is not None and type(val).__name__ == "Decimal":
row[key] = float(val)
return rows
def ensure_findings_table() -> None:
try:
execute(
"""
CREATE TABLE IF NOT EXISTS analytics_findings (
id SERIAL PRIMARY KEY,
kind TEXT NOT NULL DEFAULT 'insight',
title TEXT NOT NULL,
body TEXT,
score DOUBLE PRECISION,
source TEXT DEFAULT 'hermes',
href TEXT,
meta JSONB DEFAULT '{}'::jsonb,
active BOOLEAN DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
)
"""
)
except Exception:
pass
def list_hermes_findings(limit: int = 20) -> list[dict[str, Any]]:
ensure_findings_table()
try:
rows = fetch_all(
"""
SELECT id, kind, title, body, score, source, href, meta, created_at
FROM analytics_findings
WHERE active = TRUE
ORDER BY created_at DESC
LIMIT %s
""",
(limit,),
)
return _iso_rows(rows or [])
except Exception:
return []
def add_finding(
*,
kind: str,
title: str,
body: str = "",
score: float | None = None,
source: str = "hermes",
href: str | None = None,
meta: dict | None = None,
) -> dict[str, Any]:
ensure_findings_table()
import json
row = fetch_one(
"""
INSERT INTO analytics_findings (kind, title, body, score, source, href, meta)
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id, kind, title, body, score, source, href, meta, created_at
""",
(
kind or "insight",
title,
body or "",
score,
source or "hermes",
href,
json.dumps(meta or {}),
),
)
return _iso_rows([row])[0] if row else {}
def _revenue_kpis() -> dict[str, Any]:
out = {
"active_projects": 0,
"margin_month": 0.0,
"margin_year": 0.0,
"vision_text": "",
"horizon_text": "",
"tagline": "",
}
try:
row = fetch_one(
"""
SELECT COUNT(*)::int AS c,
COALESCE(SUM(margin_month),0) AS mm,
COALESCE(SUM(margin_year),0) AS my
FROM revenue_projects
WHERE COALESCE(status,'') NOT IN ('lost','cancelled','done')
"""
)
if row:
out["active_projects"] = int(row.get("c") or 0)
out["margin_month"] = float(row.get("mm") or 0)
out["margin_year"] = float(row.get("my") or 0)
except Exception:
pass
try:
g = fetch_one(
"""
SELECT vision_text, horizon_text, tagline, mid_text
FROM revenue_cockpit_goals
WHERE COALESCE(is_active, TRUE) = TRUE
ORDER BY updated_at DESC NULLS LAST
LIMIT 1
"""
)
if g:
out["vision_text"] = g.get("vision_text") or ""
out["horizon_text"] = g.get("horizon_text") or ""
out["tagline"] = g.get("tagline") or ""
out["mid_text"] = g.get("mid_text") or ""
except Exception:
pass
return out
def _build_pulse(data: dict[str, Any]) -> dict[str, Any]:
k = data.get("kpis") or {}
partners = {r.get("status"): int(r.get("cnt") or 0) for r in (data.get("partnership_breakdown") or [])}
active_p = int(partners.get("active") or k.get("crm_partnerships") or 0)
none_p = int(partners.get("none") or 0)
proposal_p = int(partners.get("proposal") or 0)
stores = int(k.get("supermarkets") or 0)
cover_pct = round((active_p / stores) * 100, 2) if stores else 0.0
gap = max(stores - active_p - proposal_p, 0)
wins: list[dict[str, Any]] = []
opportunities: list[dict[str, Any]] = []
attention: list[dict[str, Any]] = []
if stores >= 1000:
wins.append({
"id": "win-coverage",
"title": "Retail footprint is massief",
"body": f"{stores:,} supermarkten in de database — sterke basis voor targeting.".replace(",", "."),
"score": 92,
"metric": f"{stores} stores",
"href": "/retail",
})
if active_p >= 10:
wins.append({
"id": "win-crm",
"title": "Actieve partnerships draaien",
"body": f"{active_p} winkels met actieve CRM-status. Voorstellen: {proposal_p}.",
"score": 78,
"metric": f"{active_p} actief",
"href": "/retail",
})
if int(k.get("rss_items") or 0) >= 200:
wins.append({
"id": "win-rss",
"title": "Markt-signalen stromen binnen",
"body": f"{int(k['rss_items'])} RSS-items gevoed — concurrentie & trends blijven zichtbaar.",
"score": 74,
"metric": f"{int(k['rss_items'])} items",
"href": "/marketing",
})
if int(k.get("promo_campaigns") or 0) >= 10:
wins.append({
"id": "win-promo",
"title": "Promo-intelligence actief",
"body": f"{int(k['promo_campaigns'])} actieve folders/campagnes gemonitord.",
"score": 70,
"metric": f"{int(k['promo_campaigns'])} campagnes",
"href": "/marketing",
})
if float(k.get("margin_month") or 0) > 0:
wins.append({
"id": "win-margin",
"title": "Revenue engine levert marge",
"body": f"Cockpit-marge ≈ €{float(k['margin_month']):,.0f}/maand over {int(k.get('active_projects') or 0)} deals.".replace(",", "."),
"score": 88,
"metric": f"€{float(k['margin_month']):,.0f}".replace(",", "."),
"href": "/revenue-cockpit",
})
if int(k.get("wholesalers") or 0) >= 100:
wins.append({
"id": "win-wholesale",
"title": "Groothandel-netwerk in kaart",
"body": f"{int(k['wholesalers'])} groothandels beschikbaar voor distributie-routes.",
"score": 68,
"metric": f"{int(k['wholesalers'])} GH",
"href": "/retail",
})
if int(k.get("agent_events") or 0) >= 500:
wins.append({
"id": "win-agents",
"title": "Agent-mesh is productief",
"body": f"{int(k['agent_events'])} agent-events gelogd — Herman & crew blijven draaien.",
"score": 66,
"metric": f"{int(k['agent_events'])} events",
"href": "/agents",
})
if gap >= 100:
opportunities.append({
"id": "opp-white-space",
"title": "Enorm witruimte in retail CRM",
"body": f"{gap:,} winkels zonder partnership. Focus eerst op top-halal scores & grote ketens.".replace(",", "."),
"score": min(98, 40 + int(gap / 100)),
"metric": f"{gap} open",
"href": "/retail",
"tag": "distributie",
})
if cover_pct < 5:
opportunities.append({
"id": "opp-cover",
"title": "Partnership-dekking is nog dun",
"body": f"Slechts {cover_pct}% van de winkels is ‘active’. Doelbeeld Cucina vraagt schaal.",
"score": 90,
"metric": f"{cover_pct}% dekking",
"href": "/retail",
"tag": "groei",
})
if int(k.get("clients_active") or 0) == 0 and int(k.get("clients_total") or 0) > 0:
opportunities.append({
"id": "opp-clients",
"title": "Klanten in intake → nog niet actief",
"body": f"{int(k['clients_total'])} klanten in CRM, 0 gemarkeerd als active. Conversie = snelle win.",
"score": 82,
"metric": "0 active",
"href": "/clients",
"tag": "crm",
})
if float(k.get("pipeline_eur") or 0) <= 0 and int(k.get("deals_total") or 0) > 0:
opportunities.append({
"id": "opp-pipeline",
"title": "Deals zonder pipeline-waarde",
"body": "Er staan deals, maar €0 open pipeline. Waardes & stages aanscherpen geeft sturing.",
"score": 76,
"metric": "€0 pipeline",
"href": "/deals",
"tag": "sales",
})
top_opp = data.get("top_opportunities") or []
if top_opp:
best = top_opp[0]
opportunities.append({
"id": "opp-halal-top",
"title": f"Halal-kans: {best.get('chain') or '?'} · {best.get('city') or '?'}",
"body": f"Score {round(float(best.get('halal_opportunity_score') or 0), 1)}/100 — sterkste retail-signaal nu.",
"score": round(float(best.get("halal_opportunity_score") or 0)),
"metric": f"{round(float(best.get('halal_opportunity_score') or 0), 1)}",
"href": "/retail",
"tag": "halal",
})
# cluster cities
cities: dict[str, int] = {}
for o in top_opp:
cities[o.get("city") or "?"] = cities.get(o.get("city") or "?", 0) + 1
top_city = sorted(cities.items(), key=lambda x: -x[1])[0]
if top_city[1] >= 2:
opportunities.append({
"id": "opp-city-cluster",
"title": f"Cluster-kans in {top_city[0]}",
"body": f"{top_city[1]} top-scores in dezelfde stad — bundel outreach / bezorgroute.",
"score": 72,
"metric": top_city[0],
"href": "/retail",
"tag": "geo",
})
if int(k.get("contacts_wholesaler") or 0) == 0 and int(k.get("wholesalers") or 0) > 0:
opportunities.append({
"id": "opp-gh-contacts",
"title": "Groothandel-contacten ontbreken",
"body": "550+ groothandels, 0 contactpersonen. Relaties = hefboom voor listingen.",
"score": 69,
"metric": "0 GH-contacten",
"href": "/retail",
"tag": "netwerk",
})
if int(k.get("pending_approvals") or 0) > 0:
attention.append({
"id": "att-approvals",
"title": "Goedkeuringen wachten",
"body": f"{int(k['pending_approvals'])} agent-items op needs_approval.",
"score": 55,
"metric": str(int(k["pending_approvals"])),
"href": "/agents",
})
if int(k.get("rss_bookmarks") or 0) == 0 and int(k.get("rss_items") or 0) > 100:
attention.append({
"id": "att-bookmarks",
"title": "Geen RSS-bookmarks",
"body": "Veel signalen, weinig vastgepind. Markeer wat telt voor de weekbriefing.",
"score": 48,
"metric": "0 bookmarks",
"href": "/marketing",
})
# Pulse score 0-100
score = 35
score += min(18, active_p) # up to +18
score += 12 if stores >= 2000 else (6 if stores >= 500 else 0)
score += 8 if int(k.get("rss_items") or 0) >= 500 else 0
score += 8 if float(k.get("margin_month") or 0) > 100000 else (4 if float(k.get("margin_month") or 0) > 0 else 0)
score += 6 if int(k.get("promo_campaigns") or 0) >= 20 else 0
score += 5 if int(k.get("nas_docs") or 0) >= 40 else 0
if cover_pct < 1:
score -= 12
elif cover_pct < 3:
score -= 6
if int(k.get("pending_approvals") or 0) >= 5:
score -= 5
if int(k.get("clients_active") or 0) == 0 and int(k.get("clients_total") or 0) > 0:
score -= 4
score = max(0, min(100, score))
if score >= 75:
label, tone = "Sterk momentum", "good"
elif score >= 55:
label, tone = "Stabiel · kansen open", "mid"
else:
label, tone = "Opbouw-fase · focus nodig", "low"
wins.sort(key=lambda x: -int(x.get("score") or 0))
opportunities.sort(key=lambda x: -int(x.get("score") or 0))
return {
"score": score,
"label": label,
"tone": tone,
"cover_pct": cover_pct,
"white_space": gap,
"wins": wins[:8],
"opportunities": opportunities[:8],
"attention": attention[:6],
"headline": "Wat goed gaat vs waar de kansen liggen",
"sub": "Live uit CRM, retail, revenue & agent-mesh — Hermes vult diepere bevindingen aan.",
}
def collect_analytics(filters: Optional[dict[str, Any]] = None) -> dict[str, Any]:
f = filters or {}
chain = f.get("chain") or None
province = f.get("province") or None
stage = f.get("stage") or None
agent = f.get("agent") or None
days = int(f.get("days") or 90)
data: dict[str, Any] = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"filters": f,
}
rev = _revenue_kpis()
data["kpis"] = {
"clients_total": _safe_count("clients"),
"clients_active": _safe_count("clients", "stage = 'active'"),
"deals_total": _safe_count("deals"),
"pipeline_eur": float(
(fetch_one("SELECT COALESCE(SUM(value),0) AS t FROM deals WHERE stage NOT IN ('won','lost')") or {}).get("t", 0)
or 0
),
"supermarkets": _safe_count("supermarkets"),
"wholesalers": _safe_count("wholesalers"),
"crm_partnerships": _safe_count("supermarkets", "partnership_status = 'active'"),
"rss_items": _safe_count("rss_items"),
"rss_bookmarks": _safe_count("rss_bookmarks"),
"agent_events": _safe_count("agent_events"),
"pending_approvals": _safe_count("agent_events", "status = 'needs_approval'"),
"promo_campaigns": _safe_count("promo_campaigns", "status = 'active'"),
"contacts_supermarket": _safe_count("supermarket_contacts"),
"contacts_wholesaler": _safe_count("wholesaler_contacts"),
"nas_docs": _safe_count("document_analytics"),
"active_projects": rev.get("active_projects") or 0,
"margin_month": rev.get("margin_month") or 0,
"margin_year": rev.get("margin_year") or 0,
}
data["goals"] = {
"vision_text": rev.get("vision_text") or "",
"horizon_text": rev.get("horizon_text") or "",
"tagline": rev.get("tagline") or "",
"mid_text": rev.get("mid_text") or "",
}
try:
data["clients_by_stage"] = fetch_all(
"SELECT stage, COUNT(*) AS cnt FROM clients GROUP BY stage ORDER BY cnt DESC"
)
except Exception:
data["clients_by_stage"] = []
try:
data["deals_by_stage"] = fetch_all(
"SELECT stage, COUNT(*) AS cnt, COALESCE(SUM(value),0) AS total FROM deals GROUP BY stage ORDER BY cnt DESC"
)
except Exception:
data["deals_by_stage"] = []
try:
data["events_by_agent"] = fetch_all(
"""SELECT agent_name, COUNT(*) AS cnt FROM agent_events
WHERE created_at >= NOW() - make_interval(days => %s)
GROUP BY agent_name ORDER BY cnt DESC LIMIT 20""",
(days,),
)
except Exception:
data["events_by_agent"] = []
store_where, store_params = [], []
if chain:
store_where.append("chain = %s")
store_params.append(chain)
if province:
store_where.append("province = %s")
store_params.append(province)
sw = (" WHERE " + " AND ".join(store_where)) if store_where else ""
try:
data["supermarkets_by_chain"] = fetch_all(
f"SELECT chain, COUNT(*) AS cnt FROM supermarkets{sw} GROUP BY chain ORDER BY cnt DESC LIMIT 15",
tuple(store_params) if store_params else None,
)
except Exception:
data["supermarkets_by_chain"] = []
try:
if store_where:
data["supermarkets_by_province"] = fetch_all(
f"SELECT province, COUNT(*) AS cnt FROM supermarkets{sw} AND province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12",
tuple(store_params),
)
else:
data["supermarkets_by_province"] = fetch_all(
"SELECT province, COUNT(*) AS cnt FROM supermarkets WHERE province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12"
)
except Exception:
data["supermarkets_by_province"] = []
try:
data["partnership_breakdown"] = fetch_all(
"SELECT COALESCE(partnership_status,'none') AS status, COUNT(*) AS cnt FROM supermarkets GROUP BY partnership_status ORDER BY cnt DESC"
)
except Exception:
data["partnership_breakdown"] = []
try:
data["wholesalers_by_province"] = fetch_all(
"SELECT province, COUNT(*) AS cnt FROM wholesalers WHERE province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12"
)
except Exception:
data["wholesalers_by_province"] = []
try:
data["rss_by_category"] = fetch_all(
"""SELECT f.category, COUNT(i.id) AS cnt FROM rss_items i
JOIN rss_feeds f ON f.id = i.feed_id GROUP BY f.category ORDER BY cnt DESC"""
)
except Exception:
data["rss_by_category"] = []
try:
data["events_timeline"] = fetch_all(
"""SELECT DATE(created_at) AS day, COUNT(*) AS cnt FROM agent_events
WHERE created_at >= NOW() - make_interval(days => %s)
GROUP BY DATE(created_at) ORDER BY day ASC""",
(days,),
)
except Exception:
data["events_timeline"] = []
try:
data["milestones_by_status"] = fetch_all(
"SELECT status, COUNT(*) AS cnt FROM sales_milestones GROUP BY status ORDER BY cnt DESC"
)
except Exception:
data["milestones_by_status"] = []
try:
data["top_opportunities"] = fetch_all(
"""SELECT s.chain, s.city, ros.halal_opportunity_score
FROM retail_opportunity_scores ros JOIN supermarkets s ON s.id = ros.supermarket_id
ORDER BY ros.halal_opportunity_score DESC LIMIT 12"""
)
except Exception:
data["top_opportunities"] = []
try:
data["sentiment_distribution"] = fetch_all(
"SELECT sentiment_label, COUNT(*) AS cnt FROM document_analytics GROUP BY sentiment_label"
)
except Exception:
data["sentiment_distribution"] = []
try:
data["top_words"] = fetch_all(
"""SELECT lemma, SUM(count) AS total FROM document_word_counts
WHERE NOT is_stopword GROUP BY lemma ORDER BY total DESC LIMIT 15"""
)
except Exception:
data["top_words"] = []
try:
data["promo_by_chain"] = fetch_all(
"SELECT chain, COUNT(*) AS cnt FROM promo_campaigns WHERE status = 'active' GROUP BY chain ORDER BY cnt DESC"
)
except Exception:
data["promo_by_chain"] = []
deal_where = ""
deal_params: tuple = ()
if stage:
deal_where = " WHERE stage = %s"
deal_params = (stage,)
try:
data["recent_deals"] = fetch_all(
f"SELECT title, stage, value, updated_at FROM deals{deal_where} ORDER BY updated_at DESC LIMIT 10",
deal_params or None,
)
except Exception:
data["recent_deals"] = []
try:
if agent:
data["recent_events"] = fetch_all(
"""SELECT agent_name, event_type, title, status, created_at FROM agent_events
WHERE created_at >= NOW() - make_interval(days => %s) AND agent_name = %s
ORDER BY created_at DESC LIMIT 25""",
(days, agent),
)
else:
data["recent_events"] = fetch_all(
"""SELECT agent_name, event_type, title, status, created_at FROM agent_events
WHERE created_at >= NOW() - make_interval(days => %s)
ORDER BY created_at DESC LIMIT 25""",
(days,),
)
except Exception:
data["recent_events"] = []
try:
data["filter_meta"] = {
"chains": fetch_all("SELECT DISTINCT chain FROM supermarkets WHERE chain IS NOT NULL ORDER BY chain"),
"provinces": fetch_all("SELECT DISTINCT province FROM supermarkets WHERE province IS NOT NULL ORDER BY province"),
"client_stages": fetch_all("SELECT DISTINCT stage FROM clients ORDER BY stage"),
"deal_stages": fetch_all("SELECT DISTINCT stage FROM deals ORDER BY stage"),
"agents": fetch_all("SELECT DISTINCT agent_name FROM agent_events ORDER BY agent_name"),
}
except Exception:
data["filter_meta"] = {}
for key in (
"clients_by_stage", "deals_by_stage", "events_by_agent", "supermarkets_by_chain",
"supermarkets_by_province", "partnership_breakdown", "wholesalers_by_province",
"rss_by_category", "events_timeline", "milestones_by_status", "top_opportunities",
"sentiment_distribution", "top_words", "promo_by_chain", "recent_deals", "recent_events",
):
if isinstance(data.get(key), list):
data[key] = _iso_rows(data[key])
data["pulse"] = _build_pulse(data)
data["hermes_findings"] = list_hermes_findings(20)
return data