foodlinkk-command-center/cockpit/app/services/analytics_data.py

"""Intelligence analytics: wins, kansen, pulse score + Hermes findings hooks."""
from __future__ import annotations

from datetime import datetime, timezone
from typing import Any, Optional

from app.db import execute, fetch_all, fetch_one


def _safe_count(table: str, where: str = "", params: tuple = ()) -> int:
    try:
        clause = f" WHERE {where}" if where else ""
        row = fetch_one(f"SELECT COUNT(*) AS c FROM {table}{clause}", params or None)
        return int(row["c"]) if row else 0
    except Exception:
        return 0


def _iso_rows(rows: list) -> list:
    for row in rows:
        for key, val in list(row.items()):
            if hasattr(val, "isoformat"):
                row[key] = val.isoformat()
            elif val is not None and type(val).__name__ == "Decimal":
                row[key] = float(val)
    return rows


def ensure_findings_table() -> None:
    try:
        execute(
            """
            CREATE TABLE IF NOT EXISTS analytics_findings (
                id SERIAL PRIMARY KEY,
                kind TEXT NOT NULL DEFAULT 'insight',
                title TEXT NOT NULL,
                body TEXT,
                score DOUBLE PRECISION,
                source TEXT DEFAULT 'hermes',
                href TEXT,
                meta JSONB DEFAULT '{}'::jsonb,
                active BOOLEAN DEFAULT TRUE,
                created_at TIMESTAMPTZ DEFAULT NOW(),
                updated_at TIMESTAMPTZ DEFAULT NOW()
            )
            """
        )
    except Exception:
        pass


def list_hermes_findings(limit: int = 20) -> list[dict[str, Any]]:
    ensure_findings_table()
    try:
        rows = fetch_all(
            """
            SELECT id, kind, title, body, score, source, href, meta, created_at
            FROM analytics_findings
            WHERE active = TRUE
            ORDER BY created_at DESC
            LIMIT %s
            """,
            (limit,),
        )
        return _iso_rows(rows or [])
    except Exception:
        return []


def add_finding(
    *,
    kind: str,
    title: str,
    body: str = "",
    score: float | None = None,
    source: str = "hermes",
    href: str | None = None,
    meta: dict | None = None,
) -> dict[str, Any]:
    ensure_findings_table()
    import json

    row = fetch_one(
        """
        INSERT INTO analytics_findings (kind, title, body, score, source, href, meta)
        VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)
        RETURNING id, kind, title, body, score, source, href, meta, created_at
        """,
        (
            kind or "insight",
            title,
            body or "",
            score,
            source or "hermes",
            href,
            json.dumps(meta or {}),
        ),
    )
    return _iso_rows([row])[0] if row else {}


def _revenue_kpis() -> dict[str, Any]:
    out = {
        "active_projects": 0,
        "margin_month": 0.0,
        "margin_year": 0.0,
        "vision_text": "",
        "horizon_text": "",
        "tagline": "",
    }
    try:
        row = fetch_one(
            """
            SELECT COUNT(*)::int AS c,
                   COALESCE(SUM(margin_month),0) AS mm,
                   COALESCE(SUM(margin_year),0) AS my
            FROM revenue_projects
            WHERE COALESCE(status,'') NOT IN ('lost','cancelled','done')
            """
        )
        if row:
            out["active_projects"] = int(row.get("c") or 0)
            out["margin_month"] = float(row.get("mm") or 0)
            out["margin_year"] = float(row.get("my") or 0)
    except Exception:
        pass
    try:
        g = fetch_one(
            """
            SELECT vision_text, horizon_text, tagline, mid_text
            FROM revenue_cockpit_goals
            WHERE COALESCE(is_active, TRUE) = TRUE
            ORDER BY updated_at DESC NULLS LAST
            LIMIT 1
            """
        )
        if g:
            out["vision_text"] = g.get("vision_text") or ""
            out["horizon_text"] = g.get("horizon_text") or ""
            out["tagline"] = g.get("tagline") or ""
            out["mid_text"] = g.get("mid_text") or ""
    except Exception:
        pass
    return out


def _build_pulse(data: dict[str, Any]) -> dict[str, Any]:
    k = data.get("kpis") or {}
    partners = {r.get("status"): int(r.get("cnt") or 0) for r in (data.get("partnership_breakdown") or [])}
    active_p = int(partners.get("active") or k.get("crm_partnerships") or 0)
    none_p = int(partners.get("none") or 0)
    proposal_p = int(partners.get("proposal") or 0)
    stores = int(k.get("supermarkets") or 0)
    cover_pct = round((active_p / stores) * 100, 2) if stores else 0.0
    gap = max(stores - active_p - proposal_p, 0)

    wins: list[dict[str, Any]] = []
    opportunities: list[dict[str, Any]] = []
    attention: list[dict[str, Any]] = []

    if stores >= 1000:
        wins.append({
            "id": "win-coverage",
            "title": "Retail footprint is massief",
            "body": f"{stores:,} supermarkten in de database — sterke basis voor targeting.".replace(",", "."),
            "score": 92,
            "metric": f"{stores} stores",
            "href": "/retail",
        })
    if active_p >= 10:
        wins.append({
            "id": "win-crm",
            "title": "Actieve partnerships draaien",
            "body": f"{active_p} winkels met actieve CRM-status. Voorstellen: {proposal_p}.",
            "score": 78,
            "metric": f"{active_p} actief",
            "href": "/retail",
        })
    if int(k.get("rss_items") or 0) >= 200:
        wins.append({
            "id": "win-rss",
            "title": "Markt-signalen stromen binnen",
            "body": f"{int(k['rss_items'])} RSS-items gevoed — concurrentie & trends blijven zichtbaar.",
            "score": 74,
            "metric": f"{int(k['rss_items'])} items",
            "href": "/marketing",
        })
    if int(k.get("promo_campaigns") or 0) >= 10:
        wins.append({
            "id": "win-promo",
            "title": "Promo-intelligence actief",
            "body": f"{int(k['promo_campaigns'])} actieve folders/campagnes gemonitord.",
            "score": 70,
            "metric": f"{int(k['promo_campaigns'])} campagnes",
            "href": "/marketing",
        })
    if float(k.get("margin_month") or 0) > 0:
        wins.append({
            "id": "win-margin",
            "title": "Revenue engine levert marge",
            "body": f"Cockpit-marge ≈ €{float(k['margin_month']):,.0f}/maand over {int(k.get('active_projects') or 0)} deals.".replace(",", "."),
            "score": 88,
            "metric": f"€{float(k['margin_month']):,.0f}".replace(",", "."),
            "href": "/revenue-cockpit",
        })
    if int(k.get("wholesalers") or 0) >= 100:
        wins.append({
            "id": "win-wholesale",
            "title": "Groothandel-netwerk in kaart",
            "body": f"{int(k['wholesalers'])} groothandels beschikbaar voor distributie-routes.",
            "score": 68,
            "metric": f"{int(k['wholesalers'])} GH",
            "href": "/retail",
        })
    if int(k.get("agent_events") or 0) >= 500:
        wins.append({
            "id": "win-agents",
            "title": "Agent-mesh is productief",
            "body": f"{int(k['agent_events'])} agent-events gelogd — Herman & crew blijven draaien.",
            "score": 66,
            "metric": f"{int(k['agent_events'])} events",
            "href": "/agents",
        })

    if gap >= 100:
        opportunities.append({
            "id": "opp-white-space",
            "title": "Enorm witruimte in retail CRM",
            "body": f"{gap:,} winkels zonder partnership. Focus eerst op top-halal scores & grote ketens.".replace(",", "."),
            "score": min(98, 40 + int(gap / 100)),
            "metric": f"{gap} open",
            "href": "/retail",
            "tag": "distributie",
        })
    if cover_pct < 5:
        opportunities.append({
            "id": "opp-cover",
            "title": "Partnership-dekking is nog dun",
            "body": f"Slechts {cover_pct}% van de winkels is ‘active’. Doelbeeld Cucina vraagt schaal.",
            "score": 90,
            "metric": f"{cover_pct}% dekking",
            "href": "/retail",
            "tag": "groei",
        })
    if int(k.get("clients_active") or 0) == 0 and int(k.get("clients_total") or 0) > 0:
        opportunities.append({
            "id": "opp-clients",
            "title": "Klanten in intake → nog niet actief",
            "body": f"{int(k['clients_total'])} klanten in CRM, 0 gemarkeerd als active. Conversie = snelle win.",
            "score": 82,
            "metric": "0 active",
            "href": "/clients",
            "tag": "crm",
        })
    if float(k.get("pipeline_eur") or 0) <= 0 and int(k.get("deals_total") or 0) > 0:
        opportunities.append({
            "id": "opp-pipeline",
            "title": "Deals zonder pipeline-waarde",
            "body": "Er staan deals, maar €0 open pipeline. Waardes & stages aanscherpen geeft sturing.",
            "score": 76,
            "metric": "€0 pipeline",
            "href": "/deals",
            "tag": "sales",
        })

    top_opp = data.get("top_opportunities") or []
    if top_opp:
        best = top_opp[0]
        opportunities.append({
            "id": "opp-halal-top",
            "title": f"Halal-kans: {best.get('chain') or '?'} · {best.get('city') or '?'}",
            "body": f"Score {round(float(best.get('halal_opportunity_score') or 0), 1)}/100 — sterkste retail-signaal nu.",
            "score": round(float(best.get("halal_opportunity_score") or 0)),
            "metric": f"{round(float(best.get('halal_opportunity_score') or 0), 1)}",
            "href": "/retail",
            "tag": "halal",
        })
        # cluster cities
        cities: dict[str, int] = {}
        for o in top_opp:
            cities[o.get("city") or "?"] = cities.get(o.get("city") or "?", 0) + 1
        top_city = sorted(cities.items(), key=lambda x: -x[1])[0]
        if top_city[1] >= 2:
            opportunities.append({
                "id": "opp-city-cluster",
                "title": f"Cluster-kans in {top_city[0]}",
                "body": f"{top_city[1]} top-scores in dezelfde stad — bundel outreach / bezorgroute.",
                "score": 72,
                "metric": top_city[0],
                "href": "/retail",
                "tag": "geo",
            })

    if int(k.get("contacts_wholesaler") or 0) == 0 and int(k.get("wholesalers") or 0) > 0:
        opportunities.append({
            "id": "opp-gh-contacts",
            "title": "Groothandel-contacten ontbreken",
            "body": "550+ groothandels, 0 contactpersonen. Relaties = hefboom voor listingen.",
            "score": 69,
            "metric": "0 GH-contacten",
            "href": "/retail",
            "tag": "netwerk",
        })

    if int(k.get("pending_approvals") or 0) > 0:
        attention.append({
            "id": "att-approvals",
            "title": "Goedkeuringen wachten",
            "body": f"{int(k['pending_approvals'])} agent-items op needs_approval.",
            "score": 55,
            "metric": str(int(k["pending_approvals"])),
            "href": "/agents",
        })
    if int(k.get("rss_bookmarks") or 0) == 0 and int(k.get("rss_items") or 0) > 100:
        attention.append({
            "id": "att-bookmarks",
            "title": "Geen RSS-bookmarks",
            "body": "Veel signalen, weinig vastgepind. Markeer wat telt voor de weekbriefing.",
            "score": 48,
            "metric": "0 bookmarks",
            "href": "/marketing",
        })

    # Pulse score 0-100
    score = 35
    score += min(18, active_p)  # up to +18
    score += 12 if stores >= 2000 else (6 if stores >= 500 else 0)
    score += 8 if int(k.get("rss_items") or 0) >= 500 else 0
    score += 8 if float(k.get("margin_month") or 0) > 100000 else (4 if float(k.get("margin_month") or 0) > 0 else 0)
    score += 6 if int(k.get("promo_campaigns") or 0) >= 20 else 0
    score += 5 if int(k.get("nas_docs") or 0) >= 40 else 0
    if cover_pct < 1:
        score -= 12
    elif cover_pct < 3:
        score -= 6
    if int(k.get("pending_approvals") or 0) >= 5:
        score -= 5
    if int(k.get("clients_active") or 0) == 0 and int(k.get("clients_total") or 0) > 0:
        score -= 4
    score = max(0, min(100, score))

    if score >= 75:
        label, tone = "Sterk momentum", "good"
    elif score >= 55:
        label, tone = "Stabiel · kansen open", "mid"
    else:
        label, tone = "Opbouw-fase · focus nodig", "low"

    wins.sort(key=lambda x: -int(x.get("score") or 0))
    opportunities.sort(key=lambda x: -int(x.get("score") or 0))

    return {
        "score": score,
        "label": label,
        "tone": tone,
        "cover_pct": cover_pct,
        "white_space": gap,
        "wins": wins[:8],
        "opportunities": opportunities[:8],
        "attention": attention[:6],
        "headline": "Wat goed gaat vs waar de kansen liggen",
        "sub": "Live uit CRM, retail, revenue & agent-mesh — Hermes vult diepere bevindingen aan.",
    }


def collect_analytics(filters: Optional[dict[str, Any]] = None) -> dict[str, Any]:
    f = filters or {}
    chain = f.get("chain") or None
    province = f.get("province") or None
    stage = f.get("stage") or None
    agent = f.get("agent") or None
    days = int(f.get("days") or 90)

    data: dict[str, Any] = {
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "filters": f,
    }

    rev = _revenue_kpis()
    data["kpis"] = {
        "clients_total": _safe_count("clients"),
        "clients_active": _safe_count("clients", "stage = 'active'"),
        "deals_total": _safe_count("deals"),
        "pipeline_eur": float(
            (fetch_one("SELECT COALESCE(SUM(value),0) AS t FROM deals WHERE stage NOT IN ('won','lost')") or {}).get("t", 0)
            or 0
        ),
        "supermarkets": _safe_count("supermarkets"),
        "wholesalers": _safe_count("wholesalers"),
        "crm_partnerships": _safe_count("supermarkets", "partnership_status = 'active'"),
        "rss_items": _safe_count("rss_items"),
        "rss_bookmarks": _safe_count("rss_bookmarks"),
        "agent_events": _safe_count("agent_events"),
        "pending_approvals": _safe_count("agent_events", "status = 'needs_approval'"),
        "promo_campaigns": _safe_count("promo_campaigns", "status = 'active'"),
        "contacts_supermarket": _safe_count("supermarket_contacts"),
        "contacts_wholesaler": _safe_count("wholesaler_contacts"),
        "nas_docs": _safe_count("document_analytics"),
        "active_projects": rev.get("active_projects") or 0,
        "margin_month": rev.get("margin_month") or 0,
        "margin_year": rev.get("margin_year") or 0,
    }
    data["goals"] = {
        "vision_text": rev.get("vision_text") or "",
        "horizon_text": rev.get("horizon_text") or "",
        "tagline": rev.get("tagline") or "",
        "mid_text": rev.get("mid_text") or "",
    }

    try:
        data["clients_by_stage"] = fetch_all(
            "SELECT stage, COUNT(*) AS cnt FROM clients GROUP BY stage ORDER BY cnt DESC"
        )
    except Exception:
        data["clients_by_stage"] = []

    try:
        data["deals_by_stage"] = fetch_all(
            "SELECT stage, COUNT(*) AS cnt, COALESCE(SUM(value),0) AS total FROM deals GROUP BY stage ORDER BY cnt DESC"
        )
    except Exception:
        data["deals_by_stage"] = []

    try:
        data["events_by_agent"] = fetch_all(
            """SELECT agent_name, COUNT(*) AS cnt FROM agent_events
               WHERE created_at >= NOW() - make_interval(days => %s)
               GROUP BY agent_name ORDER BY cnt DESC LIMIT 20""",
            (days,),
        )
    except Exception:
        data["events_by_agent"] = []

    store_where, store_params = [], []
    if chain:
        store_where.append("chain = %s")
        store_params.append(chain)
    if province:
        store_where.append("province = %s")
        store_params.append(province)
    sw = (" WHERE " + " AND ".join(store_where)) if store_where else ""

    try:
        data["supermarkets_by_chain"] = fetch_all(
            f"SELECT chain, COUNT(*) AS cnt FROM supermarkets{sw} GROUP BY chain ORDER BY cnt DESC LIMIT 15",
            tuple(store_params) if store_params else None,
        )
    except Exception:
        data["supermarkets_by_chain"] = []

    try:
        if store_where:
            data["supermarkets_by_province"] = fetch_all(
                f"SELECT province, COUNT(*) AS cnt FROM supermarkets{sw} AND province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12",
                tuple(store_params),
            )
        else:
            data["supermarkets_by_province"] = fetch_all(
                "SELECT province, COUNT(*) AS cnt FROM supermarkets WHERE province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12"
            )
    except Exception:
        data["supermarkets_by_province"] = []

    try:
        data["partnership_breakdown"] = fetch_all(
            "SELECT COALESCE(partnership_status,'none') AS status, COUNT(*) AS cnt FROM supermarkets GROUP BY partnership_status ORDER BY cnt DESC"
        )
    except Exception:
        data["partnership_breakdown"] = []

    try:
        data["wholesalers_by_province"] = fetch_all(
            "SELECT province, COUNT(*) AS cnt FROM wholesalers WHERE province IS NOT NULL GROUP BY province ORDER BY cnt DESC LIMIT 12"
        )
    except Exception:
        data["wholesalers_by_province"] = []

    try:
        data["rss_by_category"] = fetch_all(
            """SELECT f.category, COUNT(i.id) AS cnt FROM rss_items i
               JOIN rss_feeds f ON f.id = i.feed_id GROUP BY f.category ORDER BY cnt DESC"""
        )
    except Exception:
        data["rss_by_category"] = []

    try:
        data["events_timeline"] = fetch_all(
            """SELECT DATE(created_at) AS day, COUNT(*) AS cnt FROM agent_events
               WHERE created_at >= NOW() - make_interval(days => %s)
               GROUP BY DATE(created_at) ORDER BY day ASC""",
            (days,),
        )
    except Exception:
        data["events_timeline"] = []

    try:
        data["milestones_by_status"] = fetch_all(
            "SELECT status, COUNT(*) AS cnt FROM sales_milestones GROUP BY status ORDER BY cnt DESC"
        )
    except Exception:
        data["milestones_by_status"] = []

    try:
        data["top_opportunities"] = fetch_all(
            """SELECT s.chain, s.city, ros.halal_opportunity_score
               FROM retail_opportunity_scores ros JOIN supermarkets s ON s.id = ros.supermarket_id
               ORDER BY ros.halal_opportunity_score DESC LIMIT 12"""
        )
    except Exception:
        data["top_opportunities"] = []

    try:
        data["sentiment_distribution"] = fetch_all(
            "SELECT sentiment_label, COUNT(*) AS cnt FROM document_analytics GROUP BY sentiment_label"
        )
    except Exception:
        data["sentiment_distribution"] = []

    try:
        data["top_words"] = fetch_all(
            """SELECT lemma, SUM(count) AS total FROM document_word_counts
               WHERE NOT is_stopword GROUP BY lemma ORDER BY total DESC LIMIT 15"""
        )
    except Exception:
        data["top_words"] = []

    try:
        data["promo_by_chain"] = fetch_all(
            "SELECT chain, COUNT(*) AS cnt FROM promo_campaigns WHERE status = 'active' GROUP BY chain ORDER BY cnt DESC"
        )
    except Exception:
        data["promo_by_chain"] = []

    deal_where = ""
    deal_params: tuple = ()
    if stage:
        deal_where = " WHERE stage = %s"
        deal_params = (stage,)

    try:
        data["recent_deals"] = fetch_all(
            f"SELECT title, stage, value, updated_at FROM deals{deal_where} ORDER BY updated_at DESC LIMIT 10",
            deal_params or None,
        )
    except Exception:
        data["recent_deals"] = []

    try:
        if agent:
            data["recent_events"] = fetch_all(
                """SELECT agent_name, event_type, title, status, created_at FROM agent_events
                   WHERE created_at >= NOW() - make_interval(days => %s) AND agent_name = %s
                   ORDER BY created_at DESC LIMIT 25""",
                (days, agent),
            )
        else:
            data["recent_events"] = fetch_all(
                """SELECT agent_name, event_type, title, status, created_at FROM agent_events
                   WHERE created_at >= NOW() - make_interval(days => %s)
                   ORDER BY created_at DESC LIMIT 25""",
                (days,),
            )
    except Exception:
        data["recent_events"] = []

    try:
        data["filter_meta"] = {
            "chains": fetch_all("SELECT DISTINCT chain FROM supermarkets WHERE chain IS NOT NULL ORDER BY chain"),
            "provinces": fetch_all("SELECT DISTINCT province FROM supermarkets WHERE province IS NOT NULL ORDER BY province"),
            "client_stages": fetch_all("SELECT DISTINCT stage FROM clients ORDER BY stage"),
            "deal_stages": fetch_all("SELECT DISTINCT stage FROM deals ORDER BY stage"),
            "agents": fetch_all("SELECT DISTINCT agent_name FROM agent_events ORDER BY agent_name"),
        }
    except Exception:
        data["filter_meta"] = {}

    for key in (
        "clients_by_stage", "deals_by_stage", "events_by_agent", "supermarkets_by_chain",
        "supermarkets_by_province", "partnership_breakdown", "wholesalers_by_province",
        "rss_by_category", "events_timeline", "milestones_by_status", "top_opportunities",
        "sentiment_distribution", "top_words", "promo_by_chain", "recent_deals", "recent_events",
    ):
        if isinstance(data.get(key), list):
            data[key] = _iso_rows(data[key])

    data["pulse"] = _build_pulse(data)
    data["hermes_findings"] = list_hermes_findings(20)
    return data