"""Retail intelligence API routes.""" from __future__ import annotations from typing import Any, Optional from fastapi import APIRouter, HTTPException, Query from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from app.db import execute, execute_returning, fetch_all, fetch_one from app.middleware import log_agent_event from app import retail_scrapers from app import retail_enrichment from app import retail_crm from app import retail_opportunities from app.connectors import halal_registry, trends_feed router = APIRouter(prefix="/retail", tags=["retail"]) FIELD_SCHEMA = { "locatie": ["id", "name", "chain", "address", "postcode", "city", "province", "store_type", "latitude", "longitude"], "contact": ["phone", "email", "website", "manager_name", "employee_count"], "halal": ["halal_certified", "halal_certifier", "has_halal_section", "halal_certificate_number", "halal_expiry_date"], "crm": ["partnership_status", "client_id", "deal_id", "halal_opportunity_score"], "cbs": ["area_population", "area_avg_income", "area_households", "muslim_proxy_pct", "area_data_source"], "meta": ["data_source", "external_id", "last_updated", "enrichment_score"], } class CrmLinkIn(BaseModel): client_id: int deal_id: Optional[int] = None relationship_type: str = "prospect" partnership_status: Optional[str] = None notes: Optional[str] = None class SupermarketCreateIn(BaseModel): name: str chain: str address: str postcode: str = "0000AA" city: str province: Optional[str] = None phone: Optional[str] = None email: Optional[str] = None website: Optional[str] = None manager_name: Optional[str] = None employee_count: Optional[int] = None store_type: Optional[str] = None partnership_status: str = "none" halal_certified: bool = False has_halal_section: bool = False halal_certifier: Optional[str] = None data_source: str = "manual" class SupermarketPatchIn(BaseModel): name: Optional[str] = None chain: Optional[str] = None address: Optional[str] = None postcode: Optional[str] = None city: Optional[str] = None province: Optional[str] = None phone: Optional[str] = None email: Optional[str] = None website: Optional[str] = None manager_name: Optional[str] = None employee_count: Optional[int] = None store_type: Optional[str] = None size_m2: Optional[int] = None partnership_status: Optional[str] = None halal_certified: Optional[bool] = None has_halal_section: Optional[bool] = None halal_certifier: Optional[str] = None halal_certificate_number: Optional[str] = None organic_section: Optional[bool] = None alcohol_section: Optional[bool] = None STORE_SELECT = """ SELECT s.*, a.population AS area_population, a.avg_income AS area_avg_income, a.households AS area_households, a.religious_composition AS area_religious, a.ethnic_composition AS area_ethnic, a.data_source AS area_data_source, ros.halal_opportunity_score AS opp_halal_score, ros.market_potential_score AS opp_market_score, sp.manager_name AS profile_manager, sp.manager_phone AS profile_manager_phone, sp.manager_email AS profile_manager_email, sp.staff_count_estimate, sp.data_completeness AS profile_completeness FROM supermarkets s LEFT JOIN area_analysis a ON a.postcode = s.postcode LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id """ def _row(row: dict | None) -> dict[str, Any]: if not row: raise HTTPException(404, "Not found") out: dict[str, Any] = {} for k, v in row.items(): if hasattr(v, "isoformat"): out[k] = v.isoformat() elif v is not None and hasattr(v, "__float__") and type(v).__name__ == "Decimal": out[k] = float(v) else: out[k] = v if out.get("area_religious") and isinstance(out["area_religious"], dict): out["muslim_proxy_pct"] = out["area_religious"].get("muslim_proxy_pct") return out def _build_filters( chain: Optional[str] = None, province: Optional[str] = None, city: Optional[str] = None, partnership: Optional[str] = None, halal_certified: Optional[bool] = None, has_halal_section: Optional[bool] = None, store_type: Optional[str] = None, postcode_prefix: Optional[str] = None, q: Optional[str] = None, min_population: Optional[int] = None, max_population: Optional[int] = None, min_avg_income: Optional[float] = None, max_avg_income: Optional[float] = None, min_muslim_pct: Optional[float] = None, has_area_data: Optional[bool] = None, min_halal_opportunity: Optional[float] = None, max_halal_opportunity: Optional[float] = None, has_phone: Optional[bool] = None, has_email: Optional[bool] = None, has_manager: Optional[bool] = None, halal_gap_only: Optional[bool] = None, linked_to_crm: Optional[bool] = None, has_halal_cert_registry: Optional[bool] = None, ) -> tuple[list[str], list[Any]]: clauses: list[str] = ["s.postcode <> '0000AA'"] params: list[Any] = [] if chain: clauses.append("s.chain ILIKE %s") params.append(f"%{chain}%") if province: clauses.append("s.province ILIKE %s") params.append(f"%{province}%") if city: clauses.append("s.city ILIKE %s") params.append(f"%{city}%") if partnership: clauses.append("s.partnership_status = %s") params.append(partnership) if halal_certified is not None: clauses.append("s.halal_certified = %s") params.append(halal_certified) if has_halal_section is not None: clauses.append("s.has_halal_section = %s") params.append(has_halal_section) if store_type: clauses.append("s.store_type ILIKE %s") params.append(f"%{store_type}%") if postcode_prefix: clauses.append("s.postcode LIKE %s") params.append(f"{postcode_prefix.upper()}%") if q: clauses.append("(s.name ILIKE %s OR s.address ILIKE %s OR s.city ILIKE %s)") like = f"%{q}%" params.extend([like, like, like]) if min_population is not None: clauses.append("a.population >= %s") params.append(min_population) if max_population is not None: clauses.append("a.population <= %s") params.append(max_population) if min_avg_income is not None: clauses.append("a.avg_income >= %s") params.append(min_avg_income) if max_avg_income is not None: clauses.append("a.avg_income <= %s") params.append(max_avg_income) if min_muslim_pct is not None: clauses.append("(a.religious_composition->>'muslim_proxy_pct')::float >= %s") params.append(min_muslim_pct) if has_area_data is True: clauses.append("a.id IS NOT NULL") elif has_area_data is False: clauses.append("a.id IS NULL") if min_halal_opportunity is not None: clauses.append("COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score, 0) >= %s") params.append(min_halal_opportunity) if max_halal_opportunity is not None: clauses.append("COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score, 0) <= %s") params.append(max_halal_opportunity) if has_phone is True: clauses.append("(s.phone IS NOT NULL AND s.phone <> '')") elif has_phone is False: clauses.append("(s.phone IS NULL OR s.phone = '')") if has_email is True: clauses.append("(s.email IS NOT NULL AND s.email <> '')") elif has_email is False: clauses.append("(s.email IS NULL OR s.email = '')") if has_manager is True: clauses.append("(s.manager_name IS NOT NULL OR sp.manager_name IS NOT NULL)") elif has_manager is False: clauses.append("(s.manager_name IS NULL AND sp.manager_name IS NULL)") if halal_gap_only: clauses.append("s.halal_certified = FALSE AND s.has_halal_section = FALSE") clauses.append("(a.religious_composition->>'muslim_proxy_pct')::float >= 5") if linked_to_crm is True: clauses.append("s.client_id IS NOT NULL") elif linked_to_crm is False: clauses.append("s.client_id IS NULL") if has_halal_cert_registry: clauses.append( "EXISTS (SELECT 1 FROM halal_certifications h WHERE h.supermarket_id = s.id AND h.status = 'active')" ) return clauses, params @router.get("/filters") def retail_filters() -> dict[str, Any]: chains = fetch_all( "SELECT chain, COUNT(*) AS n FROM supermarkets GROUP BY chain ORDER BY n DESC" ) provinces = fetch_all( """SELECT COALESCE(province, 'Onbekend') AS province, COUNT(*) AS n FROM supermarkets GROUP BY province ORDER BY n DESC""" ) partnerships = fetch_all( "SELECT partnership_status, COUNT(*) AS n FROM supermarkets GROUP BY partnership_status" ) status = retail_enrichment.enrichment_status() income = fetch_one( """SELECT MIN(avg_income) AS min_income, MAX(avg_income) AS max_income, MIN(population) AS min_pop, MAX(population) AS max_pop FROM area_analysis WHERE avg_income IS NOT NULL""" ) return { "chains": [dict(r) for r in chains], "provinces": [dict(r) for r in provinces], "partnerships": [dict(r) for r in partnerships], "enrichment": status, "ranges": { "min_income": float(income["min_income"]) if income and income.get("min_income") else None, "max_income": float(income["max_income"]) if income and income.get("max_income") else None, "min_population": int(income["min_pop"]) if income and income.get("min_pop") else None, "max_population": int(income["max_pop"]) if income and income.get("max_pop") else None, }, } @router.get("/supermarkets") def list_supermarkets( chain: Optional[str] = None, province: Optional[str] = None, city: Optional[str] = None, partnership: Optional[str] = None, halal_certified: Optional[bool] = None, has_halal_section: Optional[bool] = None, store_type: Optional[str] = None, postcode_prefix: Optional[str] = None, q: Optional[str] = None, min_population: Optional[int] = Query(None, ge=0), max_population: Optional[int] = Query(None, ge=0), min_avg_income: Optional[float] = Query(None, ge=0), max_avg_income: Optional[float] = Query(None, ge=0), min_muslim_pct: Optional[float] = Query(None, ge=0, le=100), has_area_data: Optional[bool] = None, min_halal_opportunity: Optional[float] = Query(None, ge=0, le=100), max_halal_opportunity: Optional[float] = Query(None, ge=0, le=100), has_phone: Optional[bool] = None, has_email: Optional[bool] = None, has_manager: Optional[bool] = None, halal_gap_only: Optional[bool] = None, linked_to_crm: Optional[bool] = None, has_halal_cert_registry: Optional[bool] = None, sort: Optional[str] = Query("name", pattern="^(name|halal_opportunity|population|chain)$"), limit: int = Query(2000, ge=1, le=5000), offset: int = Query(0, ge=0), ) -> dict[str, Any]: clauses, params = _build_filters( chain, province, city, partnership, halal_certified, has_halal_section, store_type, postcode_prefix, q, min_population, max_population, min_avg_income, max_avg_income, min_muslim_pct, has_area_data, min_halal_opportunity, max_halal_opportunity, has_phone, has_email, has_manager, halal_gap_only, linked_to_crm, has_halal_cert_registry, ) where = " WHERE " + " AND ".join(clauses) order = { "halal_opportunity": "COALESCE(ros.halal_opportunity_score,0) DESC, s.name", "population": "COALESCE(a.population,0) DESC, s.name", "chain": "s.chain, s.name", "name": "s.chain, s.name", }.get(sort or "name", "s.chain, s.name") rows = fetch_all( f"{STORE_SELECT}{where} ORDER BY {order} LIMIT %s OFFSET %s", tuple(params + [limit, offset]), ) total = fetch_one( f"""SELECT COUNT(*) AS n FROM supermarkets s LEFT JOIN area_analysis a ON a.postcode = s.postcode LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id {where}""", tuple(params), ) return { "items": [_row(r) for r in rows], "count": len(rows), "total": int((total or {}).get("n") or 0), } @router.get("/map") def map_points( chain: Optional[str] = None, province: Optional[str] = None, partnership: Optional[str] = None, halal_certified: Optional[bool] = None, has_halal_section: Optional[bool] = None, min_muslim_pct: Optional[float] = None, min_population: Optional[int] = None, min_halal_opportunity: Optional[float] = None, halal_gap_only: Optional[bool] = None, linked_to_crm: Optional[bool] = None, q: Optional[str] = None, limit: int = Query(5000, ge=1, le=5000), ) -> dict[str, Any]: clauses, params = _build_filters( chain, province, None, partnership, halal_certified, has_halal_section, None, None, q, min_population, None, None, None, min_muslim_pct, None, min_halal_opportunity, None, None, None, None, halal_gap_only, linked_to_crm, None, ) where = " WHERE " + " AND ".join(clauses) + " AND s.latitude IS NOT NULL AND s.longitude IS NOT NULL" rows = fetch_all( f"""SELECT s.id, s.name, s.chain, s.city, s.province, s.postcode, s.latitude, s.longitude, s.partnership_status, s.halal_certified, s.has_halal_section, s.phone, s.manager_name, a.population AS area_population, (a.religious_composition->>'muslim_proxy_pct')::float AS muslim_proxy_pct, COALESCE(ros.halal_opportunity_score, s.halal_opportunity_score) AS halal_opportunity_score FROM supermarkets s LEFT JOIN area_analysis a ON a.postcode = s.postcode LEFT JOIN retail_opportunity_scores ros ON ros.supermarket_id = s.id LEFT JOIN supermarket_profiles sp ON sp.supermarket_id = s.id {where} LIMIT %s""", tuple(params + [limit]), ) return {"items": [_row(r) for r in rows], "count": len(rows)} @router.get("/supermarkets/{store_id}") def get_supermarket(store_id: int) -> dict[str, Any]: row = fetch_one(f"{STORE_SELECT} WHERE s.id = %s", (store_id,)) data = _row(row) area = fetch_one("SELECT * FROM area_analysis WHERE postcode = %s", (data.get("postcode"),)) if area: data["area_analysis"] = _row(area) weather = fetch_all( "SELECT * FROM weather_data WHERE city ILIKE %s ORDER BY date DESC LIMIT 3", (f"%{data.get('city', '')}%",), ) data["weather"] = [_row(w) for w in weather] recs = fetch_all( """SELECT * FROM ai_recommendations WHERE related_entity_type = 'supermarket' AND related_entity_id = %s ORDER BY created_at DESC LIMIT 3""", (store_id,), ) data["recommendations"] = [_row(r) for r in recs] nearby = fetch_all( """ SELECT id, name, chain, partnership_status, distance_km FROM ( SELECT id, name, chain, partnership_status, (6371 * acos( LEAST(1.0, cos(radians(%s)) * cos(radians(latitude)) * cos(radians(longitude) - radians(%s)) + sin(radians(%s)) * sin(radians(latitude))) )) AS distance_km FROM supermarkets WHERE id <> %s AND latitude IS NOT NULL AND longitude IS NOT NULL ) nearby_q WHERE distance_km < 3 ORDER BY distance_km LIMIT 8 """, ( data.get("latitude"), data.get("longitude"), data.get("latitude"), store_id, ), ) data["nearby_stores"] = [_row(n) for n in nearby] data["crm"] = retail_crm.get_store_crm_context(store_id) opp = fetch_one("SELECT * FROM retail_opportunity_scores WHERE supermarket_id = %s", (store_id,)) if opp: data["opportunity"] = _row(opp) return data @router.post("/supermarkets") def create_supermarket(payload: SupermarketCreateIn) -> dict[str, Any]: row = execute_returning( """INSERT INTO supermarkets ( name, chain, address, postcode, city, province, phone, email, website, manager_name, employee_count, store_type, partnership_status, halal_certified, has_halal_section, halal_certifier, data_source, last_updated ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,NOW()) RETURNING id""", ( payload.name.strip(), payload.chain.strip(), payload.address.strip(), payload.postcode.strip().upper(), payload.city.strip(), payload.province, payload.phone, payload.email, payload.website, payload.manager_name, payload.employee_count, payload.store_type, payload.partnership_status, payload.halal_certified, payload.has_halal_section, payload.halal_certifier, payload.data_source, ), ) store_id = int(row["id"]) log_agent_event( agent_name="retail_crm", event_type="create", title=f"Handmatig filiaal toegevoegd: {payload.name} ({payload.chain})", ) return {"ok": True, "item": _row(fetch_one(f"{STORE_SELECT} WHERE s.id = %s", (store_id,)))} @router.patch("/supermarkets/{store_id}") def patch_supermarket(store_id: int, payload: SupermarketPatchIn) -> dict[str, Any]: existing = fetch_one("SELECT id FROM supermarkets WHERE id = %s", (store_id,)) if not existing: raise HTTPException(404, "Supermarket not found") data = payload.model_dump(exclude_unset=True) if not data: raise HTTPException(400, "Geen velden om bij te werken") allowed = set(SupermarketPatchIn.model_fields.keys()) sets: list[str] = [] params: list[Any] = [] for key, val in data.items(): if key not in allowed: continue sets.append(f"{key} = %s") params.append(val) sets.append("last_updated = NOW()") params.append(store_id) execute(f"UPDATE supermarkets SET {', '.join(sets)} WHERE id = %s", tuple(params)) log_agent_event( agent_name="retail_crm", event_type="update", title=f"Supermarkt #{store_id} data aangevuld", ) return {"ok": True, "item": _row(fetch_one(f"{STORE_SELECT} WHERE s.id = %s", (store_id,)))} @router.get("/scrape/chains") def list_scrape_chains() -> dict[str, Any]: return {"chains": retail_scrapers.list_chains()} @router.post("/scrape/all") def scrape_all_chains() -> dict[str, Any]: log_agent_event(agent_name="retail_scraper", event_type="scrape", title="OSM import all chains") return retail_scrapers.import_all_chains() @router.post("/scrape/{chain_key}") def scrape_chain(chain_key: str) -> dict[str, Any]: log_agent_event( agent_name="retail_scraper", event_type="scrape", title=f"OSM import {chain_key}", ) try: return retail_scrapers.import_chain(chain_key.lower()) except ValueError as exc: raise HTTPException(400, str(exc)) from exc except RuntimeError as exc: raise HTTPException(502, str(exc)) from exc @router.post("/enrich") def enrich_areas( limit: int = Query(50, ge=1, le=200), offset: int = Query(0, ge=0), ) -> dict[str, Any]: log_agent_event( agent_name="retail_enrichment", event_type="enrich", title=f"CBS/PDOK enrichment batch limit={limit}", ) return retail_enrichment.enrich_batch(limit=limit, offset=offset) @router.get("/enrich/status") def enrich_status() -> dict[str, Any]: return retail_enrichment.enrichment_status() @router.get("/stats") def retail_stats( chain: Optional[str] = None, province: Optional[str] = None, partnership: Optional[str] = None, min_muslim_pct: Optional[float] = None, ) -> dict[str, Any]: clauses, params = _build_filters( chain, province, None, partnership, None, None, None, None, None, None, None, None, None, min_muslim_pct, None, ) where = " WHERE " + " AND ".join(clauses) row = fetch_one( f""" SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE s.partnership_status = 'active') AS active_partnerships, COUNT(*) FILTER (WHERE s.halal_certified) AS halal_certified, COUNT(*) FILTER (WHERE a.id IS NOT NULL) AS with_area_data, ROUND(AVG(a.avg_income)::numeric, 0) AS avg_area_income, ROUND(AVG((a.religious_composition->>'muslim_proxy_pct')::float)::numeric, 1) AS avg_muslim_proxy_pct FROM supermarkets s LEFT JOIN area_analysis a ON a.postcode = s.postcode {where} """, tuple(params), ) out = {k: int(v or 0) if k in ("total", "active_partnerships", "halal_certified", "with_area_data") else v for k, v in (row or {}).items()} if out.get("avg_area_income") is not None: out["avg_area_income"] = float(out["avg_area_income"]) if out.get("avg_muslim_proxy_pct") is not None: out["avg_muslim_proxy_pct"] = float(out["avg_muslim_proxy_pct"]) halal_n = fetch_one("SELECT COUNT(*) AS n FROM supermarkets WHERE halal_certified = TRUE") out["halal_certified_count"] = int((halal_n or {}).get("n") or 0) return out @router.get("/schema") def retail_schema() -> dict[str, Any]: return {"groups": FIELD_SCHEMA, "all_fields": [f for fields in FIELD_SCHEMA.values() for f in fields]} @router.get("/crm/options") def crm_options() -> dict[str, Any]: return retail_crm.list_crm_options() @router.post("/supermarkets/{store_id}/link") def link_store_crm(store_id: int, payload: CrmLinkIn) -> dict[str, Any]: try: row = retail_crm.link_client_to_store( store_id, payload.client_id, payload.deal_id, payload.relationship_type, payload.partnership_status, payload.notes, ) log_agent_event(agent_name="retail_crm", event_type="link", title=f"Linked store {store_id} to client {payload.client_id}") return {"link": row} except ValueError as exc: raise HTTPException(400, str(exc)) from exc @router.delete("/supermarkets/{store_id}/link/{client_id}") def unlink_store_crm(store_id: int, client_id: int) -> dict[str, Any]: ok = retail_crm.unlink_client_from_store(store_id, client_id) return {"unlinked": ok} @router.get("/halal") def list_halal_stores(limit: int = Query(500, ge=1, le=2000)) -> dict[str, Any]: rows = halal_registry.list_halal_certified(limit) return {"items": [_row(r) for r in rows], "count": len(rows)} @router.get("/opportunities") def list_opportunities( limit: int = Query(50, ge=1, le=500), min_score: float = Query(30, ge=0, le=100), chain: Optional[str] = None, province: Optional[str] = None, ) -> dict[str, Any]: rows = retail_opportunities.top_opportunities(limit, min_score, chain, province) return {"items": [_row(r) for r in rows], "count": len(rows)} @router.get("/trends") def list_trends(limit: int = Query(20, ge=1, le=100)) -> dict[str, Any]: rows = trends_feed.list_live_trends(limit) return {"items": [_row(r) for r in rows], "count": len(rows)} @router.post("/sync/halal") def sync_halal() -> dict[str, Any]: log_agent_event(agent_name="halal_registry", event_type="sync", title="Halal OSM sync") return halal_registry.sync_osm_halal_tags() @router.post("/sync/contacts") def sync_contacts(limit: int = Query(100, ge=1, le=300)) -> dict[str, Any]: log_agent_event(agent_name="branch_scraper", event_type="sync", title=f"OSM contacts limit={limit}") return halal_registry.sync_osm_contact_tags(limit) @router.post("/sync/trends") def sync_trends() -> dict[str, Any]: return trends_feed.refresh_trends_from_social() @router.post("/compute-opportunities") def compute_opportunities(limit: int = Query(5000, ge=100, le=10000)) -> dict[str, Any]: log_agent_event(agent_name="retail_intel", event_type="score", title="Compute halal opportunity scores") return retail_opportunities.compute_all_scores(limit) @router.get("/export") def export_csv( chain: Optional[str] = None, province: Optional[str] = None, halal_certified: Optional[bool] = None, min_halal_opportunity: Optional[float] = None, limit: int = Query(5000, ge=1, le=5000), ): clauses, params = _build_filters( chain, province, None, None, halal_certified, None, None, None, None, None, None, None, None, None, None, min_halal_opportunity, None, None, None, None, None, None, None, ) where = " WHERE " + " AND ".join(clauses) rows = fetch_all( f"{STORE_SELECT}{where} ORDER BY s.chain, s.name LIMIT %s", tuple(params + [limit]), ) def generate(): headers = ["id", "name", "chain", "city", "province", "postcode", "phone", "email", "manager_name", "halal_certified", "has_halal_section", "partnership_status", "muslim_proxy_pct", "area_population", "area_avg_income", "halal_opportunity_score"] yield ",".join(headers) + "\n" for r in rows: rel = r.get("area_religious") or {} if isinstance(rel, str): rel = {} vals = [ r.get("id"), r.get("name"), r.get("chain"), r.get("city"), r.get("province"), r.get("postcode"), r.get("phone"), r.get("email"), r.get("manager_name"), r.get("halal_certified"), r.get("has_halal_section"), r.get("partnership_status"), rel.get("muslim_proxy_pct") if isinstance(rel, dict) else None, r.get("area_population"), r.get("area_avg_income"), r.get("opp_halal_score") or r.get("halal_opportunity_score"), ] yield ",".join('"' + str(v or "").replace('"', '""') + '"' for v in vals) + "\n" return StreamingResponse(generate(), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=retail_export.csv"})