"""Research pipeline: data providers, snapshots, briefs.""" from __future__ import annotations import json import urllib.request from datetime import date, datetime, timedelta, timezone from typing import Any from fastapi import APIRouter, HTTPException from app.db import execute, execute_returning, fetch_all, fetch_one, json_param from app.middleware import log_agent_event router = APIRouter(prefix="/research", tags=["research"]) def _json_safe(obj: Any) -> Any: if isinstance(obj, dict): return {k: _json_safe(v) for k, v in obj.items()} if isinstance(obj, list): return [_json_safe(x) for x in obj] if hasattr(obj, "isoformat"): return obj.isoformat() if type(obj).__name__ == "Decimal": return float(obj) return obj def _http_get(url: str, timeout: int = 30) -> dict: with urllib.request.urlopen(url, timeout=timeout) as resp: return json.loads(resp.read().decode()) @router.get("/providers") def list_providers() -> dict[str, Any]: rows = fetch_all("SELECT * FROM data_providers ORDER BY name") return {"items": [dict(r) for r in rows]} @router.post("/providers/{provider_id}/refresh") def refresh_provider(provider_id: int) -> dict[str, Any]: prov = fetch_one("SELECT * FROM data_providers WHERE id = %s", (provider_id,)) if not prov: raise HTTPException(404, "Provider not found") name = prov["name"] payload: dict[str, Any] = {} count = 0 if name == "crm": clients = fetch_all("SELECT id, name, stage FROM clients ORDER BY updated_at DESC LIMIT 20") deals = fetch_all( "SELECT id, title, value, stage, next_action, deadline FROM deals ORDER BY updated_at DESC LIMIT 20" ) payload = { "clients": _json_safe([dict(c) for c in clients]), "deals": _json_safe([dict(d) for d in deals]), } count = len(clients) + len(deals) elif name == "weather": url = ( "https://api.open-meteo.com/v1/forecast?" "latitude=52.37&longitude=4.89&daily=temperature_2m_max,precipitation_sum&timezone=Europe%2FAmsterdam&forecast_days=7" ) payload = _http_get(url) count = len(payload.get("daily", {}).get("time", [])) for i, day in enumerate(payload.get("daily", {}).get("time", [])[:7]): temps = payload["daily"].get("temperature_2m_max", []) prec = payload["daily"].get("precipitation_sum", []) execute( """INSERT INTO weather_data (region, city, date, temperature_c, precipitation_mm, weather_condition, data_source) VALUES ('Noord-Holland', 'Amsterdam', %s, %s, %s, 'forecast', 'open-meteo')""", (day, temps[i] if i < len(temps) else None, prec[i] if i < len(prec) else None), ) elif name == "social": rows = fetch_all( "SELECT platform, text, sentiment_score, created_at FROM social_mentions ORDER BY created_at DESC LIMIT 30" ) payload = _json_safe({"mentions": [dict(r) for r in rows]}) count = len(rows) elif name == "retail_manual": rows = fetch_all("SELECT id, name, chain, city, partnership_status FROM supermarkets ORDER BY id") payload = _json_safe({"stores": [dict(r) for r in rows]}) count = len(rows) elif name == "cbs": from app import retail_enrichment status = retail_enrichment.enrichment_status() batch = retail_enrichment.enrich_batch(limit=30, offset=0) payload = _json_safe({"status": status, "batch": batch}) count = batch.get("ok", 0) elif name == "pdok": from app.connectors import pdok as pdok_conn sample = fetch_all( "SELECT DISTINCT postcode FROM supermarkets WHERE postcode <> '0000AA' LIMIT 5" ) lookups = [pdok_conn.lookup_postcode(r["postcode"]) for r in sample] payload = _json_safe({"lookups": [x for x in lookups if x]}) count = len(payload.get("lookups", [])) else: payload = {"status": "noop"} snap = execute_returning( """INSERT INTO data_snapshots (provider_id, payload, record_count) VALUES (%s, %s, %s) RETURNING id, fetched_at""", (provider_id, json_param(payload), count), ) execute( "UPDATE data_providers SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s", (provider_id,), ) log_agent_event( agent_name="research", event_type="data_refresh", title=f"Provider {name} refreshed", metadata={"provider_id": provider_id, "records": count}, ) return {"provider": name, "snapshot_id": snap["id"], "record_count": count} @router.get("/briefs") def list_briefs(limit: int = 20) -> dict[str, Any]: rows = fetch_all( "SELECT * FROM research_briefs ORDER BY generated_at DESC LIMIT %s", (limit,), ) return {"items": [dict(r) for r in rows]} def _build_brief(domain: str, title: str, summary: str, findings: list[dict]) -> dict: row = execute_returning( """INSERT INTO research_briefs (domain, title, summary, key_findings, expires_at) VALUES (%s, %s, %s, %s, %s) RETURNING *""", ( domain, title, summary, json_param(findings), datetime.now(timezone.utc) + timedelta(days=1), ), ) return dict(row) @router.post("/run") def run_research() -> dict[str, Any]: providers = fetch_all("SELECT id, name FROM data_providers WHERE is_active = true") snapshot_ids = [] for p in providers: result = refresh_provider(int(p["id"])) snapshot_ids.append(result.get("snapshot_id")) clients_n = fetch_one("SELECT COUNT(*) AS c FROM clients")["c"] deals = fetch_all("SELECT title, value, stage FROM deals WHERE stage NOT IN ('won','lost')") pipeline = sum(float(d.get("value") or 0) for d in deals) stores = fetch_one("SELECT COUNT(*) AS c FROM supermarkets")["c"] mentions = fetch_one("SELECT COUNT(*) AS c FROM social_mentions WHERE created_at > NOW() - interval '7 days'")["c"] crm_brief = _build_brief( "crm", f"CRM snapshot {date.today()}", f"{clients_n} klanten, pipeline €{pipeline:,.0f}, {len(deals)} actieve deals.", [{"finding": f"Pipeline €{pipeline:,.0f}", "relevance": "high"}], ) retail_brief = _build_brief( "retail", f"Retail NL {date.today()}", f"{stores} supermarkten in database, partnerships actief/proposal gemapt.", [{"finding": f"{stores} locaties geladen", "relevance": "medium"}], ) social_brief = _build_brief( "social", f"Social week {date.today()}", f"{mentions} mentions afgelopen 7 dagen.", [{"finding": f"{mentions} mentions", "relevance": "medium"}], ) log_agent_event( agent_name="research", event_type="research_run", title="Full research cycle completed", metadata={"briefs": 3, "snapshots": len(snapshot_ids)}, ) from app.connectors.project_assets import register_agent_output for brief, domain in [ (crm_brief, "crm"), (retail_brief, "retail"), (social_brief, "social"), ]: try: register_agent_output( asset_type="research_brief", title=brief.get("title") or f"Research {domain}", ref_id=str(brief.get("id")), payload={"domain": domain, "summary": brief.get("summary"), "brief_id": brief.get("id")}, source_agent="research", ) except Exception: pass return { "snapshots": snapshot_ids, "briefs": [crm_brief["id"], retail_brief["id"], social_brief["id"]], }