sysops
203 lines
7.6 KiB
Python
203 lines
7.6 KiB
Python
"""Research pipeline: data providers, snapshots, briefs."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.request
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
|
|
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
|
|
from app.middleware import log_agent_event
|
|
|
|
router = APIRouter(prefix="/research", tags=["research"])
|
|
|
|
|
|
def _json_safe(obj: Any) -> Any:
|
|
if isinstance(obj, dict):
|
|
return {k: _json_safe(v) for k, v in obj.items()}
|
|
if isinstance(obj, list):
|
|
return [_json_safe(x) for x in obj]
|
|
if hasattr(obj, "isoformat"):
|
|
return obj.isoformat()
|
|
if type(obj).__name__ == "Decimal":
|
|
return float(obj)
|
|
return obj
|
|
|
|
|
|
def _http_get(url: str, timeout: int = 30) -> dict:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode())
|
|
|
|
|
|
@router.get("/providers")
|
|
def list_providers() -> dict[str, Any]:
|
|
rows = fetch_all("SELECT * FROM data_providers ORDER BY name")
|
|
return {"items": [dict(r) for r in rows]}
|
|
|
|
|
|
@router.post("/providers/{provider_id}/refresh")
|
|
def refresh_provider(provider_id: int) -> dict[str, Any]:
|
|
prov = fetch_one("SELECT * FROM data_providers WHERE id = %s", (provider_id,))
|
|
if not prov:
|
|
raise HTTPException(404, "Provider not found")
|
|
name = prov["name"]
|
|
payload: dict[str, Any] = {}
|
|
count = 0
|
|
if name == "crm":
|
|
clients = fetch_all("SELECT id, name, stage FROM clients ORDER BY updated_at DESC LIMIT 20")
|
|
deals = fetch_all(
|
|
"SELECT id, title, value, stage, next_action, deadline FROM deals ORDER BY updated_at DESC LIMIT 20"
|
|
)
|
|
payload = {
|
|
"clients": _json_safe([dict(c) for c in clients]),
|
|
"deals": _json_safe([dict(d) for d in deals]),
|
|
}
|
|
count = len(clients) + len(deals)
|
|
elif name == "weather":
|
|
url = (
|
|
"https://api.open-meteo.com/v1/forecast?"
|
|
"latitude=52.37&longitude=4.89&daily=temperature_2m_max,precipitation_sum&timezone=Europe%2FAmsterdam&forecast_days=7"
|
|
)
|
|
payload = _http_get(url)
|
|
count = len(payload.get("daily", {}).get("time", []))
|
|
for i, day in enumerate(payload.get("daily", {}).get("time", [])[:7]):
|
|
temps = payload["daily"].get("temperature_2m_max", [])
|
|
prec = payload["daily"].get("precipitation_sum", [])
|
|
execute(
|
|
"""INSERT INTO weather_data (region, city, date, temperature_c, precipitation_mm, weather_condition, data_source)
|
|
VALUES ('Noord-Holland', 'Amsterdam', %s, %s, %s, 'forecast', 'open-meteo')""",
|
|
(day, temps[i] if i < len(temps) else None, prec[i] if i < len(prec) else None),
|
|
)
|
|
elif name == "social":
|
|
rows = fetch_all(
|
|
"SELECT platform, text, sentiment_score, created_at FROM social_mentions ORDER BY created_at DESC LIMIT 30"
|
|
)
|
|
payload = _json_safe({"mentions": [dict(r) for r in rows]})
|
|
count = len(rows)
|
|
elif name == "retail_manual":
|
|
rows = fetch_all("SELECT id, name, chain, city, partnership_status FROM supermarkets ORDER BY id")
|
|
payload = _json_safe({"stores": [dict(r) for r in rows]})
|
|
count = len(rows)
|
|
elif name == "cbs":
|
|
from app import retail_enrichment
|
|
status = retail_enrichment.enrichment_status()
|
|
batch = retail_enrichment.enrich_batch(limit=30, offset=0)
|
|
payload = _json_safe({"status": status, "batch": batch})
|
|
count = batch.get("ok", 0)
|
|
elif name == "pdok":
|
|
from app.connectors import pdok as pdok_conn
|
|
sample = fetch_all(
|
|
"SELECT DISTINCT postcode FROM supermarkets WHERE postcode <> '0000AA' LIMIT 5"
|
|
)
|
|
lookups = [pdok_conn.lookup_postcode(r["postcode"]) for r in sample]
|
|
payload = _json_safe({"lookups": [x for x in lookups if x]})
|
|
count = len(payload.get("lookups", []))
|
|
else:
|
|
payload = {"status": "noop"}
|
|
snap = execute_returning(
|
|
"""INSERT INTO data_snapshots (provider_id, payload, record_count)
|
|
VALUES (%s, %s, %s) RETURNING id, fetched_at""",
|
|
(provider_id, json_param(payload), count),
|
|
)
|
|
execute(
|
|
"UPDATE data_providers SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
|
|
(provider_id,),
|
|
)
|
|
log_agent_event(
|
|
agent_name="research",
|
|
event_type="data_refresh",
|
|
title=f"Provider {name} refreshed",
|
|
metadata={"provider_id": provider_id, "records": count},
|
|
)
|
|
return {"provider": name, "snapshot_id": snap["id"], "record_count": count}
|
|
|
|
|
|
@router.get("/briefs")
|
|
def list_briefs(limit: int = 20) -> dict[str, Any]:
|
|
rows = fetch_all(
|
|
"SELECT * FROM research_briefs ORDER BY generated_at DESC LIMIT %s",
|
|
(limit,),
|
|
)
|
|
return {"items": [dict(r) for r in rows]}
|
|
|
|
|
|
def _build_brief(domain: str, title: str, summary: str, findings: list[dict]) -> dict:
|
|
row = execute_returning(
|
|
"""INSERT INTO research_briefs (domain, title, summary, key_findings, expires_at)
|
|
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
|
|
(
|
|
domain,
|
|
title,
|
|
summary,
|
|
json_param(findings),
|
|
datetime.now(timezone.utc) + timedelta(days=1),
|
|
),
|
|
)
|
|
return dict(row)
|
|
|
|
|
|
@router.post("/run")
|
|
def run_research() -> dict[str, Any]:
|
|
providers = fetch_all("SELECT id, name FROM data_providers WHERE is_active = true")
|
|
snapshot_ids = []
|
|
for p in providers:
|
|
result = refresh_provider(int(p["id"]))
|
|
snapshot_ids.append(result.get("snapshot_id"))
|
|
|
|
clients_n = fetch_one("SELECT COUNT(*) AS c FROM clients")["c"]
|
|
deals = fetch_all("SELECT title, value, stage FROM deals WHERE stage NOT IN ('won','lost')")
|
|
pipeline = sum(float(d.get("value") or 0) for d in deals)
|
|
stores = fetch_one("SELECT COUNT(*) AS c FROM supermarkets")["c"]
|
|
mentions = fetch_one("SELECT COUNT(*) AS c FROM social_mentions WHERE created_at > NOW() - interval '7 days'")["c"]
|
|
|
|
crm_brief = _build_brief(
|
|
"crm",
|
|
f"CRM snapshot {date.today()}",
|
|
f"{clients_n} klanten, pipeline €{pipeline:,.0f}, {len(deals)} actieve deals.",
|
|
[{"finding": f"Pipeline €{pipeline:,.0f}", "relevance": "high"}],
|
|
)
|
|
retail_brief = _build_brief(
|
|
"retail",
|
|
f"Retail NL {date.today()}",
|
|
f"{stores} supermarkten in database, partnerships actief/proposal gemapt.",
|
|
[{"finding": f"{stores} locaties geladen", "relevance": "medium"}],
|
|
)
|
|
social_brief = _build_brief(
|
|
"social",
|
|
f"Social week {date.today()}",
|
|
f"{mentions} mentions afgelopen 7 dagen.",
|
|
[{"finding": f"{mentions} mentions", "relevance": "medium"}],
|
|
)
|
|
|
|
log_agent_event(
|
|
agent_name="research",
|
|
event_type="research_run",
|
|
title="Full research cycle completed",
|
|
metadata={"briefs": 3, "snapshots": len(snapshot_ids)},
|
|
)
|
|
|
|
from app.connectors.project_assets import register_agent_output
|
|
|
|
for brief, domain in [
|
|
(crm_brief, "crm"),
|
|
(retail_brief, "retail"),
|
|
(social_brief, "social"),
|
|
]:
|
|
try:
|
|
register_agent_output(
|
|
asset_type="research_brief",
|
|
title=brief.get("title") or f"Research {domain}",
|
|
ref_id=str(brief.get("id")),
|
|
payload={"domain": domain, "summary": brief.get("summary"), "brief_id": brief.get("id")},
|
|
source_agent="research",
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"snapshots": snapshot_ids,
|
|
"briefs": [crm_brief["id"], retail_brief["id"], social_brief["id"]],
|
|
}
|