Platform bundle: marketing publish, IT ops, packaging, agents mesh.
Volledige Foodlinkk Command Center uitbreiding met social automatisering, reclamefolder filters, Proxmox monitoring en documentatie.
This commit is contained in:
Aissa
@@ -0,0 +1,183 @@
|
||||
"""Research pipeline: data providers, snapshots, briefs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
|
||||
from app.middleware import log_agent_event
|
||||
|
||||
router = APIRouter(prefix="/research", tags=["research"])
|
||||
|
||||
|
||||
def _json_safe(obj: Any) -> Any:
|
||||
if isinstance(obj, dict):
|
||||
return {k: _json_safe(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_json_safe(x) for x in obj]
|
||||
if hasattr(obj, "isoformat"):
|
||||
return obj.isoformat()
|
||||
if type(obj).__name__ == "Decimal":
|
||||
return float(obj)
|
||||
return obj
|
||||
|
||||
|
||||
def _http_get(url: str, timeout: int = 30) -> dict:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
@router.get("/providers")
|
||||
def list_providers() -> dict[str, Any]:
|
||||
rows = fetch_all("SELECT * FROM data_providers ORDER BY name")
|
||||
return {"items": [dict(r) for r in rows]}
|
||||
|
||||
|
||||
@router.post("/providers/{provider_id}/refresh")
|
||||
def refresh_provider(provider_id: int) -> dict[str, Any]:
|
||||
prov = fetch_one("SELECT * FROM data_providers WHERE id = %s", (provider_id,))
|
||||
if not prov:
|
||||
raise HTTPException(404, "Provider not found")
|
||||
name = prov["name"]
|
||||
payload: dict[str, Any] = {}
|
||||
count = 0
|
||||
if name == "crm":
|
||||
clients = fetch_all("SELECT id, name, stage FROM clients ORDER BY updated_at DESC LIMIT 20")
|
||||
deals = fetch_all(
|
||||
"SELECT id, title, value, stage, next_action, deadline FROM deals ORDER BY updated_at DESC LIMIT 20"
|
||||
)
|
||||
payload = {
|
||||
"clients": _json_safe([dict(c) for c in clients]),
|
||||
"deals": _json_safe([dict(d) for d in deals]),
|
||||
}
|
||||
count = len(clients) + len(deals)
|
||||
elif name == "weather":
|
||||
url = (
|
||||
"https://api.open-meteo.com/v1/forecast?"
|
||||
"latitude=52.37&longitude=4.89&daily=temperature_2m_max,precipitation_sum&timezone=Europe%2FAmsterdam&forecast_days=7"
|
||||
)
|
||||
payload = _http_get(url)
|
||||
count = len(payload.get("daily", {}).get("time", []))
|
||||
for i, day in enumerate(payload.get("daily", {}).get("time", [])[:7]):
|
||||
temps = payload["daily"].get("temperature_2m_max", [])
|
||||
prec = payload["daily"].get("precipitation_sum", [])
|
||||
execute(
|
||||
"""INSERT INTO weather_data (region, city, date, temperature_c, precipitation_mm, weather_condition, data_source)
|
||||
VALUES ('Noord-Holland', 'Amsterdam', %s, %s, %s, 'forecast', 'open-meteo')""",
|
||||
(day, temps[i] if i < len(temps) else None, prec[i] if i < len(prec) else None),
|
||||
)
|
||||
elif name == "social":
|
||||
rows = fetch_all(
|
||||
"SELECT platform, text, sentiment_score, created_at FROM social_mentions ORDER BY created_at DESC LIMIT 30"
|
||||
)
|
||||
payload = _json_safe({"mentions": [dict(r) for r in rows]})
|
||||
count = len(rows)
|
||||
elif name == "retail_manual":
|
||||
rows = fetch_all("SELECT id, name, chain, city, partnership_status FROM supermarkets ORDER BY id")
|
||||
payload = _json_safe({"stores": [dict(r) for r in rows]})
|
||||
count = len(rows)
|
||||
elif name == "cbs":
|
||||
from app import retail_enrichment
|
||||
status = retail_enrichment.enrichment_status()
|
||||
batch = retail_enrichment.enrich_batch(limit=30, offset=0)
|
||||
payload = _json_safe({"status": status, "batch": batch})
|
||||
count = batch.get("ok", 0)
|
||||
elif name == "pdok":
|
||||
from app.connectors import pdok as pdok_conn
|
||||
sample = fetch_all(
|
||||
"SELECT DISTINCT postcode FROM supermarkets WHERE postcode <> '0000AA' LIMIT 5"
|
||||
)
|
||||
lookups = [pdok_conn.lookup_postcode(r["postcode"]) for r in sample]
|
||||
payload = _json_safe({"lookups": [x for x in lookups if x]})
|
||||
count = len(payload.get("lookups", []))
|
||||
else:
|
||||
payload = {"status": "noop"}
|
||||
snap = execute_returning(
|
||||
"""INSERT INTO data_snapshots (provider_id, payload, record_count)
|
||||
VALUES (%s, %s, %s) RETURNING id, fetched_at""",
|
||||
(provider_id, json_param(payload), count),
|
||||
)
|
||||
execute(
|
||||
"UPDATE data_providers SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
|
||||
(provider_id,),
|
||||
)
|
||||
log_agent_event(
|
||||
agent_name="research",
|
||||
event_type="data_refresh",
|
||||
title=f"Provider {name} refreshed",
|
||||
metadata={"provider_id": provider_id, "records": count},
|
||||
)
|
||||
return {"provider": name, "snapshot_id": snap["id"], "record_count": count}
|
||||
|
||||
|
||||
@router.get("/briefs")
|
||||
def list_briefs(limit: int = 20) -> dict[str, Any]:
|
||||
rows = fetch_all(
|
||||
"SELECT * FROM research_briefs ORDER BY generated_at DESC LIMIT %s",
|
||||
(limit,),
|
||||
)
|
||||
return {"items": [dict(r) for r in rows]}
|
||||
|
||||
|
||||
def _build_brief(domain: str, title: str, summary: str, findings: list[dict]) -> dict:
|
||||
row = execute_returning(
|
||||
"""INSERT INTO research_briefs (domain, title, summary, key_findings, expires_at)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
|
||||
(
|
||||
domain,
|
||||
title,
|
||||
summary,
|
||||
json_param(findings),
|
||||
datetime.now(timezone.utc) + timedelta(days=1),
|
||||
),
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
def run_research() -> dict[str, Any]:
|
||||
providers = fetch_all("SELECT id, name FROM data_providers WHERE is_active = true")
|
||||
snapshot_ids = []
|
||||
for p in providers:
|
||||
result = refresh_provider(int(p["id"]))
|
||||
snapshot_ids.append(result.get("snapshot_id"))
|
||||
|
||||
clients_n = fetch_one("SELECT COUNT(*) AS c FROM clients")["c"]
|
||||
deals = fetch_all("SELECT title, value, stage FROM deals WHERE stage NOT IN ('won','lost')")
|
||||
pipeline = sum(float(d.get("value") or 0) for d in deals)
|
||||
stores = fetch_one("SELECT COUNT(*) AS c FROM supermarkets")["c"]
|
||||
mentions = fetch_one("SELECT COUNT(*) AS c FROM social_mentions WHERE created_at > NOW() - interval '7 days'")["c"]
|
||||
|
||||
crm_brief = _build_brief(
|
||||
"crm",
|
||||
f"CRM snapshot {date.today()}",
|
||||
f"{clients_n} klanten, pipeline €{pipeline:,.0f}, {len(deals)} actieve deals.",
|
||||
[{"finding": f"Pipeline €{pipeline:,.0f}", "relevance": "high"}],
|
||||
)
|
||||
retail_brief = _build_brief(
|
||||
"retail",
|
||||
f"Retail NL {date.today()}",
|
||||
f"{stores} supermarkten in database, partnerships actief/proposal gemapt.",
|
||||
[{"finding": f"{stores} locaties geladen", "relevance": "medium"}],
|
||||
)
|
||||
social_brief = _build_brief(
|
||||
"social",
|
||||
f"Social week {date.today()}",
|
||||
f"{mentions} mentions afgelopen 7 dagen.",
|
||||
[{"finding": f"{mentions} mentions", "relevance": "medium"}],
|
||||
)
|
||||
|
||||
log_agent_event(
|
||||
agent_name="research",
|
||||
event_type="research_run",
|
||||
title="Full research cycle completed",
|
||||
metadata={"briefs": 3, "snapshots": len(snapshot_ids)},
|
||||
)
|
||||
return {
|
||||
"snapshots": snapshot_ids,
|
||||
"briefs": [crm_brief["id"], retail_brief["id"], social_brief["id"]],
|
||||
}
|
||||
Reference in New Issue
Block a user