87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
|
"""Full-system data export for Reports hub."""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import csv
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from app.db import fetch_all
|
||
|
|
|
||
|
|
EXPORT_DATASETS: dict[str, dict[str, str]] = {
|
||
|
|
"clients": {"label": "CRM Klanten", "table": "clients", "order": "updated_at DESC"},
|
||
|
|
"deals": {"label": "CRM Deals", "table": "deals", "order": "updated_at DESC"},
|
||
|
|
"supermarkets": {"label": "Supermarkten", "table": "supermarkets", "order": "name ASC"},
|
||
|
|
"wholesalers": {"label": "Groothandels", "table": "wholesalers", "order": "name ASC"},
|
||
|
|
"supermarket_contacts": {"label": "Supermarkt contacten", "table": "supermarket_contacts", "order": "id ASC"},
|
||
|
|
"wholesaler_contacts": {"label": "Groothandel contacten", "table": "wholesaler_contacts", "order": "id ASC"},
|
||
|
|
"rss_items": {"label": "RSS items", "table": "rss_items", "order": "published_at DESC NULLS LAST"},
|
||
|
|
"rss_bookmarks": {"label": "RSS bookmarks", "table": "rss_bookmarks", "order": "created_at DESC"},
|
||
|
|
"agent_events": {"label": "Agent events", "table": "agent_events", "order": "created_at DESC"},
|
||
|
|
"sales_milestones": {"label": "Sales milestones", "table": "sales_milestones", "order": "created_at DESC"},
|
||
|
|
"promo_campaigns": {"label": "Promo / reclame", "table": "promo_campaigns", "order": "created_at DESC"},
|
||
|
|
"daily_briefings": {"label": "Dagrapporten", "table": "daily_briefings", "order": "created_at DESC"},
|
||
|
|
"document_analytics": {"label": "NAS documenten", "table": "document_analytics", "order": "analyzed_at DESC NULLS LAST"},
|
||
|
|
"products": {"label": "Producten", "table": "products", "order": "name ASC"},
|
||
|
|
"suppliers": {"label": "Leveranciers", "table": "suppliers", "order": "name ASC"},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _serialize(val: Any) -> Any:
|
||
|
|
if hasattr(val, "isoformat"):
|
||
|
|
return val.isoformat()
|
||
|
|
if isinstance(val, (dict, list)):
|
||
|
|
return json.dumps(val, default=str)
|
||
|
|
if val is not None and type(val).__name__ == "Decimal":
|
||
|
|
return float(val)
|
||
|
|
return val
|
||
|
|
|
||
|
|
|
||
|
|
def list_datasets() -> list[dict[str, Any]]:
|
||
|
|
out = []
|
||
|
|
for key, meta in EXPORT_DATASETS.items():
|
||
|
|
count = 0
|
||
|
|
try:
|
||
|
|
from app.db import fetch_one
|
||
|
|
row = fetch_one(f"SELECT COUNT(*) AS c FROM {meta['table']}")
|
||
|
|
count = int(row["c"]) if row else 0
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
out.append({"id": key, "label": meta["label"], "count": count})
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_dataset(name: str, limit: int = 10000) -> list[dict[str, Any]]:
|
||
|
|
meta = EXPORT_DATASETS.get(name)
|
||
|
|
if not meta:
|
||
|
|
raise ValueError(f"Unknown dataset: {name}")
|
||
|
|
rows = fetch_all(f"SELECT * FROM {meta['table']} ORDER BY {meta['order']} LIMIT %s", (limit,))
|
||
|
|
for row in rows:
|
||
|
|
for k, v in list(row.items()):
|
||
|
|
row[k] = _serialize(v)
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def to_csv(rows: list[dict[str, Any]]) -> str:
|
||
|
|
if not rows:
|
||
|
|
return ""
|
||
|
|
buf = io.StringIO()
|
||
|
|
writer = csv.DictWriter(buf, fieldnames=list(rows[0].keys()), extrasaction="ignore")
|
||
|
|
writer.writeheader()
|
||
|
|
writer.writerows(rows)
|
||
|
|
return buf.getvalue()
|
||
|
|
|
||
|
|
|
||
|
|
def export_all_json(limit: int = 5000) -> dict[str, Any]:
|
||
|
|
bundle: dict[str, Any] = {
|
||
|
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||
|
|
"datasets": {},
|
||
|
|
}
|
||
|
|
for key in EXPORT_DATASETS:
|
||
|
|
try:
|
||
|
|
bundle["datasets"][key] = fetch_dataset(key, limit=min(limit, 5000))
|
||
|
|
except Exception as exc:
|
||
|
|
bundle["datasets"][key] = {"error": str(exc)}
|
||
|
|
return bundle
|