Speed up Wereldexport map loading dramatically.
Limit world overview payload, optimize SQL joins, cache bundle and tiles, reuse map instance, and show loading state.
This commit is contained in:
Aissa
+113
-25
@@ -3,6 +3,9 @@ from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
@@ -67,6 +70,48 @@ from app.export_intel_sync import (
|
||||
sync_world_regions,
|
||||
)
|
||||
|
||||
|
||||
|
||||
_MAP_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
_MAP_CACHE_TTL = 120.0
|
||||
|
||||
|
||||
def _map_entity_limit(
|
||||
country: Optional[str],
|
||||
region: Optional[str],
|
||||
q: Optional[str],
|
||||
entity_type: Optional[str],
|
||||
entity_types: Optional[str],
|
||||
halal_min: Optional[float],
|
||||
favorite_only: bool,
|
||||
crm_linked: Optional[bool],
|
||||
) -> int:
|
||||
if q or entity_type or entity_types or halal_min is not None or favorite_only or crm_linked is not None:
|
||||
return 4000
|
||||
if country:
|
||||
return 12000
|
||||
if region:
|
||||
return 8000
|
||||
return 2500
|
||||
|
||||
|
||||
def _map_cache_get(key: str) -> Optional[dict[str, Any]]:
|
||||
hit = _MAP_CACHE.get(key)
|
||||
if not hit:
|
||||
return None
|
||||
ts, payload = hit
|
||||
if time.time() - ts > _MAP_CACHE_TTL:
|
||||
_MAP_CACHE.pop(key, None)
|
||||
return None
|
||||
return payload
|
||||
|
||||
|
||||
def _map_cache_set(key: str, payload: dict[str, Any]) -> None:
|
||||
if len(_MAP_CACHE) > 48:
|
||||
oldest = min(_MAP_CACHE.items(), key=lambda x: x[1][0])[0]
|
||||
_MAP_CACHE.pop(oldest, None)
|
||||
_MAP_CACHE[key] = (time.time(), payload)
|
||||
|
||||
router = APIRouter(prefix="/export-intel", tags=["export-intel"])
|
||||
|
||||
DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent")
|
||||
@@ -500,26 +545,58 @@ def map_bundle(
|
||||
where += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
|
||||
params.extend([f"%{q}%", f"%{q}%"])
|
||||
|
||||
row_limit = _map_entity_limit(
|
||||
country, region, q, entity_type, entity_types, halal_min, favorite_only, crm_linked
|
||||
)
|
||||
cache_key = hashlib.md5(
|
||||
json.dumps(
|
||||
{
|
||||
"country": country,
|
||||
"region": region,
|
||||
"entity_type": entity_type,
|
||||
"entity_types": entity_types,
|
||||
"q": q,
|
||||
"halal_min": halal_min,
|
||||
"favorite_only": favorite_only,
|
||||
"crm_linked": crm_linked,
|
||||
"limit": row_limit,
|
||||
},
|
||||
sort_keys=True,
|
||||
default=str,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
cached = _map_cache_get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
entities = fetch_all(
|
||||
f"""
|
||||
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
|
||||
e.confidence, e.pipeline_stage, e.city, e.address_line,
|
||||
e.email, e.phone, e.website, e.volume_band, e.source,
|
||||
e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at,
|
||||
e.product_interest, e.halal_cert_notes,
|
||||
(SELECT c.email FROM export_entity_contacts c
|
||||
WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> ''
|
||||
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email,
|
||||
(SELECT c.phone FROM export_entity_contacts c
|
||||
WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> ''
|
||||
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone,
|
||||
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
|
||||
e.is_favorite, e.client_id, e.deal_id,
|
||||
cp.contact_email, cp.contact_phone,
|
||||
COALESCE(cc.contact_count, 0) AS contact_count
|
||||
FROM export_market_entities e
|
||||
{join}
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT c.email AS contact_email,
|
||||
COALESCE(NULLIF(c.phone, ''), NULLIF(c.mobile, '')) AS contact_phone
|
||||
FROM export_entity_contacts c
|
||||
WHERE c.entity_id = e.id
|
||||
ORDER BY c.is_primary DESC NULLS LAST, c.id
|
||||
LIMIT 1
|
||||
) cp ON TRUE
|
||||
LEFT JOIN (
|
||||
SELECT entity_id, COUNT(*)::int AS contact_count
|
||||
FROM export_entity_contacts
|
||||
GROUP BY entity_id
|
||||
) cc ON cc.entity_id = e.id
|
||||
WHERE {where}
|
||||
LIMIT 20000
|
||||
ORDER BY e.confidence DESC NULLS LAST, e.id
|
||||
LIMIT %s
|
||||
""",
|
||||
tuple(params) if params else None,
|
||||
(tuple(params) + (row_limit,)) if params else (row_limit,),
|
||||
)
|
||||
|
||||
scored: list[tuple[dict[str, Any], float, list[str]]] = []
|
||||
@@ -558,22 +635,12 @@ def map_bundle(
|
||||
"name": row["name"],
|
||||
"entity_type": row["entity_type"],
|
||||
"country_iso2": row["country_iso2"],
|
||||
"confidence": row["confidence"],
|
||||
"pipeline_stage": row["pipeline_stage"],
|
||||
"city": row.get("city"),
|
||||
"address_line": row.get("address_line"),
|
||||
"email": row.get("email") or row.get("contact_email"),
|
||||
"phone": row.get("phone") or row.get("contact_phone"),
|
||||
"website": row.get("website"),
|
||||
"volume_band": row.get("volume_band"),
|
||||
"source": row.get("source"),
|
||||
"contact_count": int(row.get("contact_count") or 0),
|
||||
"halal_score": h_score,
|
||||
"halal_tier": halal_pin_tier(h_score),
|
||||
"halal_reasons": h_reasons[:4],
|
||||
"is_favorite": bool(row.get("is_favorite")),
|
||||
"client_id": row.get("client_id"),
|
||||
"deal_id": row.get("deal_id"),
|
||||
},
|
||||
}
|
||||
for row, h_score, h_reasons in scored
|
||||
@@ -614,22 +681,43 @@ def map_bundle(
|
||||
)
|
||||
trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows}
|
||||
|
||||
stats = export_stats(country=country, region=region)
|
||||
return {
|
||||
sidebar_entities = [
|
||||
{
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"entity_type": row["entity_type"],
|
||||
"country_iso2": row["country_iso2"],
|
||||
"city": row.get("city"),
|
||||
"email": row.get("email") or row.get("contact_email"),
|
||||
"phone": row.get("phone") or row.get("contact_phone"),
|
||||
"halal_score": h_score,
|
||||
"has_contact": bool(
|
||||
row.get("email") or row.get("contact_email")
|
||||
or row.get("phone") or row.get("contact_phone")
|
||||
or int(row.get("contact_count") or 0) > 0
|
||||
),
|
||||
}
|
||||
for row, h_score, h_reasons in scored[:200]
|
||||
]
|
||||
|
||||
payload = {
|
||||
"entities": {"type": "FeatureCollection", "features": features},
|
||||
"trade_choropleth": trade_choropleth,
|
||||
"halal_by_country": halal_by_country,
|
||||
"meta": {
|
||||
"entity_count": len(features),
|
||||
"map_limit": row_limit,
|
||||
"truncated": len(scored) >= row_limit,
|
||||
"countries_with_trade": len(trade_choropleth),
|
||||
"halal_min": halal_min,
|
||||
"top_markets": top_markets,
|
||||
"top_halal_entities": top_entities,
|
||||
"sidebar_entities": sidebar_entities,
|
||||
"phase": "B",
|
||||
"note": "Halal-score: type + land + naam/signaal + contact",
|
||||
},
|
||||
"stats": stats,
|
||||
}
|
||||
_map_cache_set(cache_key, payload)
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/halal/markets")
|
||||
|
||||
Reference in New Issue
Block a user