From 6856d09cd18a9df30d7d4803c61f11f102eaafb9 Mon Sep 17 00:00:00 2001 From: Aissa Date: Sun, 19 Jul 2026 18:42:47 +0000 Subject: [PATCH] Speed up Wereldexport map loading dramatically. Limit world overview payload, optimize SQL joins, cache bundle and tiles, reuse map instance, and show loading state. --- cockpit/app/routes/export_intel.py | 47 +++++++-- cockpit/static/css/export-intel.css | 17 ++++ cockpit/static/js/export-intel-map.js | 4 + cockpit/static/js/export-intel.js | 28 ++++-- cockpit/templates/export_intel.html | 9 +- tools-api/app/export_intel.py | 138 +++++++++++++++++++++----- 6 files changed, 198 insertions(+), 45 deletions(-) diff --git a/cockpit/app/routes/export_intel.py b/cockpit/app/routes/export_intel.py index 70b7a5a..4a5027b 100644 --- a/cockpit/app/routes/export_intel.py +++ b/cockpit/app/routes/export_intel.py @@ -2,6 +2,7 @@ from __future__ import annotations import os +import time from typing import Any, Optional import httpx @@ -15,6 +16,14 @@ BASE = Path(__file__).resolve().parent.parent.parent templates = Jinja2Templates(directory=str(BASE / "templates")) TOOLS = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/") +_TILE_CACHE: dict[str, tuple[float, bytes]] = {} +_TILE_CACHE_TTL = 86400.0 +_TILE_CACHE_MAX = 900 + +_BUNDLE_CACHE: dict[str, tuple[float, bytes]] = {} +_BUNDLE_CACHE_TTL = 120.0 + + async def _proxy(method: str, path: str, **kwargs) -> Any: url = f"{TOOLS}/export-intel{path}" @@ -161,7 +170,12 @@ async def api_gov_sources(country: Optional[str] = None): @router.get("/api/map/tiles/{z}/{x}/{y}.png") async def map_tile_proxy(z: int, x: int, y: int) -> Response: - """Proxy Carto dark basemap tiles via cockpit (no client internet needed).""" + """Proxy Carto dark basemap tiles via cockpit (cached, no client internet needed).""" + key = f"{z}/{x}/{y}" + hit = _TILE_CACHE.get(key) + if hit and time.time() - hit[0] < _TILE_CACHE_TTL: + return Response(content=hit[1], media_type="image/png", headers={"Cache-Control": "public, max-age=604800"}) + host = ("a", "b", "c", "d")[(x + y) % 4] url = f"https://{host}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png" try: @@ -169,11 +183,12 @@ async def map_tile_proxy(z: int, x: int, y: int) -> Response: r = await client.get(url) if r.status_code != 200: return Response(status_code=502, content=b"") - return Response( - content=r.content, - media_type="image/png", - headers={"Cache-Control": "public, max-age=604800"}, - ) + body = r.content + if len(_TILE_CACHE) >= _TILE_CACHE_MAX: + oldest = min(_TILE_CACHE.items(), key=lambda item: item[1][0])[0] + _TILE_CACHE.pop(oldest, None) + _TILE_CACHE[key] = (time.time(), body) + return Response(content=body, media_type="image/png", headers={"Cache-Control": "public, max-age=604800"}) except Exception: return Response(status_code=502, content=b"") @@ -193,7 +208,25 @@ async def api_map_bundle( "entity_types": entity_types, "q": q, "halal_min": halal_min, "favorite_only": favorite_only, "crm_linked": crm_linked, }.items() if v is not None and v is not False} - return await _proxy("GET", "/map/bundle", params=params) + import hashlib + import json + + cache_key = hashlib.md5(json.dumps(params, sort_keys=True, default=str).encode()).hexdigest() + hit = _BUNDLE_CACHE.get(cache_key) + if hit and time.time() - hit[0] < _BUNDLE_CACHE_TTL: + return Response(content=hit[1], media_type="application/json") + + url = f"{TOOLS}/export-intel/map/bundle" + async with httpx.AsyncClient(timeout=60.0) as client: + r = await client.get(url, params=params) + if r.status_code >= 400: + return {"error": r.text, "status": r.status_code} + body = r.content + if len(_BUNDLE_CACHE) > 32: + oldest = min(_BUNDLE_CACHE.items(), key=lambda item: item[1][0])[0] + _BUNDLE_CACHE.pop(oldest, None) + _BUNDLE_CACHE[cache_key] = (time.time(), body) + return Response(content=body, media_type="application/json") @router.post("/api/export-intel/entities/favorites") diff --git a/cockpit/static/css/export-intel.css b/cockpit/static/css/export-intel.css index 1034c1c..135b860 100644 --- a/cockpit/static/css/export-intel.css +++ b/cockpit/static/css/export-intel.css @@ -42,6 +42,23 @@ .ei-filter-clear { font-size: 0.72rem; opacity: 0.85; } .ei-split { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; min-height: 420px; } @media (max-width: 900px) { .ei-split { grid-template-columns: 1fr; } } + +.ei-map-wrap { position: relative; } +.ei-map-loading { + position: absolute; inset: 0; z-index: 500; + display: flex; align-items: center; justify-content: center; gap: 0.65rem; + background: rgba(8, 12, 18, 0.72); color: #94a3b8; font-size: 0.85rem; + pointer-events: none; +} +.ei-map-spinner { + width: 18px; height: 18px; border-radius: 50%; + border: 2px solid rgba(56, 189, 248, 0.25); + border-top-color: #38bdf8; + animation: ei-spin 0.8s linear infinite; +} +@keyframes ei-spin { to { transform: rotate(360deg); } } +.ei-map-truncated { margin: 0 0 0.65rem; font-size: 0.75rem; color: #64748b; } + .ei-map-error { margin: 0 0 0.65rem; padding: 0.65rem 0.85rem; border-radius: 10px; border: 1px solid rgba(251,113,133,0.35); background: rgba(127,29,29,0.25); color: #fecdd3; font-size: 0.82rem; } .ei-map-wrap { border-radius: 14px; overflow: hidden; border: 1px solid rgba(14,165,233,0.2); height: 420px; min-height: 420px; background: #0a0e14; } .ei-map-wrap #ei-map { height: 420px; width: 100%; } diff --git a/cockpit/static/js/export-intel-map.js b/cockpit/static/js/export-intel-map.js index 1b17b73..613e76e 100644 --- a/cockpit/static/js/export-intel-map.js +++ b/cockpit/static/js/export-intel-map.js @@ -170,6 +170,10 @@ window.ExportIntelMap = (function () { function init(containerId) { var el = document.getElementById(containerId); if (!el || !window.L) return null; + if (map && map.getContainer() === el) { + invalidateMapSize(); + return map; + } if (map) { map.remove(); map = null; diff --git a/cockpit/static/js/export-intel.js b/cockpit/static/js/export-intel.js index 14aaafd..fa4b128 100644 --- a/cockpit/static/js/export-intel.js +++ b/cockpit/static/js/export-intel.js @@ -26,6 +26,8 @@ function exportIntelApp() { mapHalalMode: false, mapHalalMin: 55, mapError: '', + mapLoading: false, + mapTruncated: false, halalTopMarkets: [], halalTopEntities: [], listFocusId: null, @@ -242,13 +244,7 @@ function exportIntelApp() { var base = '/api/export-intel'; var fq = this.filterQs(); if (this.tab === 'map') { - var mapQs = this.filterQs(); - var mt = this.mapTypesParam(); - if (mt.entity_type) mapQs += '&entity_type=' + encodeURIComponent(mt.entity_type); - else if (mt.entity_types) mapQs += '&entity_types=' + encodeURIComponent(mt.entity_types); - var rm = await fetch(base + '/entities?limit=200' + mapQs).then((x) => x.json()); - this.entities = rm.items || []; - this.resultCount = rm.total != null ? rm.total : this.entities.length; + return; } else if (this.tab === 'distributors') { var types = 'distributor,wholesaler,importer,logistics'; var et = this.entityTypeFilter ? '&entity_type=' + encodeURIComponent(this.entityTypeFilter) : ''; @@ -307,6 +303,8 @@ function exportIntelApp() { async loadMap() { this.clearListFocus(); + this.mapLoading = true; + this.mapError = ''; var url = '/api/export-intel/map/bundle'; var qs = []; if (this.country) qs.push('country=' + encodeURIComponent(this.country)); @@ -320,16 +318,27 @@ function exportIntelApp() { if (this.filterCrmStatus === 'linked') qs.push('crm_linked=true'); if (this.filterCrmStatus === 'not_linked') qs.push('crm_linked=false'); if (qs.length) url += '?' + qs.join('&'); - var bundle = await fetch(url).then((r) => r.json()); + var bundle; + try { + bundle = await fetch(url).then((r) => r.json()); + } catch (e) { + this.mapLoading = false; + this.mapError = 'Kaart laden mislukt. Probeer opnieuw.'; + return; + } this.halalTopMarkets = (bundle.meta && bundle.meta.top_markets) || []; this.halalTopEntities = (bundle.meta && bundle.meta.top_halal_entities) || []; + this.entities = (bundle.meta && bundle.meta.sidebar_entities) || this.halalTopEntities || []; + this.resultCount = (bundle.meta && bundle.meta.entity_count) || this.entities.length; + this.mapTruncated = !!(bundle.meta && bundle.meta.truncated); if (!window.L || !window.ExportIntelMap) { this.mapError = 'Kaart kon niet laden. Vernieuw de pagina (Ctrl+F5).'; + this.mapLoading = false; return; } this.mapError = ''; if (window.ExportIntelMap) { - if (!document.getElementById('ei-map')) return; + if (!document.getElementById('ei-map')) { this.mapLoading = false; return; } var self = this; window.ExportIntelMap.init('ei-map'); window.ExportIntelMap.setOnEntityClick(function (id) { self.openEntity(id); }); @@ -340,6 +349,7 @@ function exportIntelApp() { if (t && t.lat && t.lon) window.ExportIntelMap.flyTo(t.lat, t.lon, t.map_zoom || 6); } } + this.mapLoading = false; }, setTab(id, el) { diff --git a/cockpit/templates/export_intel.html b/cockpit/templates/export_intel.html index 6533fba..b0d28c3 100644 --- a/cockpit/templates/export_intel.html +++ b/cockpit/templates/export_intel.html @@ -4,11 +4,11 @@ - + - + {% endblock %} {% block content %}
@@ -237,7 +237,8 @@
-
+

Wereldoverzicht toont de belangrijkste ~2.500 locaties. Filter op regio of land voor meer detail.

+
Kaart laden…

Klik rij of kaart-pin voor volledig profiel.

@@ -797,5 +798,5 @@
{% endblock %} {% block scripts %} - + {% endblock %} diff --git a/tools-api/app/export_intel.py b/tools-api/app/export_intel.py index a78138c..0a0961b 100644 --- a/tools-api/app/export_intel.py +++ b/tools-api/app/export_intel.py @@ -3,6 +3,9 @@ from __future__ import annotations import csv import io +import hashlib +import json +import time from typing import Any, Optional from fastapi import APIRouter, HTTPException, Query @@ -67,6 +70,48 @@ from app.export_intel_sync import ( sync_world_regions, ) + + +_MAP_CACHE: dict[str, tuple[float, dict[str, Any]]] = {} +_MAP_CACHE_TTL = 120.0 + + +def _map_entity_limit( + country: Optional[str], + region: Optional[str], + q: Optional[str], + entity_type: Optional[str], + entity_types: Optional[str], + halal_min: Optional[float], + favorite_only: bool, + crm_linked: Optional[bool], +) -> int: + if q or entity_type or entity_types or halal_min is not None or favorite_only or crm_linked is not None: + return 4000 + if country: + return 12000 + if region: + return 8000 + return 2500 + + +def _map_cache_get(key: str) -> Optional[dict[str, Any]]: + hit = _MAP_CACHE.get(key) + if not hit: + return None + ts, payload = hit + if time.time() - ts > _MAP_CACHE_TTL: + _MAP_CACHE.pop(key, None) + return None + return payload + + +def _map_cache_set(key: str, payload: dict[str, Any]) -> None: + if len(_MAP_CACHE) > 48: + oldest = min(_MAP_CACHE.items(), key=lambda x: x[1][0])[0] + _MAP_CACHE.pop(oldest, None) + _MAP_CACHE[key] = (time.time(), payload) + router = APIRouter(prefix="/export-intel", tags=["export-intel"]) DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent") @@ -500,26 +545,58 @@ def map_bundle( where += " AND (e.name ILIKE %s OR e.city ILIKE %s)" params.extend([f"%{q}%", f"%{q}%"]) + row_limit = _map_entity_limit( + country, region, q, entity_type, entity_types, halal_min, favorite_only, crm_linked + ) + cache_key = hashlib.md5( + json.dumps( + { + "country": country, + "region": region, + "entity_type": entity_type, + "entity_types": entity_types, + "q": q, + "halal_min": halal_min, + "favorite_only": favorite_only, + "crm_linked": crm_linked, + "limit": row_limit, + }, + sort_keys=True, + default=str, + ).encode() + ).hexdigest() + cached = _map_cache_get(cache_key) + if cached is not None: + return cached + entities = fetch_all( f""" SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon, e.confidence, e.pipeline_stage, e.city, e.address_line, e.email, e.phone, e.website, e.volume_band, e.source, - e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at, - e.product_interest, e.halal_cert_notes, - (SELECT c.email FROM export_entity_contacts c - WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> '' - ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email, - (SELECT c.phone FROM export_entity_contacts c - WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> '' - ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone, - (SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count + e.is_favorite, e.client_id, e.deal_id, + cp.contact_email, cp.contact_phone, + COALESCE(cc.contact_count, 0) AS contact_count FROM export_market_entities e {join} + LEFT JOIN LATERAL ( + SELECT c.email AS contact_email, + COALESCE(NULLIF(c.phone, ''), NULLIF(c.mobile, '')) AS contact_phone + FROM export_entity_contacts c + WHERE c.entity_id = e.id + ORDER BY c.is_primary DESC NULLS LAST, c.id + LIMIT 1 + ) cp ON TRUE + LEFT JOIN ( + SELECT entity_id, COUNT(*)::int AS contact_count + FROM export_entity_contacts + GROUP BY entity_id + ) cc ON cc.entity_id = e.id WHERE {where} - LIMIT 20000 + ORDER BY e.confidence DESC NULLS LAST, e.id + LIMIT %s """, - tuple(params) if params else None, + (tuple(params) + (row_limit,)) if params else (row_limit,), ) scored: list[tuple[dict[str, Any], float, list[str]]] = [] @@ -558,22 +635,12 @@ def map_bundle( "name": row["name"], "entity_type": row["entity_type"], "country_iso2": row["country_iso2"], - "confidence": row["confidence"], - "pipeline_stage": row["pipeline_stage"], "city": row.get("city"), - "address_line": row.get("address_line"), "email": row.get("email") or row.get("contact_email"), "phone": row.get("phone") or row.get("contact_phone"), - "website": row.get("website"), - "volume_band": row.get("volume_band"), - "source": row.get("source"), "contact_count": int(row.get("contact_count") or 0), "halal_score": h_score, "halal_tier": halal_pin_tier(h_score), - "halal_reasons": h_reasons[:4], - "is_favorite": bool(row.get("is_favorite")), - "client_id": row.get("client_id"), - "deal_id": row.get("deal_id"), }, } for row, h_score, h_reasons in scored @@ -614,22 +681,43 @@ def map_bundle( ) trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows} - stats = export_stats(country=country, region=region) - return { + sidebar_entities = [ + { + "id": row["id"], + "name": row["name"], + "entity_type": row["entity_type"], + "country_iso2": row["country_iso2"], + "city": row.get("city"), + "email": row.get("email") or row.get("contact_email"), + "phone": row.get("phone") or row.get("contact_phone"), + "halal_score": h_score, + "has_contact": bool( + row.get("email") or row.get("contact_email") + or row.get("phone") or row.get("contact_phone") + or int(row.get("contact_count") or 0) > 0 + ), + } + for row, h_score, h_reasons in scored[:200] + ] + + payload = { "entities": {"type": "FeatureCollection", "features": features}, "trade_choropleth": trade_choropleth, - "halal_by_country": halal_by_country, "meta": { "entity_count": len(features), + "map_limit": row_limit, + "truncated": len(scored) >= row_limit, "countries_with_trade": len(trade_choropleth), "halal_min": halal_min, "top_markets": top_markets, "top_halal_entities": top_entities, + "sidebar_entities": sidebar_entities, "phase": "B", "note": "Halal-score: type + land + naam/signaal + contact", }, - "stats": stats, } + _map_cache_set(cache_key, payload) + return payload @router.get("/halal/markets")