Speed up Wereldexport map loading dramatically.

Limit world overview payload, optimize SQL joins, cache bundle and tiles, reuse map instance, and show loading state.

cockpit/app/routes/export_intel.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import os
+import time
 from typing import Any, Optional
 
 import httpx
@@ -15,6 +16,14 @@ BASE = Path(__file__).resolve().parent.parent.parent
 templates = Jinja2Templates(directory=str(BASE / "templates"))
 TOOLS = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/")
 
+_TILE_CACHE: dict[str, tuple[float, bytes]] = {}
+_TILE_CACHE_TTL = 86400.0
+_TILE_CACHE_MAX = 900
+
+_BUNDLE_CACHE: dict[str, tuple[float, bytes]] = {}
+_BUNDLE_CACHE_TTL = 120.0
+
+
 
 async def _proxy(method: str, path: str, **kwargs) -> Any:
     url = f"{TOOLS}/export-intel{path}"
@@ -161,7 +170,12 @@ async def api_gov_sources(country: Optional[str] = None):
 
 @router.get("/api/map/tiles/{z}/{x}/{y}.png")
 async def map_tile_proxy(z: int, x: int, y: int) -> Response:
-    """Proxy Carto dark basemap tiles via cockpit (no client internet needed)."""
+    """Proxy Carto dark basemap tiles via cockpit (cached, no client internet needed)."""
+    key = f"{z}/{x}/{y}"
+    hit = _TILE_CACHE.get(key)
+    if hit and time.time() - hit[0] < _TILE_CACHE_TTL:
+        return Response(content=hit[1], media_type="image/png", headers={"Cache-Control": "public, max-age=604800"})
+
     host = ("a", "b", "c", "d")[(x + y) % 4]
     url = f"https://{host}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png"
     try:
@@ -169,11 +183,12 @@ async def map_tile_proxy(z: int, x: int, y: int) -> Response:
             r = await client.get(url)
             if r.status_code != 200:
                 return Response(status_code=502, content=b"")
-            return Response(
+            body = r.content
-                content=r.content,
+            if len(_TILE_CACHE) >= _TILE_CACHE_MAX:
-                media_type="image/png",
+                oldest = min(_TILE_CACHE.items(), key=lambda item: item[1][0])[0]
-                headers={"Cache-Control": "public, max-age=604800"},
+                _TILE_CACHE.pop(oldest, None)
-            )
+            _TILE_CACHE[key] = (time.time(), body)
+            return Response(content=body, media_type="image/png", headers={"Cache-Control": "public, max-age=604800"})
     except Exception:
         return Response(status_code=502, content=b"")
 
@@ -193,7 +208,25 @@ async def api_map_bundle(
         "entity_types": entity_types, "q": q, "halal_min": halal_min,
         "favorite_only": favorite_only, "crm_linked": crm_linked,
     }.items() if v is not None and v is not False}
-    return await _proxy("GET", "/map/bundle", params=params)
+    import hashlib
+    import json
+
+    cache_key = hashlib.md5(json.dumps(params, sort_keys=True, default=str).encode()).hexdigest()
+    hit = _BUNDLE_CACHE.get(cache_key)
+    if hit and time.time() - hit[0] < _BUNDLE_CACHE_TTL:
+        return Response(content=hit[1], media_type="application/json")
+
+    url = f"{TOOLS}/export-intel/map/bundle"
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        r = await client.get(url, params=params)
+        if r.status_code >= 400:
+            return {"error": r.text, "status": r.status_code}
+        body = r.content
+        if len(_BUNDLE_CACHE) > 32:
+            oldest = min(_BUNDLE_CACHE.items(), key=lambda item: item[1][0])[0]
+            _BUNDLE_CACHE.pop(oldest, None)
+        _BUNDLE_CACHE[cache_key] = (time.time(), body)
+        return Response(content=body, media_type="application/json")
 
 
 @router.post("/api/export-intel/entities/favorites")

cockpit/static/css/export-intel.css

@@ -42,6 +42,23 @@
 .ei-filter-clear { font-size: 0.72rem; opacity: 0.85; }
 .ei-split { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; min-height: 420px; }
 @media (max-width: 900px) { .ei-split { grid-template-columns: 1fr; } }
+
+.ei-map-wrap { position: relative; }
+.ei-map-loading {
+  position: absolute; inset: 0; z-index: 500;
+  display: flex; align-items: center; justify-content: center; gap: 0.65rem;
+  background: rgba(8, 12, 18, 0.72); color: #94a3b8; font-size: 0.85rem;
+  pointer-events: none;
+}
+.ei-map-spinner {
+  width: 18px; height: 18px; border-radius: 50%;
+  border: 2px solid rgba(56, 189, 248, 0.25);
+  border-top-color: #38bdf8;
+  animation: ei-spin 0.8s linear infinite;
+}
+@keyframes ei-spin { to { transform: rotate(360deg); } }
+.ei-map-truncated { margin: 0 0 0.65rem; font-size: 0.75rem; color: #64748b; }
+
 .ei-map-error { margin: 0 0 0.65rem; padding: 0.65rem 0.85rem; border-radius: 10px; border: 1px solid rgba(251,113,133,0.35); background: rgba(127,29,29,0.25); color: #fecdd3; font-size: 0.82rem; }
 .ei-map-wrap { border-radius: 14px; overflow: hidden; border: 1px solid rgba(14,165,233,0.2); height: 420px; min-height: 420px; background: #0a0e14; }
 .ei-map-wrap #ei-map { height: 420px; width: 100%; }

cockpit/static/js/export-intel-map.js

@@ -170,6 +170,10 @@ window.ExportIntelMap = (function () {
   function init(containerId) {
     var el = document.getElementById(containerId);
     if (!el || !window.L) return null;
+    if (map && map.getContainer() === el) {
+      invalidateMapSize();
+      return map;
+    }
     if (map) {
       map.remove();
       map = null;

cockpit/static/js/export-intel.js

@@ -26,6 +26,8 @@ function exportIntelApp() {
     mapHalalMode: false,
     mapHalalMin: 55,
     mapError: '',
+    mapLoading: false,
+    mapTruncated: false,
     halalTopMarkets: [],
     halalTopEntities: [],
     listFocusId: null,
@@ -242,13 +244,7 @@ function exportIntelApp() {
       var base = '/api/export-intel';
       var fq = this.filterQs();
       if (this.tab === 'map') {
-        var mapQs = this.filterQs();
+        return;
-        var mt = this.mapTypesParam();
-        if (mt.entity_type) mapQs += '&entity_type=' + encodeURIComponent(mt.entity_type);
-        else if (mt.entity_types) mapQs += '&entity_types=' + encodeURIComponent(mt.entity_types);
-        var rm = await fetch(base + '/entities?limit=200' + mapQs).then((x) => x.json());
-        this.entities = rm.items || [];
-        this.resultCount = rm.total != null ? rm.total : this.entities.length;
       } else if (this.tab === 'distributors') {
         var types = 'distributor,wholesaler,importer,logistics';
         var et = this.entityTypeFilter ? '&entity_type=' + encodeURIComponent(this.entityTypeFilter) : '';
@@ -307,6 +303,8 @@ function exportIntelApp() {
 
     async loadMap() {
       this.clearListFocus();
+      this.mapLoading = true;
+      this.mapError = '';
       var url = '/api/export-intel/map/bundle';
       var qs = [];
       if (this.country) qs.push('country=' + encodeURIComponent(this.country));
@@ -320,16 +318,27 @@ function exportIntelApp() {
       if (this.filterCrmStatus === 'linked') qs.push('crm_linked=true');
       if (this.filterCrmStatus === 'not_linked') qs.push('crm_linked=false');
       if (qs.length) url += '?' + qs.join('&');
-      var bundle = await fetch(url).then((r) => r.json());
+      var bundle;
+      try {
+        bundle = await fetch(url).then((r) => r.json());
+      } catch (e) {
+        this.mapLoading = false;
+        this.mapError = 'Kaart laden mislukt. Probeer opnieuw.';
+        return;
+      }
       this.halalTopMarkets = (bundle.meta && bundle.meta.top_markets) || [];
       this.halalTopEntities = (bundle.meta && bundle.meta.top_halal_entities) || [];
+      this.entities = (bundle.meta && bundle.meta.sidebar_entities) || this.halalTopEntities || [];
+      this.resultCount = (bundle.meta && bundle.meta.entity_count) || this.entities.length;
+      this.mapTruncated = !!(bundle.meta && bundle.meta.truncated);
       if (!window.L || !window.ExportIntelMap) {
         this.mapError = 'Kaart kon niet laden. Vernieuw de pagina (Ctrl+F5).';
+        this.mapLoading = false;
         return;
       }
       this.mapError = '';
       if (window.ExportIntelMap) {
-        if (!document.getElementById('ei-map')) return;
+        if (!document.getElementById('ei-map')) { this.mapLoading = false; return; }
         var self = this;
         window.ExportIntelMap.init('ei-map');
         window.ExportIntelMap.setOnEntityClick(function (id) { self.openEntity(id); });
@@ -340,6 +349,7 @@ function exportIntelApp() {
           if (t && t.lat && t.lon) window.ExportIntelMap.flyTo(t.lat, t.lon, t.map_zoom || 6);
         }
       }
+      this.mapLoading = false;
     },
 
     setTab(id, el) {

cockpit/templates/export_intel.html

@@ -4,11 +4,11 @@
 <link rel="stylesheet" href="/static/vendor/leaflet/leaflet.css?v=1" />
 <link rel="stylesheet" href="/static/vendor/leaflet.markercluster/MarkerCluster.css?v=1" />
 <link rel="stylesheet" href="/static/vendor/leaflet.markercluster/MarkerCluster.Default.css?v=1" />
-<link rel="stylesheet" href="/static/css/export-intel.css?v=13" />
+<link rel="stylesheet" href="/static/css/export-intel.css?v=14" />
 <link rel="stylesheet" href="/static/css/export-intel-tabs.css?v=10" />
 <script src="/static/vendor/leaflet/leaflet.js?v=1"></script>
 <script src="/static/vendor/leaflet.markercluster/leaflet.markercluster.js?v=1"></script>
-<script src="/static/js/export-intel-map.js?v=9"></script>
+<script src="/static/js/export-intel-map.js?v=10"></script>
 {% endblock %}
 {% block content %}
 <div class="ei-page" x-data="exportIntelApp()" x-init="init()">
@@ -237,7 +237,8 @@
         </div>
         <div class="ei-split">
           <div class="ei-map-error" x-show="mapError" x-cloak x-text="mapError"></div>
-          <div class="ei-map-wrap"><div id="ei-map"></div></div>
+            <p class="ei-map-truncated" x-show="mapTruncated && !mapLoading" x-cloak>Wereldoverzicht toont de belangrijkste ~2.500 locaties. Filter op regio of land voor meer detail.</p>
+          <div class="ei-map-wrap"><div class="ei-map-loading" x-show="mapLoading" x-cloak><span class="ei-map-spinner"></span> Kaart laden…</div><div id="ei-map"></div></div>
           <div class="ei-panel">
             <h3 style="margin:0 0 0.5rem;font-size:0.9rem" x-text="mapHalalMode ? 'Top halal-kansen' : 'Entities op kaart'"></h3>
             <p style="font-size:0.75rem;color:#64748b;margin:0 0 0.75rem" x-show="!mapHalalMode">Klik rij of kaart-pin voor volledig profiel.</p>
@@ -797,5 +798,5 @@
 </div>
 {% endblock %}
 {% block scripts %}
-<script src="/static/js/export-intel.js?v=12"></script>
+<script src="/static/js/export-intel.js?v=13"></script>
 {% endblock %}

tools-api/app/export_intel.py

@@ -3,6 +3,9 @@ from __future__ import annotations
 
 import csv
 import io
+import hashlib
+import json
+import time
 from typing import Any, Optional
 
 from fastapi import APIRouter, HTTPException, Query
@@ -67,6 +70,48 @@ from app.export_intel_sync import (
     sync_world_regions,
 )
 
+
+
+_MAP_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
+_MAP_CACHE_TTL = 120.0
+
+
+def _map_entity_limit(
+    country: Optional[str],
+    region: Optional[str],
+    q: Optional[str],
+    entity_type: Optional[str],
+    entity_types: Optional[str],
+    halal_min: Optional[float],
+    favorite_only: bool,
+    crm_linked: Optional[bool],
+) -> int:
+    if q or entity_type or entity_types or halal_min is not None or favorite_only or crm_linked is not None:
+        return 4000
+    if country:
+        return 12000
+    if region:
+        return 8000
+    return 2500
+
+
+def _map_cache_get(key: str) -> Optional[dict[str, Any]]:
+    hit = _MAP_CACHE.get(key)
+    if not hit:
+        return None
+    ts, payload = hit
+    if time.time() - ts > _MAP_CACHE_TTL:
+        _MAP_CACHE.pop(key, None)
+        return None
+    return payload
+
+
+def _map_cache_set(key: str, payload: dict[str, Any]) -> None:
+    if len(_MAP_CACHE) > 48:
+        oldest = min(_MAP_CACHE.items(), key=lambda x: x[1][0])[0]
+        _MAP_CACHE.pop(oldest, None)
+    _MAP_CACHE[key] = (time.time(), payload)
+
 router = APIRouter(prefix="/export-intel", tags=["export-intel"])
 
 DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent")
@@ -500,26 +545,58 @@ def map_bundle(
         where += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
         params.extend([f"%{q}%", f"%{q}%"])
 
+    row_limit = _map_entity_limit(
+        country, region, q, entity_type, entity_types, halal_min, favorite_only, crm_linked
+    )
+    cache_key = hashlib.md5(
+        json.dumps(
+            {
+                "country": country,
+                "region": region,
+                "entity_type": entity_type,
+                "entity_types": entity_types,
+                "q": q,
+                "halal_min": halal_min,
+                "favorite_only": favorite_only,
+                "crm_linked": crm_linked,
+                "limit": row_limit,
+            },
+            sort_keys=True,
+            default=str,
+        ).encode()
+    ).hexdigest()
+    cached = _map_cache_get(cache_key)
+    if cached is not None:
+        return cached
+
     entities = fetch_all(
         f"""
         SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
                e.confidence, e.pipeline_stage, e.city, e.address_line,
                e.email, e.phone, e.website, e.volume_band, e.source,
-               e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at,
+               e.is_favorite, e.client_id, e.deal_id,
-               e.product_interest, e.halal_cert_notes,
+               cp.contact_email, cp.contact_phone,
-               (SELECT c.email FROM export_entity_contacts c
+               COALESCE(cc.contact_count, 0) AS contact_count
-                WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> ''
-                ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email,
-               (SELECT c.phone FROM export_entity_contacts c
-                WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> ''
-                ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone,
-               (SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
         FROM export_market_entities e
         {join}
+        LEFT JOIN LATERAL (
+            SELECT c.email AS contact_email,
+                   COALESCE(NULLIF(c.phone, ''), NULLIF(c.mobile, '')) AS contact_phone
+            FROM export_entity_contacts c
+            WHERE c.entity_id = e.id
+            ORDER BY c.is_primary DESC NULLS LAST, c.id
+            LIMIT 1
+        ) cp ON TRUE
+        LEFT JOIN (
+            SELECT entity_id, COUNT(*)::int AS contact_count
+            FROM export_entity_contacts
+            GROUP BY entity_id
+        ) cc ON cc.entity_id = e.id
         WHERE {where}
-        LIMIT 20000
+        ORDER BY e.confidence DESC NULLS LAST, e.id
+        LIMIT %s
         """,
-        tuple(params) if params else None,
+        (tuple(params) + (row_limit,)) if params else (row_limit,),
     )
 
     scored: list[tuple[dict[str, Any], float, list[str]]] = []
@@ -558,22 +635,12 @@ def map_bundle(
                 "name": row["name"],
                 "entity_type": row["entity_type"],
                 "country_iso2": row["country_iso2"],
-                "confidence": row["confidence"],
-                "pipeline_stage": row["pipeline_stage"],
                 "city": row.get("city"),
-                "address_line": row.get("address_line"),
                 "email": row.get("email") or row.get("contact_email"),
                 "phone": row.get("phone") or row.get("contact_phone"),
-                "website": row.get("website"),
-                "volume_band": row.get("volume_band"),
-                "source": row.get("source"),
                 "contact_count": int(row.get("contact_count") or 0),
                 "halal_score": h_score,
                 "halal_tier": halal_pin_tier(h_score),
-                "halal_reasons": h_reasons[:4],
-                "is_favorite": bool(row.get("is_favorite")),
-                "client_id": row.get("client_id"),
-                "deal_id": row.get("deal_id"),
             },
         }
         for row, h_score, h_reasons in scored
@@ -614,22 +681,43 @@ def map_bundle(
     )
     trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows}
 
-    stats = export_stats(country=country, region=region)
+    sidebar_entities = [
-    return {
+        {
+            "id": row["id"],
+            "name": row["name"],
+            "entity_type": row["entity_type"],
+            "country_iso2": row["country_iso2"],
+            "city": row.get("city"),
+            "email": row.get("email") or row.get("contact_email"),
+            "phone": row.get("phone") or row.get("contact_phone"),
+            "halal_score": h_score,
+            "has_contact": bool(
+                row.get("email") or row.get("contact_email")
+                or row.get("phone") or row.get("contact_phone")
+                or int(row.get("contact_count") or 0) > 0
+            ),
+        }
+        for row, h_score, h_reasons in scored[:200]
+    ]
+
+    payload = {
         "entities": {"type": "FeatureCollection", "features": features},
         "trade_choropleth": trade_choropleth,
-        "halal_by_country": halal_by_country,
         "meta": {
             "entity_count": len(features),
+            "map_limit": row_limit,
+            "truncated": len(scored) >= row_limit,
             "countries_with_trade": len(trade_choropleth),
             "halal_min": halal_min,
             "top_markets": top_markets,
             "top_halal_entities": top_entities,
+            "sidebar_entities": sidebar_entities,
             "phase": "B",
             "note": "Halal-score: type + land + naam/signaal + contact",
         },
-        "stats": stats,
     }
+    _map_cache_set(cache_key, payload)
+    return payload
 
 
 @router.get("/halal/markets")