food-cockpit/foodlinkk-command-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed up Wereldexport map loading dramatically.

Browse Source 
Limit world overview payload, optimize SQL joins, cache bundle and tiles, reuse map instance, and show loading state.
This commit is contained in:
Aissa
2026-07-19 18:42:47 +00:00
parent 7289b770e5
commit 6856d09cd1
6 changed files with 198 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cockpit/app/routes/export_intel.py
+40 -7
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import os
import time
from typing import Any, Optional
import httpx
@@ -15,6 +16,14 @@ BASE = Path(__file__).resolve().parent.parent.parent
templates = Jinja2Templates(directory=str(BASE / "templates"))
TOOLS = os.getenv("TOOLS_API_URL", "http://tools-api:8700").rstrip("/")
_TILE_CACHE: dict[str, tuple[float, bytes]] = {}
_TILE_CACHE_TTL = 86400.0
_TILE_CACHE_MAX = 900
_BUNDLE_CACHE: dict[str, tuple[float, bytes]] = {}
_BUNDLE_CACHE_TTL = 120.0
async def _proxy(method: str, path: str, **kwargs) -> Any:
url = f"{TOOLS}/export-intel{path}"
@@ -161,7 +170,12 @@ async def api_gov_sources(country: Optional[str] = None):
@router.get("/api/map/tiles/{z}/{x}/{y}.png")
async def map_tile_proxy(z: int, x: int, y: int) -> Response:
"""Proxy Carto dark basemap tiles via cockpit (no client internet needed)."""
"""Proxy Carto dark basemap tiles via cockpit (cached, no client internet needed)."""
key = f"{z}/{x}/{y}"
hit = _TILE_CACHE.get(key)
if hit and time.time() - hit[0] < _TILE_CACHE_TTL:
return Response(content=hit[1], media_type="image/png", headers={"Cache-Control": "public, max-age=604800"})
host = ("a", "b", "c", "d")[(x + y) % 4]
url = f"https://{host}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png"
try:
@@ -169,11 +183,12 @@ async def map_tile_proxy(z: int, x: int, y: int) -> Response:
r = await client.get(url)
if r.status_code != 200:
return Response(status_code=502, content=b"")
return Response(
content=r.content,
media_type="image/png",
headers={"Cache-Control": "public, max-age=604800"},
)
body = r.content
if len(_TILE_CACHE) >= _TILE_CACHE_MAX:
oldest = min(_TILE_CACHE.items(), key=lambda item: item[1][0])[0]
_TILE_CACHE.pop(oldest, None)
_TILE_CACHE[key] = (time.time(), body)
return Response(content=body, media_type="image/png", headers={"Cache-Control": "public, max-age=604800"})
except Exception:
return Response(status_code=502, content=b"")
@@ -193,7 +208,25 @@ async def api_map_bundle(
"entity_types": entity_types, "q": q, "halal_min": halal_min,
"favorite_only": favorite_only, "crm_linked": crm_linked,
}.items() if v is not None and v is not False}
return await _proxy("GET", "/map/bundle", params=params)
import hashlib
import json
cache_key = hashlib.md5(json.dumps(params, sort_keys=True, default=str).encode()).hexdigest()
hit = _BUNDLE_CACHE.get(cache_key)
if hit and time.time() - hit[0] < _BUNDLE_CACHE_TTL:
return Response(content=hit[1], media_type="application/json")
url = f"{TOOLS}/export-intel/map/bundle"
async with httpx.AsyncClient(timeout=60.0) as client:
r = await client.get(url, params=params)
if r.status_code >= 400:
return {"error": r.text, "status": r.status_code}
body = r.content
if len(_BUNDLE_CACHE) > 32:
oldest = min(_BUNDLE_CACHE.items(), key=lambda item: item[1][0])[0]
_BUNDLE_CACHE.pop(oldest, None)
_BUNDLE_CACHE[cache_key] = (time.time(), body)
return Response(content=body, media_type="application/json")
@router.post("/api/export-intel/entities/favorites")
cockpit/static/css/export-intel.css
+17
View File
@@ -42,6 +42,23 @@
.ei-filter-clear { font-size: 0.72rem; opacity: 0.85; }
.ei-split { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; min-height: 420px; }
@media (max-width: 900px) { .ei-split { grid-template-columns: 1fr; } }
.ei-map-wrap { position: relative; }
.ei-map-loading {
position: absolute; inset: 0; z-index: 500;
display: flex; align-items: center; justify-content: center; gap: 0.65rem;
background: rgba(8, 12, 18, 0.72); color: #94a3b8; font-size: 0.85rem;
pointer-events: none;
}
.ei-map-spinner {
width: 18px; height: 18px; border-radius: 50%;
border: 2px solid rgba(56, 189, 248, 0.25);
border-top-color: #38bdf8;
animation: ei-spin 0.8s linear infinite;
}
@keyframes ei-spin { to { transform: rotate(360deg); } }
.ei-map-truncated { margin: 0 0 0.65rem; font-size: 0.75rem; color: #64748b; }
.ei-map-error { margin: 0 0 0.65rem; padding: 0.65rem 0.85rem; border-radius: 10px; border: 1px solid rgba(251,113,133,0.35); background: rgba(127,29,29,0.25); color: #fecdd3; font-size: 0.82rem; }
.ei-map-wrap { border-radius: 14px; overflow: hidden; border: 1px solid rgba(14,165,233,0.2); height: 420px; min-height: 420px; background: #0a0e14; }
.ei-map-wrap #ei-map { height: 420px; width: 100%; }
cockpit/static/js/export-intel-map.js
+4
View File
@@ -170,6 +170,10 @@ window.ExportIntelMap = (function () {
function init(containerId) {
var el = document.getElementById(containerId);
if (!el || !window.L) return null;
if (map && map.getContainer() === el) {
invalidateMapSize();
return map;
}
if (map) {
map.remove();
map = null;
cockpit/static/js/export-intel.js
+19 -9
View File
@@ -26,6 +26,8 @@ function exportIntelApp() {
mapHalalMode: false,
mapHalalMin: 55,
mapError: '',
mapLoading: false,
mapTruncated: false,
halalTopMarkets: [],
halalTopEntities: [],
listFocusId: null,
@@ -242,13 +244,7 @@ function exportIntelApp() {
var base = '/api/export-intel';
var fq = this.filterQs();
if (this.tab === 'map') {
var mapQs = this.filterQs();
var mt = this.mapTypesParam();
if (mt.entity_type) mapQs += '&entity_type=' + encodeURIComponent(mt.entity_type);
else if (mt.entity_types) mapQs += '&entity_types=' + encodeURIComponent(mt.entity_types);
var rm = await fetch(base + '/entities?limit=200' + mapQs).then((x) => x.json());
this.entities = rm.items || [];
this.resultCount = rm.total != null ? rm.total : this.entities.length;
return;
} else if (this.tab === 'distributors') {
var types = 'distributor,wholesaler,importer,logistics';
var et = this.entityTypeFilter ? '&entity_type=' + encodeURIComponent(this.entityTypeFilter) : '';
@@ -307,6 +303,8 @@ function exportIntelApp() {
async loadMap() {
this.clearListFocus();
this.mapLoading = true;
this.mapError = '';
var url = '/api/export-intel/map/bundle';
var qs = [];
if (this.country) qs.push('country=' + encodeURIComponent(this.country));
@@ -320,16 +318,27 @@ function exportIntelApp() {
if (this.filterCrmStatus === 'linked') qs.push('crm_linked=true');
if (this.filterCrmStatus === 'not_linked') qs.push('crm_linked=false');
if (qs.length) url += '?' + qs.join('&');
var bundle = await fetch(url).then((r) => r.json());
var bundle;
try {
bundle = await fetch(url).then((r) => r.json());
} catch (e) {
this.mapLoading = false;
this.mapError = 'Kaart laden mislukt. Probeer opnieuw.';
return;
}
this.halalTopMarkets = (bundle.meta && bundle.meta.top_markets) || [];
this.halalTopEntities = (bundle.meta && bundle.meta.top_halal_entities) || [];
this.entities = (bundle.meta && bundle.meta.sidebar_entities) || this.halalTopEntities || [];
this.resultCount = (bundle.meta && bundle.meta.entity_count) || this.entities.length;
this.mapTruncated = !!(bundle.meta && bundle.meta.truncated);
if (!window.L || !window.ExportIntelMap) {
this.mapError = 'Kaart kon niet laden. Vernieuw de pagina (Ctrl+F5).';
this.mapLoading = false;
return;
}
this.mapError = '';
if (window.ExportIntelMap) {
if (!document.getElementById('ei-map')) return;
if (!document.getElementById('ei-map')) { this.mapLoading = false; return; }
var self = this;
window.ExportIntelMap.init('ei-map');
window.ExportIntelMap.setOnEntityClick(function (id) { self.openEntity(id); });
@@ -340,6 +349,7 @@ function exportIntelApp() {
if (t && t.lat && t.lon) window.ExportIntelMap.flyTo(t.lat, t.lon, t.map_zoom || 6);
}
}
this.mapLoading = false;
},
setTab(id, el) {
cockpit/templates/export_intel.html
+5 -4
View File
@@ -4,11 +4,11 @@
<link rel="stylesheet" href="/static/vendor/leaflet/leaflet.css?v=1" />
<link rel="stylesheet" href="/static/vendor/leaflet.markercluster/MarkerCluster.css?v=1" />
<link rel="stylesheet" href="/static/vendor/leaflet.markercluster/MarkerCluster.Default.css?v=1" />
<link rel="stylesheet" href="/static/css/export-intel.css?v=13" />
<link rel="stylesheet" href="/static/css/export-intel.css?v=14" />
<link rel="stylesheet" href="/static/css/export-intel-tabs.css?v=10" />
<script src="/static/vendor/leaflet/leaflet.js?v=1"></script>
<script src="/static/vendor/leaflet.markercluster/leaflet.markercluster.js?v=1"></script>
<script src="/static/js/export-intel-map.js?v=9"></script>
<script src="/static/js/export-intel-map.js?v=10"></script>
{% endblock %}
{% block content %}
<div class="ei-page" x-data="exportIntelApp()" x-init="init()">
@@ -237,7 +237,8 @@
</div>
<div class="ei-split">
<div class="ei-map-error" x-show="mapError" x-cloak x-text="mapError"></div>
<div class="ei-map-wrap"><div id="ei-map"></div></div>
<p class="ei-map-truncated" x-show="mapTruncated && !mapLoading" x-cloak>Wereldoverzicht toont de belangrijkste ~2.500 locaties. Filter op regio of land voor meer detail.</p>
<div class="ei-map-wrap"><div class="ei-map-loading" x-show="mapLoading" x-cloak><span class="ei-map-spinner"></span> Kaart laden…</div><div id="ei-map"></div></div>
<div class="ei-panel">
<h3 style="margin:0 0 0.5rem;font-size:0.9rem" x-text="mapHalalMode ? 'Top halal-kansen' : 'Entities op kaart'"></h3>
<p style="font-size:0.75rem;color:#64748b;margin:0 0 0.75rem" x-show="!mapHalalMode">Klik rij of kaart-pin voor volledig profiel.</p>
@@ -797,5 +798,5 @@
</div>
{% endblock %}
{% block scripts %}
<script src="/static/js/export-intel.js?v=12"></script>
<script src="/static/js/export-intel.js?v=13"></script>
{% endblock %}
tools-api/app/export_intel.py
+113 -25
View File
@@ -3,6 +3,9 @@ from __future__ import annotations
import csv
import io
import hashlib
import json
import time
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
@@ -67,6 +70,48 @@ from app.export_intel_sync import (
sync_world_regions,
)
_MAP_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
_MAP_CACHE_TTL = 120.0
def _map_entity_limit(
country: Optional[str],
region: Optional[str],
q: Optional[str],
entity_type: Optional[str],
entity_types: Optional[str],
halal_min: Optional[float],
favorite_only: bool,
crm_linked: Optional[bool],
) -> int:
if q or entity_type or entity_types or halal_min is not None or favorite_only or crm_linked is not None:
return 4000
if country:
return 12000
if region:
return 8000
return 2500
def _map_cache_get(key: str) -> Optional[dict[str, Any]]:
hit = _MAP_CACHE.get(key)
if not hit:
return None
ts, payload = hit
if time.time() - ts > _MAP_CACHE_TTL:
_MAP_CACHE.pop(key, None)
return None
return payload
def _map_cache_set(key: str, payload: dict[str, Any]) -> None:
if len(_MAP_CACHE) > 48:
oldest = min(_MAP_CACHE.items(), key=lambda x: x[1][0])[0]
_MAP_CACHE.pop(oldest, None)
_MAP_CACHE[key] = (time.time(), payload)
router = APIRouter(prefix="/export-intel", tags=["export-intel"])
DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent")
@@ -500,26 +545,58 @@ def map_bundle(
where += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
params.extend([f"%{q}%", f"%{q}%"])
row_limit = _map_entity_limit(
country, region, q, entity_type, entity_types, halal_min, favorite_only, crm_linked
)
cache_key = hashlib.md5(
json.dumps(
{
"country": country,
"region": region,
"entity_type": entity_type,
"entity_types": entity_types,
"q": q,
"halal_min": halal_min,
"favorite_only": favorite_only,
"crm_linked": crm_linked,
"limit": row_limit,
},
sort_keys=True,
default=str,
).encode()
).hexdigest()
cached = _map_cache_get(cache_key)
if cached is not None:
return cached
entities = fetch_all(
f"""
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
e.confidence, e.pipeline_stage, e.city, e.address_line,
e.email, e.phone, e.website, e.volume_band, e.source,
e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at,
e.product_interest, e.halal_cert_notes,
(SELECT c.email FROM export_entity_contacts c
WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> ''
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email,
(SELECT c.phone FROM export_entity_contacts c
WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> ''
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone,
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
e.is_favorite, e.client_id, e.deal_id,
cp.contact_email, cp.contact_phone,
COALESCE(cc.contact_count, 0) AS contact_count
FROM export_market_entities e
{join}
LEFT JOIN LATERAL (
SELECT c.email AS contact_email,
COALESCE(NULLIF(c.phone, ''), NULLIF(c.mobile, '')) AS contact_phone
FROM export_entity_contacts c
WHERE c.entity_id = e.id
ORDER BY c.is_primary DESC NULLS LAST, c.id
LIMIT 1
) cp ON TRUE
LEFT JOIN (
SELECT entity_id, COUNT(*)::int AS contact_count
FROM export_entity_contacts
GROUP BY entity_id
) cc ON cc.entity_id = e.id
WHERE {where}
LIMIT 20000
ORDER BY e.confidence DESC NULLS LAST, e.id
LIMIT %s
""",
tuple(params) if params else None,
(tuple(params) + (row_limit,)) if params else (row_limit,),
)
scored: list[tuple[dict[str, Any], float, list[str]]] = []
@@ -558,22 +635,12 @@ def map_bundle(
"name": row["name"],
"entity_type": row["entity_type"],
"country_iso2": row["country_iso2"],
"confidence": row["confidence"],
"pipeline_stage": row["pipeline_stage"],
"city": row.get("city"),
"address_line": row.get("address_line"),
"email": row.get("email") or row.get("contact_email"),
"phone": row.get("phone") or row.get("contact_phone"),
"website": row.get("website"),
"volume_band": row.get("volume_band"),
"source": row.get("source"),
"contact_count": int(row.get("contact_count") or 0),
"halal_score": h_score,
"halal_tier": halal_pin_tier(h_score),
"halal_reasons": h_reasons[:4],
"is_favorite": bool(row.get("is_favorite")),
"client_id": row.get("client_id"),
"deal_id": row.get("deal_id"),
},
}
for row, h_score, h_reasons in scored
@@ -614,22 +681,43 @@ def map_bundle(
)
trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows}
stats = export_stats(country=country, region=region)
return {
sidebar_entities = [
{
"id": row["id"],
"name": row["name"],
"entity_type": row["entity_type"],
"country_iso2": row["country_iso2"],
"city": row.get("city"),
"email": row.get("email") or row.get("contact_email"),
"phone": row.get("phone") or row.get("contact_phone"),
"halal_score": h_score,
"has_contact": bool(
row.get("email") or row.get("contact_email")
or row.get("phone") or row.get("contact_phone")
or int(row.get("contact_count") or 0) > 0
),
}
for row, h_score, h_reasons in scored[:200]
]
payload = {
"entities": {"type": "FeatureCollection", "features": features},
"trade_choropleth": trade_choropleth,
"halal_by_country": halal_by_country,
"meta": {
"entity_count": len(features),
"map_limit": row_limit,
"truncated": len(scored) >= row_limit,
"countries_with_trade": len(trade_choropleth),
"halal_min": halal_min,
"top_markets": top_markets,
"top_halal_entities": top_entities,
"sidebar_entities": sidebar_entities,
"phase": "B",
"note": "Halal-score: type + land + naam/signaal + contact",
},
"stats": stats,
}
_map_cache_set(cache_key, payload)
return payload
@router.get("/halal/markets")