SysOps: voice-agy-webbuilder-backup — 2026-06-23 10:04 UTC

This commit is contained in:
sysops
2026-06-23 10:04:23 +00:00
parent 3bf15c4850
commit 26fe76afdd
165 changed files with 47427 additions and 1264 deletions
+22
View File
@@ -0,0 +1,22 @@
"""Normalize legacy agent_name values to agent_souls keys."""
from __future__ import annotations
AGENT_NAME_MAP: dict[str, str] = {
"retail_360": "retail",
"retail_scraper": "retail",
"retail_crm": "retail",
"retail_intel": "retail",
"rss_feeds": "marketing",
"wholesale_scraper": "sourcing",
"halal_registry": "halal",
"branch_scraper": "sourcing",
"hermes": "herman",
"herman_delegate": "herman",
}
def normalize_agent_key(name: str | None) -> str:
key = (name or "").strip().lower()
if not key:
return ""
return AGENT_NAME_MAP.get(key, key)
+1
View File
@@ -0,0 +1 @@
+109
View File
@@ -0,0 +1,109 @@
"""Halal meat sales opportunity scoring for export market entities."""
from __future__ import annotations
import json
import re
from typing import Any
# Land-basis: vraag naar halal vlees / moslim markt (0100 proxy)
COUNTRY_HALAL_MARKET: dict[str, float] = {
"SA": 98, "AE": 97, "QA": 97, "KW": 96, "BH": 96, "OM": 95,
"EG": 92, "MA": 90, "DZ": 90, "TN": 89, "LY": 88, "SD": 91, "SO": 93,
"TR": 85, "ID": 94, "MY": 93, "PK": 95, "BD": 94, "AF": 96, "IR": 94,
"IQ": 93, "JO": 90, "LB": 88, "SY": 89, "YE": 95, "PS": 91,
"NG": 82, "SN": 92, "ML": 93, "NE": 95, "BF": 90, "CI": 85, "GH": 78,
"KE": 72, "TZ": 78, "UG": 70, "ET": 68, "ZA": 55, "GB": 52, "FR": 54,
"DE": 50, "NL": 58, "BE": 56, "SE": 42, "NO": 38, "DK": 40, "FI": 35,
"AT": 48, "CH": 45, "IT": 46, "ES": 48, "PT": 42, "GR": 45, "CY": 55,
"BA": 82, "AL": 85, "XK": 88, "MK": 75, "RS": 72, "BG": 58, "RO": 55,
"US": 45, "CA": 48, "AU": 38, "NZ": 35, "MX": 40, "BR": 42, "AR": 38,
"SG": 72, "TH": 55, "PH": 65, "IN": 70, "CN": 35, "JP": 30, "KR": 32,
"RU": 48, "KZ": 62, "UZ": 75, "AZ": 80, "GE": 55,
}
TYPE_BASE: dict[str, float] = {
"butcher": 92,
"doner_shoarma": 90,
"wholesaler": 86,
"distributor": 85,
"importer": 84,
"contract_caterer": 78,
"restaurant": 72,
"foodservice": 65,
"logistics": 45,
}
HALAL_KEYWORDS = re.compile(
r"halal|helal|kebab|döner|doner|shoarma|shawarma|moslim|muslim|islamic|"
r"middle\s*east|turkish|türk|arab|pakistani|moroccan|marok|syrian|lebanese|"
r"peri\s*peri|grill|bbq|meat|vlees|slager|butcher|wholesale|groothandel|"
r"cash\s*&\s*carry|horeca|ethnic|kosher",
re.I,
)
EXPLICIT_HALAL = re.compile(r"\bhalal\b|\bhelal\b", re.I)
def _product_interest_text(val: Any) -> str:
if not val:
return ""
if isinstance(val, str):
try:
val = json.loads(val)
except json.JSONDecodeError:
return val
if isinstance(val, list):
return " ".join(str(x) for x in val)
return str(val)
def score_entity(row: dict[str, Any]) -> tuple[float, list[str]]:
"""Return (score 0100, reason tags)."""
reasons: list[str] = []
country = (row.get("country_iso2") or "").upper()
etype = row.get("entity_type") or ""
name = row.get("name") or ""
notes = row.get("halal_cert_notes") or ""
interest = _product_interest_text(row.get("product_interest"))
score = TYPE_BASE.get(etype, 50)
reasons.append(f"type:{etype}")
market = COUNTRY_HALAL_MARKET.get(country, 40)
score = score * 0.55 + market * 0.45
if market >= 80:
reasons.append("sterke_halal_markt")
blob = f"{name} {notes} {interest}".lower()
if EXPLICIT_HALAL.search(blob):
score += 18
reasons.append("halal_in_naam")
elif HALAL_KEYWORDS.search(blob):
score += 10
reasons.append("halal_signaal")
if etype in ("butcher", "doner_shoarma"):
reasons.append("direct_vlees_kanaal")
elif etype in ("wholesaler", "distributor", "importer"):
reasons.append("B2B_kanaal")
conf = int(row.get("confidence") or 50)
score += (conf - 50) * 0.08
if row.get("email") or row.get("contact_email"):
score += 3
if int(row.get("contact_count") or 0) > 0:
score += 4
reasons.append("heeft_contact")
return round(min(100, max(0, score)), 1), reasons
def halal_pin_tier(score: float) -> str:
if score >= 75:
return "hot"
if score >= 55:
return "warm"
if score >= 40:
return "mild"
return "low"
+842
View File
@@ -0,0 +1,842 @@
"""Export Intel API — Foodlinkk B2B global export intelligence."""
from __future__ import annotations
import csv
import io
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from app.db import execute, execute_returning, fetch_all, fetch_one
from app.export_halal_scoring import COUNTRY_HALAL_MARKET, halal_pin_tier, score_entity
from app.export_intel_crm import push_entities_to_crm
HALAL_REASON_NL: dict[str, str] = {
"sterke_halal_markt": "Sterke halal-markt in dit land",
"halal_in_naam": "Halal expliciet in bedrijfsnaam",
"halal_signaal": "Kebab/döner/vlees-signaal in naam",
"direct_vlees_kanaal": "Direct vlees-verkoopkanaal",
"B2B_kanaal": "B2B inkoop / distributie",
"heeft_contact": "Contactpersoon in registry",
}
def _halal_reason_labels(reasons: list[str]) -> list[str]:
out: list[str] = []
for r in reasons:
if r in HALAL_REASON_NL:
out.append(HALAL_REASON_NL[r])
elif r.startswith("type:"):
out.append("Type: " + r.split(":", 1)[1].replace("_", " "))
else:
out.append(r.replace("_", " "))
return out
def _enrich_entity(row: dict[str, Any], contacts: list[dict[str, Any]]) -> dict[str, Any]:
row["contacts"] = contacts
row["contact_count"] = len(contacts)
for c in contacts:
if not row.get("email") and c.get("email"):
row["email"] = c["email"]
if not row.get("phone"):
phone = c.get("phone") or c.get("mobile")
if phone:
row["phone"] = phone
primary = contacts[0] if contacts else None
row["primary_contact"] = primary
h_score, h_reasons = score_entity(row)
row["halal_score"] = h_score
row["halal_reasons"] = h_reasons
row["halal_reason_labels"] = _halal_reason_labels(h_reasons)
if row.get("lat") and row.get("lon"):
row["maps_url"] = f"https://www.google.com/maps?q={row['lat']},{row['lon']}"
return row
from app.export_intel_sync import (
enrich_contacts_country,
materialize_caterer_presence,
sync_all_priority,
sync_caterers,
sync_contacts,
sync_customers,
sync_distributors,
sync_osm_country,
sync_ted_tenders,
sync_world_regions,
)
router = APIRouter(prefix="/export-intel", tags=["export-intel"])
DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent")
CUSTOMER_TYPES = ("restaurant", "doner_shoarma", "butcher", "foodservice")
CATERER_TYPES = ("contract_caterer")
class ContactCreateIn(BaseModel):
role: str = "general"
name: Optional[str] = None
email: Optional[str] = None
phone: Optional[str] = None
mobile: Optional[str] = None
is_primary: bool = False
class SyncRequestIn(BaseModel):
country_iso2: Optional[str] = None
region_code: Optional[str] = None
max_priority: int = 2
entity_types: Optional[list[str]] = None
class EntityIdsIn(BaseModel):
entity_ids: list[int]
class FavoritesIn(BaseModel):
entity_ids: list[int]
favorite: bool = True
class CrmPushIn(BaseModel):
entity_ids: list[int]
create_deals: bool = True
client_stage: str = "intake"
deal_stage: str = "lead"
def _entity_filters(
country: Optional[str],
region: Optional[str],
entity_type: Optional[str],
entity_types: Optional[str],
favorite_only: bool = False,
crm_linked: Optional[bool] = None,
) -> tuple[str, list[Any]]:
clauses: list[str] = ["1=1"]
params: list[Any] = []
if country:
clauses.append("e.country_iso2 = %s")
params.append(country.upper())
if region:
clauses.append("t.region_code = %s")
params.append(region)
clauses.append("e.territory_code = t.code")
if entity_type:
clauses.append("e.entity_type = %s")
params.append(entity_type)
if entity_types:
types = [t.strip() for t in entity_types.split(",") if t.strip()]
if types:
clauses.append("e.entity_type = ANY(%s)")
params.append(types)
if favorite_only:
clauses.append("e.is_favorite = TRUE")
if crm_linked is True:
clauses.append("e.client_id IS NOT NULL")
elif crm_linked is False:
clauses.append("e.client_id IS NULL")
return " AND ".join(clauses), params
@router.get("/stats")
def export_stats(
country: Optional[str] = Query(None),
region: Optional[str] = Query(None),
) -> dict[str, Any]:
entity_where = ""
params: list[Any] = []
if country:
entity_where = " WHERE country_iso2 = %s"
params = [country.upper()]
elif region:
entity_where = " WHERE territory_code IN (SELECT code FROM export_territories WHERE region_code = %s)"
params = [region]
def count(sql_suffix: str, extra_params: tuple[Any, ...] = ()) -> int:
w = entity_where
if sql_suffix:
w = f"{entity_where} AND {sql_suffix}" if entity_where else f" WHERE {sql_suffix}"
row = fetch_one(
f"SELECT COUNT(*) AS n FROM export_market_entities{w}",
tuple(params) + extra_params if params or extra_params else None,
)
return int(row["n"] or 0)
entities_n = count("")
distributors_n = count("entity_type IN ('distributor','wholesaler','importer','logistics')")
caterers_n = count("entity_type = 'contract_caterer'")
contacts = fetch_one("SELECT COUNT(*) AS n FROM export_entity_contacts")
with_email = fetch_one(
"SELECT COUNT(DISTINCT entity_id) AS n FROM export_entity_contacts WHERE email IS NOT NULL AND email <> ''"
)
territories = fetch_one("SELECT COUNT(*) AS n FROM export_territories WHERE is_active = TRUE")
presence = fetch_one("SELECT COUNT(*) AS n FROM export_caterer_presence WHERE is_active = TRUE")
tenders_open = 0
try:
row = fetch_one("SELECT COUNT(*) AS n FROM export_tenders WHERE status = 'open'")
tenders_open = int(row["n"] or 0) if row else 0
except Exception:
pass
fav_row = fetch_one("SELECT COUNT(*) AS n FROM export_market_entities WHERE is_favorite = TRUE")
crm_row = fetch_one("SELECT COUNT(*) AS n FROM export_market_entities WHERE client_id IS NOT NULL")
return {
"entities": entities_n,
"distributors": distributors_n,
"caterers": caterers_n,
"contacts": int(contacts["n"] or 0),
"entities_with_email": int(with_email["n"] or 0),
"territories": int(territories["n"] or 0),
"caterer_presence": int(presence["n"] or 0),
"tenders_open": tenders_open,
"favorites": int(fav_row["n"] or 0) if fav_row else 0,
"crm_linked": int(crm_row["n"] or 0) if crm_row else 0,
"phase": "B",
"enrichment_status": "active",
}
@router.get("/regions")
def list_regions() -> list[dict[str, Any]]:
return fetch_all("SELECT * FROM export_regions ORDER BY sort_order, code")
@router.get("/territories")
def list_territories(region: Optional[str] = Query(None)) -> list[dict[str, Any]]:
if region:
return fetch_all(
"SELECT * FROM export_territories WHERE is_active = TRUE AND region_code = %s ORDER BY sync_priority, name_nl",
(region,),
)
return fetch_all(
"SELECT * FROM export_territories WHERE is_active = TRUE ORDER BY sync_priority, name_nl"
)
@router.get("/entities")
def list_entities(
country: Optional[str] = Query(None),
region: Optional[str] = Query(None),
entity_type: Optional[str] = Query(None),
entity_types: Optional[str] = Query(None),
q: Optional[str] = Query(None),
favorite_only: bool = Query(False),
crm_linked: Optional[bool] = Query(None),
limit: int = Query(200, ge=1, le=500),
offset: int = Query(0, ge=0),
) -> dict[str, Any]:
join = ""
if region:
join = "JOIN export_territories t ON e.territory_code = t.code"
where_sql, params = _entity_filters(
country, region, entity_type, entity_types, favorite_only, crm_linked
)
if q:
where_sql += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
params.extend([f"%{q}%", f"%{q}%"])
rows = fetch_all(
f"""
SELECT e.*,
cl.name AS crm_client_name,
(SELECT COUNT(*) FROM export_entity_contacts ec WHERE ec.entity_id = e.id) AS contact_count,
(SELECT ec.email FROM export_entity_contacts ec WHERE ec.entity_id = e.id AND ec.is_primary = TRUE LIMIT 1) AS primary_email,
(SELECT ec.phone FROM export_entity_contacts ec WHERE ec.entity_id = e.id AND ec.is_primary = TRUE LIMIT 1) AS primary_phone
FROM export_market_entities e
LEFT JOIN clients cl ON cl.id = e.client_id
{join}
WHERE {where_sql}
ORDER BY e.confidence DESC, e.name
LIMIT %s OFFSET %s
""",
tuple(params) + (limit, offset),
)
total = fetch_one(
f"SELECT COUNT(*) AS n FROM export_market_entities e LEFT JOIN clients cl ON cl.id = e.client_id {join} WHERE {where_sql}",
tuple(params),
)
return {"items": rows, "total": int(total["n"] or 0), "limit": limit, "offset": offset}
@router.get("/entities/{entity_id}")
def get_entity(entity_id: int) -> dict[str, Any]:
row = fetch_one(
"""
SELECT e.*, cl.name AS crm_client_name, d.title AS crm_deal_title, d.stage AS crm_deal_stage
FROM export_market_entities e
LEFT JOIN clients cl ON cl.id = e.client_id
LEFT JOIN deals d ON d.id = e.deal_id
WHERE e.id = %s
""",
(entity_id,),
)
if not row:
raise HTTPException(404, "Entity not found")
contacts = fetch_all(
"SELECT * FROM export_entity_contacts WHERE entity_id = %s ORDER BY is_primary DESC, confidence DESC",
(entity_id,),
)
return _enrich_entity(row, contacts)
@router.get("/contacts")
def list_contacts(
country: Optional[str] = Query(None),
entity_type: Optional[str] = Query(None),
has_email: Optional[bool] = Query(None),
q: Optional[str] = Query(None),
limit: int = Query(200, ge=1, le=500),
offset: int = Query(0, ge=0),
) -> dict[str, Any]:
clauses = ["1=1"]
params: list[Any] = []
if country:
clauses.append("e.country_iso2 = %s")
params.append(country.upper())
if entity_type:
clauses.append("e.entity_type = %s")
params.append(entity_type)
if has_email:
clauses.append("c.email IS NOT NULL AND c.email <> ''")
if q:
clauses.append(
"(e.name ILIKE %s OR c.email ILIKE %s OR c.name ILIKE %s OR e.city ILIKE %s)"
)
like = f"%{q}%"
params.extend([like, like, like, like])
where = " AND ".join(clauses)
rows = fetch_all(
f"""
SELECT c.*, e.name AS entity_name, e.entity_type, e.country_iso2, e.city AS entity_city
FROM export_entity_contacts c
JOIN export_market_entities e ON e.id = c.entity_id
WHERE {where}
ORDER BY e.name, c.is_primary DESC
LIMIT %s OFFSET %s
""",
tuple(params) + (limit, offset),
)
total = fetch_one(
f"""
SELECT COUNT(*) AS n FROM export_entity_contacts c
JOIN export_market_entities e ON e.id = c.entity_id
WHERE {where}
""",
tuple(params),
)
return {"items": rows, "total": int(total["n"] or 0)}
@router.get("/contacts/export.csv")
def export_contacts_csv(
country: Optional[str] = Query(None),
entity_type: Optional[str] = Query(None),
) -> StreamingResponse:
data = list_contacts(country=country, entity_type=entity_type, limit=5000, offset=0)
buf = io.StringIO()
writer = csv.writer(buf)
writer.writerow(
["entity_name", "entity_type", "country", "role", "name", "email", "phone", "mobile", "source", "confidence"]
)
for r in data["items"]:
writer.writerow(
[
r.get("entity_name"),
r.get("entity_type"),
r.get("country_iso2"),
r.get("role"),
r.get("name"),
r.get("email"),
r.get("phone"),
r.get("mobile"),
r.get("source"),
r.get("confidence"),
]
)
buf.seek(0)
return StreamingResponse(
iter([buf.getvalue()]),
media_type="text/csv",
headers={"Content-Disposition": "attachment; filename=export-intel-contacts.csv"},
)
@router.post("/entities/{entity_id}/contacts")
def add_contact(entity_id: int, payload: ContactCreateIn) -> dict[str, Any]:
ent = fetch_one("SELECT id FROM export_market_entities WHERE id = %s", (entity_id,))
if not ent:
raise HTTPException(404, "Entity not found")
row = execute_returning(
"""
INSERT INTO export_entity_contacts (entity_id, role, name, email, phone, mobile, is_primary, source, confidence)
VALUES (%s, %s, %s, %s, %s, %s, %s, 'manual', 80)
RETURNING *
""",
(
entity_id,
payload.role,
payload.name,
payload.email,
payload.phone,
payload.mobile,
payload.is_primary,
),
)
return dict(row)
@router.get("/caterers/brands")
def list_caterer_brands() -> list[dict[str, Any]]:
return fetch_all(
"""
SELECT b.*,
(SELECT COUNT(*) FROM export_caterer_presence p WHERE p.brand_code = b.code AND p.is_active) AS country_count
FROM export_caterer_brands b
ORDER BY b.tier, b.name
"""
)
@router.get("/caterers/presence")
def list_caterer_presence(country: Optional[str] = Query(None)) -> list[dict[str, Any]]:
if country:
return fetch_all(
"""
SELECT p.*, b.name AS brand_name, b.tier, b.website AS brand_website
FROM export_caterer_presence p
JOIN export_caterer_brands b ON b.code = p.brand_code
WHERE p.country_iso2 = %s AND p.is_active = TRUE
ORDER BY b.tier, b.name
""",
(country.upper(),),
)
return fetch_all(
"""
SELECT p.*, b.name AS brand_name, b.tier, b.website AS brand_website
FROM export_caterer_presence p
JOIN export_caterer_brands b ON b.code = p.brand_code
WHERE p.is_active = TRUE
ORDER BY p.country_iso2, b.tier, b.name
"""
)
@router.get("/gov-sources")
def list_gov_sources(country: Optional[str] = Query(None)) -> list[dict[str, Any]]:
if country:
return fetch_all(
"SELECT * FROM export_gov_data_sources WHERE is_active = TRUE AND country_iso2 = %s ORDER BY category, name",
(country.upper(),),
)
return fetch_all(
"SELECT * FROM export_gov_data_sources WHERE is_active = TRUE ORDER BY country_iso2, category, name"
)
@router.get("/tenders")
def list_tenders(
country: Optional[str] = Query(None),
status: Optional[str] = Query("open"),
limit: int = Query(100, ge=1, le=200),
) -> dict[str, Any]:
clauses = ["1=1"]
params: list[Any] = []
if country:
clauses.append("country_iso2 = %s")
params.append(country.upper())
if status:
clauses.append("status = %s")
params.append(status)
where = " AND ".join(clauses)
rows = fetch_all(
f"SELECT * FROM export_tenders WHERE {where} ORDER BY created_at DESC LIMIT %s",
tuple(params) + (limit,),
)
total = fetch_one(f"SELECT COUNT(*) AS n FROM export_tenders WHERE {where}", tuple(params))
return {"items": rows, "total": int(total["n"] or 0)}
@router.get("/map/bundle")
def map_bundle(
country: Optional[str] = Query(None),
region: Optional[str] = Query(None),
entity_type: Optional[str] = Query(None),
entity_types: Optional[str] = Query(None),
q: Optional[str] = Query(None),
halal_min: Optional[float] = Query(None, ge=0, le=100),
favorite_only: bool = Query(False),
crm_linked: Optional[bool] = Query(None),
) -> dict[str, Any]:
join = ""
if region:
join = "JOIN export_territories t ON e.territory_code = t.code"
filter_sql, params = _entity_filters(
country, region, entity_type, entity_types, favorite_only, crm_linked
)
where = f"e.lat IS NOT NULL AND e.lon IS NOT NULL AND {filter_sql}"
if q:
where += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
params.extend([f"%{q}%", f"%{q}%"])
entities = fetch_all(
f"""
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
e.confidence, e.pipeline_stage, e.city, e.address_line,
e.email, e.phone, e.website, e.volume_band, e.source,
e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at,
e.product_interest, e.halal_cert_notes,
(SELECT c.email FROM export_entity_contacts c
WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> ''
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email,
(SELECT c.phone FROM export_entity_contacts c
WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> ''
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone,
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
FROM export_market_entities e
{join}
WHERE {where}
LIMIT 20000
""",
tuple(params) if params else None,
)
scored: list[tuple[dict[str, Any], float, list[str]]] = []
halal_by_country: dict[str, dict[str, Any]] = {}
for row in entities:
if row.get("lat") is None or row.get("lon") is None:
continue
h_score, h_reasons = score_entity(row)
iso = row.get("country_iso2") or ""
bucket = halal_by_country.setdefault(
iso,
{"country_iso2": iso, "market_score": COUNTRY_HALAL_MARKET.get(iso, 40), "entity_count": 0, "avg_halal_score": 0.0, "hot_count": 0},
)
bucket["entity_count"] += 1
bucket["avg_halal_score"] += h_score
if h_score >= 75:
bucket["hot_count"] += 1
scored.append((row, h_score, h_reasons))
for bucket in halal_by_country.values():
n = bucket["entity_count"] or 1
bucket["avg_halal_score"] = round(bucket["avg_halal_score"] / n, 1)
bucket["combined_score"] = round(bucket["market_score"] * 0.4 + bucket["avg_halal_score"] * 0.6, 1)
if halal_min is not None:
scored = [s for s in scored if s[1] >= halal_min]
scored.sort(key=lambda x: x[1], reverse=True)
features = [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [row["lon"], row["lat"]]},
"properties": {
"id": row["id"],
"name": row["name"],
"entity_type": row["entity_type"],
"country_iso2": row["country_iso2"],
"confidence": row["confidence"],
"pipeline_stage": row["pipeline_stage"],
"city": row.get("city"),
"address_line": row.get("address_line"),
"email": row.get("email") or row.get("contact_email"),
"phone": row.get("phone") or row.get("contact_phone"),
"website": row.get("website"),
"volume_band": row.get("volume_band"),
"source": row.get("source"),
"contact_count": int(row.get("contact_count") or 0),
"halal_score": h_score,
"halal_tier": halal_pin_tier(h_score),
"halal_reasons": h_reasons[:4],
"is_favorite": bool(row.get("is_favorite")),
"client_id": row.get("client_id"),
"deal_id": row.get("deal_id"),
},
}
for row, h_score, h_reasons in scored
]
top_markets = sorted(halal_by_country.values(), key=lambda x: x["combined_score"], reverse=True)[:15]
top_entities = [
{
"id": row["id"],
"name": row["name"],
"entity_type": row["entity_type"],
"country_iso2": row["country_iso2"],
"city": row.get("city"),
"address_line": row.get("address_line"),
"email": row.get("email") or row.get("contact_email"),
"phone": row.get("phone") or row.get("contact_phone"),
"website": row.get("website"),
"halal_score": h_score,
"halal_reason_labels": _halal_reason_labels(h_reasons),
"is_favorite": bool(row.get("is_favorite")),
"client_id": row.get("client_id"),
"has_contact": bool(
row.get("email") or row.get("contact_email")
or row.get("phone") or row.get("contact_phone")
or int(row.get("contact_count") or 0) > 0
),
}
for row, h_score, h_reasons in scored[:25]
]
density_rows = fetch_all(
"""
SELECT country_iso2, COUNT(*) AS n
FROM export_market_entities
WHERE lat IS NOT NULL AND lon IS NOT NULL
GROUP BY country_iso2
"""
)
trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows}
stats = export_stats(country=country, region=region)
return {
"entities": {"type": "FeatureCollection", "features": features},
"trade_choropleth": trade_choropleth,
"halal_by_country": halal_by_country,
"meta": {
"entity_count": len(features),
"countries_with_trade": len(trade_choropleth),
"halal_min": halal_min,
"top_markets": top_markets,
"top_halal_entities": top_entities,
"phase": "B",
"note": "Halal-score: type + land + naam/signaal + contact",
},
"stats": stats,
}
@router.get("/halal/markets")
def halal_markets(
region: Optional[str] = Query(None),
country: Optional[str] = Query(None),
limit: int = Query(30, ge=1, le=100),
) -> dict[str, Any]:
join = ""
where = "e.lat IS NOT NULL"
params: list[Any] = []
if region:
join = "JOIN export_territories t ON e.territory_code = t.code"
where += " AND t.region_code = %s"
params.append(region)
if country:
where += " AND e.country_iso2 = %s"
params.append(country.upper())
rows = fetch_all(
f"""
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.city,
e.confidence, e.product_interest, e.halal_cert_notes, e.email,
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
FROM export_market_entities e
{join}
WHERE {where}
LIMIT 15000
""",
tuple(params) if params else None,
)
by_country: dict[str, dict[str, Any]] = {}
top_entities: list[dict[str, Any]] = []
for row in rows:
h_score, reasons = score_entity(row)
iso = row.get("country_iso2") or ""
bucket = by_country.setdefault(
iso,
{"country_iso2": iso, "market_score": COUNTRY_HALAL_MARKET.get(iso, 40), "entities": 0, "score_sum": 0.0, "hot": 0},
)
bucket["entities"] += 1
bucket["score_sum"] += h_score
if h_score >= 75:
bucket["hot"] += 1
top_entities.append({
"id": row["id"],
"name": row["name"],
"entity_type": row["entity_type"],
"country_iso2": iso,
"city": row.get("city"),
"halal_score": h_score,
"halal_reasons": reasons[:3],
})
markets = []
for iso, b in by_country.items():
n = b["entities"] or 1
avg = round(b["score_sum"] / n, 1)
markets.append({
"country_iso2": iso,
"market_score": b["market_score"],
"entity_count": b["entities"],
"avg_halal_score": avg,
"hot_entities": b["hot"],
"combined_score": round(b["market_score"] * 0.4 + avg * 0.6, 1),
})
markets.sort(key=lambda x: x["combined_score"], reverse=True)
top_entities.sort(key=lambda x: x["halal_score"], reverse=True)
return {
"markets": markets[:limit],
"top_entities": top_entities[:50],
"total_entities": len(rows),
}
@router.post("/entities/favorites")
def set_favorites(payload: FavoritesIn) -> dict[str, Any]:
if not payload.entity_ids:
raise HTTPException(400, "entity_ids required")
fav_val = payload.favorite
execute(
"""
UPDATE export_market_entities
SET is_favorite = %s,
favorited_at = CASE WHEN %s THEN NOW() ELSE NULL END,
updated_at = NOW()
WHERE id = ANY(%s)
""",
(fav_val, fav_val, payload.entity_ids),
)
return {"ok": True, "updated": len(payload.entity_ids), "favorite": fav_val}
@router.get("/entities/favorites")
def list_favorite_entities(
limit: int = Query(200, ge=1, le=500),
country: Optional[str] = Query(None),
region: Optional[str] = Query(None),
) -> dict[str, Any]:
return list_entities(
country=country,
region=region,
favorite_only=True,
limit=limit,
offset=0,
)
@router.post("/crm/push")
def crm_push(payload: CrmPushIn) -> dict[str, Any]:
if not payload.entity_ids:
raise HTTPException(400, "entity_ids required")
if len(payload.entity_ids) > 100:
raise HTTPException(400, "max 100 entities per request")
return push_entities_to_crm(
payload.entity_ids,
create_deals=payload.create_deals,
client_stage=payload.client_stage,
deal_stage=payload.deal_stage,
)
@router.get("/crm/pipeline")
def crm_pipeline(
limit: int = Query(100, ge=1, le=500),
country: Optional[str] = Query(None),
region: Optional[str] = Query(None),
) -> dict[str, Any]:
join = ""
if region:
join = "JOIN export_territories t ON e.territory_code = t.code"
where_sql, params = _entity_filters(country, region, None, None, False, True)
rows = fetch_all(
f"""
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.city,
e.pipeline_stage, e.crm_pushed_at, e.is_favorite,
e.client_id, e.deal_id, e.email, e.phone,
cl.name AS crm_client_name, cl.stage AS crm_client_stage,
d.title AS crm_deal_title, d.stage AS crm_deal_stage
FROM export_market_entities e
LEFT JOIN clients cl ON cl.id = e.client_id
LEFT JOIN deals d ON d.id = e.deal_id
{join}
WHERE {where_sql}
ORDER BY e.crm_pushed_at DESC NULLS LAST, e.updated_at DESC
LIMIT %s
""",
tuple(params) + (limit,),
)
return {"items": rows, "total": len(rows)}
@router.post("/sync/contacts")
def api_sync_contacts(payload: SyncRequestIn) -> dict[str, Any]:
try:
result = sync_contacts(payload.country_iso2, payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/caterers")
def api_sync_caterers(payload: SyncRequestIn) -> dict[str, Any]:
try:
result = sync_caterers(payload.country_iso2, payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/distributors")
def api_sync_distributors(payload: SyncRequestIn) -> dict[str, Any]:
try:
result = sync_distributors(payload.country_iso2, payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/customers")
def api_sync_customers(payload: SyncRequestIn) -> dict[str, Any]:
if not payload.country_iso2 and not payload.region_code:
raise HTTPException(400, "country_iso2 or region_code required")
try:
result = sync_customers(payload.country_iso2, payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/tenders")
def api_sync_tenders() -> dict[str, Any]:
try:
result = sync_ted_tenders()
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/all")
def api_sync_all(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
try:
result = sync_all_priority(max_priority=payload.max_priority, region_code=payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/world")
def api_sync_world(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
"""Sync Europe, Middle East, Africa, Americas — long running."""
try:
result = sync_world_regions(max_priority=payload.max_priority)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/region")
def api_sync_region(payload: SyncRequestIn) -> dict[str, Any]:
if not payload.region_code:
raise HTTPException(400, "region_code required")
try:
result = sync_all_priority(max_priority=payload.max_priority, region_code=payload.region_code)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
+193
View File
@@ -0,0 +1,193 @@
"""Push export market entities into Foodlinkk CRM (clients + deals)."""
from __future__ import annotations
from typing import Any, Optional
from app.db import execute, execute_returning, fetch_one
from app.export_halal_scoring import score_entity
SECTOR_BY_TYPE: dict[str, str] = {
"distributor": "Export · Distributie",
"wholesaler": "Export · Groothandel",
"importer": "Export · Import",
"logistics": "Export · Logistiek",
"contract_caterer": "Export · Catering",
"restaurant": "Export · Horeca",
"doner_shoarma": "Export · Horeca",
"butcher": "Export · Slagerij",
"foodservice": "Export · Foodservice",
}
def _sector(entity_type: str) -> str:
return SECTOR_BY_TYPE.get(entity_type or "", "Export · B2B")
def _contact_name(contacts: list[dict[str, Any]]) -> Optional[str]:
for c in contacts:
if c.get("name"):
return c["name"]
return None
def _build_notes(ent: dict[str, Any], contacts: list[dict[str, Any]], halal_score: float) -> str:
lines = [
f"Wereldexport entity #{ent['id']}",
f"Type: {ent.get('entity_type')}",
f"Land: {ent.get('country_iso2')}",
]
if ent.get("city"):
lines.append(f"Stad: {ent['city']}")
if ent.get("address_line"):
lines.append(f"Adres: {ent['address_line']}")
if ent.get("website"):
lines.append(f"Website: {ent['website']}")
if ent.get("phone"):
lines.append(f"Telefoon: {ent['phone']}")
lines.append(f"Halal-kans score: {halal_score}/100")
lines.append(f"OSM bron: {ent.get('source') or ''}")
if contacts:
lines.append(f"Contacten in registry: {len(contacts)}")
lines.append("")
lines.append("Aangemaakt vanuit Wereldexport · Foodlinkk B2B")
return "\n".join(lines)
def push_entity_to_crm(
entity_id: int,
*,
create_deal: bool = True,
client_stage: str = "intake",
deal_stage: str = "lead",
) -> dict[str, Any]:
ent = fetch_one("SELECT * FROM export_market_entities WHERE id = %s", (entity_id,))
if not ent:
return {"entity_id": entity_id, "status": "not_found"}
if ent.get("client_id"):
client = fetch_one("SELECT id, name FROM clients WHERE id = %s", (ent["client_id"],))
return {
"entity_id": entity_id,
"status": "already_linked",
"client_id": ent["client_id"],
"deal_id": ent.get("deal_id"),
"client_name": (client or {}).get("name"),
}
from app.db import fetch_all
contacts = fetch_all(
"SELECT * FROM export_entity_contacts WHERE entity_id = %s ORDER BY is_primary DESC, confidence DESC",
(entity_id,),
)
email = ent.get("email")
phone = ent.get("phone")
for c in contacts:
if not email and c.get("email"):
email = c["email"]
if not phone:
phone = c.get("phone") or c.get("mobile")
client_id: Optional[int] = None
status = "created"
if email:
existing = fetch_one(
"SELECT id, name FROM clients WHERE LOWER(email) = LOWER(%s) LIMIT 1",
(email,),
)
if existing:
client_id = existing["id"]
status = "linked_existing_email"
if client_id is None:
contact_label = _contact_name(contacts) or phone or email
halal_score, _ = score_entity({**ent, "contact_count": len(contacts)})
notes = _build_notes(ent, contacts, halal_score)
row = execute_returning(
"""
INSERT INTO clients (name, contact, email, stage, sector, notes, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, NOW())
RETURNING id
""",
(
ent["name"][:255],
(contact_label or "")[:255] or None,
(email or "")[:255] or None,
client_stage,
_sector(ent.get("entity_type") or ""),
notes,
),
)
if not row:
return {"entity_id": entity_id, "status": "error", "error": "client_insert_failed"}
client_id = row["id"]
deal_id: Optional[int] = None
if create_deal and not ent.get("deal_id"):
country = ent.get("country_iso2") or ""
title = f"Export · {ent['name'][:180]} ({country})"
deal_row = execute_returning(
"""
INSERT INTO deals (client_id, title, value, stage, agent_owner, next_action, updated_at)
VALUES (%s, %s, 0, %s, 'herman', %s, NOW())
RETURNING id
""",
(
client_id,
title[:255],
deal_stage,
f"Eerste outreach — Wereldexport entity #{entity_id}",
),
)
if deal_row:
deal_id = deal_row["id"]
execute(
"""
UPDATE export_market_entities
SET client_id = %s,
deal_id = COALESCE(%s, deal_id),
pipeline_stage = 'crm_linked',
crm_pushed_at = NOW(),
updated_at = NOW()
WHERE id = %s
""",
(client_id, deal_id, entity_id),
)
client = fetch_one("SELECT id, name FROM clients WHERE id = %s", (client_id,))
return {
"entity_id": entity_id,
"status": status,
"client_id": client_id,
"deal_id": deal_id,
"client_name": (client or {}).get("name"),
}
def push_entities_to_crm(
entity_ids: list[int],
*,
create_deals: bool = True,
client_stage: str = "intake",
deal_stage: str = "lead",
) -> dict[str, Any]:
results = [
push_entity_to_crm(
eid,
create_deal=create_deals,
client_stage=client_stage,
deal_stage=deal_stage,
)
for eid in entity_ids
]
created = sum(1 for r in results if r.get("status") == "created")
linked = sum(1 for r in results if r.get("status") in ("linked_existing_email", "already_linked"))
return {
"results": results,
"total": len(results),
"created": created,
"linked_existing": linked,
"pushed": created + sum(1 for r in results if r.get("status") == "linked_existing_email"),
}
+183
View File
@@ -0,0 +1,183 @@
"""Google Places sync for Export Intel (optional — requires GOOGLE_MAPS_API_KEY)."""
from __future__ import annotations
import json
import os
import time
from typing import Any, Optional
import httpx
from app.db import fetch_one
from app.export_intel_sync import _get_territory, _upsert_contact
PLACES_URL = "https://maps.googleapis.com/maps/api/place/textsearch/json"
DETAILS_URL = "https://maps.googleapis.com/maps/api/place/details/json"
QUERIES = [
"meat wholesaler",
"halal meat distributor",
"poultry distributor",
"frozen food supplier",
"contract catering",
"food importer",
"halal butcher",
]
LARGE_COUNTRIES = frozenset({"US", "CA", "BR", "MX", "AR", "AU", "RU", "CN", "IN", "CD", "DZ", "EG", "IR", "SA"})
def _api_key() -> Optional[str]:
return os.getenv("GOOGLE_MAPS_API_KEY") or os.getenv("GOOGLE_PLACES_API_KEY")
def _places_search(query: str, lat: float, lon: float, radius_m: int = 80000) -> list[dict[str, Any]]:
key = _api_key()
if not key:
return []
params = {
"query": query,
"location": f"{lat},{lon}",
"radius": radius_m,
"key": key,
}
with httpx.Client(timeout=30.0) as client:
resp = client.get(PLACES_URL, params=params)
if resp.status_code >= 400:
return []
data = resp.json()
if data.get("status") not in ("OK", "ZERO_RESULTS"):
return []
return data.get("results", [])[:12]
def _place_details(place_id: str) -> dict[str, Any]:
key = _api_key()
if not key:
return {}
with httpx.Client(timeout=20.0) as client:
resp = client.get(
DETAILS_URL,
params={
"place_id": place_id,
"fields": "name,formatted_phone_number,website,formatted_address,geometry,international_phone_number",
"key": key,
},
)
if resp.status_code >= 400:
return {}
return resp.json().get("result", {})
def _bbox_points(bbox: Any, grid: int = 3) -> list[tuple[float, float]]:
if isinstance(bbox, str):
bbox = json.loads(bbox)
if not bbox or len(bbox) != 4:
return []
south, west, north, east = bbox
points: list[tuple[float, float]] = []
for i in range(grid):
for j in range(grid):
lat = south + (north - south) * (i + 0.5) / grid
lon = west + (east - west) * (j + 0.5) / grid
points.append((lat, lon))
return points
def _upsert_place(place: dict[str, Any], details: dict[str, Any], country_iso2: str, territory_code: str, query: str) -> tuple[str, int]:
from app.db import execute, execute_returning
pid = place.get("place_id")
name = details.get("name") or place.get("name") or "Unknown"
geo = details.get("geometry", {}).get("location", place.get("geometry", {}).get("location", {}))
plat = geo.get("lat")
plon = geo.get("lng")
phone = details.get("formatted_phone_number") or details.get("international_phone_number")
website = details.get("website")
entity_type = "contract_caterer" if "catering" in query else "distributor"
existing = fetch_one(
"SELECT id FROM export_market_entities WHERE metadata->>'google_place_id' = %s",
(pid,),
)
if existing:
execute(
"""
UPDATE export_market_entities SET
phone = COALESCE(%s, phone), website = COALESCE(%s, website),
lat = COALESCE(%s, lat), lon = COALESCE(%s, lon),
confidence = GREATEST(confidence, 65), updated_at = NOW()
WHERE id = %s
""",
(phone, website, plat, plon, existing["id"]),
)
entity_id = existing["id"]
action = "updated"
else:
row = execute_returning(
"""
INSERT INTO export_market_entities (
name, entity_type, country_iso2, territory_code, lat, lon,
phone, website, google_place_id, source, sources, confidence, metadata
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'google_places','["google_places"]'::jsonb,65,%s)
RETURNING id
""",
(
name[:255], entity_type, country_iso2.upper(), territory_code,
plat, plon, phone, website, pid,
json.dumps({"google_place_id": pid}),
),
)
entity_id = row["id"]
action = "inserted"
if phone:
_upsert_contact(entity_id, None, phone, "google_places", 70, role="sales", name="Google Places")
return action, entity_id
def sync_places_country(country_iso2: str) -> dict[str, Any]:
territory = _get_territory(country_iso2)
if not territory or not territory.get("lat"):
return {"skipped": True, "reason": "no territory"}
if not _api_key():
return {"skipped": True, "reason": "GOOGLE_MAPS_API_KEY not set"}
territory_code = territory["code"]
inserted = updated = 0
seen: set[str] = set()
iso = country_iso2.upper()
if iso in LARGE_COUNTRIES and territory.get("bbox"):
search_points = _bbox_points(territory["bbox"], grid=3)
else:
search_points = [(territory["lat"], territory["lon"])]
radius = 60000 if iso in LARGE_COUNTRIES else 80000
for lat, lon in search_points:
for q in QUERIES:
for place in _places_search(q, lat, lon, radius_m=radius):
pid = place.get("place_id")
if not pid or pid in seen:
continue
seen.add(pid)
details = _place_details(pid)
action, _ = _upsert_place(place, details, iso, territory_code, q)
if action == "inserted":
inserted += 1
else:
updated += 1
time.sleep(0.15)
time.sleep(0.5)
return {"inserted": inserted, "updated": updated, "queries": len(QUERIES), "search_points": len(search_points)}
def sync_places_countries(countries: list[str]) -> dict[str, Any]:
if not _api_key():
return {"skipped": True, "reason": "GOOGLE_MAPS_API_KEY not set"}
out: dict[str, Any] = {}
for c in countries:
out[c] = sync_places_country(c)
time.sleep(1)
return out
+583
View File
@@ -0,0 +1,583 @@
"""Export Intel sync — OSM Overpass, contact enrichment, tenders, fusion."""
from __future__ import annotations
import json
import re
import time
import xml.etree.ElementTree as ET
from datetime import datetime
from typing import Any, Optional
from urllib.parse import urljoin, urlparse
import httpx
from app.db import execute, execute_returning, fetch_all, fetch_one
OVERPASS_URLS = [
"https://overpass.kumi.systems/api/interpreter",
"https://overpass-api.de/api/interpreter",
]
TED_RSS_URLS = [
"https://ted.europa.eu/en/simap/rss-feed/-/rss/search/15",
"https://ted.europa.eu/en/simap/rss-feed/-/rss/search/teeq",
]
from app.export_territory_seeds import BBOX_ONLY_ISO2
EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
SKIP_EMAIL_SUFFIX = ("example.com", "sentry.io", "wixpress.com", "png", "jpg", "jpeg")
DISTRIBUTOR_QUERIES = [
'node["shop"="wholesale"]',
'way["shop"="wholesale"]',
'node["wholesale"]',
'way["wholesale"]',
'node["shop"="cash_and_carry"]',
'way["shop"="cash_and_carry"]',
]
CUSTOMER_QUERIES = [
'node["shop"="butcher"]',
'way["shop"="butcher"]',
'node["amenity"="restaurant"]',
'way["amenity"="restaurant"]',
'node["amenity"="fast_food"]',
'way["amenity"="fast_food"]',
]
CATERER_QUERIES = [
'node["name"~"catering",i]',
'way["name"~"catering",i]',
'node["name"~"cateraar",i]',
'way["name"~"cateraar",i]',
'node["name"~"food service",i]',
'way["name"~"food service",i]',
]
def _fetch_overpass(query: str) -> list[dict[str, Any]]:
last_err: Optional[str] = None
for url in OVERPASS_URLS:
for attempt in range(3):
try:
with httpx.Client(timeout=180.0) as client:
resp = client.post(url, data={"data": query})
if resp.status_code == 429:
time.sleep(12 * (attempt + 1))
continue
resp.raise_for_status()
return resp.json().get("elements", [])
except Exception as exc: # noqa: BLE001
last_err = str(exc)
time.sleep(5 * (attempt + 1))
raise RuntimeError(f"Overpass failed: {last_err}")
def _get_territory(country_iso2: str) -> Optional[dict[str, Any]]:
return fetch_one(
"""
SELECT * FROM export_territories
WHERE country_iso2 = %s AND is_active = TRUE
ORDER BY sync_priority
LIMIT 1
""",
(country_iso2.upper(),),
)
def _bbox_str(bbox: Any) -> str:
if isinstance(bbox, str):
bbox = json.loads(bbox)
if not bbox or len(bbox) != 4:
raise ValueError("Invalid bbox")
south, west, north, east = bbox
return f"{south},{west},{north},{east}"
def _coords(el: dict[str, Any]) -> tuple[Optional[float], Optional[float]]:
if el.get("type") == "node":
return el.get("lat"), el.get("lon")
center = el.get("center") or {}
return center.get("lat"), center.get("lon")
def _osm_external_id(el: dict[str, Any]) -> str:
return f"osm:{el.get('type')}:{el.get('id')}"
def _map_entity_type(tags: dict[str, Any], query_kind: str) -> str:
shop = tags.get("shop") or ""
amenity = tags.get("amenity") or ""
name = (tags.get("name") or "").lower()
if query_kind == "caterer" or "catering" in name or "cateraar" in name:
return "contract_caterer"
if shop in ("wholesale", "cash_and_carry") or tags.get("wholesale"):
return "wholesaler"
if shop == "butcher":
return "butcher"
if amenity == "fast_food":
return "doner_shoarma" if any(x in name for x in ("doner", "döner", "shoarma", "kebab")) else "restaurant"
if amenity == "restaurant":
return "restaurant"
return "foodservice"
def _upsert_osm_entity(
el: dict[str, Any],
country_iso2: str,
territory_code: Optional[str],
query_kind: str,
) -> tuple[str, Optional[int]]:
tags = el.get("tags") or {}
lat, lon = _coords(el)
if lat is None or lon is None:
return "skipped", None
ext = _osm_external_id(el)
name = tags.get("name") or tags.get("brand") or tags.get("operator") or "Unknown"
entity_type = _map_entity_type(tags, query_kind)
street = tags.get("addr:street") or ""
hn = tags.get("addr:housenumber") or ""
address = f"{street} {hn}".strip() or None
city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village")
phone = (tags.get("phone") or tags.get("contact:phone") or "")[:64] or None
email = (tags.get("email") or tags.get("contact:email") or "")[:255] or None
website = (tags.get("website") or tags.get("contact:website") or "")[:512] or None
existing = fetch_one(
"SELECT id FROM export_market_entities WHERE metadata->>'osm_external_id' = %s",
(ext,),
)
if existing:
execute(
"""
UPDATE export_market_entities SET
name = %s, entity_type = %s, lat = %s, lon = %s,
address_line = COALESCE(%s, address_line),
city = COALESCE(%s, city),
phone = COALESCE(%s, phone), email = COALESCE(%s, email),
website = COALESCE(%s, website),
sources = COALESCE(sources, '[]'::jsonb) || '["osm"]'::jsonb,
confidence = GREATEST(confidence, 55),
updated_at = NOW()
WHERE id = %s
""",
(name[:255], entity_type, lat, lon, address, city, phone, email, website, existing["id"]),
)
entity_id = existing["id"]
action = "updated"
else:
row = execute_returning(
"""
INSERT INTO export_market_entities (
name, entity_type, country_iso2, territory_code, lat, lon,
address_line, city, phone, email, website, source, sources, confidence,
metadata
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'osm','["osm"]'::jsonb,55,%s)
RETURNING id
""",
(
name[:255], entity_type, country_iso2.upper(), territory_code,
lat, lon, address, city, phone, email, website,
json.dumps({"osm_external_id": ext}),
),
)
entity_id = row["id"]
action = "inserted"
if email:
_upsert_contact(entity_id, email, phone, "osm", 60)
return action, entity_id
def _upsert_contact(
entity_id: int,
email: Optional[str],
phone: Optional[str],
source: str,
confidence: int,
role: str = "general",
name: Optional[str] = None,
) -> None:
if email:
exists = fetch_one(
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s",
(entity_id, email.lower()),
)
if not exists:
execute(
"""
INSERT INTO export_entity_contacts (entity_id, role, name, email, phone, source, confidence, is_primary)
VALUES (%s, %s, %s, %s, %s, %s, %s, FALSE)
""",
(entity_id, role, name, email.lower(), phone, source, confidence),
)
elif phone:
exists = fetch_one(
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND phone = %s",
(entity_id, phone),
)
if not exists:
execute(
"""
INSERT INTO export_entity_contacts (entity_id, role, name, phone, source, confidence)
VALUES (%s, %s, %s, %s, %s, %s)
""",
(entity_id, role, name, phone, source, confidence),
)
def _build_area_query(parts: list[str], country_iso2: str) -> str:
body = "".join(f"{p}(area.c);" for p in parts)
return f'[out:json][timeout:180];area["ISO3166-1"="{country_iso2}"]->.c;({body});out center tags;'
def _build_bbox_query(parts: list[str], bbox: str) -> str:
body = "".join(f"{p}({bbox});" for p in parts)
return f"[out:json][timeout:180];({body});out center tags;"
def _countries_for_sync(
country_iso2: Optional[str] = None,
region_code: Optional[str] = None,
max_priority: int = 2,
) -> list[str]:
if country_iso2:
return [country_iso2.upper()]
clauses = ["is_active = TRUE", "sync_priority <= %s"]
params: list[Any] = [max_priority]
if region_code:
clauses.append("region_code = %s")
params.append(region_code)
rows = fetch_all(
f"""
SELECT country_iso2 FROM export_territories
WHERE {' AND '.join(clauses)}
GROUP BY country_iso2
ORDER BY MIN(sync_priority), country_iso2
""",
tuple(params),
)
return [r["country_iso2"] for r in rows]
def _build_country_query(parts: list[str], country_iso2: str, bbox: str) -> str:
if country_iso2.upper() in BBOX_ONLY_ISO2:
return _build_bbox_query(parts, bbox)
return _build_area_query(parts, country_iso2.upper())
def sync_osm_country(country_iso2: str, kinds: Optional[list[str]] = None) -> dict[str, Any]:
territory = _get_territory(country_iso2)
if not territory or not territory.get("bbox"):
raise ValueError(f"No territory/bbox for {country_iso2}")
bbox = _bbox_str(territory["bbox"])
territory_code = territory["code"]
kinds = kinds or ["distributors", "customers", "caterers"]
stats = {"inserted": 0, "updated": 0, "fetched": 0, "kinds": {}}
mapping = {
"distributors": DISTRIBUTOR_QUERIES,
"customers": CUSTOMER_QUERIES,
"caterers": CATERER_QUERIES,
}
for kind in kinds:
parts = mapping.get(kind, [])
if not parts:
continue
query = _build_country_query(parts, country_iso2, bbox)
elements = _fetch_overpass(query)
ki = {"fetched": len(elements), "inserted": 0, "updated": 0}
seen: set[str] = set()
for el in elements:
ext = _osm_external_id(el)
if ext in seen:
continue
seen.add(ext)
action, _ = _upsert_osm_entity(el, country_iso2, territory_code, kind.replace("s", "")[:7])
if action == "inserted":
ki["inserted"] += 1
stats["inserted"] += 1
elif action == "updated":
ki["updated"] += 1
stats["updated"] += 1
stats["fetched"] += ki["fetched"]
stats["kinds"][kind] = ki
time.sleep(2)
materialize_caterer_presence(country_iso2)
return stats
def materialize_caterer_presence(country_iso2: str) -> int:
"""Create entity rows from caterer_presence + brands for country."""
rows = fetch_all(
"""
SELECT p.*, b.name AS brand_name, b.website
FROM export_caterer_presence p
JOIN export_caterer_brands b ON b.code = p.brand_code
WHERE p.country_iso2 = %s AND p.is_active = TRUE
""",
(country_iso2.upper(),),
)
territory = _get_territory(country_iso2)
territory_code = territory["code"] if territory else None
lat, lon = (territory.get("lat"), territory.get("lon")) if territory else (None, None)
created = 0
for p in rows:
name = p.get("local_legal_name") or p.get("brand_name")
exists = fetch_one(
"SELECT id FROM export_market_entities WHERE brand_code = %s AND country_iso2 = %s",
(p["brand_code"], country_iso2.upper()),
)
if exists:
if p.get("entity_id") is None:
execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (exists["id"], p["id"]))
continue
row = execute_returning(
"""
INSERT INTO export_market_entities (
name, entity_type, country_iso2, territory_code, lat, lon, website, brand_code,
source, sources, confidence, product_interest
) VALUES (%s,'contract_caterer',%s,%s,%s,%s,%s,%s,'caterer_registry','["caterer_registry"]'::jsonb,75,'["chicken","shoarma"]'::jsonb)
RETURNING id
""",
(
name, country_iso2.upper(), territory_code, lat, lon,
(p.get("website") or "")[:512] or None, p["brand_code"],
),
)
execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (row["id"], p["id"]))
created += 1
return created
def _scrape_emails(url: str) -> list[str]:
if not url:
return []
if not url.startswith("http"):
url = "https://" + url
try:
with httpx.Client(timeout=15.0, follow_redirects=True) as client:
resp = client.get(url, headers={"User-Agent": "FoodlinkkExportIntel/1.0"})
if resp.status_code >= 400:
return []
text = resp.text[:500000]
except Exception:
return []
found: set[str] = set()
for m in EMAIL_RE.findall(text):
low = m.lower()
if any(low.endswith(s) or s in low for s in SKIP_EMAIL_SUFFIX):
continue
if low.startswith("noreply") or low.startswith("no-reply"):
continue
found.add(low)
return list(found)[:5]
def enrich_contacts_country(country_iso2: str) -> dict[str, Any]:
entities = fetch_all(
"""
SELECT id, name, website, email, phone FROM export_market_entities
WHERE country_iso2 = %s AND website IS NOT NULL AND website <> ''
""",
(country_iso2.upper(),),
)
added = 0
scraped = 0
for ent in entities:
emails = _scrape_emails(ent["website"])
scraped += 1
for em in emails:
role = "procurement" if any(x in em for x in ("inkoop", "procurement", "sales", "info", "order")) else "general"
before = fetch_one(
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s",
(ent["id"], em),
)
if not before:
_upsert_contact(ent["id"], em, None, "website_scrape", 65, role=role, name="Website")
added += 1
if not ent.get("email") and emails:
execute(
"UPDATE export_market_entities SET email = %s, updated_at = NOW() WHERE id = %s",
(emails[0], ent["id"]),
)
time.sleep(0.3)
return {"entities_scraped": scraped, "contacts_added": added}
def sync_ted_tenders(limit: int = 50) -> dict[str, Any]:
"""TED Search API v3 — food / catering / meat tenders."""
inserted = updated = 0
queries = [
"BT-21-Procedure ~ catering",
"BT-21-Procedure ~ food",
"BT-21-Procedure ~ meat",
]
ted_url = "https://api.ted.europa.eu/v3/notices/search"
fields = [
"BT-21-Procedure",
"organisation-name-buyer",
"publication-number",
"organisation-country-buyer",
]
seen: set[str] = set()
try:
with httpx.Client(timeout=45.0) as client:
for q in queries:
resp = client.post(
ted_url,
json={"query": q, "limit": min(limit, 25), "page": 1, "fields": fields},
)
if resp.status_code >= 400:
continue
data = resp.json()
for notice in data.get("notices", []):
pub = notice.get("publication-number") or ""
if not pub or pub in seen:
continue
seen.add(pub)
title_raw = notice.get("BT-21-Procedure")
if isinstance(title_raw, list):
title = title_raw[0] if title_raw else f"TED {pub}"
else:
title = str(title_raw or f"TED notice {pub}")
country_raw = notice.get("organisation-country-buyer")
country_iso = None
if isinstance(country_raw, list) and country_raw:
c = country_raw[0]
country_iso = c[:2] if len(c) >= 2 else None
link = f"https://ted.europa.eu/en/notice/{pub}"
ext = f"ted:{pub}"
existing = fetch_one("SELECT id FROM export_tenders WHERE external_id = %s", (ext,))
if existing:
execute(
"""
UPDATE export_tenders SET title=%s, url=%s, country_iso2=%s, updated_at=NOW()
WHERE external_id=%s
""",
(title[:512], link, country_iso, ext),
)
updated += 1
else:
execute(
"""
INSERT INTO export_tenders (external_id, title, url, country_iso2, source, status)
VALUES (%s,%s,%s,%s,'ted','open')
""",
(ext, title[:512], link, country_iso),
)
inserted += 1
time.sleep(1)
except Exception as exc:
return {"error": str(exc), "inserted": inserted, "updated": updated}
return {"inserted": inserted, "updated": updated, "source": "ted_api_v3", "seen": len(seen)}
def sync_all_priority(max_priority: int = 2, region_code: Optional[str] = None) -> dict[str, Any]:
"""Full OSM + contacts + Places sync for active territories."""
iso_list = _countries_for_sync(region_code=region_code, max_priority=max_priority)
report: dict[str, Any] = {
"countries": iso_list,
"country_count": len(iso_list),
"region": region_code,
"max_priority": max_priority,
"steps": {},
}
report["steps"]["ted"] = sync_ted_tenders(40)
dist: dict[str, Any] = {}
cater: dict[str, Any] = {}
cust: dict[str, Any] = {}
contacts: dict[str, Any] = {}
for c in iso_list:
try:
materialize_caterer_presence(c)
dist[c] = sync_osm_country(c, kinds=["distributors"])
cater[c] = sync_osm_country(c, kinds=["caterers"])
cust[c] = sync_osm_country(c, kinds=["customers"])
contacts[c] = enrich_contacts_country(c)
except Exception as exc:
dist[c] = {"error": str(exc)}
time.sleep(2)
report["steps"]["distributors"] = dist
report["steps"]["caterers"] = cater
report["steps"]["customers"] = cust
report["steps"]["contacts"] = contacts
from app.export_intel_places import sync_places_countries
report["steps"]["google_places"] = sync_places_countries(iso_list)
return report
def sync_world_regions(max_priority: int = 2) -> dict[str, Any]:
"""Sync all target regions sequentially (Europe, MENA, Africa, Americas)."""
regions = ["europe", "middle_east", "gcc", "africa", "americas", "caribbean"]
out: dict[str, Any] = {"regions": {}, "totals": {"countries": 0}}
for reg in regions:
countries = _countries_for_sync(region_code=reg, max_priority=max_priority)
if not countries:
out["regions"][reg] = {"skipped": True, "reason": "no countries"}
continue
out["totals"]["countries"] += len(countries)
try:
out["regions"][reg] = sync_all_priority(max_priority=max_priority, region_code=reg)
except Exception as exc:
out["regions"][reg] = {"error": str(exc)}
time.sleep(5)
return out
def sync_distributors(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
countries = _countries_for_sync(country_iso2, region_code)
results: dict[str, Any] = {}
for c in countries:
try:
results[c] = sync_osm_country(c, kinds=["distributors"])
except Exception as exc:
results[c] = {"error": str(exc)}
time.sleep(3)
enrich = {}
if country_iso2:
enrich = enrich_contacts_country(country_iso2.upper())
return {"osm": results, "contact_enrichment": enrich}
def sync_caterers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
countries = _countries_for_sync(country_iso2, region_code)
results: dict[str, Any] = {}
materialized = 0
for c in countries:
try:
materialized += materialize_caterer_presence(c)
results[c] = sync_osm_country(c, kinds=["caterers"])
except Exception as exc:
results[c] = {"error": str(exc)}
time.sleep(3)
return {"osm": results, "materialized_presence": materialized}
def sync_contacts(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
countries = _countries_for_sync(country_iso2, region_code)
out: dict[str, Any] = {}
for c in countries:
out[c] = enrich_contacts_country(c)
ted = sync_ted_tenders()
return {"contacts": out, "ted_tenders": ted}
def sync_customers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
if country_iso2:
return sync_osm_country(country_iso2, kinds=["customers"])
countries = _countries_for_sync(region_code=region_code)
results: dict[str, Any] = {}
for c in countries:
try:
results[c] = sync_osm_country(c, kinds=["customers"])
except Exception as exc:
results[c] = {"error": str(exc)}
time.sleep(3)
return {"osm": results}
+133
View File
@@ -0,0 +1,133 @@
"""Territory seeds — Europe, Middle East, Africa, Americas (bbox + centroid)."""
from __future__ import annotations
# bbox: [south, west, north, east]
WORLD_TERRITORIES: list[dict] = [
# ── Europe (extra) ──
{"code": "ie", "iso2": "IE", "region": "europe", "nl": "Ierland", "en": "Ireland", "lat": 53.4, "lon": -8.0, "zoom": 7, "bbox": [51.4, -10.5, 55.4, -5.5], "pri": 2},
{"code": "pt", "iso2": "PT", "region": "europe", "nl": "Portugal", "en": "Portugal", "lat": 39.5, "lon": -8.0, "zoom": 6, "bbox": [36.9, -9.5, 42.2, -6.2], "pri": 2},
{"code": "gr", "iso2": "GR", "region": "europe", "nl": "Griekenland", "en": "Greece", "lat": 39.0, "lon": 22.0, "zoom": 6, "bbox": [34.8, 19.4, 41.8, 29.6], "pri": 2},
{"code": "cz", "iso2": "CZ", "region": "europe", "nl": "Tsjechië", "en": "Czechia", "lat": 49.8, "lon": 15.5, "zoom": 7, "bbox": [48.5, 12.1, 51.1, 18.9], "pri": 2},
{"code": "sk", "iso2": "SK", "region": "europe", "nl": "Slowakije", "en": "Slovakia", "lat": 48.7, "lon": 19.5, "zoom": 7, "bbox": [47.7, 16.8, 49.6, 22.6], "pri": 2},
{"code": "hu", "iso2": "HU", "region": "europe", "nl": "Hongarije", "en": "Hungary", "lat": 47.2, "lon": 19.5, "zoom": 7, "bbox": [45.7, 16.1, 48.6, 22.9], "pri": 2},
{"code": "ro", "iso2": "RO", "region": "europe", "nl": "Roemenië", "en": "Romania", "lat": 45.9, "lon": 25.0, "zoom": 6, "bbox": [43.6, 20.3, 48.3, 29.7], "pri": 2},
{"code": "bg", "iso2": "BG", "region": "europe", "nl": "Bulgarije", "en": "Bulgaria", "lat": 42.7, "lon": 25.5, "zoom": 7, "bbox": [41.2, 22.4, 44.2, 28.6], "pri": 2},
{"code": "hr", "iso2": "HR", "region": "europe", "nl": "Kroatië", "en": "Croatia", "lat": 45.1, "lon": 15.2, "zoom": 7, "bbox": [42.4, 13.5, 46.5, 19.4], "pri": 2},
{"code": "si", "iso2": "SI", "region": "europe", "nl": "Slovenië", "en": "Slovenia", "lat": 46.1, "lon": 14.8, "zoom": 8, "bbox": [45.4, 13.4, 46.9, 16.6], "pri": 2},
{"code": "fi", "iso2": "FI", "region": "europe", "nl": "Finland", "en": "Finland", "lat": 64.0, "lon": 26.0, "zoom": 5, "bbox": [59.8, 20.6, 70.1, 31.6], "pri": 2},
{"code": "ee", "iso2": "EE", "region": "europe", "nl": "Estland", "en": "Estonia", "lat": 58.6, "lon": 25.0, "zoom": 7, "bbox": [57.5, 21.8, 59.7, 28.2], "pri": 2},
{"code": "lv", "iso2": "LV", "region": "europe", "nl": "Letland", "en": "Latvia", "lat": 56.9, "lon": 24.6, "zoom": 7, "bbox": [55.7, 21.0, 58.1, 28.2], "pri": 2},
{"code": "lt", "iso2": "LT", "region": "europe", "nl": "Litouwen", "en": "Lithuania", "lat": 55.2, "lon": 23.9, "zoom": 7, "bbox": [53.9, 21.0, 56.4, 26.8], "pri": 2},
{"code": "lu", "iso2": "LU", "region": "europe", "nl": "Luxemburg", "en": "Luxembourg", "lat": 49.8, "lon": 6.1, "zoom": 9, "bbox": [49.4, 5.7, 50.2, 6.5], "pri": 2},
{"code": "mt", "iso2": "MT", "region": "europe", "nl": "Malta", "en": "Malta", "lat": 35.9, "lon": 14.4, "zoom": 10, "bbox": [35.8, 14.2, 36.1, 14.6], "pri": 2},
{"code": "cy", "iso2": "CY", "region": "europe", "nl": "Cyprus", "en": "Cyprus", "lat": 35.1, "lon": 33.4, "zoom": 9, "bbox": [34.6, 32.3, 35.7, 34.6], "pri": 2},
{"code": "ua", "iso2": "UA", "region": "europe", "nl": "Oekraïne", "en": "Ukraine", "lat": 48.4, "lon": 31.2, "zoom": 5, "bbox": [44.4, 22.1, 52.4, 40.2], "pri": 2},
{"code": "rs", "iso2": "RS", "region": "europe", "nl": "Servië", "en": "Serbia", "lat": 44.0, "lon": 21.0, "zoom": 7, "bbox": [42.2, 18.8, 46.2, 23.0], "pri": 2},
{"code": "ba", "iso2": "BA", "region": "europe", "nl": "Bosnië", "en": "Bosnia", "lat": 44.0, "lon": 17.7, "zoom": 7, "bbox": [42.6, 15.7, 45.3, 19.6], "pri": 2},
{"code": "mk", "iso2": "MK", "region": "europe", "nl": "Noord-Macedonië", "en": "North Macedonia", "lat": 41.6, "lon": 21.7, "zoom": 8, "bbox": [40.8, 20.5, 42.4, 23.0], "pri": 2},
{"code": "al", "iso2": "AL", "region": "europe", "nl": "Albanië", "en": "Albania", "lat": 41.2, "lon": 20.2, "zoom": 8, "bbox": [39.6, 19.3, 42.7, 21.1], "pri": 2},
{"code": "me", "iso2": "ME", "region": "europe", "nl": "Montenegro", "en": "Montenegro", "lat": 42.7, "lon": 19.4, "zoom": 8, "bbox": [41.8, 18.4, 43.6, 20.4], "pri": 2},
{"code": "md", "iso2": "MD", "region": "europe", "nl": "Moldavië", "en": "Moldova", "lat": 47.0, "lon": 28.8, "zoom": 7, "bbox": [45.5, 26.6, 48.5, 30.2], "pri": 2},
{"code": "is", "iso2": "IS", "region": "europe", "nl": "IJsland", "en": "Iceland", "lat": 64.9, "lon": -19.0, "zoom": 6, "bbox": [63.3, -24.5, 66.5, -13.5], "pri": 2},
{"code": "li", "iso2": "LI", "region": "europe", "nl": "Liechtenstein", "en": "Liechtenstein", "lat": 47.1, "lon": 9.5, "zoom": 10, "bbox": [47.0, 9.5, 47.3, 9.7], "pri": 2},
# ── Middle East ──
{"code": "il", "iso2": "IL", "region": "middle_east", "nl": "Israël", "en": "Israel", "lat": 31.5, "lon": 34.9, "zoom": 7, "bbox": [29.5, 34.3, 33.3, 35.9], "pri": 2},
{"code": "eg", "iso2": "EG", "region": "middle_east", "nl": "Egypte", "en": "Egypt", "lat": 26.8, "lon": 30.8, "zoom": 5, "bbox": [22.0, 25.0, 31.7, 35.0], "pri": 2},
{"code": "jo", "iso2": "JO", "region": "middle_east", "nl": "Jordanië", "en": "Jordan", "lat": 31.2, "lon": 36.8, "zoom": 7, "bbox": [29.2, 34.9, 33.4, 39.3], "pri": 2},
{"code": "lb", "iso2": "LB", "region": "middle_east", "nl": "Libanon", "en": "Lebanon", "lat": 33.9, "lon": 35.9, "zoom": 8, "bbox": [33.1, 35.1, 34.7, 36.6], "pri": 2},
{"code": "iq", "iso2": "IQ", "region": "middle_east", "nl": "Irak", "en": "Iraq", "lat": 33.2, "lon": 43.7, "zoom": 6, "bbox": [29.1, 38.8, 37.4, 48.6], "pri": 2},
{"code": "ir", "iso2": "IR", "region": "middle_east", "nl": "Iran", "en": "Iran", "lat": 32.4, "lon": 53.7, "zoom": 5, "bbox": [25.0, 44.0, 39.8, 63.3], "pri": 2},
{"code": "ye", "iso2": "YE", "region": "middle_east", "nl": "Jemen", "en": "Yemen", "lat": 15.6, "lon": 48.0, "zoom": 6, "bbox": [12.1, 42.5, 19.0, 54.5], "pri": 2},
{"code": "ps", "iso2": "PS", "region": "middle_east", "nl": "Palestina", "en": "Palestine", "lat": 31.9, "lon": 35.2, "zoom": 8, "bbox": [31.2, 34.2, 32.6, 35.6], "pri": 2},
{"code": "sy", "iso2": "SY", "region": "middle_east", "nl": "Syrië", "en": "Syria", "lat": 35.0, "lon": 38.5, "zoom": 6, "bbox": [32.3, 35.7, 37.3, 42.4], "pri": 3},
{"code": "ly", "iso2": "LY", "region": "middle_east", "nl": "Libië", "en": "Libya", "lat": 26.3, "lon": 17.2, "zoom": 5, "bbox": [19.5, 9.3, 33.2, 25.2], "pri": 2},
{"code": "tn", "iso2": "TN", "region": "middle_east", "nl": "Tunesië", "en": "Tunisia", "lat": 34.0, "lon": 9.5, "zoom": 6, "bbox": [30.2, 7.5, 37.5, 11.6], "pri": 2},
{"code": "dz", "iso2": "DZ", "region": "africa", "nl": "Algerije", "en": "Algeria", "lat": 28.0, "lon": 2.6, "zoom": 5, "bbox": [19.0, -8.7, 37.1, 12.0], "pri": 2},
# ── Africa ──
{"code": "za", "iso2": "ZA", "region": "africa", "nl": "Zuid-Afrika", "en": "South Africa", "lat": -30.6, "lon": 22.9, "zoom": 5, "bbox": [-34.8, 16.5, -22.1, 32.9], "pri": 2},
{"code": "ng", "iso2": "NG", "region": "africa", "nl": "Nigeria", "en": "Nigeria", "lat": 9.1, "lon": 8.7, "zoom": 6, "bbox": [4.3, 2.7, 13.9, 14.7], "pri": 2},
{"code": "ke", "iso2": "KE", "region": "africa", "nl": "Kenia", "en": "Kenya", "lat": -0.02, "lon": 37.9, "zoom": 6, "bbox": [-4.7, 33.9, 5.0, 41.9], "pri": 2},
{"code": "gh", "iso2": "GH", "region": "africa", "nl": "Ghana", "en": "Ghana", "lat": 7.9, "lon": -1.0, "zoom": 6, "bbox": [4.7, -3.3, 11.2, 1.2], "pri": 2},
{"code": "et", "iso2": "ET", "region": "africa", "nl": "Ethiopië", "en": "Ethiopia", "lat": 9.1, "lon": 40.5, "zoom": 5, "bbox": [3.4, 33.0, 14.9, 48.0], "pri": 2},
{"code": "tz", "iso2": "TZ", "region": "africa", "nl": "Tanzania", "en": "Tanzania", "lat": -6.4, "lon": 34.9, "zoom": 6, "bbox": [-11.7, 29.3, -0.99, 40.5], "pri": 2},
{"code": "ug", "iso2": "UG", "region": "africa", "nl": "Oeganda", "en": "Uganda", "lat": 1.4, "lon": 32.3, "zoom": 7, "bbox": [-1.5, 29.6, 4.2, 35.0], "pri": 2},
{"code": "sn", "iso2": "SN", "region": "africa", "nl": "Senegal", "en": "Senegal", "lat": 14.5, "lon": -14.5, "zoom": 7, "bbox": [12.3, -17.5, 16.7, -11.4], "pri": 2},
{"code": "ci", "iso2": "CI", "region": "africa", "nl": "Ivoorkust", "en": "Côte d'Ivoire", "lat": 7.5, "lon": -5.5, "zoom": 6, "bbox": [4.4, -8.6, 10.7, -2.5], "pri": 2},
{"code": "cm", "iso2": "CM", "region": "africa", "nl": "Kameroen", "en": "Cameroon", "lat": 6.6, "lon": 12.4, "zoom": 6, "bbox": [1.7, 8.5, 13.1, 16.2], "pri": 2},
{"code": "ao", "iso2": "AO", "region": "africa", "nl": "Angola", "en": "Angola", "lat": -11.2, "lon": 17.9, "zoom": 5, "bbox": [-18.0, 11.7, -4.4, 24.1], "pri": 2},
{"code": "mz", "iso2": "MZ", "region": "africa", "nl": "Mozambique", "en": "Mozambique", "lat": -18.7, "lon": 35.5, "zoom": 5, "bbox": [-26.9, 30.2, -10.5, 40.8], "pri": 2},
{"code": "zw", "iso2": "ZW", "region": "africa", "nl": "Zimbabwe", "en": "Zimbabwe", "lat": -19.0, "lon": 29.2, "zoom": 6, "bbox": [-22.4, 25.2, -15.6, 33.1], "pri": 2},
{"code": "zm", "iso2": "ZM", "region": "africa", "nl": "Zambia", "en": "Zambia", "lat": -13.1, "lon": 27.8, "zoom": 6, "bbox": [-18.1, 22.0, -8.2, 33.7], "pri": 2},
{"code": "rw", "iso2": "RW", "region": "africa", "nl": "Rwanda", "en": "Rwanda", "lat": -1.9, "lon": 30.0, "zoom": 8, "bbox": [-2.8, 28.9, -1.0, 30.9], "pri": 2},
{"code": "sd", "iso2": "SD", "region": "africa", "nl": "Soedan", "en": "Sudan", "lat": 15.5, "lon": 30.2, "zoom": 5, "bbox": [8.7, 21.8, 22.2, 38.6], "pri": 2},
{"code": "ss", "iso2": "SS", "region": "africa", "nl": "Zuid-Soedan", "en": "South Sudan", "lat": 7.9, "lon": 30.0, "zoom": 6, "bbox": [3.5, 23.9, 12.2, 35.9], "pri": 3},
{"code": "cd", "iso2": "CD", "region": "africa", "nl": "Congo (DRC)", "en": "DR Congo", "lat": -4.0, "lon": 21.8, "zoom": 5, "bbox": [-13.5, 12.2, 5.4, 31.3], "pri": 2},
{"code": "cg", "iso2": "CG", "region": "africa", "nl": "Congo", "en": "Congo", "lat": -0.7, "lon": 15.3, "zoom": 6, "bbox": [-5.0, 11.2, 3.7, 18.6], "pri": 2},
{"code": "bf", "iso2": "BF", "region": "africa", "nl": "Burkina Faso", "en": "Burkina Faso", "lat": 12.2, "lon": -1.6, "zoom": 6, "bbox": [9.4, -5.5, 15.1, 2.4], "pri": 2},
{"code": "ml", "iso2": "ML", "region": "africa", "nl": "Mali", "en": "Mali", "lat": 17.6, "lon": -4.0, "zoom": 5, "bbox": [10.1, -12.2, 25.0, 4.3], "pri": 2},
{"code": "ne", "iso2": "NE", "region": "africa", "nl": "Niger", "en": "Niger", "lat": 17.6, "lon": 8.1, "zoom": 5, "bbox": [11.7, 0.2, 23.5, 16.0], "pri": 2},
{"code": "td", "iso2": "TD", "region": "africa", "nl": "Tsjaad", "en": "Chad", "lat": 15.5, "lon": 18.7, "zoom": 5, "bbox": [7.4, 13.5, 23.5, 24.0], "pri": 2},
{"code": "so", "iso2": "SO", "region": "africa", "nl": "Somalië", "en": "Somalia", "lat": 5.2, "lon": 46.2, "zoom": 5, "bbox": [-1.7, 41.0, 12.0, 51.4], "pri": 3},
{"code": "mg", "iso2": "MG", "region": "africa", "nl": "Madagascar", "en": "Madagascar", "lat": -18.8, "lon": 46.9, "zoom": 5, "bbox": [-25.6, 43.2, -11.9, 50.5], "pri": 2},
{"code": "mu", "iso2": "MU", "region": "africa", "nl": "Mauritius", "en": "Mauritius", "lat": -20.3, "lon": 57.6, "zoom": 9, "bbox": [-20.5, 57.3, -19.9, 63.5], "pri": 2},
{"code": "na", "iso2": "NA", "region": "africa", "nl": "Namibië", "en": "Namibia", "lat": -22.6, "lon": 17.1, "zoom": 5, "bbox": [-28.0, 11.7, -16.9, 25.3], "pri": 2},
{"code": "bw", "iso2": "BW", "region": "africa", "nl": "Botswana", "en": "Botswana", "lat": -22.3, "lon": 24.7, "zoom": 6, "bbox": [-26.9, 20.0, -17.8, 29.4], "pri": 2},
{"code": "ga", "iso2": "GA", "region": "africa", "nl": "Gabon", "en": "Gabon", "lat": -0.8, "lon": 11.6, "zoom": 6, "bbox": [-4.0, 8.7, 2.3, 14.5], "pri": 2},
{"code": "gn", "iso2": "GN", "region": "africa", "nl": "Guinee", "en": "Guinea", "lat": 9.9, "lon": -11.7, "zoom": 6, "bbox": [7.2, -15.1, 12.7, -7.6], "pri": 2},
{"code": "bj", "iso2": "BJ", "region": "africa", "nl": "Benin", "en": "Benin", "lat": 9.3, "lon": 2.3, "zoom": 7, "bbox": [6.2, 0.8, 12.4, 3.9], "pri": 2},
{"code": "tg", "iso2": "TG", "region": "africa", "nl": "Togo", "en": "Togo", "lat": 8.6, "lon": 1.0, "zoom": 7, "bbox": [6.1, -0.1, 11.1, 1.8], "pri": 2},
{"code": "lr", "iso2": "LR", "region": "africa", "nl": "Liberia", "en": "Liberia", "lat": 6.4, "lon": -9.4, "zoom": 7, "bbox": [4.3, -11.5, 8.6, -7.4], "pri": 2},
{"code": "sl", "iso2": "SL", "region": "africa", "nl": "Sierra Leone", "en": "Sierra Leone", "lat": 8.5, "lon": -11.8, "zoom": 7, "bbox": [6.9, -13.3, 10.0, -10.3], "pri": 2},
{"code": "mr", "iso2": "MR", "region": "africa", "nl": "Mauritanië", "en": "Mauritania", "lat": 21.0, "lon": -10.9, "zoom": 5, "bbox": [14.7, -17.1, 27.3, -4.8], "pri": 2},
{"code": "er", "iso2": "ER", "region": "africa", "nl": "Eritrea", "en": "Eritrea", "lat": 15.2, "lon": 39.8, "zoom": 7, "bbox": [12.4, 36.4, 18.0, 43.1], "pri": 2},
{"code": "dj", "iso2": "DJ", "region": "africa", "nl": "Djibouti", "en": "Djibouti", "lat": 11.8, "lon": 42.6, "zoom": 8, "bbox": [10.9, 41.8, 12.7, 43.4], "pri": 2},
{"code": "mw", "iso2": "MW", "region": "africa", "nl": "Malawi", "en": "Malawi", "lat": -13.3, "lon": 34.3, "zoom": 7, "bbox": [-17.1, 32.7, -9.4, 35.9], "pri": 2},
{"code": "ls", "iso2": "LS", "region": "africa", "nl": "Lesotho", "en": "Lesotho", "lat": -29.6, "lon": 28.2, "zoom": 8, "bbox": [-30.7, 27.0, -28.6, 29.5], "pri": 2},
{"code": "sz", "iso2": "SZ", "region": "africa", "nl": "Eswatini", "en": "Eswatini", "lat": -26.5, "lon": 31.5, "zoom": 8, "bbox": [-27.3, 30.8, -25.7, 32.1], "pri": 2},
{"code": "cv", "iso2": "CV", "region": "africa", "nl": "Kaapverdië", "en": "Cape Verde", "lat": 16.0, "lon": -24.0, "zoom": 8, "bbox": [14.8, -25.4, 17.2, -22.7], "pri": 2},
{"code": "gq", "iso2": "GQ", "region": "africa", "nl": "Equatoriaal-Guinea", "en": "Equatorial Guinea", "lat": 1.7, "lon": 10.3, "zoom": 8, "bbox": [0.9, 9.3, 3.8, 11.3], "pri": 2},
{"code": "st", "iso2": "ST", "region": "africa", "nl": "São Tomé", "en": "São Tomé", "lat": 0.3, "lon": 6.6, "zoom": 9, "bbox": [0.0, 6.5, 1.7, 7.5], "pri": 2},
{"code": "sc", "iso2": "SC", "region": "africa", "nl": "Seychellen", "en": "Seychelles", "lat": -4.7, "lon": 55.5, "zoom": 9, "bbox": [-10.0, 46.0, -3.7, 56.3], "pri": 2},
{"code": "km", "iso2": "KM", "region": "africa", "nl": "Comoren", "en": "Comoros", "lat": -11.9, "lon": 43.9, "zoom": 9, "bbox": [-12.4, 43.2, -11.4, 44.5], "pri": 2},
{"code": "bi", "iso2": "BI", "region": "africa", "nl": "Burundi", "en": "Burundi", "lat": -3.4, "lon": 29.9, "zoom": 8, "bbox": [-4.5, 29.0, -2.3, 30.8], "pri": 2},
{"code": "cf", "iso2": "CF", "region": "africa", "nl": "Centraal-Afrika", "en": "Central African Rep.", "lat": 6.6, "lon": 20.9, "zoom": 6, "bbox": [2.2, 14.4, 11.0, 27.5], "pri": 3},
{"code": "gm", "iso2": "GM", "region": "africa", "nl": "Gambia", "en": "Gambia", "lat": 13.4, "lon": -15.3, "zoom": 8, "bbox": [13.1, -16.8, 13.8, -13.8], "pri": 2},
{"code": "gw", "iso2": "GW", "region": "africa", "nl": "Guinee-Bissau", "en": "Guinea-Bissau", "lat": 11.8, "lon": -15.2, "zoom": 8, "bbox": [10.9, -16.7, 12.7, -13.6], "pri": 2},
# ── Americas ──
{"code": "ca", "iso2": "CA", "region": "americas", "nl": "Canada", "en": "Canada", "lat": 56.1, "lon": -96.8, "zoom": 4, "bbox": [41.7, -141.0, 83.1, -52.6], "pri": 2},
{"code": "mx", "iso2": "MX", "region": "americas", "nl": "Mexico", "en": "Mexico", "lat": 23.6, "lon": -102.6, "zoom": 5, "bbox": [14.5, -118.4, 32.7, -86.7], "pri": 2},
{"code": "br", "iso2": "BR", "region": "americas", "nl": "Brazilië", "en": "Brazil", "lat": -14.2, "lon": -51.9, "zoom": 4, "bbox": [-33.8, -73.9, 5.3, -34.8], "pri": 2},
{"code": "ar", "iso2": "AR", "region": "americas", "nl": "Argentinië", "en": "Argentina", "lat": -38.4, "lon": -63.6, "zoom": 4, "bbox": [-55.1, -73.6, -21.8, -53.6], "pri": 2},
{"code": "co", "iso2": "CO", "region": "americas", "nl": "Colombia", "en": "Colombia", "lat": 4.6, "lon": -74.3, "zoom": 5, "bbox": [-4.2, -79.0, 12.5, -66.9], "pri": 2},
{"code": "cl", "iso2": "CL", "region": "americas", "nl": "Chili", "en": "Chile", "lat": -35.7, "lon": -71.5, "zoom": 4, "bbox": [-55.9, -75.6, -17.5, -66.4], "pri": 2},
{"code": "pe", "iso2": "PE", "region": "americas", "nl": "Peru", "en": "Peru", "lat": -9.2, "lon": -75.0, "zoom": 5, "bbox": [-18.3, -81.3, -0.0, -68.7], "pri": 2},
{"code": "ve", "iso2": "VE", "region": "americas", "nl": "Venezuela", "en": "Venezuela", "lat": 6.4, "lon": -66.6, "zoom": 5, "bbox": [0.6, -73.4, 12.2, -59.8], "pri": 2},
{"code": "ec", "iso2": "EC", "region": "americas", "nl": "Ecuador", "en": "Ecuador", "lat": -1.8, "lon": -78.2, "zoom": 6, "bbox": [-5.0, -81.1, 1.5, -75.2], "pri": 2},
{"code": "bo", "iso2": "BO", "region": "americas", "nl": "Bolivia", "en": "Bolivia", "lat": -16.3, "lon": -63.6, "zoom": 5, "bbox": [-22.9, -69.6, -9.7, -57.5], "pri": 2},
{"code": "py", "iso2": "PY", "region": "americas", "nl": "Paraguay", "en": "Paraguay", "lat": -23.4, "lon": -58.4, "zoom": 6, "bbox": [-27.6, -62.6, -19.3, -54.3], "pri": 2},
{"code": "uy", "iso2": "UY", "region": "americas", "nl": "Uruguay", "en": "Uruguay", "lat": -32.5, "lon": -55.8, "zoom": 7, "bbox": [-35.0, -58.4, -30.1, -53.1], "pri": 2},
{"code": "gt", "iso2": "GT", "region": "americas", "nl": "Guatemala", "en": "Guatemala", "lat": 15.8, "lon": -90.2, "zoom": 7, "bbox": [13.7, -92.2, 17.8, -88.2], "pri": 2},
{"code": "hn", "iso2": "HN", "region": "americas", "nl": "Honduras", "en": "Honduras", "lat": 14.6, "lon": -86.2, "zoom": 7, "bbox": [12.9, -89.4, 16.5, -83.1], "pri": 2},
{"code": "sv", "iso2": "SV", "region": "americas", "nl": "El Salvador", "en": "El Salvador", "lat": 13.7, "lon": -88.9, "zoom": 8, "bbox": [13.1, -90.1, 14.4, -87.7], "pri": 2},
{"code": "ni", "iso2": "NI", "region": "americas", "nl": "Nicaragua", "en": "Nicaragua", "lat": 12.9, "lon": -85.0, "zoom": 7, "bbox": [10.7, -87.7, 15.0, -82.7], "pri": 2},
{"code": "cr", "iso2": "CR", "region": "americas", "nl": "Costa Rica", "en": "Costa Rica", "lat": 9.7, "lon": -84.0, "zoom": 7, "bbox": [8.0, -85.9, 11.2, -82.6], "pri": 2},
{"code": "pa", "iso2": "PA", "region": "americas", "nl": "Panama", "en": "Panama", "lat": 8.5, "lon": -80.8, "zoom": 7, "bbox": [7.2, -83.0, 9.6, -77.2], "pri": 2},
{"code": "bz", "iso2": "BZ", "region": "americas", "nl": "Belize", "en": "Belize", "lat": 17.2, "lon": -88.7, "zoom": 8, "bbox": [15.9, -89.2, 18.5, -87.8], "pri": 2},
{"code": "cu", "iso2": "CU", "region": "americas", "nl": "Cuba", "en": "Cuba", "lat": 21.5, "lon": -79.0, "zoom": 6, "bbox": [19.8, -84.9, 23.3, -74.1], "pri": 2},
{"code": "do", "iso2": "DO", "region": "americas", "nl": "Dominicaanse Rep.", "en": "Dominican Republic", "lat": 18.7, "lon": -70.2, "zoom": 7, "bbox": [17.5, -72.0, 19.9, -68.3], "pri": 2},
{"code": "ht", "iso2": "HT", "region": "americas", "nl": "Haïti", "en": "Haiti", "lat": 18.9, "lon": -72.3, "zoom": 8, "bbox": [18.0, -74.5, 20.1, -71.6], "pri": 2},
{"code": "pr", "iso2": "PR", "region": "americas", "nl": "Puerto Rico", "en": "Puerto Rico", "lat": 18.2, "lon": -66.5, "zoom": 8, "bbox": [17.9, -67.9, 18.5, -65.2], "pri": 2},
{"code": "bs", "iso2": "BS", "region": "caribbean", "nl": "Bahama's", "en": "Bahamas", "lat": 25.0, "lon": -77.4, "zoom": 7, "bbox": [20.9, -78.9, 27.3, -72.7], "pri": 2},
{"code": "bb", "iso2": "BB", "region": "caribbean", "nl": "Barbados", "en": "Barbados", "lat": 13.2, "lon": -59.5, "zoom": 10, "bbox": [13.0, -59.7, 13.3, -59.4], "pri": 2},
{"code": "gd", "iso2": "GD", "region": "caribbean", "nl": "Grenada", "en": "Grenada", "lat": 12.1, "lon": -61.7, "zoom": 10, "bbox": [11.9, -61.8, 12.5, -61.4], "pri": 2},
{"code": "lc", "iso2": "LC", "region": "caribbean", "nl": "Saint Lucia", "en": "Saint Lucia", "lat": 13.9, "lon": -60.9, "zoom": 10, "bbox": [13.7, -61.1, 14.1, -60.9], "pri": 2},
{"code": "gy", "iso2": "GY", "region": "americas", "nl": "Guyana", "en": "Guyana", "lat": 4.9, "lon": -58.9, "zoom": 6, "bbox": [1.2, -61.4, 8.6, -56.5], "pri": 2},
{"code": "sr", "iso2": "SR", "region": "americas", "nl": "Suriname", "en": "Suriname", "lat": 3.9, "lon": -56.0, "zoom": 7, "bbox": [1.8, -58.1, 6.0, -53.9], "pri": 2},
]
BBOX_ONLY_ISO2 = frozenset(
{
"AW", "CW", "BQ", "SX", "JM", "TT", "BH", "MT", "CY", "LU", "LI", "SG", "MC", "AD", "SM",
"BB", "GD", "LC", "VC", "AG", "DM", "KN", "ST", "KM", "SC", "GM", "PS",
}
)
+2
View File
@@ -49,6 +49,7 @@ class BrainSearchIn(BaseModel):
limit: int = Field(default=8, ge=1, le=30)
from app.export_intel import router as export_intel_router
from app.email_config import get_active_email_config
from app.comfyui import fetch_image_bytes, generate_image, get_job, start_generation, QUALITY_PRESETS
import json
@@ -64,6 +65,7 @@ app.include_router(research_router)
app.include_router(recommendations_router)
app.include_router(packaging_router)
app.include_router(ops_router)
app.include_router(export_intel_router)
class AgentEventCreate(BaseModel):
+18 -2
View File
@@ -1,7 +1,9 @@
"""Agent event logging helpers used by the Tools API."""
from typing import Any, Optional
from uuid import uuid4
from app.agent_names import normalize_agent_key
from app.db import execute_returning, json_param
@@ -17,7 +19,21 @@ def log_agent_event(
related_table: Optional[str] = None,
related_id: Optional[int] = None,
channel: str = "dashboard",
correlation_id: Optional[str] = None,
source_agent: Optional[str] = None,
target_agent: Optional[str] = None,
) -> dict[str, Any]:
meta = dict(metadata or {})
if correlation_id:
meta.setdefault("correlation_id", correlation_id)
if source_agent:
meta.setdefault("source_agent", normalize_agent_key(source_agent))
if target_agent:
meta.setdefault("target_agent", normalize_agent_key(target_agent))
if "correlation_id" not in meta:
meta.setdefault("correlation_id", str(uuid4()))
normalized = normalize_agent_key(agent_name)
row = execute_returning(
"""
INSERT INTO agent_events (
@@ -28,12 +44,12 @@ def log_agent_event(
RETURNING *
""",
(
agent_name,
normalized,
agent_type,
event_type,
title,
body,
json_param(metadata),
json_param(meta),
status,
related_table,
related_id,