SysOps: voice-agy-webbuilder-backup — 2026-06-23 10:04 UTC
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
"""Normalize legacy agent_name values to agent_souls keys."""
|
||||
from __future__ import annotations
|
||||
|
||||
AGENT_NAME_MAP: dict[str, str] = {
|
||||
"retail_360": "retail",
|
||||
"retail_scraper": "retail",
|
||||
"retail_crm": "retail",
|
||||
"retail_intel": "retail",
|
||||
"rss_feeds": "marketing",
|
||||
"wholesale_scraper": "sourcing",
|
||||
"halal_registry": "halal",
|
||||
"branch_scraper": "sourcing",
|
||||
"hermes": "herman",
|
||||
"herman_delegate": "herman",
|
||||
}
|
||||
|
||||
|
||||
def normalize_agent_key(name: str | None) -> str:
|
||||
key = (name or "").strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
return AGENT_NAME_MAP.get(key, key)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Halal meat sales opportunity scoring for export market entities."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# Land-basis: vraag naar halal vlees / moslim markt (0–100 proxy)
|
||||
COUNTRY_HALAL_MARKET: dict[str, float] = {
|
||||
"SA": 98, "AE": 97, "QA": 97, "KW": 96, "BH": 96, "OM": 95,
|
||||
"EG": 92, "MA": 90, "DZ": 90, "TN": 89, "LY": 88, "SD": 91, "SO": 93,
|
||||
"TR": 85, "ID": 94, "MY": 93, "PK": 95, "BD": 94, "AF": 96, "IR": 94,
|
||||
"IQ": 93, "JO": 90, "LB": 88, "SY": 89, "YE": 95, "PS": 91,
|
||||
"NG": 82, "SN": 92, "ML": 93, "NE": 95, "BF": 90, "CI": 85, "GH": 78,
|
||||
"KE": 72, "TZ": 78, "UG": 70, "ET": 68, "ZA": 55, "GB": 52, "FR": 54,
|
||||
"DE": 50, "NL": 58, "BE": 56, "SE": 42, "NO": 38, "DK": 40, "FI": 35,
|
||||
"AT": 48, "CH": 45, "IT": 46, "ES": 48, "PT": 42, "GR": 45, "CY": 55,
|
||||
"BA": 82, "AL": 85, "XK": 88, "MK": 75, "RS": 72, "BG": 58, "RO": 55,
|
||||
"US": 45, "CA": 48, "AU": 38, "NZ": 35, "MX": 40, "BR": 42, "AR": 38,
|
||||
"SG": 72, "TH": 55, "PH": 65, "IN": 70, "CN": 35, "JP": 30, "KR": 32,
|
||||
"RU": 48, "KZ": 62, "UZ": 75, "AZ": 80, "GE": 55,
|
||||
}
|
||||
|
||||
TYPE_BASE: dict[str, float] = {
|
||||
"butcher": 92,
|
||||
"doner_shoarma": 90,
|
||||
"wholesaler": 86,
|
||||
"distributor": 85,
|
||||
"importer": 84,
|
||||
"contract_caterer": 78,
|
||||
"restaurant": 72,
|
||||
"foodservice": 65,
|
||||
"logistics": 45,
|
||||
}
|
||||
|
||||
HALAL_KEYWORDS = re.compile(
|
||||
r"halal|helal|kebab|döner|doner|shoarma|shawarma|moslim|muslim|islamic|"
|
||||
r"middle\s*east|turkish|türk|arab|pakistani|moroccan|marok|syrian|lebanese|"
|
||||
r"peri\s*peri|grill|bbq|meat|vlees|slager|butcher|wholesale|groothandel|"
|
||||
r"cash\s*&\s*carry|horeca|ethnic|kosher",
|
||||
re.I,
|
||||
)
|
||||
|
||||
EXPLICIT_HALAL = re.compile(r"\bhalal\b|\bhelal\b", re.I)
|
||||
|
||||
|
||||
def _product_interest_text(val: Any) -> str:
|
||||
if not val:
|
||||
return ""
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
val = json.loads(val)
|
||||
except json.JSONDecodeError:
|
||||
return val
|
||||
if isinstance(val, list):
|
||||
return " ".join(str(x) for x in val)
|
||||
return str(val)
|
||||
|
||||
|
||||
def score_entity(row: dict[str, Any]) -> tuple[float, list[str]]:
|
||||
"""Return (score 0–100, reason tags)."""
|
||||
reasons: list[str] = []
|
||||
country = (row.get("country_iso2") or "").upper()
|
||||
etype = row.get("entity_type") or ""
|
||||
name = row.get("name") or ""
|
||||
notes = row.get("halal_cert_notes") or ""
|
||||
interest = _product_interest_text(row.get("product_interest"))
|
||||
|
||||
score = TYPE_BASE.get(etype, 50)
|
||||
reasons.append(f"type:{etype}")
|
||||
|
||||
market = COUNTRY_HALAL_MARKET.get(country, 40)
|
||||
score = score * 0.55 + market * 0.45
|
||||
if market >= 80:
|
||||
reasons.append("sterke_halal_markt")
|
||||
|
||||
blob = f"{name} {notes} {interest}".lower()
|
||||
if EXPLICIT_HALAL.search(blob):
|
||||
score += 18
|
||||
reasons.append("halal_in_naam")
|
||||
elif HALAL_KEYWORDS.search(blob):
|
||||
score += 10
|
||||
reasons.append("halal_signaal")
|
||||
|
||||
if etype in ("butcher", "doner_shoarma"):
|
||||
reasons.append("direct_vlees_kanaal")
|
||||
elif etype in ("wholesaler", "distributor", "importer"):
|
||||
reasons.append("B2B_kanaal")
|
||||
|
||||
conf = int(row.get("confidence") or 50)
|
||||
score += (conf - 50) * 0.08
|
||||
|
||||
if row.get("email") or row.get("contact_email"):
|
||||
score += 3
|
||||
if int(row.get("contact_count") or 0) > 0:
|
||||
score += 4
|
||||
reasons.append("heeft_contact")
|
||||
|
||||
return round(min(100, max(0, score)), 1), reasons
|
||||
|
||||
|
||||
def halal_pin_tier(score: float) -> str:
|
||||
if score >= 75:
|
||||
return "hot"
|
||||
if score >= 55:
|
||||
return "warm"
|
||||
if score >= 40:
|
||||
return "mild"
|
||||
return "low"
|
||||
@@ -0,0 +1,842 @@
|
||||
"""Export Intel API — Foodlinkk B2B global export intelligence."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.db import execute, execute_returning, fetch_all, fetch_one
|
||||
from app.export_halal_scoring import COUNTRY_HALAL_MARKET, halal_pin_tier, score_entity
|
||||
from app.export_intel_crm import push_entities_to_crm
|
||||
|
||||
HALAL_REASON_NL: dict[str, str] = {
|
||||
"sterke_halal_markt": "Sterke halal-markt in dit land",
|
||||
"halal_in_naam": "Halal expliciet in bedrijfsnaam",
|
||||
"halal_signaal": "Kebab/döner/vlees-signaal in naam",
|
||||
"direct_vlees_kanaal": "Direct vlees-verkoopkanaal",
|
||||
"B2B_kanaal": "B2B inkoop / distributie",
|
||||
"heeft_contact": "Contactpersoon in registry",
|
||||
}
|
||||
|
||||
|
||||
def _halal_reason_labels(reasons: list[str]) -> list[str]:
|
||||
out: list[str] = []
|
||||
for r in reasons:
|
||||
if r in HALAL_REASON_NL:
|
||||
out.append(HALAL_REASON_NL[r])
|
||||
elif r.startswith("type:"):
|
||||
out.append("Type: " + r.split(":", 1)[1].replace("_", " "))
|
||||
else:
|
||||
out.append(r.replace("_", " "))
|
||||
return out
|
||||
|
||||
|
||||
def _enrich_entity(row: dict[str, Any], contacts: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
row["contacts"] = contacts
|
||||
row["contact_count"] = len(contacts)
|
||||
for c in contacts:
|
||||
if not row.get("email") and c.get("email"):
|
||||
row["email"] = c["email"]
|
||||
if not row.get("phone"):
|
||||
phone = c.get("phone") or c.get("mobile")
|
||||
if phone:
|
||||
row["phone"] = phone
|
||||
primary = contacts[0] if contacts else None
|
||||
row["primary_contact"] = primary
|
||||
h_score, h_reasons = score_entity(row)
|
||||
row["halal_score"] = h_score
|
||||
row["halal_reasons"] = h_reasons
|
||||
row["halal_reason_labels"] = _halal_reason_labels(h_reasons)
|
||||
if row.get("lat") and row.get("lon"):
|
||||
row["maps_url"] = f"https://www.google.com/maps?q={row['lat']},{row['lon']}"
|
||||
return row
|
||||
from app.export_intel_sync import (
|
||||
enrich_contacts_country,
|
||||
materialize_caterer_presence,
|
||||
sync_all_priority,
|
||||
sync_caterers,
|
||||
sync_contacts,
|
||||
sync_customers,
|
||||
sync_distributors,
|
||||
sync_osm_country,
|
||||
sync_ted_tenders,
|
||||
sync_world_regions,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/export-intel", tags=["export-intel"])
|
||||
|
||||
DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold_storage", "port_agent")
|
||||
CUSTOMER_TYPES = ("restaurant", "doner_shoarma", "butcher", "foodservice")
|
||||
CATERER_TYPES = ("contract_caterer")
|
||||
|
||||
|
||||
class ContactCreateIn(BaseModel):
|
||||
role: str = "general"
|
||||
name: Optional[str] = None
|
||||
email: Optional[str] = None
|
||||
phone: Optional[str] = None
|
||||
mobile: Optional[str] = None
|
||||
is_primary: bool = False
|
||||
|
||||
|
||||
class SyncRequestIn(BaseModel):
|
||||
country_iso2: Optional[str] = None
|
||||
region_code: Optional[str] = None
|
||||
max_priority: int = 2
|
||||
entity_types: Optional[list[str]] = None
|
||||
|
||||
|
||||
class EntityIdsIn(BaseModel):
|
||||
entity_ids: list[int]
|
||||
|
||||
|
||||
class FavoritesIn(BaseModel):
|
||||
entity_ids: list[int]
|
||||
favorite: bool = True
|
||||
|
||||
|
||||
class CrmPushIn(BaseModel):
|
||||
entity_ids: list[int]
|
||||
create_deals: bool = True
|
||||
client_stage: str = "intake"
|
||||
deal_stage: str = "lead"
|
||||
|
||||
|
||||
def _entity_filters(
|
||||
country: Optional[str],
|
||||
region: Optional[str],
|
||||
entity_type: Optional[str],
|
||||
entity_types: Optional[str],
|
||||
favorite_only: bool = False,
|
||||
crm_linked: Optional[bool] = None,
|
||||
) -> tuple[str, list[Any]]:
|
||||
clauses: list[str] = ["1=1"]
|
||||
params: list[Any] = []
|
||||
if country:
|
||||
clauses.append("e.country_iso2 = %s")
|
||||
params.append(country.upper())
|
||||
if region:
|
||||
clauses.append("t.region_code = %s")
|
||||
params.append(region)
|
||||
clauses.append("e.territory_code = t.code")
|
||||
if entity_type:
|
||||
clauses.append("e.entity_type = %s")
|
||||
params.append(entity_type)
|
||||
if entity_types:
|
||||
types = [t.strip() for t in entity_types.split(",") if t.strip()]
|
||||
if types:
|
||||
clauses.append("e.entity_type = ANY(%s)")
|
||||
params.append(types)
|
||||
if favorite_only:
|
||||
clauses.append("e.is_favorite = TRUE")
|
||||
if crm_linked is True:
|
||||
clauses.append("e.client_id IS NOT NULL")
|
||||
elif crm_linked is False:
|
||||
clauses.append("e.client_id IS NULL")
|
||||
return " AND ".join(clauses), params
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def export_stats(
|
||||
country: Optional[str] = Query(None),
|
||||
region: Optional[str] = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
entity_where = ""
|
||||
params: list[Any] = []
|
||||
if country:
|
||||
entity_where = " WHERE country_iso2 = %s"
|
||||
params = [country.upper()]
|
||||
elif region:
|
||||
entity_where = " WHERE territory_code IN (SELECT code FROM export_territories WHERE region_code = %s)"
|
||||
params = [region]
|
||||
|
||||
def count(sql_suffix: str, extra_params: tuple[Any, ...] = ()) -> int:
|
||||
w = entity_where
|
||||
if sql_suffix:
|
||||
w = f"{entity_where} AND {sql_suffix}" if entity_where else f" WHERE {sql_suffix}"
|
||||
row = fetch_one(
|
||||
f"SELECT COUNT(*) AS n FROM export_market_entities{w}",
|
||||
tuple(params) + extra_params if params or extra_params else None,
|
||||
)
|
||||
return int(row["n"] or 0)
|
||||
|
||||
entities_n = count("")
|
||||
distributors_n = count("entity_type IN ('distributor','wholesaler','importer','logistics')")
|
||||
caterers_n = count("entity_type = 'contract_caterer'")
|
||||
contacts = fetch_one("SELECT COUNT(*) AS n FROM export_entity_contacts")
|
||||
with_email = fetch_one(
|
||||
"SELECT COUNT(DISTINCT entity_id) AS n FROM export_entity_contacts WHERE email IS NOT NULL AND email <> ''"
|
||||
)
|
||||
territories = fetch_one("SELECT COUNT(*) AS n FROM export_territories WHERE is_active = TRUE")
|
||||
presence = fetch_one("SELECT COUNT(*) AS n FROM export_caterer_presence WHERE is_active = TRUE")
|
||||
tenders_open = 0
|
||||
try:
|
||||
row = fetch_one("SELECT COUNT(*) AS n FROM export_tenders WHERE status = 'open'")
|
||||
tenders_open = int(row["n"] or 0) if row else 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fav_row = fetch_one("SELECT COUNT(*) AS n FROM export_market_entities WHERE is_favorite = TRUE")
|
||||
crm_row = fetch_one("SELECT COUNT(*) AS n FROM export_market_entities WHERE client_id IS NOT NULL")
|
||||
|
||||
return {
|
||||
"entities": entities_n,
|
||||
"distributors": distributors_n,
|
||||
"caterers": caterers_n,
|
||||
"contacts": int(contacts["n"] or 0),
|
||||
"entities_with_email": int(with_email["n"] or 0),
|
||||
"territories": int(territories["n"] or 0),
|
||||
"caterer_presence": int(presence["n"] or 0),
|
||||
"tenders_open": tenders_open,
|
||||
"favorites": int(fav_row["n"] or 0) if fav_row else 0,
|
||||
"crm_linked": int(crm_row["n"] or 0) if crm_row else 0,
|
||||
"phase": "B",
|
||||
"enrichment_status": "active",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/regions")
|
||||
def list_regions() -> list[dict[str, Any]]:
|
||||
return fetch_all("SELECT * FROM export_regions ORDER BY sort_order, code")
|
||||
|
||||
|
||||
@router.get("/territories")
|
||||
def list_territories(region: Optional[str] = Query(None)) -> list[dict[str, Any]]:
|
||||
if region:
|
||||
return fetch_all(
|
||||
"SELECT * FROM export_territories WHERE is_active = TRUE AND region_code = %s ORDER BY sync_priority, name_nl",
|
||||
(region,),
|
||||
)
|
||||
return fetch_all(
|
||||
"SELECT * FROM export_territories WHERE is_active = TRUE ORDER BY sync_priority, name_nl"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/entities")
|
||||
def list_entities(
|
||||
country: Optional[str] = Query(None),
|
||||
region: Optional[str] = Query(None),
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_types: Optional[str] = Query(None),
|
||||
q: Optional[str] = Query(None),
|
||||
favorite_only: bool = Query(False),
|
||||
crm_linked: Optional[bool] = Query(None),
|
||||
limit: int = Query(200, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
join = ""
|
||||
if region:
|
||||
join = "JOIN export_territories t ON e.territory_code = t.code"
|
||||
where_sql, params = _entity_filters(
|
||||
country, region, entity_type, entity_types, favorite_only, crm_linked
|
||||
)
|
||||
if q:
|
||||
where_sql += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
|
||||
params.extend([f"%{q}%", f"%{q}%"])
|
||||
|
||||
rows = fetch_all(
|
||||
f"""
|
||||
SELECT e.*,
|
||||
cl.name AS crm_client_name,
|
||||
(SELECT COUNT(*) FROM export_entity_contacts ec WHERE ec.entity_id = e.id) AS contact_count,
|
||||
(SELECT ec.email FROM export_entity_contacts ec WHERE ec.entity_id = e.id AND ec.is_primary = TRUE LIMIT 1) AS primary_email,
|
||||
(SELECT ec.phone FROM export_entity_contacts ec WHERE ec.entity_id = e.id AND ec.is_primary = TRUE LIMIT 1) AS primary_phone
|
||||
FROM export_market_entities e
|
||||
LEFT JOIN clients cl ON cl.id = e.client_id
|
||||
{join}
|
||||
WHERE {where_sql}
|
||||
ORDER BY e.confidence DESC, e.name
|
||||
LIMIT %s OFFSET %s
|
||||
""",
|
||||
tuple(params) + (limit, offset),
|
||||
)
|
||||
total = fetch_one(
|
||||
f"SELECT COUNT(*) AS n FROM export_market_entities e LEFT JOIN clients cl ON cl.id = e.client_id {join} WHERE {where_sql}",
|
||||
tuple(params),
|
||||
)
|
||||
return {"items": rows, "total": int(total["n"] or 0), "limit": limit, "offset": offset}
|
||||
|
||||
|
||||
@router.get("/entities/{entity_id}")
|
||||
def get_entity(entity_id: int) -> dict[str, Any]:
|
||||
row = fetch_one(
|
||||
"""
|
||||
SELECT e.*, cl.name AS crm_client_name, d.title AS crm_deal_title, d.stage AS crm_deal_stage
|
||||
FROM export_market_entities e
|
||||
LEFT JOIN clients cl ON cl.id = e.client_id
|
||||
LEFT JOIN deals d ON d.id = e.deal_id
|
||||
WHERE e.id = %s
|
||||
""",
|
||||
(entity_id,),
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(404, "Entity not found")
|
||||
contacts = fetch_all(
|
||||
"SELECT * FROM export_entity_contacts WHERE entity_id = %s ORDER BY is_primary DESC, confidence DESC",
|
||||
(entity_id,),
|
||||
)
|
||||
return _enrich_entity(row, contacts)
|
||||
|
||||
|
||||
@router.get("/contacts")
|
||||
def list_contacts(
|
||||
country: Optional[str] = Query(None),
|
||||
entity_type: Optional[str] = Query(None),
|
||||
has_email: Optional[bool] = Query(None),
|
||||
q: Optional[str] = Query(None),
|
||||
limit: int = Query(200, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
clauses = ["1=1"]
|
||||
params: list[Any] = []
|
||||
if country:
|
||||
clauses.append("e.country_iso2 = %s")
|
||||
params.append(country.upper())
|
||||
if entity_type:
|
||||
clauses.append("e.entity_type = %s")
|
||||
params.append(entity_type)
|
||||
if has_email:
|
||||
clauses.append("c.email IS NOT NULL AND c.email <> ''")
|
||||
if q:
|
||||
clauses.append(
|
||||
"(e.name ILIKE %s OR c.email ILIKE %s OR c.name ILIKE %s OR e.city ILIKE %s)"
|
||||
)
|
||||
like = f"%{q}%"
|
||||
params.extend([like, like, like, like])
|
||||
|
||||
where = " AND ".join(clauses)
|
||||
rows = fetch_all(
|
||||
f"""
|
||||
SELECT c.*, e.name AS entity_name, e.entity_type, e.country_iso2, e.city AS entity_city
|
||||
FROM export_entity_contacts c
|
||||
JOIN export_market_entities e ON e.id = c.entity_id
|
||||
WHERE {where}
|
||||
ORDER BY e.name, c.is_primary DESC
|
||||
LIMIT %s OFFSET %s
|
||||
""",
|
||||
tuple(params) + (limit, offset),
|
||||
)
|
||||
total = fetch_one(
|
||||
f"""
|
||||
SELECT COUNT(*) AS n FROM export_entity_contacts c
|
||||
JOIN export_market_entities e ON e.id = c.entity_id
|
||||
WHERE {where}
|
||||
""",
|
||||
tuple(params),
|
||||
)
|
||||
return {"items": rows, "total": int(total["n"] or 0)}
|
||||
|
||||
|
||||
@router.get("/contacts/export.csv")
|
||||
def export_contacts_csv(
|
||||
country: Optional[str] = Query(None),
|
||||
entity_type: Optional[str] = Query(None),
|
||||
) -> StreamingResponse:
|
||||
data = list_contacts(country=country, entity_type=entity_type, limit=5000, offset=0)
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf)
|
||||
writer.writerow(
|
||||
["entity_name", "entity_type", "country", "role", "name", "email", "phone", "mobile", "source", "confidence"]
|
||||
)
|
||||
for r in data["items"]:
|
||||
writer.writerow(
|
||||
[
|
||||
r.get("entity_name"),
|
||||
r.get("entity_type"),
|
||||
r.get("country_iso2"),
|
||||
r.get("role"),
|
||||
r.get("name"),
|
||||
r.get("email"),
|
||||
r.get("phone"),
|
||||
r.get("mobile"),
|
||||
r.get("source"),
|
||||
r.get("confidence"),
|
||||
]
|
||||
)
|
||||
buf.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([buf.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=export-intel-contacts.csv"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/entities/{entity_id}/contacts")
|
||||
def add_contact(entity_id: int, payload: ContactCreateIn) -> dict[str, Any]:
|
||||
ent = fetch_one("SELECT id FROM export_market_entities WHERE id = %s", (entity_id,))
|
||||
if not ent:
|
||||
raise HTTPException(404, "Entity not found")
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO export_entity_contacts (entity_id, role, name, email, phone, mobile, is_primary, source, confidence)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'manual', 80)
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
entity_id,
|
||||
payload.role,
|
||||
payload.name,
|
||||
payload.email,
|
||||
payload.phone,
|
||||
payload.mobile,
|
||||
payload.is_primary,
|
||||
),
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
|
||||
@router.get("/caterers/brands")
|
||||
def list_caterer_brands() -> list[dict[str, Any]]:
|
||||
return fetch_all(
|
||||
"""
|
||||
SELECT b.*,
|
||||
(SELECT COUNT(*) FROM export_caterer_presence p WHERE p.brand_code = b.code AND p.is_active) AS country_count
|
||||
FROM export_caterer_brands b
|
||||
ORDER BY b.tier, b.name
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@router.get("/caterers/presence")
|
||||
def list_caterer_presence(country: Optional[str] = Query(None)) -> list[dict[str, Any]]:
|
||||
if country:
|
||||
return fetch_all(
|
||||
"""
|
||||
SELECT p.*, b.name AS brand_name, b.tier, b.website AS brand_website
|
||||
FROM export_caterer_presence p
|
||||
JOIN export_caterer_brands b ON b.code = p.brand_code
|
||||
WHERE p.country_iso2 = %s AND p.is_active = TRUE
|
||||
ORDER BY b.tier, b.name
|
||||
""",
|
||||
(country.upper(),),
|
||||
)
|
||||
return fetch_all(
|
||||
"""
|
||||
SELECT p.*, b.name AS brand_name, b.tier, b.website AS brand_website
|
||||
FROM export_caterer_presence p
|
||||
JOIN export_caterer_brands b ON b.code = p.brand_code
|
||||
WHERE p.is_active = TRUE
|
||||
ORDER BY p.country_iso2, b.tier, b.name
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@router.get("/gov-sources")
|
||||
def list_gov_sources(country: Optional[str] = Query(None)) -> list[dict[str, Any]]:
|
||||
if country:
|
||||
return fetch_all(
|
||||
"SELECT * FROM export_gov_data_sources WHERE is_active = TRUE AND country_iso2 = %s ORDER BY category, name",
|
||||
(country.upper(),),
|
||||
)
|
||||
return fetch_all(
|
||||
"SELECT * FROM export_gov_data_sources WHERE is_active = TRUE ORDER BY country_iso2, category, name"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tenders")
|
||||
def list_tenders(
|
||||
country: Optional[str] = Query(None),
|
||||
status: Optional[str] = Query("open"),
|
||||
limit: int = Query(100, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
clauses = ["1=1"]
|
||||
params: list[Any] = []
|
||||
if country:
|
||||
clauses.append("country_iso2 = %s")
|
||||
params.append(country.upper())
|
||||
if status:
|
||||
clauses.append("status = %s")
|
||||
params.append(status)
|
||||
where = " AND ".join(clauses)
|
||||
rows = fetch_all(
|
||||
f"SELECT * FROM export_tenders WHERE {where} ORDER BY created_at DESC LIMIT %s",
|
||||
tuple(params) + (limit,),
|
||||
)
|
||||
total = fetch_one(f"SELECT COUNT(*) AS n FROM export_tenders WHERE {where}", tuple(params))
|
||||
return {"items": rows, "total": int(total["n"] or 0)}
|
||||
|
||||
|
||||
@router.get("/map/bundle")
|
||||
def map_bundle(
|
||||
country: Optional[str] = Query(None),
|
||||
region: Optional[str] = Query(None),
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_types: Optional[str] = Query(None),
|
||||
q: Optional[str] = Query(None),
|
||||
halal_min: Optional[float] = Query(None, ge=0, le=100),
|
||||
favorite_only: bool = Query(False),
|
||||
crm_linked: Optional[bool] = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
join = ""
|
||||
if region:
|
||||
join = "JOIN export_territories t ON e.territory_code = t.code"
|
||||
filter_sql, params = _entity_filters(
|
||||
country, region, entity_type, entity_types, favorite_only, crm_linked
|
||||
)
|
||||
where = f"e.lat IS NOT NULL AND e.lon IS NOT NULL AND {filter_sql}"
|
||||
if q:
|
||||
where += " AND (e.name ILIKE %s OR e.city ILIKE %s)"
|
||||
params.extend([f"%{q}%", f"%{q}%"])
|
||||
|
||||
entities = fetch_all(
|
||||
f"""
|
||||
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
|
||||
e.confidence, e.pipeline_stage, e.city, e.address_line,
|
||||
e.email, e.phone, e.website, e.volume_band, e.source,
|
||||
e.is_favorite, e.client_id, e.deal_id, e.crm_pushed_at,
|
||||
e.product_interest, e.halal_cert_notes,
|
||||
(SELECT c.email FROM export_entity_contacts c
|
||||
WHERE c.entity_id = e.id AND c.email IS NOT NULL AND c.email <> ''
|
||||
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_email,
|
||||
(SELECT c.phone FROM export_entity_contacts c
|
||||
WHERE c.entity_id = e.id AND c.phone IS NOT NULL AND c.phone <> ''
|
||||
ORDER BY c.is_primary DESC NULLS LAST LIMIT 1) AS contact_phone,
|
||||
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
|
||||
FROM export_market_entities e
|
||||
{join}
|
||||
WHERE {where}
|
||||
LIMIT 20000
|
||||
""",
|
||||
tuple(params) if params else None,
|
||||
)
|
||||
|
||||
scored: list[tuple[dict[str, Any], float, list[str]]] = []
|
||||
halal_by_country: dict[str, dict[str, Any]] = {}
|
||||
for row in entities:
|
||||
if row.get("lat") is None or row.get("lon") is None:
|
||||
continue
|
||||
h_score, h_reasons = score_entity(row)
|
||||
iso = row.get("country_iso2") or ""
|
||||
bucket = halal_by_country.setdefault(
|
||||
iso,
|
||||
{"country_iso2": iso, "market_score": COUNTRY_HALAL_MARKET.get(iso, 40), "entity_count": 0, "avg_halal_score": 0.0, "hot_count": 0},
|
||||
)
|
||||
bucket["entity_count"] += 1
|
||||
bucket["avg_halal_score"] += h_score
|
||||
if h_score >= 75:
|
||||
bucket["hot_count"] += 1
|
||||
scored.append((row, h_score, h_reasons))
|
||||
|
||||
for bucket in halal_by_country.values():
|
||||
n = bucket["entity_count"] or 1
|
||||
bucket["avg_halal_score"] = round(bucket["avg_halal_score"] / n, 1)
|
||||
bucket["combined_score"] = round(bucket["market_score"] * 0.4 + bucket["avg_halal_score"] * 0.6, 1)
|
||||
|
||||
if halal_min is not None:
|
||||
scored = [s for s in scored if s[1] >= halal_min]
|
||||
|
||||
scored.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
features = [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [row["lon"], row["lat"]]},
|
||||
"properties": {
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"entity_type": row["entity_type"],
|
||||
"country_iso2": row["country_iso2"],
|
||||
"confidence": row["confidence"],
|
||||
"pipeline_stage": row["pipeline_stage"],
|
||||
"city": row.get("city"),
|
||||
"address_line": row.get("address_line"),
|
||||
"email": row.get("email") or row.get("contact_email"),
|
||||
"phone": row.get("phone") or row.get("contact_phone"),
|
||||
"website": row.get("website"),
|
||||
"volume_band": row.get("volume_band"),
|
||||
"source": row.get("source"),
|
||||
"contact_count": int(row.get("contact_count") or 0),
|
||||
"halal_score": h_score,
|
||||
"halal_tier": halal_pin_tier(h_score),
|
||||
"halal_reasons": h_reasons[:4],
|
||||
"is_favorite": bool(row.get("is_favorite")),
|
||||
"client_id": row.get("client_id"),
|
||||
"deal_id": row.get("deal_id"),
|
||||
},
|
||||
}
|
||||
for row, h_score, h_reasons in scored
|
||||
]
|
||||
|
||||
top_markets = sorted(halal_by_country.values(), key=lambda x: x["combined_score"], reverse=True)[:15]
|
||||
top_entities = [
|
||||
{
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"entity_type": row["entity_type"],
|
||||
"country_iso2": row["country_iso2"],
|
||||
"city": row.get("city"),
|
||||
"address_line": row.get("address_line"),
|
||||
"email": row.get("email") or row.get("contact_email"),
|
||||
"phone": row.get("phone") or row.get("contact_phone"),
|
||||
"website": row.get("website"),
|
||||
"halal_score": h_score,
|
||||
"halal_reason_labels": _halal_reason_labels(h_reasons),
|
||||
"is_favorite": bool(row.get("is_favorite")),
|
||||
"client_id": row.get("client_id"),
|
||||
"has_contact": bool(
|
||||
row.get("email") or row.get("contact_email")
|
||||
or row.get("phone") or row.get("contact_phone")
|
||||
or int(row.get("contact_count") or 0) > 0
|
||||
),
|
||||
}
|
||||
for row, h_score, h_reasons in scored[:25]
|
||||
]
|
||||
|
||||
density_rows = fetch_all(
|
||||
"""
|
||||
SELECT country_iso2, COUNT(*) AS n
|
||||
FROM export_market_entities
|
||||
WHERE lat IS NOT NULL AND lon IS NOT NULL
|
||||
GROUP BY country_iso2
|
||||
"""
|
||||
)
|
||||
trade_choropleth = {r["country_iso2"]: r["n"] for r in density_rows}
|
||||
|
||||
stats = export_stats(country=country, region=region)
|
||||
return {
|
||||
"entities": {"type": "FeatureCollection", "features": features},
|
||||
"trade_choropleth": trade_choropleth,
|
||||
"halal_by_country": halal_by_country,
|
||||
"meta": {
|
||||
"entity_count": len(features),
|
||||
"countries_with_trade": len(trade_choropleth),
|
||||
"halal_min": halal_min,
|
||||
"top_markets": top_markets,
|
||||
"top_halal_entities": top_entities,
|
||||
"phase": "B",
|
||||
"note": "Halal-score: type + land + naam/signaal + contact",
|
||||
},
|
||||
"stats": stats,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/halal/markets")
|
||||
def halal_markets(
|
||||
region: Optional[str] = Query(None),
|
||||
country: Optional[str] = Query(None),
|
||||
limit: int = Query(30, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
join = ""
|
||||
where = "e.lat IS NOT NULL"
|
||||
params: list[Any] = []
|
||||
if region:
|
||||
join = "JOIN export_territories t ON e.territory_code = t.code"
|
||||
where += " AND t.region_code = %s"
|
||||
params.append(region)
|
||||
if country:
|
||||
where += " AND e.country_iso2 = %s"
|
||||
params.append(country.upper())
|
||||
|
||||
rows = fetch_all(
|
||||
f"""
|
||||
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.city,
|
||||
e.confidence, e.product_interest, e.halal_cert_notes, e.email,
|
||||
(SELECT COUNT(*) FROM export_entity_contacts c WHERE c.entity_id = e.id) AS contact_count
|
||||
FROM export_market_entities e
|
||||
{join}
|
||||
WHERE {where}
|
||||
LIMIT 15000
|
||||
""",
|
||||
tuple(params) if params else None,
|
||||
)
|
||||
by_country: dict[str, dict[str, Any]] = {}
|
||||
top_entities: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
h_score, reasons = score_entity(row)
|
||||
iso = row.get("country_iso2") or ""
|
||||
bucket = by_country.setdefault(
|
||||
iso,
|
||||
{"country_iso2": iso, "market_score": COUNTRY_HALAL_MARKET.get(iso, 40), "entities": 0, "score_sum": 0.0, "hot": 0},
|
||||
)
|
||||
bucket["entities"] += 1
|
||||
bucket["score_sum"] += h_score
|
||||
if h_score >= 75:
|
||||
bucket["hot"] += 1
|
||||
top_entities.append({
|
||||
"id": row["id"],
|
||||
"name": row["name"],
|
||||
"entity_type": row["entity_type"],
|
||||
"country_iso2": iso,
|
||||
"city": row.get("city"),
|
||||
"halal_score": h_score,
|
||||
"halal_reasons": reasons[:3],
|
||||
})
|
||||
|
||||
markets = []
|
||||
for iso, b in by_country.items():
|
||||
n = b["entities"] or 1
|
||||
avg = round(b["score_sum"] / n, 1)
|
||||
markets.append({
|
||||
"country_iso2": iso,
|
||||
"market_score": b["market_score"],
|
||||
"entity_count": b["entities"],
|
||||
"avg_halal_score": avg,
|
||||
"hot_entities": b["hot"],
|
||||
"combined_score": round(b["market_score"] * 0.4 + avg * 0.6, 1),
|
||||
})
|
||||
markets.sort(key=lambda x: x["combined_score"], reverse=True)
|
||||
top_entities.sort(key=lambda x: x["halal_score"], reverse=True)
|
||||
|
||||
return {
|
||||
"markets": markets[:limit],
|
||||
"top_entities": top_entities[:50],
|
||||
"total_entities": len(rows),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/entities/favorites")
|
||||
def set_favorites(payload: FavoritesIn) -> dict[str, Any]:
|
||||
if not payload.entity_ids:
|
||||
raise HTTPException(400, "entity_ids required")
|
||||
fav_val = payload.favorite
|
||||
execute(
|
||||
"""
|
||||
UPDATE export_market_entities
|
||||
SET is_favorite = %s,
|
||||
favorited_at = CASE WHEN %s THEN NOW() ELSE NULL END,
|
||||
updated_at = NOW()
|
||||
WHERE id = ANY(%s)
|
||||
""",
|
||||
(fav_val, fav_val, payload.entity_ids),
|
||||
)
|
||||
return {"ok": True, "updated": len(payload.entity_ids), "favorite": fav_val}
|
||||
|
||||
|
||||
@router.get("/entities/favorites")
|
||||
def list_favorite_entities(
|
||||
limit: int = Query(200, ge=1, le=500),
|
||||
country: Optional[str] = Query(None),
|
||||
region: Optional[str] = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
return list_entities(
|
||||
country=country,
|
||||
region=region,
|
||||
favorite_only=True,
|
||||
limit=limit,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/crm/push")
|
||||
def crm_push(payload: CrmPushIn) -> dict[str, Any]:
|
||||
if not payload.entity_ids:
|
||||
raise HTTPException(400, "entity_ids required")
|
||||
if len(payload.entity_ids) > 100:
|
||||
raise HTTPException(400, "max 100 entities per request")
|
||||
return push_entities_to_crm(
|
||||
payload.entity_ids,
|
||||
create_deals=payload.create_deals,
|
||||
client_stage=payload.client_stage,
|
||||
deal_stage=payload.deal_stage,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/crm/pipeline")
|
||||
def crm_pipeline(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
country: Optional[str] = Query(None),
|
||||
region: Optional[str] = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
join = ""
|
||||
if region:
|
||||
join = "JOIN export_territories t ON e.territory_code = t.code"
|
||||
where_sql, params = _entity_filters(country, region, None, None, False, True)
|
||||
rows = fetch_all(
|
||||
f"""
|
||||
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.city,
|
||||
e.pipeline_stage, e.crm_pushed_at, e.is_favorite,
|
||||
e.client_id, e.deal_id, e.email, e.phone,
|
||||
cl.name AS crm_client_name, cl.stage AS crm_client_stage,
|
||||
d.title AS crm_deal_title, d.stage AS crm_deal_stage
|
||||
FROM export_market_entities e
|
||||
LEFT JOIN clients cl ON cl.id = e.client_id
|
||||
LEFT JOIN deals d ON d.id = e.deal_id
|
||||
{join}
|
||||
WHERE {where_sql}
|
||||
ORDER BY e.crm_pushed_at DESC NULLS LAST, e.updated_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
tuple(params) + (limit,),
|
||||
)
|
||||
return {"items": rows, "total": len(rows)}
|
||||
|
||||
|
||||
@router.post("/sync/contacts")
|
||||
def api_sync_contacts(payload: SyncRequestIn) -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_contacts(payload.country_iso2, payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/caterers")
|
||||
def api_sync_caterers(payload: SyncRequestIn) -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_caterers(payload.country_iso2, payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/distributors")
|
||||
def api_sync_distributors(payload: SyncRequestIn) -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_distributors(payload.country_iso2, payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/customers")
|
||||
def api_sync_customers(payload: SyncRequestIn) -> dict[str, Any]:
|
||||
if not payload.country_iso2 and not payload.region_code:
|
||||
raise HTTPException(400, "country_iso2 or region_code required")
|
||||
try:
|
||||
result = sync_customers(payload.country_iso2, payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/tenders")
|
||||
def api_sync_tenders() -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_ted_tenders()
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/all")
|
||||
def api_sync_all(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_all_priority(max_priority=payload.max_priority, region_code=payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/world")
|
||||
def api_sync_world(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
|
||||
"""Sync Europe, Middle East, Africa, Americas — long running."""
|
||||
try:
|
||||
result = sync_world_regions(max_priority=payload.max_priority)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/sync/region")
|
||||
def api_sync_region(payload: SyncRequestIn) -> dict[str, Any]:
|
||||
if not payload.region_code:
|
||||
raise HTTPException(400, "region_code required")
|
||||
try:
|
||||
result = sync_all_priority(max_priority=payload.max_priority, region_code=payload.region_code)
|
||||
return {"status": "completed", "phase": "B", "result": result}
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc)) from exc
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Push export market entities into Foodlinkk CRM (clients + deals)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.db import execute, execute_returning, fetch_one
|
||||
from app.export_halal_scoring import score_entity
|
||||
|
||||
SECTOR_BY_TYPE: dict[str, str] = {
|
||||
"distributor": "Export · Distributie",
|
||||
"wholesaler": "Export · Groothandel",
|
||||
"importer": "Export · Import",
|
||||
"logistics": "Export · Logistiek",
|
||||
"contract_caterer": "Export · Catering",
|
||||
"restaurant": "Export · Horeca",
|
||||
"doner_shoarma": "Export · Horeca",
|
||||
"butcher": "Export · Slagerij",
|
||||
"foodservice": "Export · Foodservice",
|
||||
}
|
||||
|
||||
|
||||
def _sector(entity_type: str) -> str:
|
||||
return SECTOR_BY_TYPE.get(entity_type or "", "Export · B2B")
|
||||
|
||||
|
||||
def _contact_name(contacts: list[dict[str, Any]]) -> Optional[str]:
|
||||
for c in contacts:
|
||||
if c.get("name"):
|
||||
return c["name"]
|
||||
return None
|
||||
|
||||
|
||||
def _build_notes(ent: dict[str, Any], contacts: list[dict[str, Any]], halal_score: float) -> str:
|
||||
lines = [
|
||||
f"Wereldexport entity #{ent['id']}",
|
||||
f"Type: {ent.get('entity_type')}",
|
||||
f"Land: {ent.get('country_iso2')}",
|
||||
]
|
||||
if ent.get("city"):
|
||||
lines.append(f"Stad: {ent['city']}")
|
||||
if ent.get("address_line"):
|
||||
lines.append(f"Adres: {ent['address_line']}")
|
||||
if ent.get("website"):
|
||||
lines.append(f"Website: {ent['website']}")
|
||||
if ent.get("phone"):
|
||||
lines.append(f"Telefoon: {ent['phone']}")
|
||||
lines.append(f"Halal-kans score: {halal_score}/100")
|
||||
lines.append(f"OSM bron: {ent.get('source') or '—'}")
|
||||
if contacts:
|
||||
lines.append(f"Contacten in registry: {len(contacts)}")
|
||||
lines.append("—")
|
||||
lines.append("Aangemaakt vanuit Wereldexport · Foodlinkk B2B")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def push_entity_to_crm(
|
||||
entity_id: int,
|
||||
*,
|
||||
create_deal: bool = True,
|
||||
client_stage: str = "intake",
|
||||
deal_stage: str = "lead",
|
||||
) -> dict[str, Any]:
|
||||
ent = fetch_one("SELECT * FROM export_market_entities WHERE id = %s", (entity_id,))
|
||||
if not ent:
|
||||
return {"entity_id": entity_id, "status": "not_found"}
|
||||
|
||||
if ent.get("client_id"):
|
||||
client = fetch_one("SELECT id, name FROM clients WHERE id = %s", (ent["client_id"],))
|
||||
return {
|
||||
"entity_id": entity_id,
|
||||
"status": "already_linked",
|
||||
"client_id": ent["client_id"],
|
||||
"deal_id": ent.get("deal_id"),
|
||||
"client_name": (client or {}).get("name"),
|
||||
}
|
||||
|
||||
from app.db import fetch_all
|
||||
|
||||
contacts = fetch_all(
|
||||
"SELECT * FROM export_entity_contacts WHERE entity_id = %s ORDER BY is_primary DESC, confidence DESC",
|
||||
(entity_id,),
|
||||
)
|
||||
email = ent.get("email")
|
||||
phone = ent.get("phone")
|
||||
for c in contacts:
|
||||
if not email and c.get("email"):
|
||||
email = c["email"]
|
||||
if not phone:
|
||||
phone = c.get("phone") or c.get("mobile")
|
||||
|
||||
client_id: Optional[int] = None
|
||||
status = "created"
|
||||
|
||||
if email:
|
||||
existing = fetch_one(
|
||||
"SELECT id, name FROM clients WHERE LOWER(email) = LOWER(%s) LIMIT 1",
|
||||
(email,),
|
||||
)
|
||||
if existing:
|
||||
client_id = existing["id"]
|
||||
status = "linked_existing_email"
|
||||
|
||||
if client_id is None:
|
||||
contact_label = _contact_name(contacts) or phone or email
|
||||
halal_score, _ = score_entity({**ent, "contact_count": len(contacts)})
|
||||
notes = _build_notes(ent, contacts, halal_score)
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO clients (name, contact, email, stage, sector, notes, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, NOW())
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
ent["name"][:255],
|
||||
(contact_label or "")[:255] or None,
|
||||
(email or "")[:255] or None,
|
||||
client_stage,
|
||||
_sector(ent.get("entity_type") or ""),
|
||||
notes,
|
||||
),
|
||||
)
|
||||
if not row:
|
||||
return {"entity_id": entity_id, "status": "error", "error": "client_insert_failed"}
|
||||
client_id = row["id"]
|
||||
|
||||
deal_id: Optional[int] = None
|
||||
if create_deal and not ent.get("deal_id"):
|
||||
country = ent.get("country_iso2") or ""
|
||||
title = f"Export · {ent['name'][:180]} ({country})"
|
||||
deal_row = execute_returning(
|
||||
"""
|
||||
INSERT INTO deals (client_id, title, value, stage, agent_owner, next_action, updated_at)
|
||||
VALUES (%s, %s, 0, %s, 'herman', %s, NOW())
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
client_id,
|
||||
title[:255],
|
||||
deal_stage,
|
||||
f"Eerste outreach — Wereldexport entity #{entity_id}",
|
||||
),
|
||||
)
|
||||
if deal_row:
|
||||
deal_id = deal_row["id"]
|
||||
|
||||
execute(
|
||||
"""
|
||||
UPDATE export_market_entities
|
||||
SET client_id = %s,
|
||||
deal_id = COALESCE(%s, deal_id),
|
||||
pipeline_stage = 'crm_linked',
|
||||
crm_pushed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(client_id, deal_id, entity_id),
|
||||
)
|
||||
|
||||
client = fetch_one("SELECT id, name FROM clients WHERE id = %s", (client_id,))
|
||||
return {
|
||||
"entity_id": entity_id,
|
||||
"status": status,
|
||||
"client_id": client_id,
|
||||
"deal_id": deal_id,
|
||||
"client_name": (client or {}).get("name"),
|
||||
}
|
||||
|
||||
|
||||
def push_entities_to_crm(
|
||||
entity_ids: list[int],
|
||||
*,
|
||||
create_deals: bool = True,
|
||||
client_stage: str = "intake",
|
||||
deal_stage: str = "lead",
|
||||
) -> dict[str, Any]:
|
||||
results = [
|
||||
push_entity_to_crm(
|
||||
eid,
|
||||
create_deal=create_deals,
|
||||
client_stage=client_stage,
|
||||
deal_stage=deal_stage,
|
||||
)
|
||||
for eid in entity_ids
|
||||
]
|
||||
created = sum(1 for r in results if r.get("status") == "created")
|
||||
linked = sum(1 for r in results if r.get("status") in ("linked_existing_email", "already_linked"))
|
||||
return {
|
||||
"results": results,
|
||||
"total": len(results),
|
||||
"created": created,
|
||||
"linked_existing": linked,
|
||||
"pushed": created + sum(1 for r in results if r.get("status") == "linked_existing_email"),
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Google Places sync for Export Intel (optional — requires GOOGLE_MAPS_API_KEY)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from app.db import fetch_one
|
||||
from app.export_intel_sync import _get_territory, _upsert_contact
|
||||
|
||||
PLACES_URL = "https://maps.googleapis.com/maps/api/place/textsearch/json"
|
||||
DETAILS_URL = "https://maps.googleapis.com/maps/api/place/details/json"
|
||||
|
||||
QUERIES = [
|
||||
"meat wholesaler",
|
||||
"halal meat distributor",
|
||||
"poultry distributor",
|
||||
"frozen food supplier",
|
||||
"contract catering",
|
||||
"food importer",
|
||||
"halal butcher",
|
||||
]
|
||||
|
||||
LARGE_COUNTRIES = frozenset({"US", "CA", "BR", "MX", "AR", "AU", "RU", "CN", "IN", "CD", "DZ", "EG", "IR", "SA"})
|
||||
|
||||
|
||||
def _api_key() -> Optional[str]:
|
||||
return os.getenv("GOOGLE_MAPS_API_KEY") or os.getenv("GOOGLE_PLACES_API_KEY")
|
||||
|
||||
|
||||
def _places_search(query: str, lat: float, lon: float, radius_m: int = 80000) -> list[dict[str, Any]]:
|
||||
key = _api_key()
|
||||
if not key:
|
||||
return []
|
||||
params = {
|
||||
"query": query,
|
||||
"location": f"{lat},{lon}",
|
||||
"radius": radius_m,
|
||||
"key": key,
|
||||
}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
resp = client.get(PLACES_URL, params=params)
|
||||
if resp.status_code >= 400:
|
||||
return []
|
||||
data = resp.json()
|
||||
if data.get("status") not in ("OK", "ZERO_RESULTS"):
|
||||
return []
|
||||
return data.get("results", [])[:12]
|
||||
|
||||
|
||||
def _place_details(place_id: str) -> dict[str, Any]:
|
||||
key = _api_key()
|
||||
if not key:
|
||||
return {}
|
||||
with httpx.Client(timeout=20.0) as client:
|
||||
resp = client.get(
|
||||
DETAILS_URL,
|
||||
params={
|
||||
"place_id": place_id,
|
||||
"fields": "name,formatted_phone_number,website,formatted_address,geometry,international_phone_number",
|
||||
"key": key,
|
||||
},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
return {}
|
||||
return resp.json().get("result", {})
|
||||
|
||||
|
||||
def _bbox_points(bbox: Any, grid: int = 3) -> list[tuple[float, float]]:
|
||||
if isinstance(bbox, str):
|
||||
bbox = json.loads(bbox)
|
||||
if not bbox or len(bbox) != 4:
|
||||
return []
|
||||
south, west, north, east = bbox
|
||||
points: list[tuple[float, float]] = []
|
||||
for i in range(grid):
|
||||
for j in range(grid):
|
||||
lat = south + (north - south) * (i + 0.5) / grid
|
||||
lon = west + (east - west) * (j + 0.5) / grid
|
||||
points.append((lat, lon))
|
||||
return points
|
||||
|
||||
|
||||
def _upsert_place(place: dict[str, Any], details: dict[str, Any], country_iso2: str, territory_code: str, query: str) -> tuple[str, int]:
|
||||
from app.db import execute, execute_returning
|
||||
|
||||
pid = place.get("place_id")
|
||||
name = details.get("name") or place.get("name") or "Unknown"
|
||||
geo = details.get("geometry", {}).get("location", place.get("geometry", {}).get("location", {}))
|
||||
plat = geo.get("lat")
|
||||
plon = geo.get("lng")
|
||||
phone = details.get("formatted_phone_number") or details.get("international_phone_number")
|
||||
website = details.get("website")
|
||||
entity_type = "contract_caterer" if "catering" in query else "distributor"
|
||||
|
||||
existing = fetch_one(
|
||||
"SELECT id FROM export_market_entities WHERE metadata->>'google_place_id' = %s",
|
||||
(pid,),
|
||||
)
|
||||
if existing:
|
||||
execute(
|
||||
"""
|
||||
UPDATE export_market_entities SET
|
||||
phone = COALESCE(%s, phone), website = COALESCE(%s, website),
|
||||
lat = COALESCE(%s, lat), lon = COALESCE(%s, lon),
|
||||
confidence = GREATEST(confidence, 65), updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(phone, website, plat, plon, existing["id"]),
|
||||
)
|
||||
entity_id = existing["id"]
|
||||
action = "updated"
|
||||
else:
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO export_market_entities (
|
||||
name, entity_type, country_iso2, territory_code, lat, lon,
|
||||
phone, website, google_place_id, source, sources, confidence, metadata
|
||||
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'google_places','["google_places"]'::jsonb,65,%s)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
name[:255], entity_type, country_iso2.upper(), territory_code,
|
||||
plat, plon, phone, website, pid,
|
||||
json.dumps({"google_place_id": pid}),
|
||||
),
|
||||
)
|
||||
entity_id = row["id"]
|
||||
action = "inserted"
|
||||
if phone:
|
||||
_upsert_contact(entity_id, None, phone, "google_places", 70, role="sales", name="Google Places")
|
||||
return action, entity_id
|
||||
|
||||
|
||||
def sync_places_country(country_iso2: str) -> dict[str, Any]:
|
||||
territory = _get_territory(country_iso2)
|
||||
if not territory or not territory.get("lat"):
|
||||
return {"skipped": True, "reason": "no territory"}
|
||||
if not _api_key():
|
||||
return {"skipped": True, "reason": "GOOGLE_MAPS_API_KEY not set"}
|
||||
|
||||
territory_code = territory["code"]
|
||||
inserted = updated = 0
|
||||
seen: set[str] = set()
|
||||
|
||||
iso = country_iso2.upper()
|
||||
if iso in LARGE_COUNTRIES and territory.get("bbox"):
|
||||
search_points = _bbox_points(territory["bbox"], grid=3)
|
||||
else:
|
||||
search_points = [(territory["lat"], territory["lon"])]
|
||||
|
||||
radius = 60000 if iso in LARGE_COUNTRIES else 80000
|
||||
|
||||
for lat, lon in search_points:
|
||||
for q in QUERIES:
|
||||
for place in _places_search(q, lat, lon, radius_m=radius):
|
||||
pid = place.get("place_id")
|
||||
if not pid or pid in seen:
|
||||
continue
|
||||
seen.add(pid)
|
||||
details = _place_details(pid)
|
||||
action, _ = _upsert_place(place, details, iso, territory_code, q)
|
||||
if action == "inserted":
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
time.sleep(0.15)
|
||||
time.sleep(0.5)
|
||||
|
||||
return {"inserted": inserted, "updated": updated, "queries": len(QUERIES), "search_points": len(search_points)}
|
||||
|
||||
|
||||
def sync_places_countries(countries: list[str]) -> dict[str, Any]:
|
||||
if not _api_key():
|
||||
return {"skipped": True, "reason": "GOOGLE_MAPS_API_KEY not set"}
|
||||
out: dict[str, Any] = {}
|
||||
for c in countries:
|
||||
out[c] = sync_places_country(c)
|
||||
time.sleep(1)
|
||||
return out
|
||||
@@ -0,0 +1,583 @@
|
||||
"""Export Intel sync — OSM Overpass, contact enrichment, tenders, fusion."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.db import execute, execute_returning, fetch_all, fetch_one
|
||||
|
||||
OVERPASS_URLS = [
|
||||
"https://overpass.kumi.systems/api/interpreter",
|
||||
"https://overpass-api.de/api/interpreter",
|
||||
]
|
||||
|
||||
TED_RSS_URLS = [
|
||||
"https://ted.europa.eu/en/simap/rss-feed/-/rss/search/15",
|
||||
"https://ted.europa.eu/en/simap/rss-feed/-/rss/search/teeq",
|
||||
]
|
||||
|
||||
from app.export_territory_seeds import BBOX_ONLY_ISO2
|
||||
|
||||
EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
SKIP_EMAIL_SUFFIX = ("example.com", "sentry.io", "wixpress.com", "png", "jpg", "jpeg")
|
||||
|
||||
DISTRIBUTOR_QUERIES = [
|
||||
'node["shop"="wholesale"]',
|
||||
'way["shop"="wholesale"]',
|
||||
'node["wholesale"]',
|
||||
'way["wholesale"]',
|
||||
'node["shop"="cash_and_carry"]',
|
||||
'way["shop"="cash_and_carry"]',
|
||||
]
|
||||
|
||||
CUSTOMER_QUERIES = [
|
||||
'node["shop"="butcher"]',
|
||||
'way["shop"="butcher"]',
|
||||
'node["amenity"="restaurant"]',
|
||||
'way["amenity"="restaurant"]',
|
||||
'node["amenity"="fast_food"]',
|
||||
'way["amenity"="fast_food"]',
|
||||
]
|
||||
|
||||
CATERER_QUERIES = [
|
||||
'node["name"~"catering",i]',
|
||||
'way["name"~"catering",i]',
|
||||
'node["name"~"cateraar",i]',
|
||||
'way["name"~"cateraar",i]',
|
||||
'node["name"~"food service",i]',
|
||||
'way["name"~"food service",i]',
|
||||
]
|
||||
|
||||
|
||||
def _fetch_overpass(query: str) -> list[dict[str, Any]]:
|
||||
last_err: Optional[str] = None
|
||||
for url in OVERPASS_URLS:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with httpx.Client(timeout=180.0) as client:
|
||||
resp = client.post(url, data={"data": query})
|
||||
if resp.status_code == 429:
|
||||
time.sleep(12 * (attempt + 1))
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("elements", [])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = str(exc)
|
||||
time.sleep(5 * (attempt + 1))
|
||||
raise RuntimeError(f"Overpass failed: {last_err}")
|
||||
|
||||
|
||||
def _get_territory(country_iso2: str) -> Optional[dict[str, Any]]:
|
||||
return fetch_one(
|
||||
"""
|
||||
SELECT * FROM export_territories
|
||||
WHERE country_iso2 = %s AND is_active = TRUE
|
||||
ORDER BY sync_priority
|
||||
LIMIT 1
|
||||
""",
|
||||
(country_iso2.upper(),),
|
||||
)
|
||||
|
||||
|
||||
def _bbox_str(bbox: Any) -> str:
|
||||
if isinstance(bbox, str):
|
||||
bbox = json.loads(bbox)
|
||||
if not bbox or len(bbox) != 4:
|
||||
raise ValueError("Invalid bbox")
|
||||
south, west, north, east = bbox
|
||||
return f"{south},{west},{north},{east}"
|
||||
|
||||
|
||||
def _coords(el: dict[str, Any]) -> tuple[Optional[float], Optional[float]]:
|
||||
if el.get("type") == "node":
|
||||
return el.get("lat"), el.get("lon")
|
||||
center = el.get("center") or {}
|
||||
return center.get("lat"), center.get("lon")
|
||||
|
||||
|
||||
def _osm_external_id(el: dict[str, Any]) -> str:
|
||||
return f"osm:{el.get('type')}:{el.get('id')}"
|
||||
|
||||
|
||||
def _map_entity_type(tags: dict[str, Any], query_kind: str) -> str:
|
||||
shop = tags.get("shop") or ""
|
||||
amenity = tags.get("amenity") or ""
|
||||
name = (tags.get("name") or "").lower()
|
||||
if query_kind == "caterer" or "catering" in name or "cateraar" in name:
|
||||
return "contract_caterer"
|
||||
if shop in ("wholesale", "cash_and_carry") or tags.get("wholesale"):
|
||||
return "wholesaler"
|
||||
if shop == "butcher":
|
||||
return "butcher"
|
||||
if amenity == "fast_food":
|
||||
return "doner_shoarma" if any(x in name for x in ("doner", "döner", "shoarma", "kebab")) else "restaurant"
|
||||
if amenity == "restaurant":
|
||||
return "restaurant"
|
||||
return "foodservice"
|
||||
|
||||
|
||||
def _upsert_osm_entity(
|
||||
el: dict[str, Any],
|
||||
country_iso2: str,
|
||||
territory_code: Optional[str],
|
||||
query_kind: str,
|
||||
) -> tuple[str, Optional[int]]:
|
||||
tags = el.get("tags") or {}
|
||||
lat, lon = _coords(el)
|
||||
if lat is None or lon is None:
|
||||
return "skipped", None
|
||||
ext = _osm_external_id(el)
|
||||
name = tags.get("name") or tags.get("brand") or tags.get("operator") or "Unknown"
|
||||
entity_type = _map_entity_type(tags, query_kind)
|
||||
street = tags.get("addr:street") or ""
|
||||
hn = tags.get("addr:housenumber") or ""
|
||||
address = f"{street} {hn}".strip() or None
|
||||
city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village")
|
||||
phone = (tags.get("phone") or tags.get("contact:phone") or "")[:64] or None
|
||||
email = (tags.get("email") or tags.get("contact:email") or "")[:255] or None
|
||||
website = (tags.get("website") or tags.get("contact:website") or "")[:512] or None
|
||||
|
||||
existing = fetch_one(
|
||||
"SELECT id FROM export_market_entities WHERE metadata->>'osm_external_id' = %s",
|
||||
(ext,),
|
||||
)
|
||||
if existing:
|
||||
execute(
|
||||
"""
|
||||
UPDATE export_market_entities SET
|
||||
name = %s, entity_type = %s, lat = %s, lon = %s,
|
||||
address_line = COALESCE(%s, address_line),
|
||||
city = COALESCE(%s, city),
|
||||
phone = COALESCE(%s, phone), email = COALESCE(%s, email),
|
||||
website = COALESCE(%s, website),
|
||||
sources = COALESCE(sources, '[]'::jsonb) || '["osm"]'::jsonb,
|
||||
confidence = GREATEST(confidence, 55),
|
||||
updated_at = NOW()
|
||||
WHERE id = %s
|
||||
""",
|
||||
(name[:255], entity_type, lat, lon, address, city, phone, email, website, existing["id"]),
|
||||
)
|
||||
entity_id = existing["id"]
|
||||
action = "updated"
|
||||
else:
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO export_market_entities (
|
||||
name, entity_type, country_iso2, territory_code, lat, lon,
|
||||
address_line, city, phone, email, website, source, sources, confidence,
|
||||
metadata
|
||||
) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'osm','["osm"]'::jsonb,55,%s)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
name[:255], entity_type, country_iso2.upper(), territory_code,
|
||||
lat, lon, address, city, phone, email, website,
|
||||
json.dumps({"osm_external_id": ext}),
|
||||
),
|
||||
)
|
||||
entity_id = row["id"]
|
||||
action = "inserted"
|
||||
|
||||
if email:
|
||||
_upsert_contact(entity_id, email, phone, "osm", 60)
|
||||
return action, entity_id
|
||||
|
||||
|
||||
def _upsert_contact(
|
||||
entity_id: int,
|
||||
email: Optional[str],
|
||||
phone: Optional[str],
|
||||
source: str,
|
||||
confidence: int,
|
||||
role: str = "general",
|
||||
name: Optional[str] = None,
|
||||
) -> None:
|
||||
if email:
|
||||
exists = fetch_one(
|
||||
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s",
|
||||
(entity_id, email.lower()),
|
||||
)
|
||||
if not exists:
|
||||
execute(
|
||||
"""
|
||||
INSERT INTO export_entity_contacts (entity_id, role, name, email, phone, source, confidence, is_primary)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, FALSE)
|
||||
""",
|
||||
(entity_id, role, name, email.lower(), phone, source, confidence),
|
||||
)
|
||||
elif phone:
|
||||
exists = fetch_one(
|
||||
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND phone = %s",
|
||||
(entity_id, phone),
|
||||
)
|
||||
if not exists:
|
||||
execute(
|
||||
"""
|
||||
INSERT INTO export_entity_contacts (entity_id, role, name, phone, source, confidence)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(entity_id, role, name, phone, source, confidence),
|
||||
)
|
||||
|
||||
|
||||
def _build_area_query(parts: list[str], country_iso2: str) -> str:
|
||||
body = "".join(f"{p}(area.c);" for p in parts)
|
||||
return f'[out:json][timeout:180];area["ISO3166-1"="{country_iso2}"]->.c;({body});out center tags;'
|
||||
|
||||
|
||||
def _build_bbox_query(parts: list[str], bbox: str) -> str:
|
||||
body = "".join(f"{p}({bbox});" for p in parts)
|
||||
return f"[out:json][timeout:180];({body});out center tags;"
|
||||
|
||||
|
||||
def _countries_for_sync(
|
||||
country_iso2: Optional[str] = None,
|
||||
region_code: Optional[str] = None,
|
||||
max_priority: int = 2,
|
||||
) -> list[str]:
|
||||
if country_iso2:
|
||||
return [country_iso2.upper()]
|
||||
clauses = ["is_active = TRUE", "sync_priority <= %s"]
|
||||
params: list[Any] = [max_priority]
|
||||
if region_code:
|
||||
clauses.append("region_code = %s")
|
||||
params.append(region_code)
|
||||
rows = fetch_all(
|
||||
f"""
|
||||
SELECT country_iso2 FROM export_territories
|
||||
WHERE {' AND '.join(clauses)}
|
||||
GROUP BY country_iso2
|
||||
ORDER BY MIN(sync_priority), country_iso2
|
||||
""",
|
||||
tuple(params),
|
||||
)
|
||||
return [r["country_iso2"] for r in rows]
|
||||
|
||||
|
||||
def _build_country_query(parts: list[str], country_iso2: str, bbox: str) -> str:
|
||||
if country_iso2.upper() in BBOX_ONLY_ISO2:
|
||||
return _build_bbox_query(parts, bbox)
|
||||
return _build_area_query(parts, country_iso2.upper())
|
||||
|
||||
|
||||
def sync_osm_country(country_iso2: str, kinds: Optional[list[str]] = None) -> dict[str, Any]:
|
||||
territory = _get_territory(country_iso2)
|
||||
if not territory or not territory.get("bbox"):
|
||||
raise ValueError(f"No territory/bbox for {country_iso2}")
|
||||
bbox = _bbox_str(territory["bbox"])
|
||||
territory_code = territory["code"]
|
||||
kinds = kinds or ["distributors", "customers", "caterers"]
|
||||
stats = {"inserted": 0, "updated": 0, "fetched": 0, "kinds": {}}
|
||||
|
||||
mapping = {
|
||||
"distributors": DISTRIBUTOR_QUERIES,
|
||||
"customers": CUSTOMER_QUERIES,
|
||||
"caterers": CATERER_QUERIES,
|
||||
}
|
||||
for kind in kinds:
|
||||
parts = mapping.get(kind, [])
|
||||
if not parts:
|
||||
continue
|
||||
query = _build_country_query(parts, country_iso2, bbox)
|
||||
elements = _fetch_overpass(query)
|
||||
ki = {"fetched": len(elements), "inserted": 0, "updated": 0}
|
||||
seen: set[str] = set()
|
||||
for el in elements:
|
||||
ext = _osm_external_id(el)
|
||||
if ext in seen:
|
||||
continue
|
||||
seen.add(ext)
|
||||
action, _ = _upsert_osm_entity(el, country_iso2, territory_code, kind.replace("s", "")[:7])
|
||||
if action == "inserted":
|
||||
ki["inserted"] += 1
|
||||
stats["inserted"] += 1
|
||||
elif action == "updated":
|
||||
ki["updated"] += 1
|
||||
stats["updated"] += 1
|
||||
stats["fetched"] += ki["fetched"]
|
||||
stats["kinds"][kind] = ki
|
||||
time.sleep(2)
|
||||
|
||||
materialize_caterer_presence(country_iso2)
|
||||
return stats
|
||||
|
||||
|
||||
def materialize_caterer_presence(country_iso2: str) -> int:
|
||||
"""Create entity rows from caterer_presence + brands for country."""
|
||||
rows = fetch_all(
|
||||
"""
|
||||
SELECT p.*, b.name AS brand_name, b.website
|
||||
FROM export_caterer_presence p
|
||||
JOIN export_caterer_brands b ON b.code = p.brand_code
|
||||
WHERE p.country_iso2 = %s AND p.is_active = TRUE
|
||||
""",
|
||||
(country_iso2.upper(),),
|
||||
)
|
||||
territory = _get_territory(country_iso2)
|
||||
territory_code = territory["code"] if territory else None
|
||||
lat, lon = (territory.get("lat"), territory.get("lon")) if territory else (None, None)
|
||||
created = 0
|
||||
for p in rows:
|
||||
name = p.get("local_legal_name") or p.get("brand_name")
|
||||
exists = fetch_one(
|
||||
"SELECT id FROM export_market_entities WHERE brand_code = %s AND country_iso2 = %s",
|
||||
(p["brand_code"], country_iso2.upper()),
|
||||
)
|
||||
if exists:
|
||||
if p.get("entity_id") is None:
|
||||
execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (exists["id"], p["id"]))
|
||||
continue
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO export_market_entities (
|
||||
name, entity_type, country_iso2, territory_code, lat, lon, website, brand_code,
|
||||
source, sources, confidence, product_interest
|
||||
) VALUES (%s,'contract_caterer',%s,%s,%s,%s,%s,%s,'caterer_registry','["caterer_registry"]'::jsonb,75,'["chicken","shoarma"]'::jsonb)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
name, country_iso2.upper(), territory_code, lat, lon,
|
||||
(p.get("website") or "")[:512] or None, p["brand_code"],
|
||||
),
|
||||
)
|
||||
execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (row["id"], p["id"]))
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
def _scrape_emails(url: str) -> list[str]:
|
||||
if not url:
|
||||
return []
|
||||
if not url.startswith("http"):
|
||||
url = "https://" + url
|
||||
try:
|
||||
with httpx.Client(timeout=15.0, follow_redirects=True) as client:
|
||||
resp = client.get(url, headers={"User-Agent": "FoodlinkkExportIntel/1.0"})
|
||||
if resp.status_code >= 400:
|
||||
return []
|
||||
text = resp.text[:500000]
|
||||
except Exception:
|
||||
return []
|
||||
found: set[str] = set()
|
||||
for m in EMAIL_RE.findall(text):
|
||||
low = m.lower()
|
||||
if any(low.endswith(s) or s in low for s in SKIP_EMAIL_SUFFIX):
|
||||
continue
|
||||
if low.startswith("noreply") or low.startswith("no-reply"):
|
||||
continue
|
||||
found.add(low)
|
||||
return list(found)[:5]
|
||||
|
||||
|
||||
def enrich_contacts_country(country_iso2: str) -> dict[str, Any]:
|
||||
entities = fetch_all(
|
||||
"""
|
||||
SELECT id, name, website, email, phone FROM export_market_entities
|
||||
WHERE country_iso2 = %s AND website IS NOT NULL AND website <> ''
|
||||
""",
|
||||
(country_iso2.upper(),),
|
||||
)
|
||||
added = 0
|
||||
scraped = 0
|
||||
for ent in entities:
|
||||
emails = _scrape_emails(ent["website"])
|
||||
scraped += 1
|
||||
for em in emails:
|
||||
role = "procurement" if any(x in em for x in ("inkoop", "procurement", "sales", "info", "order")) else "general"
|
||||
before = fetch_one(
|
||||
"SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s",
|
||||
(ent["id"], em),
|
||||
)
|
||||
if not before:
|
||||
_upsert_contact(ent["id"], em, None, "website_scrape", 65, role=role, name="Website")
|
||||
added += 1
|
||||
if not ent.get("email") and emails:
|
||||
execute(
|
||||
"UPDATE export_market_entities SET email = %s, updated_at = NOW() WHERE id = %s",
|
||||
(emails[0], ent["id"]),
|
||||
)
|
||||
time.sleep(0.3)
|
||||
return {"entities_scraped": scraped, "contacts_added": added}
|
||||
|
||||
|
||||
def sync_ted_tenders(limit: int = 50) -> dict[str, Any]:
|
||||
"""TED Search API v3 — food / catering / meat tenders."""
|
||||
inserted = updated = 0
|
||||
queries = [
|
||||
"BT-21-Procedure ~ catering",
|
||||
"BT-21-Procedure ~ food",
|
||||
"BT-21-Procedure ~ meat",
|
||||
]
|
||||
ted_url = "https://api.ted.europa.eu/v3/notices/search"
|
||||
fields = [
|
||||
"BT-21-Procedure",
|
||||
"organisation-name-buyer",
|
||||
"publication-number",
|
||||
"organisation-country-buyer",
|
||||
]
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
with httpx.Client(timeout=45.0) as client:
|
||||
for q in queries:
|
||||
resp = client.post(
|
||||
ted_url,
|
||||
json={"query": q, "limit": min(limit, 25), "page": 1, "fields": fields},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
continue
|
||||
data = resp.json()
|
||||
for notice in data.get("notices", []):
|
||||
pub = notice.get("publication-number") or ""
|
||||
if not pub or pub in seen:
|
||||
continue
|
||||
seen.add(pub)
|
||||
title_raw = notice.get("BT-21-Procedure")
|
||||
if isinstance(title_raw, list):
|
||||
title = title_raw[0] if title_raw else f"TED {pub}"
|
||||
else:
|
||||
title = str(title_raw or f"TED notice {pub}")
|
||||
country_raw = notice.get("organisation-country-buyer")
|
||||
country_iso = None
|
||||
if isinstance(country_raw, list) and country_raw:
|
||||
c = country_raw[0]
|
||||
country_iso = c[:2] if len(c) >= 2 else None
|
||||
link = f"https://ted.europa.eu/en/notice/{pub}"
|
||||
ext = f"ted:{pub}"
|
||||
existing = fetch_one("SELECT id FROM export_tenders WHERE external_id = %s", (ext,))
|
||||
if existing:
|
||||
execute(
|
||||
"""
|
||||
UPDATE export_tenders SET title=%s, url=%s, country_iso2=%s, updated_at=NOW()
|
||||
WHERE external_id=%s
|
||||
""",
|
||||
(title[:512], link, country_iso, ext),
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
execute(
|
||||
"""
|
||||
INSERT INTO export_tenders (external_id, title, url, country_iso2, source, status)
|
||||
VALUES (%s,%s,%s,%s,'ted','open')
|
||||
""",
|
||||
(ext, title[:512], link, country_iso),
|
||||
)
|
||||
inserted += 1
|
||||
time.sleep(1)
|
||||
except Exception as exc:
|
||||
return {"error": str(exc), "inserted": inserted, "updated": updated}
|
||||
return {"inserted": inserted, "updated": updated, "source": "ted_api_v3", "seen": len(seen)}
|
||||
|
||||
|
||||
def sync_all_priority(max_priority: int = 2, region_code: Optional[str] = None) -> dict[str, Any]:
|
||||
"""Full OSM + contacts + Places sync for active territories."""
|
||||
iso_list = _countries_for_sync(region_code=region_code, max_priority=max_priority)
|
||||
report: dict[str, Any] = {
|
||||
"countries": iso_list,
|
||||
"country_count": len(iso_list),
|
||||
"region": region_code,
|
||||
"max_priority": max_priority,
|
||||
"steps": {},
|
||||
}
|
||||
|
||||
report["steps"]["ted"] = sync_ted_tenders(40)
|
||||
|
||||
dist: dict[str, Any] = {}
|
||||
cater: dict[str, Any] = {}
|
||||
cust: dict[str, Any] = {}
|
||||
contacts: dict[str, Any] = {}
|
||||
for c in iso_list:
|
||||
try:
|
||||
materialize_caterer_presence(c)
|
||||
dist[c] = sync_osm_country(c, kinds=["distributors"])
|
||||
cater[c] = sync_osm_country(c, kinds=["caterers"])
|
||||
cust[c] = sync_osm_country(c, kinds=["customers"])
|
||||
contacts[c] = enrich_contacts_country(c)
|
||||
except Exception as exc:
|
||||
dist[c] = {"error": str(exc)}
|
||||
time.sleep(2)
|
||||
report["steps"]["distributors"] = dist
|
||||
report["steps"]["caterers"] = cater
|
||||
report["steps"]["customers"] = cust
|
||||
report["steps"]["contacts"] = contacts
|
||||
|
||||
from app.export_intel_places import sync_places_countries
|
||||
report["steps"]["google_places"] = sync_places_countries(iso_list)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def sync_world_regions(max_priority: int = 2) -> dict[str, Any]:
|
||||
"""Sync all target regions sequentially (Europe, MENA, Africa, Americas)."""
|
||||
regions = ["europe", "middle_east", "gcc", "africa", "americas", "caribbean"]
|
||||
out: dict[str, Any] = {"regions": {}, "totals": {"countries": 0}}
|
||||
for reg in regions:
|
||||
countries = _countries_for_sync(region_code=reg, max_priority=max_priority)
|
||||
if not countries:
|
||||
out["regions"][reg] = {"skipped": True, "reason": "no countries"}
|
||||
continue
|
||||
out["totals"]["countries"] += len(countries)
|
||||
try:
|
||||
out["regions"][reg] = sync_all_priority(max_priority=max_priority, region_code=reg)
|
||||
except Exception as exc:
|
||||
out["regions"][reg] = {"error": str(exc)}
|
||||
time.sleep(5)
|
||||
return out
|
||||
|
||||
|
||||
def sync_distributors(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
|
||||
countries = _countries_for_sync(country_iso2, region_code)
|
||||
results: dict[str, Any] = {}
|
||||
for c in countries:
|
||||
try:
|
||||
results[c] = sync_osm_country(c, kinds=["distributors"])
|
||||
except Exception as exc:
|
||||
results[c] = {"error": str(exc)}
|
||||
time.sleep(3)
|
||||
enrich = {}
|
||||
if country_iso2:
|
||||
enrich = enrich_contacts_country(country_iso2.upper())
|
||||
return {"osm": results, "contact_enrichment": enrich}
|
||||
|
||||
|
||||
def sync_caterers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
|
||||
countries = _countries_for_sync(country_iso2, region_code)
|
||||
results: dict[str, Any] = {}
|
||||
materialized = 0
|
||||
for c in countries:
|
||||
try:
|
||||
materialized += materialize_caterer_presence(c)
|
||||
results[c] = sync_osm_country(c, kinds=["caterers"])
|
||||
except Exception as exc:
|
||||
results[c] = {"error": str(exc)}
|
||||
time.sleep(3)
|
||||
return {"osm": results, "materialized_presence": materialized}
|
||||
|
||||
|
||||
def sync_contacts(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
|
||||
countries = _countries_for_sync(country_iso2, region_code)
|
||||
out: dict[str, Any] = {}
|
||||
for c in countries:
|
||||
out[c] = enrich_contacts_country(c)
|
||||
ted = sync_ted_tenders()
|
||||
return {"contacts": out, "ted_tenders": ted}
|
||||
|
||||
|
||||
def sync_customers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
|
||||
if country_iso2:
|
||||
return sync_osm_country(country_iso2, kinds=["customers"])
|
||||
countries = _countries_for_sync(region_code=region_code)
|
||||
results: dict[str, Any] = {}
|
||||
for c in countries:
|
||||
try:
|
||||
results[c] = sync_osm_country(c, kinds=["customers"])
|
||||
except Exception as exc:
|
||||
results[c] = {"error": str(exc)}
|
||||
time.sleep(3)
|
||||
return {"osm": results}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Territory seeds — Europe, Middle East, Africa, Americas (bbox + centroid)."""
|
||||
from __future__ import annotations
|
||||
|
||||
# bbox: [south, west, north, east]
|
||||
WORLD_TERRITORIES: list[dict] = [
|
||||
# ── Europe (extra) ──
|
||||
{"code": "ie", "iso2": "IE", "region": "europe", "nl": "Ierland", "en": "Ireland", "lat": 53.4, "lon": -8.0, "zoom": 7, "bbox": [51.4, -10.5, 55.4, -5.5], "pri": 2},
|
||||
{"code": "pt", "iso2": "PT", "region": "europe", "nl": "Portugal", "en": "Portugal", "lat": 39.5, "lon": -8.0, "zoom": 6, "bbox": [36.9, -9.5, 42.2, -6.2], "pri": 2},
|
||||
{"code": "gr", "iso2": "GR", "region": "europe", "nl": "Griekenland", "en": "Greece", "lat": 39.0, "lon": 22.0, "zoom": 6, "bbox": [34.8, 19.4, 41.8, 29.6], "pri": 2},
|
||||
{"code": "cz", "iso2": "CZ", "region": "europe", "nl": "Tsjechië", "en": "Czechia", "lat": 49.8, "lon": 15.5, "zoom": 7, "bbox": [48.5, 12.1, 51.1, 18.9], "pri": 2},
|
||||
{"code": "sk", "iso2": "SK", "region": "europe", "nl": "Slowakije", "en": "Slovakia", "lat": 48.7, "lon": 19.5, "zoom": 7, "bbox": [47.7, 16.8, 49.6, 22.6], "pri": 2},
|
||||
{"code": "hu", "iso2": "HU", "region": "europe", "nl": "Hongarije", "en": "Hungary", "lat": 47.2, "lon": 19.5, "zoom": 7, "bbox": [45.7, 16.1, 48.6, 22.9], "pri": 2},
|
||||
{"code": "ro", "iso2": "RO", "region": "europe", "nl": "Roemenië", "en": "Romania", "lat": 45.9, "lon": 25.0, "zoom": 6, "bbox": [43.6, 20.3, 48.3, 29.7], "pri": 2},
|
||||
{"code": "bg", "iso2": "BG", "region": "europe", "nl": "Bulgarije", "en": "Bulgaria", "lat": 42.7, "lon": 25.5, "zoom": 7, "bbox": [41.2, 22.4, 44.2, 28.6], "pri": 2},
|
||||
{"code": "hr", "iso2": "HR", "region": "europe", "nl": "Kroatië", "en": "Croatia", "lat": 45.1, "lon": 15.2, "zoom": 7, "bbox": [42.4, 13.5, 46.5, 19.4], "pri": 2},
|
||||
{"code": "si", "iso2": "SI", "region": "europe", "nl": "Slovenië", "en": "Slovenia", "lat": 46.1, "lon": 14.8, "zoom": 8, "bbox": [45.4, 13.4, 46.9, 16.6], "pri": 2},
|
||||
{"code": "fi", "iso2": "FI", "region": "europe", "nl": "Finland", "en": "Finland", "lat": 64.0, "lon": 26.0, "zoom": 5, "bbox": [59.8, 20.6, 70.1, 31.6], "pri": 2},
|
||||
{"code": "ee", "iso2": "EE", "region": "europe", "nl": "Estland", "en": "Estonia", "lat": 58.6, "lon": 25.0, "zoom": 7, "bbox": [57.5, 21.8, 59.7, 28.2], "pri": 2},
|
||||
{"code": "lv", "iso2": "LV", "region": "europe", "nl": "Letland", "en": "Latvia", "lat": 56.9, "lon": 24.6, "zoom": 7, "bbox": [55.7, 21.0, 58.1, 28.2], "pri": 2},
|
||||
{"code": "lt", "iso2": "LT", "region": "europe", "nl": "Litouwen", "en": "Lithuania", "lat": 55.2, "lon": 23.9, "zoom": 7, "bbox": [53.9, 21.0, 56.4, 26.8], "pri": 2},
|
||||
{"code": "lu", "iso2": "LU", "region": "europe", "nl": "Luxemburg", "en": "Luxembourg", "lat": 49.8, "lon": 6.1, "zoom": 9, "bbox": [49.4, 5.7, 50.2, 6.5], "pri": 2},
|
||||
{"code": "mt", "iso2": "MT", "region": "europe", "nl": "Malta", "en": "Malta", "lat": 35.9, "lon": 14.4, "zoom": 10, "bbox": [35.8, 14.2, 36.1, 14.6], "pri": 2},
|
||||
{"code": "cy", "iso2": "CY", "region": "europe", "nl": "Cyprus", "en": "Cyprus", "lat": 35.1, "lon": 33.4, "zoom": 9, "bbox": [34.6, 32.3, 35.7, 34.6], "pri": 2},
|
||||
{"code": "ua", "iso2": "UA", "region": "europe", "nl": "Oekraïne", "en": "Ukraine", "lat": 48.4, "lon": 31.2, "zoom": 5, "bbox": [44.4, 22.1, 52.4, 40.2], "pri": 2},
|
||||
{"code": "rs", "iso2": "RS", "region": "europe", "nl": "Servië", "en": "Serbia", "lat": 44.0, "lon": 21.0, "zoom": 7, "bbox": [42.2, 18.8, 46.2, 23.0], "pri": 2},
|
||||
{"code": "ba", "iso2": "BA", "region": "europe", "nl": "Bosnië", "en": "Bosnia", "lat": 44.0, "lon": 17.7, "zoom": 7, "bbox": [42.6, 15.7, 45.3, 19.6], "pri": 2},
|
||||
{"code": "mk", "iso2": "MK", "region": "europe", "nl": "Noord-Macedonië", "en": "North Macedonia", "lat": 41.6, "lon": 21.7, "zoom": 8, "bbox": [40.8, 20.5, 42.4, 23.0], "pri": 2},
|
||||
{"code": "al", "iso2": "AL", "region": "europe", "nl": "Albanië", "en": "Albania", "lat": 41.2, "lon": 20.2, "zoom": 8, "bbox": [39.6, 19.3, 42.7, 21.1], "pri": 2},
|
||||
{"code": "me", "iso2": "ME", "region": "europe", "nl": "Montenegro", "en": "Montenegro", "lat": 42.7, "lon": 19.4, "zoom": 8, "bbox": [41.8, 18.4, 43.6, 20.4], "pri": 2},
|
||||
{"code": "md", "iso2": "MD", "region": "europe", "nl": "Moldavië", "en": "Moldova", "lat": 47.0, "lon": 28.8, "zoom": 7, "bbox": [45.5, 26.6, 48.5, 30.2], "pri": 2},
|
||||
{"code": "is", "iso2": "IS", "region": "europe", "nl": "IJsland", "en": "Iceland", "lat": 64.9, "lon": -19.0, "zoom": 6, "bbox": [63.3, -24.5, 66.5, -13.5], "pri": 2},
|
||||
{"code": "li", "iso2": "LI", "region": "europe", "nl": "Liechtenstein", "en": "Liechtenstein", "lat": 47.1, "lon": 9.5, "zoom": 10, "bbox": [47.0, 9.5, 47.3, 9.7], "pri": 2},
|
||||
# ── Middle East ──
|
||||
{"code": "il", "iso2": "IL", "region": "middle_east", "nl": "Israël", "en": "Israel", "lat": 31.5, "lon": 34.9, "zoom": 7, "bbox": [29.5, 34.3, 33.3, 35.9], "pri": 2},
|
||||
{"code": "eg", "iso2": "EG", "region": "middle_east", "nl": "Egypte", "en": "Egypt", "lat": 26.8, "lon": 30.8, "zoom": 5, "bbox": [22.0, 25.0, 31.7, 35.0], "pri": 2},
|
||||
{"code": "jo", "iso2": "JO", "region": "middle_east", "nl": "Jordanië", "en": "Jordan", "lat": 31.2, "lon": 36.8, "zoom": 7, "bbox": [29.2, 34.9, 33.4, 39.3], "pri": 2},
|
||||
{"code": "lb", "iso2": "LB", "region": "middle_east", "nl": "Libanon", "en": "Lebanon", "lat": 33.9, "lon": 35.9, "zoom": 8, "bbox": [33.1, 35.1, 34.7, 36.6], "pri": 2},
|
||||
{"code": "iq", "iso2": "IQ", "region": "middle_east", "nl": "Irak", "en": "Iraq", "lat": 33.2, "lon": 43.7, "zoom": 6, "bbox": [29.1, 38.8, 37.4, 48.6], "pri": 2},
|
||||
{"code": "ir", "iso2": "IR", "region": "middle_east", "nl": "Iran", "en": "Iran", "lat": 32.4, "lon": 53.7, "zoom": 5, "bbox": [25.0, 44.0, 39.8, 63.3], "pri": 2},
|
||||
{"code": "ye", "iso2": "YE", "region": "middle_east", "nl": "Jemen", "en": "Yemen", "lat": 15.6, "lon": 48.0, "zoom": 6, "bbox": [12.1, 42.5, 19.0, 54.5], "pri": 2},
|
||||
{"code": "ps", "iso2": "PS", "region": "middle_east", "nl": "Palestina", "en": "Palestine", "lat": 31.9, "lon": 35.2, "zoom": 8, "bbox": [31.2, 34.2, 32.6, 35.6], "pri": 2},
|
||||
{"code": "sy", "iso2": "SY", "region": "middle_east", "nl": "Syrië", "en": "Syria", "lat": 35.0, "lon": 38.5, "zoom": 6, "bbox": [32.3, 35.7, 37.3, 42.4], "pri": 3},
|
||||
{"code": "ly", "iso2": "LY", "region": "middle_east", "nl": "Libië", "en": "Libya", "lat": 26.3, "lon": 17.2, "zoom": 5, "bbox": [19.5, 9.3, 33.2, 25.2], "pri": 2},
|
||||
{"code": "tn", "iso2": "TN", "region": "middle_east", "nl": "Tunesië", "en": "Tunisia", "lat": 34.0, "lon": 9.5, "zoom": 6, "bbox": [30.2, 7.5, 37.5, 11.6], "pri": 2},
|
||||
{"code": "dz", "iso2": "DZ", "region": "africa", "nl": "Algerije", "en": "Algeria", "lat": 28.0, "lon": 2.6, "zoom": 5, "bbox": [19.0, -8.7, 37.1, 12.0], "pri": 2},
|
||||
# ── Africa ──
|
||||
{"code": "za", "iso2": "ZA", "region": "africa", "nl": "Zuid-Afrika", "en": "South Africa", "lat": -30.6, "lon": 22.9, "zoom": 5, "bbox": [-34.8, 16.5, -22.1, 32.9], "pri": 2},
|
||||
{"code": "ng", "iso2": "NG", "region": "africa", "nl": "Nigeria", "en": "Nigeria", "lat": 9.1, "lon": 8.7, "zoom": 6, "bbox": [4.3, 2.7, 13.9, 14.7], "pri": 2},
|
||||
{"code": "ke", "iso2": "KE", "region": "africa", "nl": "Kenia", "en": "Kenya", "lat": -0.02, "lon": 37.9, "zoom": 6, "bbox": [-4.7, 33.9, 5.0, 41.9], "pri": 2},
|
||||
{"code": "gh", "iso2": "GH", "region": "africa", "nl": "Ghana", "en": "Ghana", "lat": 7.9, "lon": -1.0, "zoom": 6, "bbox": [4.7, -3.3, 11.2, 1.2], "pri": 2},
|
||||
{"code": "et", "iso2": "ET", "region": "africa", "nl": "Ethiopië", "en": "Ethiopia", "lat": 9.1, "lon": 40.5, "zoom": 5, "bbox": [3.4, 33.0, 14.9, 48.0], "pri": 2},
|
||||
{"code": "tz", "iso2": "TZ", "region": "africa", "nl": "Tanzania", "en": "Tanzania", "lat": -6.4, "lon": 34.9, "zoom": 6, "bbox": [-11.7, 29.3, -0.99, 40.5], "pri": 2},
|
||||
{"code": "ug", "iso2": "UG", "region": "africa", "nl": "Oeganda", "en": "Uganda", "lat": 1.4, "lon": 32.3, "zoom": 7, "bbox": [-1.5, 29.6, 4.2, 35.0], "pri": 2},
|
||||
{"code": "sn", "iso2": "SN", "region": "africa", "nl": "Senegal", "en": "Senegal", "lat": 14.5, "lon": -14.5, "zoom": 7, "bbox": [12.3, -17.5, 16.7, -11.4], "pri": 2},
|
||||
{"code": "ci", "iso2": "CI", "region": "africa", "nl": "Ivoorkust", "en": "Côte d'Ivoire", "lat": 7.5, "lon": -5.5, "zoom": 6, "bbox": [4.4, -8.6, 10.7, -2.5], "pri": 2},
|
||||
{"code": "cm", "iso2": "CM", "region": "africa", "nl": "Kameroen", "en": "Cameroon", "lat": 6.6, "lon": 12.4, "zoom": 6, "bbox": [1.7, 8.5, 13.1, 16.2], "pri": 2},
|
||||
{"code": "ao", "iso2": "AO", "region": "africa", "nl": "Angola", "en": "Angola", "lat": -11.2, "lon": 17.9, "zoom": 5, "bbox": [-18.0, 11.7, -4.4, 24.1], "pri": 2},
|
||||
{"code": "mz", "iso2": "MZ", "region": "africa", "nl": "Mozambique", "en": "Mozambique", "lat": -18.7, "lon": 35.5, "zoom": 5, "bbox": [-26.9, 30.2, -10.5, 40.8], "pri": 2},
|
||||
{"code": "zw", "iso2": "ZW", "region": "africa", "nl": "Zimbabwe", "en": "Zimbabwe", "lat": -19.0, "lon": 29.2, "zoom": 6, "bbox": [-22.4, 25.2, -15.6, 33.1], "pri": 2},
|
||||
{"code": "zm", "iso2": "ZM", "region": "africa", "nl": "Zambia", "en": "Zambia", "lat": -13.1, "lon": 27.8, "zoom": 6, "bbox": [-18.1, 22.0, -8.2, 33.7], "pri": 2},
|
||||
{"code": "rw", "iso2": "RW", "region": "africa", "nl": "Rwanda", "en": "Rwanda", "lat": -1.9, "lon": 30.0, "zoom": 8, "bbox": [-2.8, 28.9, -1.0, 30.9], "pri": 2},
|
||||
{"code": "sd", "iso2": "SD", "region": "africa", "nl": "Soedan", "en": "Sudan", "lat": 15.5, "lon": 30.2, "zoom": 5, "bbox": [8.7, 21.8, 22.2, 38.6], "pri": 2},
|
||||
{"code": "ss", "iso2": "SS", "region": "africa", "nl": "Zuid-Soedan", "en": "South Sudan", "lat": 7.9, "lon": 30.0, "zoom": 6, "bbox": [3.5, 23.9, 12.2, 35.9], "pri": 3},
|
||||
{"code": "cd", "iso2": "CD", "region": "africa", "nl": "Congo (DRC)", "en": "DR Congo", "lat": -4.0, "lon": 21.8, "zoom": 5, "bbox": [-13.5, 12.2, 5.4, 31.3], "pri": 2},
|
||||
{"code": "cg", "iso2": "CG", "region": "africa", "nl": "Congo", "en": "Congo", "lat": -0.7, "lon": 15.3, "zoom": 6, "bbox": [-5.0, 11.2, 3.7, 18.6], "pri": 2},
|
||||
{"code": "bf", "iso2": "BF", "region": "africa", "nl": "Burkina Faso", "en": "Burkina Faso", "lat": 12.2, "lon": -1.6, "zoom": 6, "bbox": [9.4, -5.5, 15.1, 2.4], "pri": 2},
|
||||
{"code": "ml", "iso2": "ML", "region": "africa", "nl": "Mali", "en": "Mali", "lat": 17.6, "lon": -4.0, "zoom": 5, "bbox": [10.1, -12.2, 25.0, 4.3], "pri": 2},
|
||||
{"code": "ne", "iso2": "NE", "region": "africa", "nl": "Niger", "en": "Niger", "lat": 17.6, "lon": 8.1, "zoom": 5, "bbox": [11.7, 0.2, 23.5, 16.0], "pri": 2},
|
||||
{"code": "td", "iso2": "TD", "region": "africa", "nl": "Tsjaad", "en": "Chad", "lat": 15.5, "lon": 18.7, "zoom": 5, "bbox": [7.4, 13.5, 23.5, 24.0], "pri": 2},
|
||||
{"code": "so", "iso2": "SO", "region": "africa", "nl": "Somalië", "en": "Somalia", "lat": 5.2, "lon": 46.2, "zoom": 5, "bbox": [-1.7, 41.0, 12.0, 51.4], "pri": 3},
|
||||
{"code": "mg", "iso2": "MG", "region": "africa", "nl": "Madagascar", "en": "Madagascar", "lat": -18.8, "lon": 46.9, "zoom": 5, "bbox": [-25.6, 43.2, -11.9, 50.5], "pri": 2},
|
||||
{"code": "mu", "iso2": "MU", "region": "africa", "nl": "Mauritius", "en": "Mauritius", "lat": -20.3, "lon": 57.6, "zoom": 9, "bbox": [-20.5, 57.3, -19.9, 63.5], "pri": 2},
|
||||
{"code": "na", "iso2": "NA", "region": "africa", "nl": "Namibië", "en": "Namibia", "lat": -22.6, "lon": 17.1, "zoom": 5, "bbox": [-28.0, 11.7, -16.9, 25.3], "pri": 2},
|
||||
{"code": "bw", "iso2": "BW", "region": "africa", "nl": "Botswana", "en": "Botswana", "lat": -22.3, "lon": 24.7, "zoom": 6, "bbox": [-26.9, 20.0, -17.8, 29.4], "pri": 2},
|
||||
{"code": "ga", "iso2": "GA", "region": "africa", "nl": "Gabon", "en": "Gabon", "lat": -0.8, "lon": 11.6, "zoom": 6, "bbox": [-4.0, 8.7, 2.3, 14.5], "pri": 2},
|
||||
{"code": "gn", "iso2": "GN", "region": "africa", "nl": "Guinee", "en": "Guinea", "lat": 9.9, "lon": -11.7, "zoom": 6, "bbox": [7.2, -15.1, 12.7, -7.6], "pri": 2},
|
||||
{"code": "bj", "iso2": "BJ", "region": "africa", "nl": "Benin", "en": "Benin", "lat": 9.3, "lon": 2.3, "zoom": 7, "bbox": [6.2, 0.8, 12.4, 3.9], "pri": 2},
|
||||
{"code": "tg", "iso2": "TG", "region": "africa", "nl": "Togo", "en": "Togo", "lat": 8.6, "lon": 1.0, "zoom": 7, "bbox": [6.1, -0.1, 11.1, 1.8], "pri": 2},
|
||||
{"code": "lr", "iso2": "LR", "region": "africa", "nl": "Liberia", "en": "Liberia", "lat": 6.4, "lon": -9.4, "zoom": 7, "bbox": [4.3, -11.5, 8.6, -7.4], "pri": 2},
|
||||
{"code": "sl", "iso2": "SL", "region": "africa", "nl": "Sierra Leone", "en": "Sierra Leone", "lat": 8.5, "lon": -11.8, "zoom": 7, "bbox": [6.9, -13.3, 10.0, -10.3], "pri": 2},
|
||||
{"code": "mr", "iso2": "MR", "region": "africa", "nl": "Mauritanië", "en": "Mauritania", "lat": 21.0, "lon": -10.9, "zoom": 5, "bbox": [14.7, -17.1, 27.3, -4.8], "pri": 2},
|
||||
{"code": "er", "iso2": "ER", "region": "africa", "nl": "Eritrea", "en": "Eritrea", "lat": 15.2, "lon": 39.8, "zoom": 7, "bbox": [12.4, 36.4, 18.0, 43.1], "pri": 2},
|
||||
{"code": "dj", "iso2": "DJ", "region": "africa", "nl": "Djibouti", "en": "Djibouti", "lat": 11.8, "lon": 42.6, "zoom": 8, "bbox": [10.9, 41.8, 12.7, 43.4], "pri": 2},
|
||||
{"code": "mw", "iso2": "MW", "region": "africa", "nl": "Malawi", "en": "Malawi", "lat": -13.3, "lon": 34.3, "zoom": 7, "bbox": [-17.1, 32.7, -9.4, 35.9], "pri": 2},
|
||||
{"code": "ls", "iso2": "LS", "region": "africa", "nl": "Lesotho", "en": "Lesotho", "lat": -29.6, "lon": 28.2, "zoom": 8, "bbox": [-30.7, 27.0, -28.6, 29.5], "pri": 2},
|
||||
{"code": "sz", "iso2": "SZ", "region": "africa", "nl": "Eswatini", "en": "Eswatini", "lat": -26.5, "lon": 31.5, "zoom": 8, "bbox": [-27.3, 30.8, -25.7, 32.1], "pri": 2},
|
||||
{"code": "cv", "iso2": "CV", "region": "africa", "nl": "Kaapverdië", "en": "Cape Verde", "lat": 16.0, "lon": -24.0, "zoom": 8, "bbox": [14.8, -25.4, 17.2, -22.7], "pri": 2},
|
||||
{"code": "gq", "iso2": "GQ", "region": "africa", "nl": "Equatoriaal-Guinea", "en": "Equatorial Guinea", "lat": 1.7, "lon": 10.3, "zoom": 8, "bbox": [0.9, 9.3, 3.8, 11.3], "pri": 2},
|
||||
{"code": "st", "iso2": "ST", "region": "africa", "nl": "São Tomé", "en": "São Tomé", "lat": 0.3, "lon": 6.6, "zoom": 9, "bbox": [0.0, 6.5, 1.7, 7.5], "pri": 2},
|
||||
{"code": "sc", "iso2": "SC", "region": "africa", "nl": "Seychellen", "en": "Seychelles", "lat": -4.7, "lon": 55.5, "zoom": 9, "bbox": [-10.0, 46.0, -3.7, 56.3], "pri": 2},
|
||||
{"code": "km", "iso2": "KM", "region": "africa", "nl": "Comoren", "en": "Comoros", "lat": -11.9, "lon": 43.9, "zoom": 9, "bbox": [-12.4, 43.2, -11.4, 44.5], "pri": 2},
|
||||
{"code": "bi", "iso2": "BI", "region": "africa", "nl": "Burundi", "en": "Burundi", "lat": -3.4, "lon": 29.9, "zoom": 8, "bbox": [-4.5, 29.0, -2.3, 30.8], "pri": 2},
|
||||
{"code": "cf", "iso2": "CF", "region": "africa", "nl": "Centraal-Afrika", "en": "Central African Rep.", "lat": 6.6, "lon": 20.9, "zoom": 6, "bbox": [2.2, 14.4, 11.0, 27.5], "pri": 3},
|
||||
{"code": "gm", "iso2": "GM", "region": "africa", "nl": "Gambia", "en": "Gambia", "lat": 13.4, "lon": -15.3, "zoom": 8, "bbox": [13.1, -16.8, 13.8, -13.8], "pri": 2},
|
||||
{"code": "gw", "iso2": "GW", "region": "africa", "nl": "Guinee-Bissau", "en": "Guinea-Bissau", "lat": 11.8, "lon": -15.2, "zoom": 8, "bbox": [10.9, -16.7, 12.7, -13.6], "pri": 2},
|
||||
# ── Americas ──
|
||||
{"code": "ca", "iso2": "CA", "region": "americas", "nl": "Canada", "en": "Canada", "lat": 56.1, "lon": -96.8, "zoom": 4, "bbox": [41.7, -141.0, 83.1, -52.6], "pri": 2},
|
||||
{"code": "mx", "iso2": "MX", "region": "americas", "nl": "Mexico", "en": "Mexico", "lat": 23.6, "lon": -102.6, "zoom": 5, "bbox": [14.5, -118.4, 32.7, -86.7], "pri": 2},
|
||||
{"code": "br", "iso2": "BR", "region": "americas", "nl": "Brazilië", "en": "Brazil", "lat": -14.2, "lon": -51.9, "zoom": 4, "bbox": [-33.8, -73.9, 5.3, -34.8], "pri": 2},
|
||||
{"code": "ar", "iso2": "AR", "region": "americas", "nl": "Argentinië", "en": "Argentina", "lat": -38.4, "lon": -63.6, "zoom": 4, "bbox": [-55.1, -73.6, -21.8, -53.6], "pri": 2},
|
||||
{"code": "co", "iso2": "CO", "region": "americas", "nl": "Colombia", "en": "Colombia", "lat": 4.6, "lon": -74.3, "zoom": 5, "bbox": [-4.2, -79.0, 12.5, -66.9], "pri": 2},
|
||||
{"code": "cl", "iso2": "CL", "region": "americas", "nl": "Chili", "en": "Chile", "lat": -35.7, "lon": -71.5, "zoom": 4, "bbox": [-55.9, -75.6, -17.5, -66.4], "pri": 2},
|
||||
{"code": "pe", "iso2": "PE", "region": "americas", "nl": "Peru", "en": "Peru", "lat": -9.2, "lon": -75.0, "zoom": 5, "bbox": [-18.3, -81.3, -0.0, -68.7], "pri": 2},
|
||||
{"code": "ve", "iso2": "VE", "region": "americas", "nl": "Venezuela", "en": "Venezuela", "lat": 6.4, "lon": -66.6, "zoom": 5, "bbox": [0.6, -73.4, 12.2, -59.8], "pri": 2},
|
||||
{"code": "ec", "iso2": "EC", "region": "americas", "nl": "Ecuador", "en": "Ecuador", "lat": -1.8, "lon": -78.2, "zoom": 6, "bbox": [-5.0, -81.1, 1.5, -75.2], "pri": 2},
|
||||
{"code": "bo", "iso2": "BO", "region": "americas", "nl": "Bolivia", "en": "Bolivia", "lat": -16.3, "lon": -63.6, "zoom": 5, "bbox": [-22.9, -69.6, -9.7, -57.5], "pri": 2},
|
||||
{"code": "py", "iso2": "PY", "region": "americas", "nl": "Paraguay", "en": "Paraguay", "lat": -23.4, "lon": -58.4, "zoom": 6, "bbox": [-27.6, -62.6, -19.3, -54.3], "pri": 2},
|
||||
{"code": "uy", "iso2": "UY", "region": "americas", "nl": "Uruguay", "en": "Uruguay", "lat": -32.5, "lon": -55.8, "zoom": 7, "bbox": [-35.0, -58.4, -30.1, -53.1], "pri": 2},
|
||||
{"code": "gt", "iso2": "GT", "region": "americas", "nl": "Guatemala", "en": "Guatemala", "lat": 15.8, "lon": -90.2, "zoom": 7, "bbox": [13.7, -92.2, 17.8, -88.2], "pri": 2},
|
||||
{"code": "hn", "iso2": "HN", "region": "americas", "nl": "Honduras", "en": "Honduras", "lat": 14.6, "lon": -86.2, "zoom": 7, "bbox": [12.9, -89.4, 16.5, -83.1], "pri": 2},
|
||||
{"code": "sv", "iso2": "SV", "region": "americas", "nl": "El Salvador", "en": "El Salvador", "lat": 13.7, "lon": -88.9, "zoom": 8, "bbox": [13.1, -90.1, 14.4, -87.7], "pri": 2},
|
||||
{"code": "ni", "iso2": "NI", "region": "americas", "nl": "Nicaragua", "en": "Nicaragua", "lat": 12.9, "lon": -85.0, "zoom": 7, "bbox": [10.7, -87.7, 15.0, -82.7], "pri": 2},
|
||||
{"code": "cr", "iso2": "CR", "region": "americas", "nl": "Costa Rica", "en": "Costa Rica", "lat": 9.7, "lon": -84.0, "zoom": 7, "bbox": [8.0, -85.9, 11.2, -82.6], "pri": 2},
|
||||
{"code": "pa", "iso2": "PA", "region": "americas", "nl": "Panama", "en": "Panama", "lat": 8.5, "lon": -80.8, "zoom": 7, "bbox": [7.2, -83.0, 9.6, -77.2], "pri": 2},
|
||||
{"code": "bz", "iso2": "BZ", "region": "americas", "nl": "Belize", "en": "Belize", "lat": 17.2, "lon": -88.7, "zoom": 8, "bbox": [15.9, -89.2, 18.5, -87.8], "pri": 2},
|
||||
{"code": "cu", "iso2": "CU", "region": "americas", "nl": "Cuba", "en": "Cuba", "lat": 21.5, "lon": -79.0, "zoom": 6, "bbox": [19.8, -84.9, 23.3, -74.1], "pri": 2},
|
||||
{"code": "do", "iso2": "DO", "region": "americas", "nl": "Dominicaanse Rep.", "en": "Dominican Republic", "lat": 18.7, "lon": -70.2, "zoom": 7, "bbox": [17.5, -72.0, 19.9, -68.3], "pri": 2},
|
||||
{"code": "ht", "iso2": "HT", "region": "americas", "nl": "Haïti", "en": "Haiti", "lat": 18.9, "lon": -72.3, "zoom": 8, "bbox": [18.0, -74.5, 20.1, -71.6], "pri": 2},
|
||||
{"code": "pr", "iso2": "PR", "region": "americas", "nl": "Puerto Rico", "en": "Puerto Rico", "lat": 18.2, "lon": -66.5, "zoom": 8, "bbox": [17.9, -67.9, 18.5, -65.2], "pri": 2},
|
||||
{"code": "bs", "iso2": "BS", "region": "caribbean", "nl": "Bahama's", "en": "Bahamas", "lat": 25.0, "lon": -77.4, "zoom": 7, "bbox": [20.9, -78.9, 27.3, -72.7], "pri": 2},
|
||||
{"code": "bb", "iso2": "BB", "region": "caribbean", "nl": "Barbados", "en": "Barbados", "lat": 13.2, "lon": -59.5, "zoom": 10, "bbox": [13.0, -59.7, 13.3, -59.4], "pri": 2},
|
||||
{"code": "gd", "iso2": "GD", "region": "caribbean", "nl": "Grenada", "en": "Grenada", "lat": 12.1, "lon": -61.7, "zoom": 10, "bbox": [11.9, -61.8, 12.5, -61.4], "pri": 2},
|
||||
{"code": "lc", "iso2": "LC", "region": "caribbean", "nl": "Saint Lucia", "en": "Saint Lucia", "lat": 13.9, "lon": -60.9, "zoom": 10, "bbox": [13.7, -61.1, 14.1, -60.9], "pri": 2},
|
||||
{"code": "gy", "iso2": "GY", "region": "americas", "nl": "Guyana", "en": "Guyana", "lat": 4.9, "lon": -58.9, "zoom": 6, "bbox": [1.2, -61.4, 8.6, -56.5], "pri": 2},
|
||||
{"code": "sr", "iso2": "SR", "region": "americas", "nl": "Suriname", "en": "Suriname", "lat": 3.9, "lon": -56.0, "zoom": 7, "bbox": [1.8, -58.1, 6.0, -53.9], "pri": 2},
|
||||
]
|
||||
|
||||
BBOX_ONLY_ISO2 = frozenset(
|
||||
{
|
||||
"AW", "CW", "BQ", "SX", "JM", "TT", "BH", "MT", "CY", "LU", "LI", "SG", "MC", "AD", "SM",
|
||||
"BB", "GD", "LC", "VC", "AG", "DM", "KN", "ST", "KM", "SC", "GM", "PS",
|
||||
}
|
||||
)
|
||||
@@ -49,6 +49,7 @@ class BrainSearchIn(BaseModel):
|
||||
limit: int = Field(default=8, ge=1, le=30)
|
||||
|
||||
|
||||
from app.export_intel import router as export_intel_router
|
||||
from app.email_config import get_active_email_config
|
||||
from app.comfyui import fetch_image_bytes, generate_image, get_job, start_generation, QUALITY_PRESETS
|
||||
import json
|
||||
@@ -64,6 +65,7 @@ app.include_router(research_router)
|
||||
app.include_router(recommendations_router)
|
||||
app.include_router(packaging_router)
|
||||
app.include_router(ops_router)
|
||||
app.include_router(export_intel_router)
|
||||
|
||||
|
||||
class AgentEventCreate(BaseModel):
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Agent event logging helpers used by the Tools API."""
|
||||
|
||||
from typing import Any, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from app.agent_names import normalize_agent_key
|
||||
from app.db import execute_returning, json_param
|
||||
|
||||
|
||||
@@ -17,7 +19,21 @@ def log_agent_event(
|
||||
related_table: Optional[str] = None,
|
||||
related_id: Optional[int] = None,
|
||||
channel: str = "dashboard",
|
||||
correlation_id: Optional[str] = None,
|
||||
source_agent: Optional[str] = None,
|
||||
target_agent: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
meta = dict(metadata or {})
|
||||
if correlation_id:
|
||||
meta.setdefault("correlation_id", correlation_id)
|
||||
if source_agent:
|
||||
meta.setdefault("source_agent", normalize_agent_key(source_agent))
|
||||
if target_agent:
|
||||
meta.setdefault("target_agent", normalize_agent_key(target_agent))
|
||||
if "correlation_id" not in meta:
|
||||
meta.setdefault("correlation_id", str(uuid4()))
|
||||
|
||||
normalized = normalize_agent_key(agent_name)
|
||||
row = execute_returning(
|
||||
"""
|
||||
INSERT INTO agent_events (
|
||||
@@ -28,12 +44,12 @@ def log_agent_event(
|
||||
RETURNING *
|
||||
""",
|
||||
(
|
||||
agent_name,
|
||||
normalized,
|
||||
agent_type,
|
||||
event_type,
|
||||
title,
|
||||
body,
|
||||
json_param(metadata),
|
||||
json_param(meta),
|
||||
status,
|
||||
related_table,
|
||||
related_id,
|
||||
|
||||
Reference in New Issue
Block a user