"""Export Intel sync — OSM Overpass, contact enrichment, tenders, fusion.""" from __future__ import annotations import json import re import time import xml.etree.ElementTree as ET from datetime import datetime from typing import Any, Optional from urllib.parse import urljoin, urlparse import httpx from app.db import execute, execute_returning, fetch_all, fetch_one OVERPASS_URLS = [ "https://overpass.kumi.systems/api/interpreter", "https://overpass-api.de/api/interpreter", ] TED_RSS_URLS = [ "https://ted.europa.eu/en/simap/rss-feed/-/rss/search/15", "https://ted.europa.eu/en/simap/rss-feed/-/rss/search/teeq", ] from app.export_territory_seeds import BBOX_ONLY_ISO2 EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") SKIP_EMAIL_SUFFIX = ("example.com", "sentry.io", "wixpress.com", "png", "jpg", "jpeg") DISTRIBUTOR_QUERIES = [ 'node["shop"="wholesale"]', 'way["shop"="wholesale"]', 'node["wholesale"]', 'way["wholesale"]', 'node["shop"="cash_and_carry"]', 'way["shop"="cash_and_carry"]', ] CUSTOMER_QUERIES = [ 'node["shop"="butcher"]', 'way["shop"="butcher"]', 'node["amenity"="restaurant"]', 'way["amenity"="restaurant"]', 'node["amenity"="fast_food"]', 'way["amenity"="fast_food"]', ] CATERER_QUERIES = [ 'node["name"~"catering",i]', 'way["name"~"catering",i]', 'node["name"~"cateraar",i]', 'way["name"~"cateraar",i]', 'node["name"~"food service",i]', 'way["name"~"food service",i]', ] def _fetch_overpass(query: str) -> list[dict[str, Any]]: last_err: Optional[str] = None for url in OVERPASS_URLS: for attempt in range(3): try: with httpx.Client(timeout=180.0) as client: resp = client.post(url, data={"data": query}) if resp.status_code == 429: time.sleep(12 * (attempt + 1)) continue resp.raise_for_status() return resp.json().get("elements", []) except Exception as exc: # noqa: BLE001 last_err = str(exc) time.sleep(5 * (attempt + 1)) raise RuntimeError(f"Overpass failed: {last_err}") def _get_territory(country_iso2: str) -> Optional[dict[str, Any]]: return fetch_one( """ SELECT * FROM export_territories WHERE country_iso2 = %s AND is_active = TRUE ORDER BY sync_priority LIMIT 1 """, (country_iso2.upper(),), ) def _bbox_str(bbox: Any) -> str: if isinstance(bbox, str): bbox = json.loads(bbox) if not bbox or len(bbox) != 4: raise ValueError("Invalid bbox") south, west, north, east = bbox return f"{south},{west},{north},{east}" def _coords(el: dict[str, Any]) -> tuple[Optional[float], Optional[float]]: if el.get("type") == "node": return el.get("lat"), el.get("lon") center = el.get("center") or {} return center.get("lat"), center.get("lon") def _osm_external_id(el: dict[str, Any]) -> str: return f"osm:{el.get('type')}:{el.get('id')}" def _map_entity_type(tags: dict[str, Any], query_kind: str) -> str: shop = tags.get("shop") or "" amenity = tags.get("amenity") or "" name = (tags.get("name") or "").lower() if query_kind == "caterer" or "catering" in name or "cateraar" in name: return "contract_caterer" if shop in ("wholesale", "cash_and_carry") or tags.get("wholesale"): return "wholesaler" if shop == "butcher": return "butcher" if amenity == "fast_food": return "doner_shoarma" if any(x in name for x in ("doner", "döner", "shoarma", "kebab")) else "restaurant" if amenity == "restaurant": return "restaurant" return "foodservice" def _upsert_osm_entity( el: dict[str, Any], country_iso2: str, territory_code: Optional[str], query_kind: str, ) -> tuple[str, Optional[int]]: tags = el.get("tags") or {} lat, lon = _coords(el) if lat is None or lon is None: return "skipped", None ext = _osm_external_id(el) name = tags.get("name") or tags.get("brand") or tags.get("operator") or "Unknown" entity_type = _map_entity_type(tags, query_kind) street = tags.get("addr:street") or "" hn = tags.get("addr:housenumber") or "" address = f"{street} {hn}".strip() or None city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village") phone = (tags.get("phone") or tags.get("contact:phone") or "")[:64] or None email = (tags.get("email") or tags.get("contact:email") or "")[:255] or None website = (tags.get("website") or tags.get("contact:website") or "")[:512] or None existing = fetch_one( "SELECT id FROM export_market_entities WHERE metadata->>'osm_external_id' = %s", (ext,), ) if existing: execute( """ UPDATE export_market_entities SET name = %s, entity_type = %s, lat = %s, lon = %s, address_line = COALESCE(%s, address_line), city = COALESCE(%s, city), phone = COALESCE(%s, phone), email = COALESCE(%s, email), website = COALESCE(%s, website), sources = COALESCE(sources, '[]'::jsonb) || '["osm"]'::jsonb, confidence = GREATEST(confidence, 55), updated_at = NOW() WHERE id = %s """, (name[:255], entity_type, lat, lon, address, city, phone, email, website, existing["id"]), ) entity_id = existing["id"] action = "updated" else: row = execute_returning( """ INSERT INTO export_market_entities ( name, entity_type, country_iso2, territory_code, lat, lon, address_line, city, phone, email, website, source, sources, confidence, metadata ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'osm','["osm"]'::jsonb,55,%s) RETURNING id """, ( name[:255], entity_type, country_iso2.upper(), territory_code, lat, lon, address, city, phone, email, website, json.dumps({"osm_external_id": ext}), ), ) entity_id = row["id"] action = "inserted" if email: _upsert_contact(entity_id, email, phone, "osm", 60) return action, entity_id def _upsert_contact( entity_id: int, email: Optional[str], phone: Optional[str], source: str, confidence: int, role: str = "general", name: Optional[str] = None, ) -> None: if email: exists = fetch_one( "SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s", (entity_id, email.lower()), ) if not exists: execute( """ INSERT INTO export_entity_contacts (entity_id, role, name, email, phone, source, confidence, is_primary) VALUES (%s, %s, %s, %s, %s, %s, %s, FALSE) """, (entity_id, role, name, email.lower(), phone, source, confidence), ) elif phone: exists = fetch_one( "SELECT id FROM export_entity_contacts WHERE entity_id = %s AND phone = %s", (entity_id, phone), ) if not exists: execute( """ INSERT INTO export_entity_contacts (entity_id, role, name, phone, source, confidence) VALUES (%s, %s, %s, %s, %s, %s) """, (entity_id, role, name, phone, source, confidence), ) def _build_area_query(parts: list[str], country_iso2: str) -> str: body = "".join(f"{p}(area.c);" for p in parts) return f'[out:json][timeout:180];area["ISO3166-1"="{country_iso2}"]->.c;({body});out center tags;' def _build_bbox_query(parts: list[str], bbox: str) -> str: body = "".join(f"{p}({bbox});" for p in parts) return f"[out:json][timeout:180];({body});out center tags;" def _countries_for_sync( country_iso2: Optional[str] = None, region_code: Optional[str] = None, max_priority: int = 2, ) -> list[str]: if country_iso2: return [country_iso2.upper()] clauses = ["is_active = TRUE", "sync_priority <= %s"] params: list[Any] = [max_priority] if region_code: clauses.append("region_code = %s") params.append(region_code) rows = fetch_all( f""" SELECT country_iso2 FROM export_territories WHERE {' AND '.join(clauses)} GROUP BY country_iso2 ORDER BY MIN(sync_priority), country_iso2 """, tuple(params), ) return [r["country_iso2"] for r in rows] def _build_country_query(parts: list[str], country_iso2: str, bbox: str) -> str: if country_iso2.upper() in BBOX_ONLY_ISO2: return _build_bbox_query(parts, bbox) return _build_area_query(parts, country_iso2.upper()) def sync_osm_country(country_iso2: str, kinds: Optional[list[str]] = None) -> dict[str, Any]: territory = _get_territory(country_iso2) if not territory or not territory.get("bbox"): raise ValueError(f"No territory/bbox for {country_iso2}") bbox = _bbox_str(territory["bbox"]) territory_code = territory["code"] kinds = kinds or ["distributors", "customers", "caterers"] stats = {"inserted": 0, "updated": 0, "fetched": 0, "kinds": {}} mapping = { "distributors": DISTRIBUTOR_QUERIES, "customers": CUSTOMER_QUERIES, "caterers": CATERER_QUERIES, } for kind in kinds: parts = mapping.get(kind, []) if not parts: continue query = _build_country_query(parts, country_iso2, bbox) elements = _fetch_overpass(query) ki = {"fetched": len(elements), "inserted": 0, "updated": 0} seen: set[str] = set() for el in elements: ext = _osm_external_id(el) if ext in seen: continue seen.add(ext) action, _ = _upsert_osm_entity(el, country_iso2, territory_code, kind.replace("s", "")[:7]) if action == "inserted": ki["inserted"] += 1 stats["inserted"] += 1 elif action == "updated": ki["updated"] += 1 stats["updated"] += 1 stats["fetched"] += ki["fetched"] stats["kinds"][kind] = ki time.sleep(2) materialize_caterer_presence(country_iso2) return stats def materialize_caterer_presence(country_iso2: str) -> int: """Create entity rows from caterer_presence + brands for country.""" rows = fetch_all( """ SELECT p.*, b.name AS brand_name, b.website FROM export_caterer_presence p JOIN export_caterer_brands b ON b.code = p.brand_code WHERE p.country_iso2 = %s AND p.is_active = TRUE """, (country_iso2.upper(),), ) territory = _get_territory(country_iso2) territory_code = territory["code"] if territory else None lat, lon = (territory.get("lat"), territory.get("lon")) if territory else (None, None) created = 0 for p in rows: name = p.get("local_legal_name") or p.get("brand_name") exists = fetch_one( "SELECT id FROM export_market_entities WHERE brand_code = %s AND country_iso2 = %s", (p["brand_code"], country_iso2.upper()), ) if exists: if p.get("entity_id") is None: execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (exists["id"], p["id"])) continue row = execute_returning( """ INSERT INTO export_market_entities ( name, entity_type, country_iso2, territory_code, lat, lon, website, brand_code, source, sources, confidence, product_interest ) VALUES (%s,'contract_caterer',%s,%s,%s,%s,%s,%s,'caterer_registry','["caterer_registry"]'::jsonb,75,'["chicken","shoarma"]'::jsonb) RETURNING id """, ( name, country_iso2.upper(), territory_code, lat, lon, (p.get("website") or "")[:512] or None, p["brand_code"], ), ) execute("UPDATE export_caterer_presence SET entity_id = %s WHERE id = %s", (row["id"], p["id"])) created += 1 return created def _scrape_emails(url: str) -> list[str]: if not url: return [] if not url.startswith("http"): url = "https://" + url try: with httpx.Client(timeout=15.0, follow_redirects=True) as client: resp = client.get(url, headers={"User-Agent": "FoodlinkkExportIntel/1.0"}) if resp.status_code >= 400: return [] text = resp.text[:500000] except Exception: return [] found: set[str] = set() for m in EMAIL_RE.findall(text): low = m.lower() if any(low.endswith(s) or s in low for s in SKIP_EMAIL_SUFFIX): continue if low.startswith("noreply") or low.startswith("no-reply"): continue found.add(low) return list(found)[:5] def enrich_contacts_country(country_iso2: str) -> dict[str, Any]: entities = fetch_all( """ SELECT id, name, website, email, phone FROM export_market_entities WHERE country_iso2 = %s AND website IS NOT NULL AND website <> '' """, (country_iso2.upper(),), ) added = 0 scraped = 0 for ent in entities: emails = _scrape_emails(ent["website"]) scraped += 1 for em in emails: role = "procurement" if any(x in em for x in ("inkoop", "procurement", "sales", "info", "order")) else "general" before = fetch_one( "SELECT id FROM export_entity_contacts WHERE entity_id = %s AND email = %s", (ent["id"], em), ) if not before: _upsert_contact(ent["id"], em, None, "website_scrape", 65, role=role, name="Website") added += 1 if not ent.get("email") and emails: execute( "UPDATE export_market_entities SET email = %s, updated_at = NOW() WHERE id = %s", (emails[0], ent["id"]), ) time.sleep(0.3) return {"entities_scraped": scraped, "contacts_added": added} def sync_ted_tenders(limit: int = 50) -> dict[str, Any]: """TED Search API v3 — food / catering / meat tenders.""" inserted = updated = 0 queries = [ "BT-21-Procedure ~ catering", "BT-21-Procedure ~ food", "BT-21-Procedure ~ meat", ] ted_url = "https://api.ted.europa.eu/v3/notices/search" fields = [ "BT-21-Procedure", "organisation-name-buyer", "publication-number", "organisation-country-buyer", ] seen: set[str] = set() try: with httpx.Client(timeout=45.0) as client: for q in queries: resp = client.post( ted_url, json={"query": q, "limit": min(limit, 25), "page": 1, "fields": fields}, ) if resp.status_code >= 400: continue data = resp.json() for notice in data.get("notices", []): pub = notice.get("publication-number") or "" if not pub or pub in seen: continue seen.add(pub) title_raw = notice.get("BT-21-Procedure") if isinstance(title_raw, list): title = title_raw[0] if title_raw else f"TED {pub}" else: title = str(title_raw or f"TED notice {pub}") country_raw = notice.get("organisation-country-buyer") country_iso = None if isinstance(country_raw, list) and country_raw: c = country_raw[0] country_iso = c[:2] if len(c) >= 2 else None link = f"https://ted.europa.eu/en/notice/{pub}" ext = f"ted:{pub}" existing = fetch_one("SELECT id FROM export_tenders WHERE external_id = %s", (ext,)) if existing: execute( """ UPDATE export_tenders SET title=%s, url=%s, country_iso2=%s, updated_at=NOW() WHERE external_id=%s """, (title[:512], link, country_iso, ext), ) updated += 1 else: execute( """ INSERT INTO export_tenders (external_id, title, url, country_iso2, source, status) VALUES (%s,%s,%s,%s,'ted','open') """, (ext, title[:512], link, country_iso), ) inserted += 1 time.sleep(1) except Exception as exc: return {"error": str(exc), "inserted": inserted, "updated": updated} return {"inserted": inserted, "updated": updated, "source": "ted_api_v3", "seen": len(seen)} def sync_all_priority(max_priority: int = 2, region_code: Optional[str] = None) -> dict[str, Any]: """Full OSM + contacts + Places sync for active territories.""" iso_list = _countries_for_sync(region_code=region_code, max_priority=max_priority) report: dict[str, Any] = { "countries": iso_list, "country_count": len(iso_list), "region": region_code, "max_priority": max_priority, "steps": {}, } report["steps"]["ted"] = sync_ted_tenders(40) dist: dict[str, Any] = {} cater: dict[str, Any] = {} cust: dict[str, Any] = {} contacts: dict[str, Any] = {} for c in iso_list: try: materialize_caterer_presence(c) dist[c] = sync_osm_country(c, kinds=["distributors"]) cater[c] = sync_osm_country(c, kinds=["caterers"]) cust[c] = sync_osm_country(c, kinds=["customers"]) contacts[c] = enrich_contacts_country(c) except Exception as exc: dist[c] = {"error": str(exc)} time.sleep(2) report["steps"]["distributors"] = dist report["steps"]["caterers"] = cater report["steps"]["customers"] = cust report["steps"]["contacts"] = contacts from app.export_intel_places import sync_places_countries report["steps"]["google_places"] = sync_places_countries(iso_list) return report def sync_world_regions(max_priority: int = 2) -> dict[str, Any]: """Sync all target regions sequentially (Europe, MENA, Africa, Americas).""" regions = ["europe", "middle_east", "gcc", "africa", "americas", "caribbean"] out: dict[str, Any] = {"regions": {}, "totals": {"countries": 0}} for reg in regions: countries = _countries_for_sync(region_code=reg, max_priority=max_priority) if not countries: out["regions"][reg] = {"skipped": True, "reason": "no countries"} continue out["totals"]["countries"] += len(countries) try: out["regions"][reg] = sync_all_priority(max_priority=max_priority, region_code=reg) except Exception as exc: out["regions"][reg] = {"error": str(exc)} time.sleep(5) return out def sync_distributors(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]: countries = _countries_for_sync(country_iso2, region_code) results: dict[str, Any] = {} for c in countries: try: results[c] = sync_osm_country(c, kinds=["distributors"]) except Exception as exc: results[c] = {"error": str(exc)} time.sleep(3) enrich = {} if country_iso2: enrich = enrich_contacts_country(country_iso2.upper()) return {"osm": results, "contact_enrichment": enrich} def sync_caterers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]: countries = _countries_for_sync(country_iso2, region_code) results: dict[str, Any] = {} materialized = 0 for c in countries: try: materialized += materialize_caterer_presence(c) results[c] = sync_osm_country(c, kinds=["caterers"]) except Exception as exc: results[c] = {"error": str(exc)} time.sleep(3) return {"osm": results, "materialized_presence": materialized} def sync_contacts(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]: countries = _countries_for_sync(country_iso2, region_code) out: dict[str, Any] = {} for c in countries: out[c] = enrich_contacts_country(c) ted = sync_ted_tenders() return {"contacts": out, "ted_tenders": ted} def sync_customers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]: if country_iso2: return sync_osm_country(country_iso2, kinds=["customers"]) countries = _countries_for_sync(region_code=region_code) results: dict[str, Any] = {} for c in countries: try: results[c] = sync_osm_country(c, kinds=["customers"]) except Exception as exc: results[c] = {"error": str(exc)} time.sleep(3) return {"osm": results}