SysOps: manual — 2026-07-30 09:33 UTC

tools-api/app/export_intel_sync.py

@@ -25,6 +25,15 @@ TED_RSS_URLS = [
 
 from app.export_territory_seeds import BBOX_ONLY_ISO2
 
+from app.export_intel_sync_progress import (
+    finish_job,
+    get_active_job,
+    job_step,
+    set_active_job,
+    start_job,
+    update_job,
+)
+
 EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
 SKIP_EMAIL_SUFFIX = ("example.com", "sentry.io", "wixpress.com", "png", "jpg", "jpeg")
 
@@ -140,9 +149,22 @@ def _upsert_osm_entity(
     hn = tags.get("addr:housenumber") or ""
     address = f"{street} {hn}".strip() or None
     city = tags.get("addr:city") or tags.get("addr:town") or tags.get("addr:village")
-    phone = (tags.get("phone") or tags.get("contact:phone") or "")[:64] or None
+    phone = (
+        tags.get("phone")
+        or tags.get("contact:phone")
+        or tags.get("contact:mobile")
+        or tags.get("mobile")
+        or tags.get("phone:mobile")
+        or ""
+    )[:64] or None
     email = (tags.get("email") or tags.get("contact:email") or "")[:255] or None
-    website = (tags.get("website") or tags.get("contact:website") or "")[:512] or None
+    website = (
+        tags.get("website")
+        or tags.get("contact:website")
+        or tags.get("contact:url")
+        or tags.get("url")
+        or ""
+    )[:512] or None
 
     existing = fetch_one(
         "SELECT id FROM export_market_entities WHERE metadata->>'osm_external_id' = %s",
@@ -185,7 +207,7 @@ def _upsert_osm_entity(
         entity_id = row["id"]
         action = "inserted"
 
-    if email:
+    if email or phone:
         _upsert_contact(entity_id, email, phone, "osm", 60)
     return action, entity_id
 
@@ -285,6 +307,9 @@ def sync_osm_country(country_iso2: str, kinds: Optional[list[str]] = None) -> di
         parts = mapping.get(kind, [])
         if not parts:
             continue
+        active = get_active_job()
+        if active:
+            job_step(active, message=f"OSM {kind} · {country_iso2}", country=country_iso2, step=kind)
         query = _build_country_query(parts, country_iso2, bbox)
         elements = _fetch_overpass(query)
         ki = {"fetched": len(elements), "inserted": 0, "updated": 0}
@@ -352,6 +377,46 @@ def materialize_caterer_presence(country_iso2: str) -> int:
     return created
 
 
+
+PHONE_RE = re.compile(
+    r"(?:tel:|phone|telefoon)[^\d+]{0,12}(\+?[\d\s\(\)\-\.]{8,20})|(\+31[\s\-\.]?(?:\(?\d{2}\)?[\s\-\.]?)?[\d\s\-\.]{6,12})",
+    re.I,
+)
+
+
+def _normalize_phone(raw: str) -> Optional[str]:
+    s = re.sub(r"[^\d+]", "", (raw or "").strip())
+    if len(s) < 8:
+        return None
+    if s.startswith("00"):
+        s = "+" + s[2:]
+    if s.startswith("0") and not s.startswith("+"):
+        s = "+31" + s[1:]
+    return s[:32]
+
+
+def _scrape_phones(url: str) -> list[str]:
+    if not url:
+        return []
+    if not url.startswith("http"):
+        url = "https://" + url
+    try:
+        with httpx.Client(timeout=15.0, follow_redirects=True) as client:
+            resp = client.get(url, headers={"User-Agent": "FoodlinkkExportIntel/1.0"})
+            if resp.status_code >= 400:
+                return []
+            text = resp.text[:500000]
+    except Exception:
+        return []
+    found: set[str] = set()
+    for m in PHONE_RE.finditer(text):
+        raw = (m.group(1) or m.group(2) or "").strip()
+        norm = _normalize_phone(raw)
+        if norm:
+            found.add(norm)
+    return list(found)[:3]
+
+
 def _scrape_emails(url: str) -> list[str]:
     if not url:
         return []
@@ -386,8 +451,25 @@ def enrich_contacts_country(country_iso2: str) -> dict[str, Any]:
     )
     added = 0
     scraped = 0
+    phones_added = 0
     for ent in entities:
-        emails = _scrape_emails(ent["website"])
+        site = ent["website"]
+        emails = _scrape_emails(site)
+        phones = _scrape_phones(site) if not ent.get("phone") else []
+        # probe common contact pages (gratis, geen API)
+        if not emails or not phones:
+            base = site if site.startswith("http") else ("https://" + site)
+            base = base.rstrip("/")
+            for path in ("/contact", "/contact/", "/contact-ons", "/contactus", "/over-ons", "/about"):
+                try:
+                    emails = list(dict.fromkeys(emails + _scrape_emails(base + path)))
+                    if not ent.get("phone"):
+                        phones = list(dict.fromkeys(phones + _scrape_phones(base + path)))
+                except Exception:
+                    pass
+                if emails and (ent.get("phone") or phones):
+                    break
+                time.sleep(0.15)
         scraped += 1
         for em in emails:
             role = "procurement" if any(x in em for x in ("inkoop", "procurement", "sales", "info", "order")) else "general"
@@ -398,13 +480,27 @@ def enrich_contacts_country(country_iso2: str) -> dict[str, Any]:
             if not before:
                 _upsert_contact(ent["id"], em, None, "website_scrape", 65, role=role, name="Website")
                 added += 1
+        for ph in phones:
+            before_p = fetch_one(
+                "SELECT id FROM export_entity_contacts WHERE entity_id = %s AND phone = %s",
+                (ent["id"], ph),
+            )
+            if not before_p:
+                _upsert_contact(ent["id"], None, ph, "website_scrape", 60, role="general", name="Website")
+                added += 1
+                phones_added += 1
         if not ent.get("email") and emails:
             execute(
                 "UPDATE export_market_entities SET email = %s, updated_at = NOW() WHERE id = %s",
                 (emails[0], ent["id"]),
             )
+        if not ent.get("phone") and phones:
+            execute(
+                "UPDATE export_market_entities SET phone = %s, updated_at = NOW() WHERE id = %s",
+                (phones[0], ent["id"]),
+            )
         time.sleep(0.3)
-    return {"entities_scraped": scraped, "contacts_added": added}
+    return {"entities_scraped": scraped, "contacts_added": added, "phones_added": phones_added}
 
 
 def sync_ted_tenders(limit: int = 50) -> dict[str, Any]:
@@ -476,46 +572,105 @@ def sync_ted_tenders(limit: int = 50) -> dict[str, Any]:
 
 
 def sync_all_priority(max_priority: int = 2, region_code: Optional[str] = None) -> dict[str, Any]:
-    """Full OSM + contacts + Places sync for active territories."""
+    """Full OSM + contacts sync for active territories (gratis)."""
     iso_list = _countries_for_sync(region_code=region_code, max_priority=max_priority)
+    label = f"Sync {region_code or 'alle landen'} ({len(iso_list)} landen)"
+    job_id = start_job("all", label, total=len(iso_list), meta={"region": region_code, "max_priority": max_priority})
     report: dict[str, Any] = {
         "countries": iso_list,
         "country_count": len(iso_list),
         "region": region_code,
         "max_priority": max_priority,
         "steps": {},
+        "job_id": job_id,
     }
+    try:
+        update_job(job_id, step="ted", message="TED tenders ophalen…")
+        report["steps"]["ted"] = sync_ted_tenders(40)
 
-    report["steps"]["ted"] = sync_ted_tenders(40)
+        dist: dict[str, Any] = {}
+        cater: dict[str, Any] = {}
+        cust: dict[str, Any] = {}
+        contacts: dict[str, Any] = {}
+        for i, c in enumerate(iso_list, start=1):
+            job_step(job_id, message=f"Land {i}/{len(iso_list)}: {c}", country=c, step="osm", current=i, total=len(iso_list))
+            try:
+                materialize_caterer_presence(c)
+                dist[c] = sync_osm_country(c, kinds=["distributors"])
+                cater[c] = sync_osm_country(c, kinds=["caterers"])
+                cust[c] = sync_osm_country(c, kinds=["customers"])
+                contacts[c] = enrich_contacts_country(c)
+            except Exception as exc:
+                dist[c] = {"error": str(exc)}
+            time.sleep(2)
+        report["steps"]["distributors"] = dist
+        report["steps"]["caterers"] = cater
+        report["steps"]["customers"] = cust
+        report["steps"]["contacts"] = contacts
 
+        report["steps"]["google_places"] = {"skipped": True, "reason": "gratis-modus: geen Google Places"}
+        finish_job(job_id, result={"country_count": len(iso_list), "region": region_code})
+    except Exception as exc:
+        finish_job(job_id, error=str(exc))
+        raise
+    finally:
+        set_active_job(None)
+    return report
+
+
+
+def sync_missing_countries(max_priority: int = 3) -> dict[str, Any]:
+    """OSM + contacts sync for active territories with zero entities."""
+    rows = fetch_all(
+        """
+        SELECT t.country_iso2
+        FROM export_territories t
+        WHERE t.is_active
+          AND t.sync_priority <= %s
+          AND NOT EXISTS (
+            SELECT 1 FROM export_market_entities e WHERE e.country_iso2 = t.country_iso2
+          )
+        GROUP BY t.country_iso2
+        ORDER BY MIN(t.sync_priority), t.country_iso2
+        """,
+        (max_priority,),
+    )
+    iso_list = [r["country_iso2"] for r in rows]
+    job_id = start_job("missing", f"Lege landen ({len(iso_list)})", total=len(iso_list))
+    report: dict[str, Any] = {"countries": iso_list, "country_count": len(iso_list), "steps": {}, "job_id": job_id}
     dist: dict[str, Any] = {}
     cater: dict[str, Any] = {}
     cust: dict[str, Any] = {}
     contacts: dict[str, Any] = {}
-    for c in iso_list:
-        try:
-            materialize_caterer_presence(c)
-            dist[c] = sync_osm_country(c, kinds=["distributors"])
-            cater[c] = sync_osm_country(c, kinds=["caterers"])
-            cust[c] = sync_osm_country(c, kinds=["customers"])
-            contacts[c] = enrich_contacts_country(c)
-        except Exception as exc:
-            dist[c] = {"error": str(exc)}
-        time.sleep(2)
-    report["steps"]["distributors"] = dist
-    report["steps"]["caterers"] = cater
-    report["steps"]["customers"] = cust
-    report["steps"]["contacts"] = contacts
-
-    from app.export_intel_places import sync_places_countries
-    report["steps"]["google_places"] = sync_places_countries(iso_list)
-
+    try:
+        for i, c in enumerate(iso_list, start=1):
+            job_step(job_id, message=f"Leeg land {i}/{len(iso_list)}: {c}", country=c, current=i, total=len(iso_list))
+            try:
+                materialize_caterer_presence(c)
+                dist[c] = sync_osm_country(c, kinds=["distributors"])
+                cater[c] = sync_osm_country(c, kinds=["caterers"])
+                cust[c] = sync_osm_country(c, kinds=["customers"])
+                contacts[c] = enrich_contacts_country(c)
+            except Exception as exc:
+                dist[c] = {"error": str(exc)}
+            time.sleep(2)
+        report["steps"] = {
+            "distributors": dist,
+            "caterers": cater,
+            "customers": cust,
+            "contacts": contacts,
+        }
+        finish_job(job_id, result={"country_count": len(iso_list)})
+    except Exception as exc:
+        finish_job(job_id, error=str(exc))
+        raise
+    finally:
+        set_active_job(None)
     return report
 
-
 def sync_world_regions(max_priority: int = 2) -> dict[str, Any]:
     """Sync all target regions sequentially (Europe, MENA, Africa, Americas)."""
-    regions = ["europe", "middle_east", "gcc", "africa", "americas", "caribbean"]
+    regions = ["europe", "middle_east", "gcc", "africa", "americas", "caribbean", "asia", "oceania"]
     out: dict[str, Any] = {"regions": {}, "totals": {"countries": 0}}
     for reg in regions:
         countries = _countries_for_sync(region_code=reg, max_priority=max_priority)
@@ -571,13 +726,33 @@ def sync_contacts(country_iso2: Optional[str] = None, region_code: Optional[str]
 
 def sync_customers(country_iso2: Optional[str] = None, region_code: Optional[str] = None) -> dict[str, Any]:
     if country_iso2:
-        return sync_osm_country(country_iso2, kinds=["customers"])
-    countries = _countries_for_sync(region_code=region_code)
-    results: dict[str, Any] = {}
-    for c in countries:
+        job_id = start_job("customers", f"Horeca {country_iso2.upper()}", total=1, meta={"country": country_iso2.upper()})
         try:
-            results[c] = sync_osm_country(c, kinds=["customers"])
+            job_step(job_id, message=f"Horeca OSM · {country_iso2.upper()}", country=country_iso2.upper(), current=1, total=1)
+            result = sync_osm_country(country_iso2, kinds=["customers"])
+            finish_job(job_id, result=result)
+            return result
         except Exception as exc:
-            results[c] = {"error": str(exc)}
-        time.sleep(3)
-    return {"osm": results}
+            finish_job(job_id, error=str(exc))
+            raise
+        finally:
+            set_active_job(None)
+    countries = _countries_for_sync(region_code=region_code)
+    label = f"Horeca {region_code or 'regio'} ({len(countries)} landen)"
+    job_id = start_job("customers", label, total=len(countries), meta={"region": region_code})
+    results: dict[str, Any] = {}
+    try:
+        for i, c in enumerate(countries, start=1):
+            job_step(job_id, message=f"Horeca {i}/{len(countries)}: {c}", country=c, current=i, total=len(countries))
+            try:
+                results[c] = sync_osm_country(c, kinds=["customers"])
+            except Exception as exc:
+                results[c] = {"error": str(exc)}
+            time.sleep(3)
+        finish_job(job_id, result={"countries": len(countries), "region": region_code})
+    except Exception as exc:
+        finish_job(job_id, error=str(exc))
+        raise
+    finally:
+        set_active_job(None)
+    return {"osm": results, "job_id": job_id}