SysOps: manual — 2026-07-30 09:33 UTC

tools-api/app/export_intel_places.py

@@ -1,20 +1,26 @@
-"""Google Places sync for Export Intel (optional — requires GOOGLE_MAPS_API_KEY)."""
+"""Google Places sync + entity enrichment for Export Intel."""
 from __future__ import annotations
 
 import json
+import math
 import os
 import time
 from typing import Any, Optional
 
 import httpx
 
-from app.db import fetch_one
+from app.db import execute, fetch_all, fetch_one
 from app.export_intel_sync import _get_territory, _upsert_contact
 
 PLACES_URL = "https://maps.googleapis.com/maps/api/place/textsearch/json"
 DETAILS_URL = "https://maps.googleapis.com/maps/api/place/details/json"
+FIND_URL = "https://maps.googleapis.com/maps/api/place/findplacefromtext/json"
+
+SETTINGS_KEY = "export_intel.google_maps_api_key"
 
 QUERIES = [
+    "groothandel groente fruit",
+    "wholesale food",
     "meat wholesaler",
     "halal meat distributor",
     "poultry distributor",
@@ -22,13 +28,58 @@ QUERIES = [
     "contract catering",
     "food importer",
     "halal butcher",
+    "horeca groothandel",
 ]
 
 LARGE_COUNTRIES = frozenset({"US", "CA", "BR", "MX", "AR", "AU", "RU", "CN", "IN", "CD", "DZ", "EG", "IR", "SA"})
 
 
+def get_google_api_key() -> Optional[str]:
+    env = (os.getenv("GOOGLE_MAPS_API_KEY") or os.getenv("GOOGLE_PLACES_API_KEY") or "").strip()
+    if env:
+        return env
+    row = fetch_one("SELECT value FROM app_settings WHERE key = %s", (SETTINGS_KEY,))
+    if not row or not row.get("value"):
+        return None
+    val = row["value"]
+    if isinstance(val, dict):
+        key = (val.get("api_key") or val.get("key") or "").strip()
+        return key or None
+    if isinstance(val, str):
+        try:
+            parsed = json.loads(val)
+            if isinstance(parsed, dict):
+                return (parsed.get("api_key") or parsed.get("key") or "").strip() or None
+        except Exception:
+            return val.strip() or None
+    return None
+
+
+def set_google_api_key(api_key: str) -> None:
+    key = (api_key or "").strip()
+    if not key:
+        execute("DELETE FROM app_settings WHERE key = %s", (SETTINGS_KEY,))
+        return
+    execute(
+        """
+        INSERT INTO app_settings (key, value, updated_at)
+        VALUES (%s, %s::jsonb, NOW())
+        ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()
+        """,
+        (SETTINGS_KEY, json.dumps({"api_key": key})),
+    )
+
+
+def google_api_key_status() -> dict[str, Any]:
+    key = get_google_api_key()
+    return {
+        "google_maps_api_key_set": bool(key),
+        "source": "env" if (os.getenv("GOOGLE_MAPS_API_KEY") or os.getenv("GOOGLE_PLACES_API_KEY")) else ("db" if key else None),
+    }
+
+
 def _api_key() -> Optional[str]:
-    return os.getenv("GOOGLE_MAPS_API_KEY") or os.getenv("GOOGLE_PLACES_API_KEY")
+    return get_google_api_key()
 
 
 def _places_search(query: str, lat: float, lon: float, radius_m: int = 80000) -> list[dict[str, Any]]:
@@ -51,6 +102,46 @@ def _places_search(query: str, lat: float, lon: float, radius_m: int = 80000) ->
         return data.get("results", [])[:12]
 
 
+def _find_place_id(name: str, lat: float, lon: float, radius_m: int = 3000) -> Optional[str]:
+    key = _api_key()
+    if not key or not name.strip():
+        return None
+    params = {
+        "input": name.strip()[:128],
+        "inputtype": "textquery",
+        "locationbias": f"circle:{radius_m}@{lat},{lon}",
+        "fields": "place_id,name,geometry",
+        "key": key,
+    }
+    with httpx.Client(timeout=20.0) as client:
+        resp = client.get(FIND_URL, params=params)
+        if resp.status_code >= 400:
+            return None
+        data = resp.json()
+        if data.get("status") != "OK":
+            return None
+        cands = data.get("candidates") or []
+        if not cands:
+            return None
+        best = cands[0]
+        geo = best.get("geometry", {}).get("location", {})
+        clat, clon = geo.get("lat"), geo.get("lng")
+        if clat is not None and clon is not None:
+            dist = _haversine_m(lat, lon, float(clat), float(clon))
+            if dist > max(radius_m * 2, 8000):
+                return None
+        return best.get("place_id")
+
+
+def _haversine_m(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
+    r = 6371000.0
+    p1, p2 = math.radians(lat1), math.radians(lat2)
+    dp = math.radians(lat2 - lat1)
+    dl = math.radians(lon2 - lon1)
+    a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
+    return 2 * r * math.asin(math.sqrt(a))
+
+
 def _place_details(place_id: str) -> dict[str, Any]:
     key = _api_key()
     if not key:
@@ -60,7 +151,7 @@ def _place_details(place_id: str) -> dict[str, Any]:
             DETAILS_URL,
             params={
                 "place_id": place_id,
-                "fields": "name,formatted_phone_number,website,formatted_address,geometry,international_phone_number",
+                "fields": "name,formatted_phone_number,website,formatted_address,geometry,international_phone_number,place_id",
                 "key": key,
             },
         )
@@ -84,55 +175,132 @@ def _bbox_points(bbox: Any, grid: int = 3) -> list[tuple[float, float]]:
     return points
 
 
-def _upsert_place(place: dict[str, Any], details: dict[str, Any], country_iso2: str, territory_code: str, query: str) -> tuple[str, int]:
-    from app.db import execute, execute_returning
+def _apply_place_to_entity(entity_id: int, place_id: str, details: dict[str, Any]) -> bool:
+    phone = details.get("formatted_phone_number") or details.get("international_phone_number")
+    website = details.get("website")
+    geo = details.get("geometry", {}).get("location", {})
+    plat, plon = geo.get("lat"), geo.get("lng")
+    execute(
+        """
+        UPDATE export_market_entities SET
+            phone = COALESCE(%s, phone),
+            website = COALESCE(%s, website),
+            google_place_id = COALESCE(google_place_id, %s),
+            lat = COALESCE(%s, lat),
+            lon = COALESCE(%s, lon),
+            confidence = GREATEST(confidence, 68),
+            updated_at = NOW(),
+            metadata = COALESCE(metadata, '{}'::jsonb) || %s::jsonb
+        WHERE id = %s
+        """,
+        (
+            phone,
+            website,
+            place_id,
+            plat,
+            plon,
+            json.dumps({"google_place_id": place_id, "google_enriched": True}),
+            entity_id,
+        ),
+    )
+    if phone:
+        _upsert_contact(entity_id, None, phone, "google_places", 72, role="sales", name="Google Places")
+    return bool(phone or website)
 
-    pid = place.get("place_id")
+
+def _upsert_place(place: dict[str, Any], details: dict[str, Any], country_iso2: str, territory_code: str, query: str) -> tuple[str, int]:
+    from app.db import execute_returning
+
+    pid = place.get("place_id") or details.get("place_id")
+    if not pid:
+        return "skipped", 0
     name = details.get("name") or place.get("name") or "Unknown"
     geo = details.get("geometry", {}).get("location", place.get("geometry", {}).get("location", {}))
     plat = geo.get("lat")
     plon = geo.get("lng")
     phone = details.get("formatted_phone_number") or details.get("international_phone_number")
     website = details.get("website")
-    entity_type = "contract_caterer" if "catering" in query else "distributor"
+    entity_type = "contract_caterer" if "catering" in query else "wholesaler"
 
     existing = fetch_one(
-        "SELECT id FROM export_market_entities WHERE metadata->>'google_place_id' = %s",
+        "SELECT id FROM export_market_entities WHERE google_place_id = %s",
         (pid,),
     )
     if existing:
-        execute(
-            """
-            UPDATE export_market_entities SET
-                phone = COALESCE(%s, phone), website = COALESCE(%s, website),
-                lat = COALESCE(%s, lat), lon = COALESCE(%s, lon),
-                confidence = GREATEST(confidence, 65), updated_at = NOW()
-            WHERE id = %s
-            """,
-            (phone, website, plat, plon, existing["id"]),
-        )
-        entity_id = existing["id"]
-        action = "updated"
-    else:
-        row = execute_returning(
-            """
-            INSERT INTO export_market_entities (
-                name, entity_type, country_iso2, territory_code, lat, lon,
-                phone, website, google_place_id, source, sources, confidence, metadata
-            ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'google_places','["google_places"]'::jsonb,65,%s)
-            RETURNING id
-            """,
-            (
-                name[:255], entity_type, country_iso2.upper(), territory_code,
-                plat, plon, phone, website, pid,
-                json.dumps({"google_place_id": pid}),
-            ),
-        )
-        entity_id = row["id"]
-        action = "inserted"
+        _apply_place_to_entity(int(existing["id"]), pid, details)
+        return "updated", int(existing["id"])
+
+    row = execute_returning(
+        """
+        INSERT INTO export_market_entities (
+            name, entity_type, country_iso2, territory_code, lat, lon,
+            phone, website, google_place_id, source, sources, confidence, metadata
+        ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'google_places','["google_places"]'::jsonb,65,%s)
+        RETURNING id
+        """,
+        (
+            name[:255],
+            entity_type,
+            country_iso2.upper(),
+            territory_code,
+            plat,
+            plon,
+            phone,
+            website,
+            pid,
+            json.dumps({"google_place_id": pid}),
+        ),
+    )
+    entity_id = int(row["id"])
     if phone:
         _upsert_contact(entity_id, None, phone, "google_places", 70, role="sales", name="Google Places")
-    return action, entity_id
+    return "inserted", entity_id
+
+
+def enrich_entities_places_country(country_iso2: str, limit: int = 400) -> dict[str, Any]:
+    """Match existing OSM entities to Google Places for phone/website."""
+    if not _api_key():
+        return {"skipped": True, "reason": "GOOGLE_MAPS_API_KEY not set"}
+
+    entities = fetch_all(
+        """
+        SELECT id, name, lat, lon, city, phone, website
+        FROM export_market_entities
+        WHERE country_iso2 = %s
+          AND lat IS NOT NULL AND lon IS NOT NULL
+          AND (phone IS NULL OR phone = '')
+          AND (google_place_id IS NULL OR google_place_id = '')
+        ORDER BY confidence DESC NULLS LAST, id
+        LIMIT %s
+        """,
+        (country_iso2.upper(), limit),
+    )
+    enriched = 0
+    not_found = 0
+    errors = 0
+    for ent in entities:
+        query = ent["name"] or ""
+        if ent.get("city"):
+            query = f"{query} {ent['city']}"
+        try:
+            pid = _find_place_id(query, float(ent["lat"]), float(ent["lon"]))
+            if not pid:
+                not_found += 1
+                time.sleep(0.12)
+                continue
+            details = _place_details(pid)
+            if _apply_place_to_entity(int(ent["id"]), pid, details):
+                enriched += 1
+        except Exception:
+            errors += 1
+        time.sleep(0.15)
+    return {
+        "country": country_iso2.upper(),
+        "candidates": len(entities),
+        "enriched": enriched,
+        "not_found": not_found,
+        "errors": errors,
+    }
 
 
 def sync_places_country(country_iso2: str) -> dict[str, Any]:
@@ -165,12 +333,19 @@ def sync_places_country(country_iso2: str) -> dict[str, Any]:
                 action, _ = _upsert_place(place, details, iso, territory_code, q)
                 if action == "inserted":
                     inserted += 1
-                else:
+                elif action == "updated":
                     updated += 1
                 time.sleep(0.15)
             time.sleep(0.5)
 
-    return {"inserted": inserted, "updated": updated, "queries": len(QUERIES), "search_points": len(search_points)}
+    enrich = enrich_entities_places_country(iso, limit=500)
+    return {
+        "inserted": inserted,
+        "updated": updated,
+        "queries": len(QUERIES),
+        "search_points": len(search_points),
+        "enrich_existing": enrich,
+    }
 
 
 def sync_places_countries(countries: list[str]) -> dict[str, Any]: