"""Halal certification registry sync and supermarket matching.""" from __future__ import annotations import json import re import urllib.parse import urllib.request from typing import Any, Optional from app.db import execute, execute_returning, fetch_all, fetch_one, json_param OVERPASS_URL = "https://overpass.kumi.systems/api/interpreter" # Known halal-friendly retail brands (indicative — verified via certifier when possible) HALAL_FRIENDLY_CHAINS = { "Spar": {"has_halal_section": True, "note": "chain policy varies by franchise"}, "Ekoplaza": {"halal_certified": False, "has_halal_section": True}, } def _fetch_overpass(query: str) -> list[dict[str, Any]]: data = urllib.parse.urlencode({"data": query}).encode() req = urllib.request.Request(OVERPASS_URL, data=data, method="POST") with urllib.request.urlopen(req, timeout=120) as resp: payload = json.loads(resp.read().decode()) return payload.get("elements", []) def sync_osm_halal_tags() -> dict[str, Any]: """Mark supermarkets with OSM diet:halal=yes and import certification records.""" query = ( '[out:json][timeout:120];area["ISO3166-1"="NL"]->.nl;' '(node["shop"="supermarket"]["diet:halal"="yes"](area.nl);' 'way["shop"="supermarket"]["diet:halal"="yes"](area.nl););out tags center;' ) elements = _fetch_overpass(query) matched = created = 0 for el in elements: tags = el.get("tags") or {} external_id = f"osm:{el.get('type')}:{el.get('id')}" store = fetch_one("SELECT id, name FROM supermarkets WHERE external_id = %s", (external_id,)) if not store: name = tags.get("name") or tags.get("brand") or "Unknown" store = fetch_one( "SELECT id, name FROM supermarkets WHERE name ILIKE %s LIMIT 1", (f"%{name[:40]}%",), ) if not store: continue matched += 1 execute( """UPDATE supermarkets SET halal_certified = TRUE, has_halal_section = TRUE, halal_certifier = COALESCE(halal_certifier, 'OSM diet:halal'), last_updated = NOW() WHERE id = %s""", (store["id"],), ) existing = fetch_one( "SELECT id FROM halal_certifications WHERE supermarket_id = %s AND registry_source = 'osm'", (store["id"],), ) if not existing: execute_returning( """INSERT INTO halal_certifications ( supermarket_id, certifier, business_name, status, registry_source, matched_confidence, raw_data ) VALUES (%s, 'OSM', %s, 'active', 'osm', 0.85, %s) RETURNING id""", (store["id"], store["name"], json_param(tags)), ) created += 1 return {"osm_halal_elements": len(elements), "stores_matched": matched, "certs_created": created} def sync_osm_contact_tags(limit: int = 500) -> dict[str, Any]: """Pull phone/email/website/operator from OSM for existing stores.""" stores = fetch_all( """SELECT id, external_id, phone, email, website, manager_name FROM supermarkets WHERE external_id LIKE %s AND (phone IS NULL OR email IS NULL OR manager_name IS NULL) LIMIT %s""", ("osm:%", limit), ) updated = contacts = 0 for store in stores: parts = (store.get("external_id") or "").split(":") if len(parts) != 3: continue osm_type, osm_id = parts[1], parts[2] query = f'[out:json][timeout:30];{osm_type}({osm_id});out tags;' try: elements = _fetch_overpass(query) except Exception: continue if not elements: continue tags = elements[0].get("tags") or {} phone = tags.get("phone") or tags.get("contact:phone") email = tags.get("email") or tags.get("contact:email") website = tags.get("website") or tags.get("contact:website") operator = tags.get("operator") or tags.get("contact:name") manager = tags.get("manager") or tags.get("contact:manager") or operator sets, params = [], [] if phone and not store.get("phone"): sets.append("phone = %s"); params.append(str(phone)[:20]) if email and not store.get("email"): sets.append("email = %s"); params.append(str(email)[:255]) if website and not store.get("website"): sets.append("website = %s"); params.append(str(website)[:255]) if manager and not store.get("manager_name"): sets.append("manager_name = %s"); params.append(str(manager)[:255]) if sets: params.append(store["id"]) execute(f"UPDATE supermarkets SET {', '.join(sets)}, last_updated = NOW() WHERE id = %s", tuple(params)) updated += 1 if manager or phone or email: existing = fetch_one( "SELECT id FROM supermarket_contacts WHERE supermarket_id = %s AND source = 'osm' LIMIT 1", (store["id"],), ) if not existing: execute( """INSERT INTO supermarket_contacts ( supermarket_id, role, full_name, phone, email, source, confidence ) VALUES (%s, 'manager', %s, %s, %s, 'osm', 0.6)""", (store["id"], manager, phone, email), ) contacts += 1 execute( """INSERT INTO supermarket_profiles (supermarket_id, manager_name, manager_phone, manager_email, web_data, last_scraped_at, data_completeness) VALUES (%s,%s,%s,%s,%s,NOW(),0.4) ON CONFLICT (supermarket_id) DO UPDATE SET manager_name = COALESCE(EXCLUDED.manager_name, supermarket_profiles.manager_name), manager_phone = COALESCE(EXCLUDED.manager_phone, supermarket_profiles.manager_phone), manager_email = COALESCE(EXCLUDED.manager_email, supermarket_profiles.manager_email), web_data = supermarket_profiles.web_data || EXCLUDED.web_data, last_scraped_at = NOW()""", (store["id"], manager, phone, email, json_param({"osm_tags": tags})), ) return {"scanned": len(stores), "stores_updated": updated, "contacts_added": contacts} def list_halal_certified(limit: int = 500) -> list[dict[str, Any]]: return fetch_all( """ SELECT s.*, h.certifier, h.certificate_number, h.expiry_date, h.registry_source, h.matched_confidence FROM supermarkets s LEFT JOIN halal_certifications h ON h.supermarket_id = s.id AND h.status = 'active' WHERE s.halal_certified = TRUE OR s.has_halal_section = TRUE OR h.id IS NOT NULL ORDER BY s.chain, s.city LIMIT %s """, (limit,), )