food-cockpit/foodlinkk-command-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SysOps: manual — 2026-07-30 09:33 UTC

Browse Source
This commit is contained in:
sysops
2026-07-30 09:33:03 +00:00
parent f21d58483b
commit 5dd69a1138
25 changed files with 4148 additions and 734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools-api/app/export_intel.py
+462 -9
View File
@@ -6,6 +6,7 @@ import io
import hashlib
import json
import time
from collections import defaultdict
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, Query
@@ -38,6 +39,241 @@ def _halal_reason_labels(reasons: list[str]) -> list[str]:
return out
def _normalize_url(url: Any) -> Optional[str]:
if url is None:
return None
s = str(url).strip()
if not s or s.lower() in ("null", "none", "-", "—"):
return None
if s.startswith("mailto:") or s.startswith("tel:"):
return s
if not s.startswith(("http://", "https://")):
s = "https://" + s.lstrip("/")
return s
def _display_url(url: str) -> str:
return url.replace("https://", "").replace("http://", "").rstrip("/")[:96]
def _collect_websites(
row: dict[str, Any],
contacts: Optional[list[dict[str, Any]]] = None,
brand_website: Optional[str] = None,
) -> list[dict[str, str]]:
seen: set[str] = set()
out: list[dict[str, str]] = []
def add(label: str, raw: Any) -> None:
href = _normalize_url(raw)
if not href:
return
key = href.lower().rstrip("/")
if key in seen:
return
seen.add(key)
out.append({
"label": (label or "Website")[:64],
"url": href,
"display": _display_url(href),
})
add("Organisatie", row.get("website"))
add("Brand", brand_website)
for c in contacts or []:
role = (c.get("role") or c.get("name") or "Contact").strip()
add(role, c.get("website_contact_url"))
if c.get("linkedin_url"):
add("LinkedIn", c.get("linkedin_url"))
src = c.get("source_url")
if src and "linkedin.com" not in str(src).lower():
add("Bron", src)
return out
def _clean_str(val: Any) -> Optional[str]:
if val is None:
return None
s = str(val).strip()
if not s or s.lower() in ("null", "none", "-", "—", "n/a"):
return None
return s
def _dedupe_items(items: list[dict[str, str]], key: str = "value") -> list[dict[str, str]]:
seen: set[str] = set()
out: list[dict[str, str]] = []
for item in items:
val = _clean_str(item.get(key))
if not val:
continue
k = val.lower()
if k in seen:
continue
seen.add(k)
out.append({**item, key: val})
return out
def _collect_emails(row: dict[str, Any], contacts: Optional[list[dict[str, Any]]] = None) -> list[dict[str, str]]:
items: list[dict[str, str]] = []
def add(label: str, raw: Any) -> None:
val = _clean_str(raw)
if not val or "@" not in val:
return
items.append({"label": (label or "E-mail")[:64], "value": val, "href": f"mailto:{val}"})
add("Organisatie", row.get("email"))
for c in contacts or []:
who = _clean_str(c.get("name")) or _clean_str(c.get("role")) or "Contact"
add(who, c.get("email"))
alt = _clean_str(c.get("email_alt"))
if alt and alt != _clean_str(c.get("email")):
add(f"{who} (alt)", alt)
return _dedupe_items(items)
def _collect_phones(row: dict[str, Any], contacts: Optional[list[dict[str, Any]]] = None) -> list[dict[str, str]]:
items: list[dict[str, str]] = []
def add(label: str, raw: Any, kind: str = "tel") -> None:
val = _clean_str(raw)
if not val:
return
digits = "".join(ch for ch in val if ch.isdigit() or ch == "+")
href = f"https://wa.me/{digits.lstrip('+')}" if kind == "wa" and digits else f"tel:{val}"
items.append({"label": (label or "Telefoon")[:64], "value": val, "href": href, "kind": kind})
add("Organisatie", row.get("phone"))
for c in contacts or []:
who = _clean_str(c.get("name")) or _clean_str(c.get("role")) or "Contact"
add(who, c.get("phone"))
add(f"{who} mobiel", c.get("mobile"))
add(f"{who} WhatsApp", c.get("whatsapp"), "wa")
add(f"{who} fax", c.get("fax"))
return _dedupe_items(items)
def _collect_addresses(row: dict[str, Any], contacts: Optional[list[dict[str, Any]]] = None) -> list[dict[str, str]]:
items: list[dict[str, str]] = []
def add(label: str, line: Any, city: Any = None, postal: Any = None) -> None:
parts = [_clean_str(line), _clean_str(postal), _clean_str(city)]
parts = [p for p in parts if p]
if not parts:
return
val = ", ".join(parts)
items.append({"label": (label or "Adres")[:64], "value": val})
add("Locatie", row.get("address_line"), row.get("city"), row.get("postal_code"))
for c in contacts or []:
who = _clean_str(c.get("name")) or _clean_str(c.get("role")) or "Contact"
add(who, c.get("address_line"), c.get("city"), c.get("postal_code"))
return _dedupe_items(items)
def _collect_social(row: dict[str, Any], contacts: Optional[list[dict[str, Any]]] = None) -> list[dict[str, str]]:
items: list[dict[str, str]] = []
def add(label: str, raw: Any) -> None:
href = _normalize_url(raw)
if not href:
return
items.append({"label": label[:64], "value": _display_url(href), "href": href})
for c in contacts or []:
who = _clean_str(c.get("name")) or _clean_str(c.get("role")) or "Contact"
if c.get("linkedin_url"):
add(f"LinkedIn · {who}", c.get("linkedin_url"))
return _dedupe_items(items, key="href")
def _public_contacts(contacts: Optional[list[dict[str, Any]]], limit: int = 8) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for c in (contacts or [])[:limit]:
out.append({
"name": _clean_str(c.get("name")),
"role": _clean_str(c.get("role")),
"title": _clean_str(c.get("title")),
"department": _clean_str(c.get("department")),
"email": _clean_str(c.get("email")),
"email_alt": _clean_str(c.get("email_alt")),
"phone": _clean_str(c.get("phone")),
"mobile": _clean_str(c.get("mobile")),
"whatsapp": _clean_str(c.get("whatsapp")),
"fax": _clean_str(c.get("fax")),
"linkedin_url": _normalize_url(c.get("linkedin_url")),
"website": _normalize_url(c.get("website_contact_url")),
"is_primary": bool(c.get("is_primary")),
"source": _clean_str(c.get("source")),
})
return out
def _build_reach(
row: dict[str, Any],
contacts: Optional[list[dict[str, Any]]] = None,
brand_website: Optional[str] = None,
) -> dict[str, Any]:
lat, lon = row.get("lat"), row.get("lon")
maps_url = row.get("maps_url")
if not maps_url and lat is not None and lon is not None:
maps_url = f"https://www.google.com/maps?q={lat},{lon}"
place_id = _clean_str(row.get("google_place_id"))
if place_id and not maps_url:
maps_url = f"https://www.google.com/maps/search/?api=1&query_place_id={place_id}"
reach = {
"emails": _collect_emails(row, contacts),
"phones": _collect_phones(row, contacts),
"websites": _collect_websites(row, contacts, brand_website),
"addresses": _collect_addresses(row, contacts),
"social": _collect_social(row, contacts),
"maps_url": maps_url,
"google_place_id": place_id,
"postal_code": _clean_str(row.get("postal_code")),
"contact_people": _public_contacts(contacts),
}
reach["has_any"] = bool(
reach["emails"] or reach["phones"] or reach["websites"]
or reach["addresses"] or reach["social"] or reach["maps_url"]
)
return reach
def _map_contacts_batch(entity_ids: list[int]) -> dict[int, list[dict[str, Any]]]:
if not entity_ids:
return {}
rows = fetch_all(
"""
SELECT entity_id, role, name, title, department, email, email_alt,
phone, mobile, whatsapp, fax, linkedin_url, website_contact_url,
source_url, address_line, city, postal_code, is_primary, confidence, source
FROM export_entity_contacts
WHERE entity_id = ANY(%s)
ORDER BY is_primary DESC NULLS LAST, confidence DESC NULLS LAST, id
""",
(entity_ids,),
)
grouped: dict[int, list[dict[str, Any]]] = defaultdict(list)
for row in rows:
grouped[int(row["entity_id"])].append(row)
return grouped
def _brand_websites(codes: list[str]) -> dict[str, Optional[str]]:
if not codes:
return {}
rows = fetch_all(
"SELECT code, website FROM export_caterer_brands WHERE code = ANY(%s)",
(codes,),
)
return {r["code"]: r.get("website") for r in rows}
def _enrich_entity(row: dict[str, Any], contacts: list[dict[str, Any]]) -> dict[str, Any]:
row["contacts"] = contacts
row["contact_count"] = len(contacts)
@@ -50,6 +286,15 @@ def _enrich_entity(row: dict[str, Any], contacts: list[dict[str, Any]]) -> dict[
row["phone"] = phone
primary = contacts[0] if contacts else None
row["primary_contact"] = primary
brand_website = row.pop("brand_website", None)
row["reach"] = _build_reach(row, contacts, brand_website)
row["websites"] = row["reach"]["websites"]
if row["websites"] and not row.get("website"):
row["website"] = row["websites"][0]["url"]
if row["reach"]["emails"] and not row.get("email"):
row["email"] = row["reach"]["emails"][0]["value"]
if row["reach"]["phones"] and not row.get("phone"):
row["phone"] = row["reach"]["phones"][0]["value"]
h_score, h_reasons = score_entity(row)
row["halal_score"] = h_score
row["halal_reasons"] = h_reasons
@@ -87,7 +332,7 @@ def _map_entity_limit(
crm_linked: Optional[bool],
) -> int:
if q or entity_type or entity_types or halal_min is not None or favorite_only or crm_linked is not None:
return 4000
return 8000
if country:
return 12000
if region:
@@ -118,6 +363,49 @@ DISTRIBUTOR_TYPES = ("distributor", "wholesaler", "importer", "logistics", "cold
CUSTOMER_TYPES = ("restaurant", "doner_shoarma", "butcher", "foodservice")
CATERER_TYPES = ("contract_caterer")
TYPE_FILTER_ALIASES: dict[str, tuple[str, ...]] = {
"b2b": DISTRIBUTOR_TYPES,
"distributor": DISTRIBUTOR_TYPES,
"importer": DISTRIBUTOR_TYPES,
"logistics": DISTRIBUTOR_TYPES,
"horeca": CUSTOMER_TYPES,
"restaurant": CUSTOMER_TYPES,
"doner_shoarma": CUSTOMER_TYPES,
"butcher": CUSTOMER_TYPES,
"foodservice": CUSTOMER_TYPES,
"caterer": CATERER_TYPES,
"contract_caterer": CATERER_TYPES,
}
def _expand_entity_types(
entity_type: Optional[str],
entity_types: Optional[str],
) -> tuple[Optional[str], Optional[str]]:
expanded: list[str] = []
if entity_types:
for raw in entity_types.split(","):
key = raw.strip()
if not key:
continue
alias = TYPE_FILTER_ALIASES.get(key)
expanded.extend(alias if alias else (key,))
if entity_type:
key = entity_type.strip()
alias = TYPE_FILTER_ALIASES.get(key)
expanded.extend(alias if alias else (key,))
if not expanded:
return entity_type, entity_types
deduped: list[str] = []
seen: set[str] = set()
for key in expanded:
if key not in seen:
seen.add(key)
deduped.append(key)
if len(deduped) == 1:
return deduped[0], None
return None, ",".join(deduped)
class ContactCreateIn(BaseModel):
role: str = "general"
@@ -186,6 +474,7 @@ def _entity_filters(
clauses.append("t.region_code = %s")
params.append(region)
clauses.append("e.territory_code = t.code")
entity_type, entity_types = _expand_entity_types(entity_type, entity_types)
if entity_type:
clauses.append("e.entity_type = %s")
params.append(entity_type)
@@ -228,7 +517,24 @@ def export_stats(
return int(row["n"] or 0)
entities_n = count("")
distributors_n = count("entity_type IN ('distributor','wholesaler','importer','logistics')")
type_geo_rows = fetch_all(
"""
SELECT entity_type, COUNT(*) AS n
FROM export_market_entities
WHERE lat IS NOT NULL AND lon IS NOT NULL
GROUP BY entity_type
"""
)
entity_types_geo = {r["entity_type"]: int(r["n"] or 0) for r in type_geo_rows}
country_count_rows = fetch_all(
"""
SELECT country_iso2, COUNT(*) AS n
FROM export_market_entities
GROUP BY country_iso2
"""
)
country_entity_counts = {r["country_iso2"]: int(r["n"] or 0) for r in country_count_rows}
distributors_n = count("entity_type IN ('distributor','wholesaler','importer','logistics','cold_storage','port_agent')")
caterers_n = count("entity_type = 'contract_caterer'")
contacts = fetch_one("SELECT COUNT(*) AS n FROM export_entity_contacts")
with_email = fetch_one(
@@ -259,6 +565,8 @@ def export_stats(
"crm_linked": int(crm_row["n"] or 0) if crm_row else 0,
"phase": "B",
"enrichment_status": "active",
"entity_types_geo": entity_types_geo,
"country_entity_counts": country_entity_counts,
}
@@ -267,16 +575,31 @@ def list_regions() -> list[dict[str, Any]]:
return fetch_all("SELECT * FROM export_regions ORDER BY sort_order, code")
def _dedupe_territories(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
seen: set[str] = set()
out: list[dict[str, Any]] = []
for row in rows:
iso = row.get("country_iso2")
if iso and iso in seen:
continue
if iso:
seen.add(iso)
out.append(row)
return out
@router.get("/territories")
def list_territories(region: Optional[str] = Query(None)) -> list[dict[str, Any]]:
if region:
return fetch_all(
rows = fetch_all(
"SELECT * FROM export_territories WHERE is_active = TRUE AND region_code = %s ORDER BY sync_priority, name_nl",
(region,),
)
return fetch_all(
"SELECT * FROM export_territories WHERE is_active = TRUE ORDER BY sync_priority, name_nl"
)
else:
rows = fetch_all(
"SELECT * FROM export_territories WHERE is_active = TRUE ORDER BY sync_priority, name_nl"
)
return _dedupe_territories(rows)
@router.get("/entities")
@@ -328,10 +651,12 @@ def list_entities(
def get_entity(entity_id: int) -> dict[str, Any]:
row = fetch_one(
"""
SELECT e.*, cl.name AS crm_client_name, d.title AS crm_deal_title, d.stage AS crm_deal_stage
SELECT e.*, cl.name AS crm_client_name, d.title AS crm_deal_title, d.stage AS crm_deal_stage,
b.website AS brand_website
FROM export_market_entities e
LEFT JOIN clients cl ON cl.id = e.client_id
LEFT JOIN deals d ON d.id = e.deal_id
LEFT JOIN export_caterer_brands b ON b.code = e.brand_code
WHERE e.id = %s
""",
(entity_id,),
@@ -614,9 +939,9 @@ def map_bundle(
entities = fetch_all(
f"""
SELECT e.id, e.name, e.entity_type, e.country_iso2, e.lat, e.lon,
e.confidence, e.pipeline_stage, e.city, e.address_line,
e.confidence, e.pipeline_stage, e.city, e.address_line, e.postal_code,
e.email, e.phone, e.website, e.volume_band, e.source,
e.is_favorite, e.client_id, e.deal_id,
e.brand_code, e.is_favorite, e.client_id, e.deal_id,
cp.contact_email, cp.contact_phone,
COALESCE(cc.contact_count, 0) AS contact_count
FROM export_market_entities e
@@ -641,6 +966,11 @@ def map_bundle(
(tuple(params) + (row_limit,)) if params else (row_limit,),
)
entity_ids = [int(r["id"]) for r in entities]
contacts_by_entity = _map_contacts_batch(entity_ids)
brand_codes = list({r["brand_code"] for r in entities if r.get("brand_code")})
brands_by_code = _brand_websites(brand_codes)
scored: list[tuple[dict[str, Any], float, list[str]]] = []
halal_by_country: dict[str, dict[str, Any]] = {}
for row in entities:
@@ -678,8 +1008,23 @@ def map_bundle(
"entity_type": row["entity_type"],
"country_iso2": row["country_iso2"],
"city": row.get("city"),
"address_line": row.get("address_line"),
"email": row.get("email") or row.get("contact_email"),
"phone": row.get("phone") or row.get("contact_phone"),
"website": row.get("website"),
"reach": _build_reach(
row,
contacts_by_entity.get(int(row["id"]), []),
brands_by_code.get(row.get("brand_code")),
),
"websites": _build_reach(
row,
contacts_by_entity.get(int(row["id"]), []),
brands_by_code.get(row.get("brand_code")),
)["websites"],
"volume_band": row.get("volume_band"),
"confidence": row.get("confidence"),
"pipeline_stage": row.get("pipeline_stage"),
"contact_count": int(row.get("contact_count") or 0),
"halal_score": h_score,
"halal_tier": halal_pin_tier(h_score),
@@ -971,6 +1316,114 @@ def api_sync_all(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
raise HTTPException(500, str(exc)) from exc
@router.post("/sync/missing")
def api_sync_missing(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
try:
from app.export_intel_sync import sync_missing_countries
result = sync_missing_countries(max_priority=payload.max_priority or 3)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
raise HTTPException(500, str(exc)) from exc
class ExportIntelSettingsIn(BaseModel):
google_maps_api_key: Optional[str] = None
@router.get("/settings")
def export_intel_settings_get() -> dict[str, Any]:
from app.export_intel_places import google_api_key_status
return google_api_key_status()
@router.put("/settings")
def export_intel_settings_put(payload: ExportIntelSettingsIn) -> dict[str, Any]:
from app.export_intel_places import google_api_key_status, set_google_api_key
if payload.google_maps_api_key is not None:
set_google_api_key(payload.google_maps_api_key)
return google_api_key_status()
@router.post("/sync/places")
def api_sync_places(payload: SyncRequestIn) -> dict[str, Any]:
from app.export_intel_places import enrich_entities_places_country, sync_places_country
from app.export_intel_sync import _countries_for_sync
from app.export_intel_sync_progress import finish_job, job_step, start_job, set_active_job
if payload.country_iso2:
job_id = start_job("places", f"Google Places · {payload.country_iso2.upper()}", total=1)
try:
job_step(job_id, message=f"Google Places {payload.country_iso2.upper()}", current=1, total=1)
result = sync_places_country(payload.country_iso2.upper())
finish_job(job_id, result=result)
return {"status": "completed", "phase": "B", "result": result}
except Exception as exc:
finish_job(job_id, error=str(exc))
raise HTTPException(500, str(exc)) from exc
finally:
set_active_job(None)
countries = _countries_for_sync(region_code=payload.region_code, max_priority=payload.max_priority or 3)
if not countries:
raise HTTPException(400, "country_iso2 or region_code required")
job_id = start_job("places", f"Google Places · {len(countries)} landen", total=len(countries))
results: dict[str, Any] = {}
try:
for i, c in enumerate(countries, start=1):
job_step(job_id, message=f"Google Places {i}/{len(countries)}: {c}", country=c, current=i, total=len(countries))
results[c] = sync_places_country(c)
time.sleep(1)
finish_job(job_id, result={"countries": len(countries)})
except Exception as exc:
finish_job(job_id, error=str(exc))
raise HTTPException(500, str(exc)) from exc
finally:
set_active_job(None)
return {"status": "completed", "phase": "B", "result": results}
@router.get("/sync/status")
def sync_status() -> dict[str, Any]:
from app.export_intel_sync_progress import get_monitor_snapshot
total_row = fetch_one("SELECT COUNT(*) AS n FROM export_market_entities")
terr_row = fetch_one("SELECT COUNT(*) AS n FROM export_territories WHERE is_active = TRUE")
with_data = fetch_one(
"""
SELECT COUNT(DISTINCT country_iso2) AS n FROM export_market_entities
WHERE country_iso2 IS NOT NULL
"""
)
zero_rows = fetch_all(
"""
SELECT t.country_iso2, t.name_nl, t.region_code
FROM export_territories t
WHERE t.is_active
AND NOT EXISTS (
SELECT 1 FROM export_market_entities e WHERE e.country_iso2 = t.country_iso2
)
ORDER BY t.region_code, t.name_nl
"""
)
type_rows = fetch_all(
"""
SELECT entity_type, COUNT(*) AS n FROM export_market_entities GROUP BY entity_type ORDER BY n DESC
"""
)
coverage = {
"entities_total": int(total_row["n"] or 0) if total_row else 0,
"territories_total": int(terr_row["n"] or 0) if terr_row else 0,
"countries_with_data": int(with_data["n"] or 0) if with_data else 0,
"countries_empty": len(zero_rows),
"countries_empty_list": zero_rows[:30],
"entity_types": {r["entity_type"]: int(r["n"] or 0) for r in type_rows},
}
snap = get_monitor_snapshot(coverage)
return snap
@router.post("/sync/world")
def api_sync_world(payload: SyncRequestIn = SyncRequestIn()) -> dict[str, Any]:
"""Sync Europe, Middle East, Africa, Americas — long running."""