Platform bundle: marketing publish, IT ops, packaging, agents mesh.
Volledige Foodlinkk Command Center uitbreiding met social automatisering, reclamefolder filters, Proxmox monitoring en documentatie.
This commit is contained in:
Aissa
@@ -0,0 +1,142 @@
|
||||
"""CBS Open Data — gemeente demografie via OData."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
|
||||
CBS_BASE = "https://opendata.cbs.nl/ODataApi/OData"
|
||||
_GEMEENTE_CACHE: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _int_val(raw: Any) -> Optional[int]:
|
||||
if raw is None:
|
||||
return None
|
||||
s = str(raw).strip().replace(".", "")
|
||||
if not s or s == ".":
|
||||
return None
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _float_val(raw: Any) -> Optional[float]:
|
||||
if raw is None:
|
||||
return None
|
||||
s = str(raw).strip()
|
||||
if not s or s == ".":
|
||||
return None
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_untyped(dataset: str, filter_expr: str, top: int = 1) -> list[dict[str, Any]]:
|
||||
params = urllib.parse.urlencode(
|
||||
{"$filter": filter_expr, "$top": str(top), "$format": "json"},
|
||||
quote_via=urllib.parse.quote,
|
||||
)
|
||||
url = f"{CBS_BASE}/{dataset}/UntypedDataSet?{params}"
|
||||
with urllib.request.urlopen(url, timeout=45) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
return data.get("value", [])
|
||||
|
||||
|
||||
def _normalize_gm(code: str) -> str:
|
||||
code = (code or "").strip().upper()
|
||||
if code.startswith("GM"):
|
||||
return code
|
||||
digits = re.sub(r"\D", "", code)
|
||||
return f"GM{digits}" if digits else code
|
||||
|
||||
|
||||
def fetch_gemeente_stats(gemeente_code: str) -> Optional[dict[str, Any]]:
|
||||
gm = _normalize_gm(gemeente_code)
|
||||
if not gm:
|
||||
return None
|
||||
if gm in _GEMEENTE_CACHE:
|
||||
return _GEMEENTE_CACHE[gm]
|
||||
|
||||
pop_rows = _fetch_untyped(
|
||||
"03759ned",
|
||||
f"RegioS eq '{gm}' and Leeftijd eq '10000' and Geslacht eq 'T001038' "
|
||||
f"and BurgerlijkeStaat eq 'T001019' and substringof('2024',Perioden)",
|
||||
)
|
||||
income_rows = _fetch_untyped(
|
||||
"86005NED",
|
||||
f"RegioS eq '{gm}' and substringof('2023',Perioden) and Geslacht eq 'T001038'",
|
||||
)
|
||||
area_rows = _fetch_untyped(
|
||||
"84583NED",
|
||||
f"startswith(WijkenEnBuurten,'{gm}') and SoortRegio_2 eq 'Gemeente '",
|
||||
)
|
||||
|
||||
population = _int_val(pop_rows[0].get("BevolkingOp1Januari_1")) if pop_rows else None
|
||||
avg_income = None
|
||||
median_income = None
|
||||
if income_rows:
|
||||
avg_income = _float_val(income_rows[0].get("GemiddeldPersoonlijkInkomen_6"))
|
||||
median_income = _float_val(income_rows[0].get("MediaanPersoonlijkInkomen_7"))
|
||||
if avg_income:
|
||||
avg_income *= 1000
|
||||
if median_income:
|
||||
median_income *= 1000
|
||||
|
||||
area = area_rows[0] if area_rows else {}
|
||||
pop_area = _int_val(area.get("AantalInwoners_5")) or population
|
||||
households = _int_val(area.get("HuishoudensTotaal_28"))
|
||||
niet_westers = _int_val(area.get("NietWestersTotaal_18"))
|
||||
marokko = _int_val(area.get("Marokko_19"))
|
||||
turkije = _int_val(area.get("Turkije_22"))
|
||||
suriname = _int_val(area.get("Suriname_21"))
|
||||
avg_hh_size = _float_val(area.get("GemiddeldeHuishoudensgrootte_32"))
|
||||
income_per_inhabitant = _float_val(area.get("GemiddeldInkomenPerInwoner_72"))
|
||||
if income_per_inhabitant and not avg_income:
|
||||
avg_income = income_per_inhabitant * 1000
|
||||
|
||||
muslim_proxy_pct = None
|
||||
niet_westers_pct = None
|
||||
if pop_area and pop_area > 0:
|
||||
if marokko is not None and turkije is not None:
|
||||
muslim_proxy_pct = round((marokko + turkije) / pop_area * 100, 2)
|
||||
if niet_westers is not None:
|
||||
niet_westers_pct = round(niet_westers / pop_area * 100, 2)
|
||||
|
||||
stats = {
|
||||
"gemeente_code": gm,
|
||||
"population": pop_area,
|
||||
"households": households,
|
||||
"avg_household_size": avg_hh_size,
|
||||
"avg_income": avg_income,
|
||||
"median_income": median_income,
|
||||
"unemployment_rate": None,
|
||||
"ethnic_composition": {
|
||||
"niet_westers_totaal": niet_westers,
|
||||
"niet_westers_pct": niet_westers_pct,
|
||||
"marokko": marokko,
|
||||
"turkije": turkije,
|
||||
"suriname": suriname,
|
||||
},
|
||||
"religious_composition": {
|
||||
"muslim_proxy_pct": muslim_proxy_pct,
|
||||
"note": "Indicatief: Marokko+Turkije / bevolking (CBS Kerncijfers wijken en buurten)",
|
||||
},
|
||||
"education_level": {
|
||||
"laag": _int_val(area.get("OpleidingsniveauLaag_64")),
|
||||
"middelbaar": _int_val(area.get("OpleidingsniveauMiddelbaar_65")),
|
||||
"hoog": _int_val(area.get("OpleidingsniveauHoog_66")),
|
||||
},
|
||||
"housing_type": {
|
||||
"koop_pct": _float_val(area.get("Koopwoningen_40")),
|
||||
"huur_pct": _float_val(area.get("HuurwoningenTotaal_41")),
|
||||
},
|
||||
"car_ownership": _float_val(area.get("PersonenautoSPerHuishouden_102")),
|
||||
"data_granularity": "gemeente",
|
||||
"data_source": "cbs+pdok",
|
||||
}
|
||||
_GEMEENTE_CACHE[gm] = stats
|
||||
return stats
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Halal vlees trends, top gerechten en food concept seeds."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db import fetch_all
|
||||
|
||||
HALAL_MEAT_KEYWORDS = (
|
||||
"halal vlees", "halal meat", "halal kip", "halal lam", "halal rund",
|
||||
"halal gehakt", "halal chicken", "halal beef", "halal slacht",
|
||||
"halal certific", "vleesvervanger halal",
|
||||
)
|
||||
|
||||
TOP_DISH_KEYWORDS = (
|
||||
"kant-en-klaar", "kant en klaar", "ready meal", "maaltijd", "gerecht",
|
||||
"curry", "stamppot", "biryani", "tagine", "lasagne", "schotel",
|
||||
"meal prep", "microwave meal", "diepvries maaltijd",
|
||||
)
|
||||
|
||||
CONCEPT_SEEDS = [
|
||||
"Halal {dish} single-serve voor {chain} schappen in regio's met score >{score}",
|
||||
"Premium halal {meat} maaltijdlijn — inspelen op trend: {trend}",
|
||||
"Seizoens {dish} tray (4-portions) voor Plus/Jumbo non-listed partnership pitch",
|
||||
"AH {dish} variant — benchmark tegen {competitor} koers momentum ({pct}%)",
|
||||
"Halal-gap fill: {dish} + {meat} combo voor filialen zonder halal schap",
|
||||
]
|
||||
|
||||
|
||||
def _match_keywords(text: str, keywords: tuple[str, ...]) -> bool:
|
||||
blob = (text or "").lower()
|
||||
return any(k in blob for k in keywords)
|
||||
|
||||
|
||||
def fetch_halal_meat_trends(limit: int = 15) -> list[dict[str, Any]]:
|
||||
rows = fetch_all(
|
||||
"""SELECT i.title, i.link, i.summary, i.published_at, f.name AS feed_name, f.url AS feed_url
|
||||
FROM rss_items i JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
|
||||
ORDER BY i.published_at DESC NULLS LAST LIMIT 200"""
|
||||
)
|
||||
out = []
|
||||
for r in rows:
|
||||
title = r.get("title") or ""
|
||||
summary = r.get("summary") or ""
|
||||
if not _match_keywords(f"{title} {summary}", HALAL_MEAT_KEYWORDS):
|
||||
continue
|
||||
out.append({
|
||||
"title": title,
|
||||
"link": r.get("link"),
|
||||
"summary": (summary or "")[:280],
|
||||
"feed_name": r.get("feed_name"),
|
||||
"feed_url": r.get("feed_url"),
|
||||
"published_at": r.get("published_at").isoformat() if r.get("published_at") else None,
|
||||
"category": "halal_vlees",
|
||||
"source_url": r.get("link"),
|
||||
})
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def fetch_top_dishes(limit: int = 12) -> list[dict[str, Any]]:
|
||||
rows = fetch_all(
|
||||
"""SELECT i.title, i.link, i.summary, i.published_at, f.name AS feed_name, f.url AS feed_url
|
||||
FROM rss_items i JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
|
||||
ORDER BY i.published_at DESC NULLS LAST LIMIT 250"""
|
||||
)
|
||||
scored: dict[str, dict[str, Any]] = {}
|
||||
for r in rows:
|
||||
title = (r.get("title") or "").lower()
|
||||
summary = (r.get("summary") or "").lower()
|
||||
blob = f"{title} {summary}"
|
||||
if not _match_keywords(blob, TOP_DISH_KEYWORDS):
|
||||
continue
|
||||
for kw in TOP_DISH_KEYWORDS:
|
||||
if kw in blob:
|
||||
key = kw.strip()
|
||||
if key not in scored:
|
||||
scored[key] = {
|
||||
"dish_keyword": key,
|
||||
"mentions": 0,
|
||||
"latest_title": r.get("title"),
|
||||
"latest_link": r.get("link"),
|
||||
"feed_name": r.get("feed_name"),
|
||||
"source_url": r.get("link"),
|
||||
}
|
||||
scored[key]["mentions"] += 1
|
||||
break
|
||||
items = sorted(scored.values(), key=lambda x: x["mentions"], reverse=True)[:limit]
|
||||
return items
|
||||
|
||||
|
||||
def fetch_market_trend_rows(limit: int = 8) -> list[dict[str, Any]]:
|
||||
rows = fetch_all(
|
||||
"""SELECT trend_name, description, opportunity_score, source, category, updated_at
|
||||
FROM market_trends ORDER BY updated_at DESC NULLS LAST LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
out = []
|
||||
for r in rows:
|
||||
out.append({
|
||||
"trend_name": r.get("trend_name"),
|
||||
"description": r.get("description"),
|
||||
"opportunity_score": float(r.get("opportunity_score") or 0),
|
||||
"source": r.get("source") or "Foodlinkk trends",
|
||||
"source_url": "/retail",
|
||||
"category": r.get("category") or "markt",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def generate_concepts(
|
||||
dishes: list[dict[str, Any]] | None = None,
|
||||
halal_items: list[dict[str, Any]] | None = None,
|
||||
market_best: dict[str, Any] | None = None,
|
||||
limit: int = 6,
|
||||
) -> list[dict[str, Any]]:
|
||||
dishes = dishes or fetch_top_dishes(5)
|
||||
halal_items = halal_items or fetch_halal_meat_trends(5)
|
||||
best_chain = (market_best or {}).get("chains", ["Albert Heijn"])[0]
|
||||
pct = (market_best or {}).get("change_pct", 0)
|
||||
competitor = (market_best or {}).get("name", "Ahold Delhaize")
|
||||
|
||||
concepts = []
|
||||
for i, tmpl in enumerate(CONCEPT_SEEDS[:limit]):
|
||||
dish = dishes[i % len(dishes)]["dish_keyword"] if dishes else "kant-en-klaar maaltijd"
|
||||
meat = "halal kip" if halal_items else "halal vlees"
|
||||
trend = halal_items[i % len(halal_items)]["title"][:60] if halal_items else "groei halal convenience"
|
||||
text = tmpl.format(
|
||||
dish=dish,
|
||||
meat=meat,
|
||||
chain=best_chain,
|
||||
score=75,
|
||||
trend=trend,
|
||||
competitor=competitor,
|
||||
pct=pct,
|
||||
)
|
||||
concepts.append({
|
||||
"id": i + 1,
|
||||
"concept": text,
|
||||
"based_on": {
|
||||
"dish": dish,
|
||||
"halal_trend": trend,
|
||||
"market_signal": f"{competitor} {pct:+.1f}%" if market_best else "retail DB",
|
||||
},
|
||||
"source_urls": [
|
||||
u for u in [
|
||||
dishes[i % len(dishes)].get("source_url") if dishes else None,
|
||||
halal_items[i % len(halal_items)].get("source_url") if halal_items else None,
|
||||
(market_best or {}).get("source_url"),
|
||||
] if u
|
||||
],
|
||||
})
|
||||
return concepts
|
||||
|
||||
|
||||
def food_trends_dashboard() -> dict[str, Any]:
|
||||
halal = fetch_halal_meat_trends()
|
||||
dishes = fetch_top_dishes()
|
||||
trends = fetch_market_trend_rows()
|
||||
return {
|
||||
"halal_meat_trends": halal,
|
||||
"top_dishes": dishes,
|
||||
"market_trends": trends,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Halal certification registry sync and supermarket matching."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
|
||||
|
||||
OVERPASS_URL = "https://overpass.kumi.systems/api/interpreter"
|
||||
|
||||
# Known halal-friendly retail brands (indicative — verified via certifier when possible)
|
||||
HALAL_FRIENDLY_CHAINS = {
|
||||
"Spar": {"has_halal_section": True, "note": "chain policy varies by franchise"},
|
||||
"Ekoplaza": {"halal_certified": False, "has_halal_section": True},
|
||||
}
|
||||
|
||||
|
||||
def _fetch_overpass(query: str) -> list[dict[str, Any]]:
|
||||
data = urllib.parse.urlencode({"data": query}).encode()
|
||||
req = urllib.request.Request(OVERPASS_URL, data=data, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
payload = json.loads(resp.read().decode())
|
||||
return payload.get("elements", [])
|
||||
|
||||
|
||||
def sync_osm_halal_tags() -> dict[str, Any]:
|
||||
"""Mark supermarkets with OSM diet:halal=yes and import certification records."""
|
||||
query = (
|
||||
'[out:json][timeout:120];area["ISO3166-1"="NL"]->.nl;'
|
||||
'(node["shop"="supermarket"]["diet:halal"="yes"](area.nl);'
|
||||
'way["shop"="supermarket"]["diet:halal"="yes"](area.nl););out tags center;'
|
||||
)
|
||||
elements = _fetch_overpass(query)
|
||||
matched = created = 0
|
||||
for el in elements:
|
||||
tags = el.get("tags") or {}
|
||||
external_id = f"osm:{el.get('type')}:{el.get('id')}"
|
||||
store = fetch_one("SELECT id, name FROM supermarkets WHERE external_id = %s", (external_id,))
|
||||
if not store:
|
||||
name = tags.get("name") or tags.get("brand") or "Unknown"
|
||||
store = fetch_one(
|
||||
"SELECT id, name FROM supermarkets WHERE name ILIKE %s LIMIT 1",
|
||||
(f"%{name[:40]}%",),
|
||||
)
|
||||
if not store:
|
||||
continue
|
||||
matched += 1
|
||||
execute(
|
||||
"""UPDATE supermarkets SET halal_certified = TRUE, has_halal_section = TRUE,
|
||||
halal_certifier = COALESCE(halal_certifier, 'OSM diet:halal'),
|
||||
last_updated = NOW() WHERE id = %s""",
|
||||
(store["id"],),
|
||||
)
|
||||
existing = fetch_one(
|
||||
"SELECT id FROM halal_certifications WHERE supermarket_id = %s AND registry_source = 'osm'",
|
||||
(store["id"],),
|
||||
)
|
||||
if not existing:
|
||||
execute_returning(
|
||||
"""INSERT INTO halal_certifications (
|
||||
supermarket_id, certifier, business_name, status, registry_source,
|
||||
matched_confidence, raw_data
|
||||
) VALUES (%s, 'OSM', %s, 'active', 'osm', 0.85, %s) RETURNING id""",
|
||||
(store["id"], store["name"], json_param(tags)),
|
||||
)
|
||||
created += 1
|
||||
return {"osm_halal_elements": len(elements), "stores_matched": matched, "certs_created": created}
|
||||
|
||||
|
||||
def sync_osm_contact_tags(limit: int = 500) -> dict[str, Any]:
|
||||
"""Pull phone/email/website/operator from OSM for existing stores."""
|
||||
stores = fetch_all(
|
||||
"""SELECT id, external_id, phone, email, website, manager_name
|
||||
FROM supermarkets WHERE external_id LIKE %s
|
||||
AND (phone IS NULL OR email IS NULL OR manager_name IS NULL)
|
||||
LIMIT %s""",
|
||||
("osm:%", limit),
|
||||
)
|
||||
updated = contacts = 0
|
||||
for store in stores:
|
||||
parts = (store.get("external_id") or "").split(":")
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
osm_type, osm_id = parts[1], parts[2]
|
||||
query = f'[out:json][timeout:30];{osm_type}({osm_id});out tags;'
|
||||
try:
|
||||
elements = _fetch_overpass(query)
|
||||
except Exception:
|
||||
continue
|
||||
if not elements:
|
||||
continue
|
||||
tags = elements[0].get("tags") or {}
|
||||
phone = tags.get("phone") or tags.get("contact:phone")
|
||||
email = tags.get("email") or tags.get("contact:email")
|
||||
website = tags.get("website") or tags.get("contact:website")
|
||||
operator = tags.get("operator") or tags.get("contact:name")
|
||||
manager = tags.get("manager") or tags.get("contact:manager") or operator
|
||||
|
||||
sets, params = [], []
|
||||
if phone and not store.get("phone"):
|
||||
sets.append("phone = %s"); params.append(str(phone)[:20])
|
||||
if email and not store.get("email"):
|
||||
sets.append("email = %s"); params.append(str(email)[:255])
|
||||
if website and not store.get("website"):
|
||||
sets.append("website = %s"); params.append(str(website)[:255])
|
||||
if manager and not store.get("manager_name"):
|
||||
sets.append("manager_name = %s"); params.append(str(manager)[:255])
|
||||
if sets:
|
||||
params.append(store["id"])
|
||||
execute(f"UPDATE supermarkets SET {', '.join(sets)}, last_updated = NOW() WHERE id = %s", tuple(params))
|
||||
updated += 1
|
||||
|
||||
if manager or phone or email:
|
||||
existing = fetch_one(
|
||||
"SELECT id FROM supermarket_contacts WHERE supermarket_id = %s AND source = 'osm' LIMIT 1",
|
||||
(store["id"],),
|
||||
)
|
||||
if not existing:
|
||||
execute(
|
||||
"""INSERT INTO supermarket_contacts (
|
||||
supermarket_id, role, full_name, phone, email, source, confidence
|
||||
) VALUES (%s, 'manager', %s, %s, %s, 'osm', 0.6)""",
|
||||
(store["id"], manager, phone, email),
|
||||
)
|
||||
contacts += 1
|
||||
execute(
|
||||
"""INSERT INTO supermarket_profiles (supermarket_id, manager_name, manager_phone,
|
||||
manager_email, web_data, last_scraped_at, data_completeness)
|
||||
VALUES (%s,%s,%s,%s,%s,NOW(),0.4)
|
||||
ON CONFLICT (supermarket_id) DO UPDATE SET
|
||||
manager_name = COALESCE(EXCLUDED.manager_name, supermarket_profiles.manager_name),
|
||||
manager_phone = COALESCE(EXCLUDED.manager_phone, supermarket_profiles.manager_phone),
|
||||
manager_email = COALESCE(EXCLUDED.manager_email, supermarket_profiles.manager_email),
|
||||
web_data = supermarket_profiles.web_data || EXCLUDED.web_data,
|
||||
last_scraped_at = NOW()""",
|
||||
(store["id"], manager, phone, email, json_param({"osm_tags": tags})),
|
||||
)
|
||||
return {"scanned": len(stores), "stores_updated": updated, "contacts_added": contacts}
|
||||
|
||||
|
||||
def list_halal_certified(limit: int = 500) -> list[dict[str, Any]]:
|
||||
return fetch_all(
|
||||
"""
|
||||
SELECT s.*, h.certifier, h.certificate_number, h.expiry_date, h.registry_source,
|
||||
h.matched_confidence
|
||||
FROM supermarkets s
|
||||
LEFT JOIN halal_certifications h ON h.supermarket_id = s.id AND h.status = 'active'
|
||||
WHERE s.halal_certified = TRUE OR s.has_halal_section = TRUE OR h.id IS NOT NULL
|
||||
ORDER BY s.chain, s.city LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Live supermarket & food-retail stock quotes — Yahoo Finance."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
USER_AGENT = "Foodlinkk-MarketIntel/1.0"
|
||||
DATA_SOURCE = "Yahoo Finance"
|
||||
SOURCE_BASE = "https://finance.yahoo.com/quote/"
|
||||
|
||||
# Beursgenoteerde supermarkt / food-retail ketens (geen FMCG zoals Unilever)
|
||||
LISTED_SUPERMARKET_STOCKS = [
|
||||
{
|
||||
"symbol": "AD.AS",
|
||||
"name": "Ahold Delhaize",
|
||||
"chains": ["Albert Heijn", "Gall & Gall", "Etos", "Bol"],
|
||||
"country": "NL/EU",
|
||||
"exchange": "Euronext Amsterdam",
|
||||
"listed": True,
|
||||
"color": "#0066cc",
|
||||
},
|
||||
{
|
||||
"symbol": "CAR.PA",
|
||||
"name": "Carrefour",
|
||||
"chains": ["Carrefour", "Carrefour Express"],
|
||||
"country": "EU",
|
||||
"exchange": "Euronext Paris",
|
||||
"listed": True,
|
||||
"color": "#005baa",
|
||||
},
|
||||
{
|
||||
"symbol": "TSCO.L",
|
||||
"name": "Tesco",
|
||||
"chains": ["Tesco", "Tesco Express"],
|
||||
"country": "UK",
|
||||
"exchange": "London Stock Exchange",
|
||||
"listed": True,
|
||||
"color": "#0050aa",
|
||||
},
|
||||
{
|
||||
"symbol": "SBRY.L",
|
||||
"name": "Sainsbury's",
|
||||
"chains": ["Sainsbury's", "Argos food"],
|
||||
"country": "UK",
|
||||
"exchange": "London Stock Exchange",
|
||||
"listed": True,
|
||||
"color": "#f06c00",
|
||||
},
|
||||
{
|
||||
"symbol": "MRW.L",
|
||||
"name": "Morrisons",
|
||||
"chains": ["Morrisons"],
|
||||
"country": "UK",
|
||||
"exchange": "London Stock Exchange",
|
||||
"listed": True,
|
||||
"color": "#f5c518",
|
||||
},
|
||||
{
|
||||
"symbol": "MKS.L",
|
||||
"name": "Marks & Spencer",
|
||||
"chains": ["M&S Food"],
|
||||
"country": "UK",
|
||||
"exchange": "London Stock Exchange",
|
||||
"listed": True,
|
||||
"color": "#00663d",
|
||||
},
|
||||
{
|
||||
"symbol": "COLR.BR",
|
||||
"name": "Colruyt Group",
|
||||
"chains": ["Colruyt", "Bio-Planet", "OKay"],
|
||||
"country": "BE/EU",
|
||||
"exchange": "Euronext Brussels",
|
||||
"listed": True,
|
||||
"color": "#e30613",
|
||||
},
|
||||
{
|
||||
"symbol": "ICA-B.ST",
|
||||
"name": "ICA Gruppen",
|
||||
"chains": ["ICA", "Maxi", "Rimi"],
|
||||
"country": "Nordics",
|
||||
"exchange": "Nasdaq Stockholm",
|
||||
"listed": True,
|
||||
"color": "#e30613",
|
||||
},
|
||||
{
|
||||
"symbol": "KR",
|
||||
"name": "Kroger",
|
||||
"chains": ["Kroger", "Albertsons merger context"],
|
||||
"country": "USA",
|
||||
"exchange": "NYSE",
|
||||
"listed": True,
|
||||
"color": "#004b87",
|
||||
},
|
||||
{
|
||||
"symbol": "WMT",
|
||||
"name": "Walmart",
|
||||
"chains": ["Walmart", "Sam's Club"],
|
||||
"country": "USA",
|
||||
"exchange": "NYSE",
|
||||
"listed": True,
|
||||
"color": "#0071ce",
|
||||
},
|
||||
{
|
||||
"symbol": "COST",
|
||||
"name": "Costco",
|
||||
"chains": ["Costco Wholesale"],
|
||||
"country": "USA/Global",
|
||||
"exchange": "NASDAQ",
|
||||
"listed": True,
|
||||
"color": "#e31837",
|
||||
},
|
||||
]
|
||||
|
||||
# NL supermarkten — niet beursgenoteerd (transparantie voor CEO)
|
||||
UNLISTED_NL_CHAINS = [
|
||||
{
|
||||
"symbol": None,
|
||||
"name": "Jumbo",
|
||||
"chains": ["Jumbo", "Jumbo City"],
|
||||
"country": "NL",
|
||||
"exchange": "Familiebedrijf · niet beursgenoteerd",
|
||||
"listed": False,
|
||||
"parent": "Van Eerd familie",
|
||||
"color": "#ffcc00",
|
||||
"info_url": "https://www.jumbo.com/over-jumbo",
|
||||
},
|
||||
{
|
||||
"symbol": None,
|
||||
"name": "Plus",
|
||||
"chains": ["Plus", "Plus Compact"],
|
||||
"country": "NL",
|
||||
"exchange": "Coöperatief · niet beursgenoteerd",
|
||||
"listed": False,
|
||||
"parent": "Plus Retail (coöperatie)",
|
||||
"color": "#008040",
|
||||
"info_url": "https://www.plus.nl",
|
||||
},
|
||||
{
|
||||
"symbol": None,
|
||||
"name": "Dirk van den Broek",
|
||||
"chains": ["Dirk", "Dekamarkt"],
|
||||
"country": "NL",
|
||||
"exchange": "Privé · niet beursgenoteerd",
|
||||
"listed": False,
|
||||
"parent": "Schuitema / Dirk van den Broek",
|
||||
"color": "#e30613",
|
||||
"info_url": "https://www.dirk.nl",
|
||||
},
|
||||
{
|
||||
"symbol": None,
|
||||
"name": "Lidl",
|
||||
"chains": ["Lidl"],
|
||||
"country": "NL/EU",
|
||||
"exchange": "Schwarz Group · privé",
|
||||
"listed": False,
|
||||
"parent": "Schwarz Gruppe (DE)",
|
||||
"color": "#0050aa",
|
||||
"info_url": "https://www.lidl.nl",
|
||||
},
|
||||
{
|
||||
"symbol": None,
|
||||
"name": "ALDI",
|
||||
"chains": ["ALDI", "ALDI Nord/Süd"],
|
||||
"country": "NL/EU",
|
||||
"exchange": "Privé · niet beursgenoteerd",
|
||||
"listed": False,
|
||||
"parent": "Aldi Süd / Aldi Nord",
|
||||
"color": "#0066b3",
|
||||
"info_url": "https://www.aldi.nl",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _yahoo_url(symbol: str) -> str:
|
||||
return f"{SOURCE_BASE}{quote(symbol, safe='')}"
|
||||
|
||||
|
||||
def _fetch_chart(symbol: str) -> dict[str, Any]:
|
||||
url = (
|
||||
f"https://query1.finance.yahoo.com/v8/finance/chart/{quote(symbol, safe='')}"
|
||||
f"?interval=1d&range=1mo&includePrePost=false"
|
||||
)
|
||||
req = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urlopen(req, timeout=14) as resp:
|
||||
payload = json.loads(resp.read().decode())
|
||||
result = (payload.get("chart") or {}).get("result") or []
|
||||
if not result:
|
||||
return {}
|
||||
meta = result[0].get("meta") or {}
|
||||
closes = (result[0].get("indicators") or {}).get("quote") or [{}]
|
||||
close_series = closes[0].get("close") or []
|
||||
valid = [c for c in close_series if c is not None]
|
||||
sparkline = valid[-14:] if len(valid) >= 14 else valid
|
||||
prev = valid[-2] if len(valid) >= 2 else None
|
||||
last = valid[-1] if valid else meta.get("regularMarketPrice")
|
||||
change_pct = meta.get("regularMarketChangePercent")
|
||||
if change_pct is None and prev and last and prev:
|
||||
change_pct = ((last - prev) / prev) * 100
|
||||
return {
|
||||
"price": meta.get("regularMarketPrice") or last,
|
||||
"currency": meta.get("currency") or "EUR",
|
||||
"change_pct": round(float(change_pct or 0), 2),
|
||||
"change_abs": meta.get("regularMarketChange"),
|
||||
"sparkline": [round(float(v), 2) for v in sparkline],
|
||||
"market_state": meta.get("marketState") or "CLOSED",
|
||||
"exchange_name": meta.get("exchangeName") or meta.get("fullExchangeName"),
|
||||
"quote_time": meta.get("regularMarketTime"),
|
||||
}
|
||||
|
||||
|
||||
def fetch_supermarket_quotes() -> list[dict[str, Any]]:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
for stock in LISTED_SUPERMARKET_STOCKS:
|
||||
row = dict(stock)
|
||||
sym = stock["symbol"]
|
||||
row["source"] = DATA_SOURCE
|
||||
row["source_url"] = _yahoo_url(sym)
|
||||
row["chart_api"] = f"https://query1.finance.yahoo.com/v8/finance/chart/{sym}"
|
||||
row["fetched_at"] = now
|
||||
try:
|
||||
chart = _fetch_chart(sym)
|
||||
row.update(chart)
|
||||
row["trend"] = "up" if (row.get("change_pct") or 0) >= 0 else "down"
|
||||
row["live"] = row.get("price") is not None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
row["error"] = str(exc)[:100]
|
||||
row["price"] = None
|
||||
row["change_pct"] = 0
|
||||
row["sparkline"] = []
|
||||
row["trend"] = "flat"
|
||||
row["live"] = False
|
||||
items.append(row)
|
||||
|
||||
for chain in UNLISTED_NL_CHAINS:
|
||||
row = dict(chain)
|
||||
row["source"] = "Foodlinkk Intel"
|
||||
row["source_url"] = chain.get("info_url")
|
||||
row["fetched_at"] = now
|
||||
row["live"] = False
|
||||
row["price"] = None
|
||||
row["change_pct"] = None
|
||||
row["note"] = "Niet beursgenoteerd — geen live koers beschikbaar"
|
||||
items.append(row)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def fetch_retail_quotes() -> list[dict[str, Any]]:
|
||||
"""Back-compat — alleen beursgenoteerde supermarkt-aandelen."""
|
||||
return [q for q in fetch_supermarket_quotes() if q.get("listed")]
|
||||
|
||||
|
||||
def market_summary(quotes: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
quotes = quotes or fetch_retail_quotes()
|
||||
valid = [q for q in quotes if q.get("price") is not None]
|
||||
avg_change = sum(float(q.get("change_pct") or 0) for q in valid) / len(valid) if valid else 0
|
||||
best = max(valid, key=lambda q: float(q.get("change_pct") or 0), default=None)
|
||||
worst = min(valid, key=lambda q: float(q.get("change_pct") or 0), default=None)
|
||||
return {
|
||||
"avg_change_pct": round(avg_change, 2),
|
||||
"best_performer": best,
|
||||
"worst_performer": worst,
|
||||
"quote_count": len(valid),
|
||||
"listed_count": len(LISTED_SUPERMARKET_STOCKS),
|
||||
"unlisted_nl_count": len(UNLISTED_NL_CHAINS),
|
||||
"data_source": DATA_SOURCE,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
"""PDOK Locatieserver — postcode geocoding."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
|
||||
PDOK_URL = "https://api.pdok.nl/bzk/locatieserver/search/v3_1/free"
|
||||
POSTCODE_RE = re.compile(r"^(\d{4})\s?([A-Za-z]{2})$")
|
||||
|
||||
|
||||
def normalize_postcode(raw: str) -> str:
|
||||
cleaned = (raw or "").strip().upper().replace(" ", "")
|
||||
m = POSTCODE_RE.match(cleaned)
|
||||
if m:
|
||||
return f"{m.group(1)}{m.group(2)}"
|
||||
return cleaned[:6] if cleaned else ""
|
||||
|
||||
|
||||
def _parse_point(value: Optional[str]) -> tuple[Optional[float], Optional[float]]:
|
||||
if not value or "POINT" not in value:
|
||||
return None, None
|
||||
nums = re.findall(r"[-+]?\d*\.?\d+", value)
|
||||
if len(nums) >= 2:
|
||||
return float(nums[1]), float(nums[0]) # lat, lon
|
||||
return None, None
|
||||
|
||||
|
||||
def lookup_postcode(postcode: str) -> Optional[dict[str, Any]]:
|
||||
pc = normalize_postcode(postcode)
|
||||
if len(pc) < 6:
|
||||
return None
|
||||
q = urllib.parse.urlencode({"q": pc, "rows": 1, "fq": "type:postcode"})
|
||||
with urllib.request.urlopen(f"{PDOK_URL}?{q}", timeout=20) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
docs = data.get("response", {}).get("docs", [])
|
||||
if not docs:
|
||||
return None
|
||||
doc = docs[0]
|
||||
lat, lon = _parse_point(doc.get("centroide_ll"))
|
||||
gemeente_code = (doc.get("gemeentecode") or "").strip()
|
||||
if gemeente_code and not gemeente_code.startswith("GM"):
|
||||
gemeente_code = f"GM{gemeente_code}"
|
||||
return {
|
||||
"postcode": pc,
|
||||
"city": (doc.get("woonplaatsnaam") or "").strip(),
|
||||
"province": (doc.get("provincienaam") or "").strip(),
|
||||
"province_code": (doc.get("provinciecode") or "").strip(),
|
||||
"municipality": (doc.get("gemeentenaam") or "").strip(),
|
||||
"municipality_code": gemeente_code,
|
||||
"street": (doc.get("straatnaam") or "").strip(),
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from app.db import execute, fetch_all, fetch_one
|
||||
|
||||
PROXMOX_HOST = "10.4.7.14"
|
||||
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
|
||||
SSH_USER = "aissa"
|
||||
SSH_PASSWORD = "Foodlinkk#2026"
|
||||
|
||||
VM_105_IP = "10.4.7.19"
|
||||
VM_106_IP = "10.4.7.18"
|
||||
|
||||
SERVICE_LAYOUT: list[dict[str, Any]] = [
|
||||
{"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
|
||||
{"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
|
||||
{"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
|
||||
{"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
|
||||
{"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
|
||||
]
|
||||
|
||||
|
||||
def _iso_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
|
||||
ssh_cmd = [
|
||||
"sshpass",
|
||||
"-p",
|
||||
SSH_PASSWORD,
|
||||
"ssh",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
"-o",
|
||||
"ConnectTimeout=7",
|
||||
f"{SSH_USER}@{host}",
|
||||
command,
|
||||
]
|
||||
try:
|
||||
proc = subprocess.run( # noqa: S603
|
||||
ssh_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
return {"ok": False, "error": f"ssh tooling missing: {exc}"}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"ok": False, "error": "ssh timeout"}
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"code": proc.returncode,
|
||||
"stdout": (proc.stdout or "").strip(),
|
||||
"stderr": (proc.stderr or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]:
|
||||
req = Request(url, headers=headers or {})
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310
|
||||
payload = resp.read().decode("utf-8")
|
||||
return json.loads(payload)
|
||||
|
||||
|
||||
def _build_token_header(token_value: str) -> str:
|
||||
val = token_value.strip()
|
||||
if val.startswith("PVEAPIToken="):
|
||||
return val
|
||||
return f"PVEAPIToken={val}"
|
||||
|
||||
|
||||
def _create_api_token_via_ssh() -> str | None:
|
||||
token_name = f"ops{int(time.time())}"
|
||||
cmd = (
|
||||
f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} "
|
||||
"--privsep 0 --expire 0 --output-format json"
|
||||
)
|
||||
result = _run_ssh(PROXMOX_HOST, cmd, timeout=15)
|
||||
if not result.get("ok"):
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(result.get("stdout") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
tokenid = parsed.get("full-tokenid")
|
||||
secret = parsed.get("value")
|
||||
if tokenid and secret:
|
||||
return f"PVEAPIToken={tokenid}={secret}"
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
|
||||
token = os.getenv("PROXMOX_TOKEN")
|
||||
tried = []
|
||||
if token:
|
||||
tried.append("env-token")
|
||||
try:
|
||||
data = _http_get_json(
|
||||
f"{PROXMOX_API_URL}/nodes",
|
||||
headers={"Authorization": _build_token_header(token)},
|
||||
)
|
||||
return data.get("data") or [], "api-token-env", None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
tried.append(f"env-failed:{exc}")
|
||||
created = _create_api_token_via_ssh()
|
||||
if created:
|
||||
tried.append("ssh-created-token")
|
||||
try:
|
||||
data = _http_get_json(
|
||||
f"{PROXMOX_API_URL}/nodes",
|
||||
headers={"Authorization": created},
|
||||
)
|
||||
return data.get("data") or [], "api-token-ssh", None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
tried.append(f"ssh-token-failed:{exc}")
|
||||
return [], "none", ", ".join(tried) if tried else "no-token"
|
||||
|
||||
|
||||
def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
|
||||
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
|
||||
if pvesh.get("ok"):
|
||||
try:
|
||||
return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
qm = _run_ssh(PROXMOX_HOST, "qm list")
|
||||
rows: list[dict[str, Any]] = []
|
||||
if qm.get("ok") and qm.get("stdout"):
|
||||
lines = (qm["stdout"] or "").splitlines()
|
||||
for line in lines[1:]:
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
vmid = parts[0]
|
||||
rows.append(
|
||||
{
|
||||
"node": "pve",
|
||||
"type": "qemu",
|
||||
"id": f"qemu/{vmid}",
|
||||
"vmid": int(vmid) if vmid.isdigit() else vmid,
|
||||
"status": parts[2] if len(parts) > 2 else "unknown",
|
||||
}
|
||||
)
|
||||
return rows, "ssh-qm-list", None
|
||||
err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
|
||||
return [], "none", err
|
||||
|
||||
|
||||
def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
return sock.connect_ex((host, port)) == 0
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def check_docker_services() -> dict[str, Any]:
|
||||
result = _run_ssh(VM_106_IP, "docker ps --format json")
|
||||
method = "docker-ps-json"
|
||||
if not result.get("ok"):
|
||||
result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
|
||||
method = "docker-ps-template-json"
|
||||
if not result.get("ok"):
|
||||
return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}
|
||||
|
||||
containers: list[dict[str, Any]] = []
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(line)
|
||||
containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
|
||||
except json.JSONDecodeError:
|
||||
containers.append({"raw": line})
|
||||
return {"ok": True, "source": method, "containers": containers}
|
||||
|
||||
|
||||
def get_topology() -> dict[str, Any]:
|
||||
api_nodes, api_source, api_error = _fetch_nodes_via_api()
|
||||
ssh_nodes: list[dict[str, Any]] = []
|
||||
ssh_source = "none"
|
||||
ssh_error: str | None = None
|
||||
if not api_nodes:
|
||||
ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()
|
||||
|
||||
api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
|
||||
host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
|
||||
host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
|
||||
host_status = api_node.get("status") if api_node else "unknown"
|
||||
if host_status == "unknown" and ssh_nodes:
|
||||
host_status = "online"
|
||||
|
||||
vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
|
||||
source_rows = api_nodes or ssh_nodes
|
||||
for row in source_rows:
|
||||
vmid = str(row.get("vmid") or "").strip()
|
||||
if vmid in vm_states:
|
||||
vm_states[vmid] = str(row.get("status") or "unknown")
|
||||
|
||||
docker_state = check_docker_services()
|
||||
docker_names = {
|
||||
str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
|
||||
}
|
||||
|
||||
vm105_children: list[dict[str, Any]] = []
|
||||
vm106_children: list[dict[str, Any]] = []
|
||||
for svc in SERVICE_LAYOUT:
|
||||
up = _port_health(str(svc["host"]), int(svc["port"]))
|
||||
hinted = "unknown"
|
||||
for name, details in docker_names.items():
|
||||
if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
|
||||
hinted = str(details.get("State") or details.get("Status") or "running")
|
||||
break
|
||||
item = {
|
||||
"id": svc["id"],
|
||||
"label": svc["label"],
|
||||
"type": "service",
|
||||
"status": "online" if up else "offline",
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"host": svc["host"],
|
||||
"hint": hinted,
|
||||
"children": [],
|
||||
}
|
||||
if svc["parent"] == "vm105-hermes":
|
||||
vm105_children.append(item)
|
||||
else:
|
||||
vm106_children.append(item)
|
||||
|
||||
topology_nodes = [
|
||||
{
|
||||
"id": "proxmox-host",
|
||||
"label": f"proxmox-host ({PROXMOX_HOST})",
|
||||
"type": "proxmox",
|
||||
"status": host_status,
|
||||
"cpu": host_cpu,
|
||||
"mem": host_mem,
|
||||
"children": [
|
||||
{
|
||||
"id": "vm105-hermes",
|
||||
"label": f"vm105-hermes ({VM_105_IP})",
|
||||
"type": "vm",
|
||||
"status": vm_states["105"],
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"children": vm105_children,
|
||||
},
|
||||
{
|
||||
"id": "vm106-command",
|
||||
"label": f"vm106-command ({VM_106_IP})",
|
||||
"type": "vm",
|
||||
"status": vm_states["106"],
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"children": vm106_children,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
return {
|
||||
"generated_at": _iso_now(),
|
||||
"nodes": topology_nodes,
|
||||
"meta": {
|
||||
"proxmox_host": PROXMOX_HOST,
|
||||
"api_source": api_source,
|
||||
"api_error": api_error,
|
||||
"ssh_source": ssh_source,
|
||||
"ssh_error": ssh_error,
|
||||
"docker_source": docker_state.get("source"),
|
||||
"docker_ok": docker_state.get("ok", False),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_status_summary() -> dict[str, Any]:
|
||||
topo = get_topology()
|
||||
flat: list[dict[str, Any]] = []
|
||||
|
||||
def _collect(node: dict[str, Any]) -> None:
|
||||
flat.append(node)
|
||||
for child in node.get("children") or []:
|
||||
_collect(child)
|
||||
|
||||
for root in topo.get("nodes") or []:
|
||||
_collect(root)
|
||||
|
||||
total = len(flat)
|
||||
online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"})
|
||||
degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"})
|
||||
offline = max(0, total - online - degraded)
|
||||
return {
|
||||
"generated_at": topo.get("generated_at"),
|
||||
"health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"),
|
||||
"counts": {"total": total, "online": online, "degraded": degraded, "offline": offline},
|
||||
"sources": topo.get("meta", {}),
|
||||
"topology": topo,
|
||||
}
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
row = fetch_one(
|
||||
"""
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = %s
|
||||
) AS ok
|
||||
""",
|
||||
(table_name,),
|
||||
)
|
||||
return bool(row and row.get("ok"))
|
||||
|
||||
|
||||
def poll_and_snapshot() -> dict[str, Any]:
|
||||
status = get_status_summary()
|
||||
if not _table_exists("infra_snapshots"):
|
||||
return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status}
|
||||
|
||||
cols = fetch_all(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'infra_snapshots'
|
||||
ORDER BY ordinal_position
|
||||
"""
|
||||
)
|
||||
colset = {c.get("column_name") for c in cols}
|
||||
payload = {
|
||||
"source": "proxmox",
|
||||
"topology": status.get("topology"),
|
||||
"summary": {k: v for k, v in status.items() if k != "topology"},
|
||||
"generated_at": status.get("generated_at"),
|
||||
}
|
||||
|
||||
value_map: dict[str, Any] = {}
|
||||
if "source" in colset:
|
||||
value_map["source"] = "proxmox"
|
||||
if "provider" in colset:
|
||||
value_map["provider"] = "proxmox"
|
||||
if "snapshot" in colset:
|
||||
value_map["snapshot"] = json.dumps(payload)
|
||||
if "payload" in colset:
|
||||
value_map["payload"] = json.dumps(payload)
|
||||
if "topology" in colset:
|
||||
value_map["topology"] = json.dumps(status.get("topology"))
|
||||
if "summary" in colset:
|
||||
value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"})
|
||||
if "created_at" in colset:
|
||||
value_map["created_at"] = datetime.now(timezone.utc)
|
||||
|
||||
if not value_map:
|
||||
return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}
|
||||
|
||||
columns = list(value_map.keys())
|
||||
placeholders = ", ".join(["%s"] * len(columns))
|
||||
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
|
||||
execute(sql, tuple(value_map[c] for c in columns))
|
||||
return {"ok": True, "saved": True, "columns": columns, "status": status}
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Fetch live supermarket folders from reclamefolder.nl — all chains."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import unquote
|
||||
from urllib.request import HTTPCookieProcessor, Request, build_opener
|
||||
|
||||
from app.db import execute, fetch_all, fetch_one
|
||||
|
||||
BASE = "https://www.reclamefolder.nl"
|
||||
SUPERMARKT_URL = f"{BASE}/categorieen/supermarkt/"
|
||||
SITEMAP_RETAILERS = f"{BASE}/sitemap/retailers.xml"
|
||||
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
|
||||
SUPERMARKT_KEYWORDS = (
|
||||
"supermarkt", "markt", "ah", "jumbo", "lidl", "aldi", "plus", "dirk",
|
||||
"coop", "boni", "vomar", "deka", "ekoplaza", "mitra", "spar", "hoogvliet",
|
||||
"food", "gall", "poiesz", "nettorama",
|
||||
)
|
||||
|
||||
_opener = None
|
||||
|
||||
|
||||
def _get_opener():
|
||||
global _opener
|
||||
if _opener is None:
|
||||
_opener = build_opener(HTTPCookieProcessor())
|
||||
return _opener
|
||||
|
||||
|
||||
def _fetch_html(url: str) -> str:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept-Language": "nl-NL,nl;q=0.9"}
|
||||
html = _get_opener().open(Request(url, headers=headers), timeout=35).read().decode("utf-8", "ignore")
|
||||
if "__NEXT_DATA__" not in html:
|
||||
cb = re.search(r"decodeURIComponent\('([^']+)'\)", html)
|
||||
if cb:
|
||||
html = _get_opener().open(Request(unquote(cb.group(1)), headers=headers), timeout=35).read().decode("utf-8", "ignore")
|
||||
return html
|
||||
|
||||
|
||||
def _fetch_page_props(url: str) -> dict[str, Any]:
|
||||
html = _fetch_html(url)
|
||||
match = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.+?)</script>', html)
|
||||
if not match:
|
||||
return {}
|
||||
data = json.loads(match.group(1))
|
||||
return data.get("props", {}).get("pageProps", {}) or {}
|
||||
|
||||
|
||||
def _parse_day(raw: Optional[str]) -> Optional[date]:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00")).date()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _folder_item(row: dict[str, Any], source: str = "category") -> Optional[dict[str, Any]]:
|
||||
retailer = row.get("retailer") or {}
|
||||
if source == "retailer_page":
|
||||
chain = retailer.get("name") or row.get("name", "")
|
||||
edition_id = row.get("id")
|
||||
valid_label = ""
|
||||
cover = row.get("cover") or {}
|
||||
folder_name = row.get("name") or "Folder"
|
||||
else:
|
||||
chain = retailer.get("name")
|
||||
edition_id = row.get("editionId") or row.get("id")
|
||||
valid_label = row.get("validLabel") or ""
|
||||
cover = row.get("cover") or {}
|
||||
folder_name = valid_label or "Folder"
|
||||
|
||||
if not edition_id or not chain:
|
||||
return None
|
||||
|
||||
valid_to = _parse_day(row.get("validThru"))
|
||||
today = datetime.now(timezone.utc).date()
|
||||
if valid_to and valid_to < today:
|
||||
return None
|
||||
|
||||
permaname = retailer.get("permaname") or ""
|
||||
url = f"{BASE}/f/folders/{edition_id}/"
|
||||
title = f"{chain} — {folder_name}" if folder_name != chain else f"{chain} folder ({valid_label})".strip(" ()")
|
||||
|
||||
return {
|
||||
"chain": chain,
|
||||
"title": title,
|
||||
"folder_path": url,
|
||||
"folder_label": valid_label or folder_name or chain,
|
||||
"description": f"Reclamefolder.nl · {valid_label or folder_name}".strip(" ·"),
|
||||
"valid_from": _parse_day(row.get("validFrom")),
|
||||
"valid_to": valid_to,
|
||||
"image_url": cover.get("imageUrl") if isinstance(cover, dict) else None,
|
||||
"status": "active",
|
||||
"promo_type": row.get("name") or "folder",
|
||||
"source": "reclamefolder.nl",
|
||||
"metadata": {
|
||||
"edition_id": str(edition_id),
|
||||
"version_id": str(row.get("versionId") or row.get("id") or ""),
|
||||
"retailer_permaname": permaname,
|
||||
"retailer_url": f"{BASE}/winkels/{permaname}/" if permaname else "",
|
||||
"folder_type": row.get("name") or "folder",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def fetch_retailer_slugs() -> list[str]:
|
||||
try:
|
||||
req = Request(SITEMAP_RETAILERS, headers={"User-Agent": USER_AGENT})
|
||||
data = _get_opener().open(req, timeout=45).read()
|
||||
text = data.decode("utf-8", "ignore")
|
||||
slugs = set()
|
||||
for m in re.finditer(r"https://www\.reclamefolder\.nl/winkels/([a-z0-9-]+)/", text):
|
||||
slug = m.group(1)
|
||||
if "vestiging" in slug:
|
||||
continue
|
||||
if any(k in slug for k in SUPERMARKET_KEYWORDS):
|
||||
slugs.add(slug)
|
||||
return sorted(slugs)
|
||||
except Exception:
|
||||
return [
|
||||
"albert-heijn", "jumbo", "lidl", "plus", "dirk", "aldi", "dekamarkt",
|
||||
"ekoplaza", "vomar", "coop-supermarkten", "boni-supermarkt", "mitra",
|
||||
]
|
||||
|
||||
|
||||
def fetch_supermarkt_folders(include_all_retailers: bool = True) -> list[dict[str, Any]]:
|
||||
seen: set[str] = set()
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
props = _fetch_page_props(SUPERMARKT_URL)
|
||||
for row in props.get("foldersFromProps") or []:
|
||||
item = _folder_item(row, "category")
|
||||
if item and item["metadata"]["edition_id"] not in seen:
|
||||
seen.add(item["metadata"]["edition_id"])
|
||||
items.append(item)
|
||||
|
||||
if include_all_retailers:
|
||||
for slug in fetch_retailer_slugs():
|
||||
try:
|
||||
rprops = _fetch_page_props(f"{BASE}/winkels/{slug}/")
|
||||
retailer = rprops.get("retailer") or {}
|
||||
for folder in retailer.get("folders") or []:
|
||||
folder = dict(folder)
|
||||
folder["retailer"] = retailer
|
||||
item = _folder_item(folder, "retailer_page")
|
||||
if item and item["metadata"]["edition_id"] not in seen:
|
||||
seen.add(item["metadata"]["edition_id"])
|
||||
items.append(item)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
items.sort(key=lambda x: (x.get("chain") or "", x.get("valid_to") or date.max))
|
||||
return items
|
||||
|
||||
|
||||
def sync_to_db() -> dict[str, Any]:
|
||||
folders = fetch_supermarkt_folders()
|
||||
execute(
|
||||
"UPDATE promo_campaigns SET status = 'expired', updated_at = NOW() WHERE source = 'reclamefolder.nl'"
|
||||
)
|
||||
inserted = 0
|
||||
chains: set[str] = set()
|
||||
for f in folders:
|
||||
fetch_one(
|
||||
"""INSERT INTO promo_campaigns
|
||||
(chain, title, folder_path, folder_label, description, valid_from, valid_to,
|
||||
status, promo_type, image_url, source, metadata)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
|
||||
RETURNING id""",
|
||||
(
|
||||
f["chain"], f["title"], f["folder_path"], f["folder_label"],
|
||||
f["description"], f["valid_from"], f["valid_to"],
|
||||
f["status"], f["promo_type"], f.get("image_url"),
|
||||
f["source"], json.dumps(f["metadata"]),
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
chains.add(f["chain"])
|
||||
return {
|
||||
"ok": True,
|
||||
"source": "reclamefolder.nl",
|
||||
"synced": inserted,
|
||||
"chains": len(chains),
|
||||
"items": folders,
|
||||
}
|
||||
|
||||
|
||||
def list_cached(limit: int = 200) -> list[dict[str, Any]]:
|
||||
rows = fetch_all(
|
||||
"""SELECT * FROM promo_campaigns
|
||||
WHERE source = 'reclamefolder.nl' AND status = 'active'
|
||||
ORDER BY valid_to ASC NULLS LAST, chain ASC
|
||||
LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def list_chains() -> list[str]:
|
||||
rows = fetch_all(
|
||||
"""SELECT DISTINCT chain FROM promo_campaigns
|
||||
WHERE source = 'reclamefolder.nl' AND status = 'active' AND chain IS NOT NULL
|
||||
ORDER BY chain"""
|
||||
)
|
||||
return [r["chain"] for r in rows if r.get("chain")]
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Retail 360 workspace API — notes, media, milestones, RSS, wholesalers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.db import fetch_all, fetch_one
|
||||
from app.middleware import log_agent_event
|
||||
from app import retail_360
|
||||
from app import wholesaler_scrapers
|
||||
from app.connectors import market_stocks, rss_feeds
|
||||
|
||||
router = APIRouter(prefix="/retail", tags=["retail-360"])
|
||||
|
||||
|
||||
class NoteIn(BaseModel):
|
||||
body: str = Field(..., min_length=1)
|
||||
title: Optional[str] = None
|
||||
note_type: str = "general"
|
||||
|
||||
|
||||
class MilestoneIn(BaseModel):
|
||||
title: str
|
||||
milestone_type: str = "custom"
|
||||
client_id: Optional[int] = None
|
||||
deal_id: Optional[int] = None
|
||||
target_date: Optional[str] = None
|
||||
value_eur: Optional[float] = None
|
||||
notes: Optional[str] = None
|
||||
|
||||
|
||||
class OwnershipIn(BaseModel):
|
||||
new_owner: str
|
||||
previous_owner: Optional[str] = None
|
||||
change_type: str = "acquisition"
|
||||
effective_date: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
notes: Optional[str] = None
|
||||
|
||||
|
||||
class CalendarIn(BaseModel):
|
||||
title: str
|
||||
starts_at: str
|
||||
description: Optional[str] = None
|
||||
ends_at: Optional[str] = None
|
||||
client_id: Optional[int] = None
|
||||
deal_id: Optional[int] = None
|
||||
location: Optional[str] = None
|
||||
|
||||
|
||||
class MediaIn(BaseModel):
|
||||
filename: str
|
||||
storage_path: str
|
||||
content_type: str = "image/jpeg"
|
||||
caption: Optional[str] = None
|
||||
|
||||
|
||||
def _row(row: dict | None) -> dict[str, Any]:
|
||||
if not row:
|
||||
raise HTTPException(404, "Not found")
|
||||
out: dict[str, Any] = {}
|
||||
for k, v in row.items():
|
||||
if hasattr(v, "isoformat"):
|
||||
out[k] = v.isoformat()
|
||||
elif v is not None and hasattr(v, "__float__") and type(v).__name__ == "Decimal":
|
||||
out[k] = float(v)
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
def _fetch_weather_forecast(lat: float, lon: float) -> list[dict[str, Any]]:
|
||||
url = (
|
||||
f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}"
|
||||
f"&daily=temperature_2m_max,precipitation_sum,weathercode"
|
||||
f"&timezone=Europe%2FAmsterdam&forecast_days=7"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
days = data.get("daily", {}).get("time", [])
|
||||
temps = data.get("daily", {}).get("temperature_2m_max", [])
|
||||
prec = data.get("daily", {}).get("precipitation_sum", [])
|
||||
return [
|
||||
{"date": days[i], "temperature_c": temps[i] if i < len(temps) else None,
|
||||
"precipitation_mm": prec[i] if i < len(prec) else None, "source": "open-meteo-live"}
|
||||
for i in range(len(days))
|
||||
]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
@router.get("/360/{store_id}")
|
||||
def get_360_view(store_id: int) -> dict[str, Any]:
|
||||
try:
|
||||
data = retail_360.get_store_360(store_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(404, str(exc)) from exc
|
||||
store = data["store"]
|
||||
if store.get("latitude") and store.get("longitude"):
|
||||
live = _fetch_weather_forecast(float(store["latitude"]), float(store["longitude"]))
|
||||
if live:
|
||||
data["weather_forecast"] = live
|
||||
for key in ("notes", "media", "milestones", "ownership_changes", "calendar", "weather"):
|
||||
data[key] = [_row(x) for x in data.get(key, [])]
|
||||
if data.get("area_analysis"):
|
||||
data["area_analysis"] = _row(data["area_analysis"])
|
||||
return data
|
||||
|
||||
|
||||
@router.post("/360/{store_id}/notes")
|
||||
def add_store_note(store_id: int, payload: NoteIn) -> dict[str, Any]:
|
||||
note = retail_360.add_note("supermarket", store_id, payload.body, payload.title, payload.note_type)
|
||||
log_agent_event(agent_name="retail_360", event_type="note", title=f"Note on store {store_id}")
|
||||
return {"note": _row(note)}
|
||||
|
||||
|
||||
@router.post("/360/{store_id}/milestones")
|
||||
def add_store_milestone(store_id: int, payload: MilestoneIn) -> dict[str, Any]:
|
||||
ms = retail_360.add_milestone(store_id, payload.title, payload.milestone_type, **payload.model_dump(exclude={"title", "milestone_type"}))
|
||||
return {"milestone": _row(ms)}
|
||||
|
||||
|
||||
@router.post("/360/{store_id}/ownership")
|
||||
def add_store_ownership(store_id: int, payload: OwnershipIn) -> dict[str, Any]:
|
||||
store = fetch_one("SELECT chain FROM supermarkets WHERE id = %s", (store_id,))
|
||||
row = retail_360.add_ownership(entity_id=store_id, chain=store.get("chain") if store else None, **payload.model_dump())
|
||||
return {"ownership": _row(row)}
|
||||
|
||||
|
||||
@router.post("/360/{store_id}/calendar")
|
||||
def add_store_calendar(store_id: int, payload: CalendarIn) -> dict[str, Any]:
|
||||
ev = retail_360.add_calendar_event(store_id, payload.title, payload.starts_at, **payload.model_dump(exclude={"title", "starts_at"}))
|
||||
return {"event": _row(ev)}
|
||||
|
||||
|
||||
@router.post("/360/{store_id}/media")
|
||||
def register_store_media(store_id: int, payload: MediaIn) -> dict[str, Any]:
|
||||
media = retail_360.register_media("supermarket", store_id, payload.filename, payload.storage_path, payload.content_type, payload.caption)
|
||||
return {"media": _row(media)}
|
||||
|
||||
|
||||
@router.get("/cities")
|
||||
def list_cities(limit: int = Query(200, ge=1, le=1000)) -> dict[str, Any]:
|
||||
rows = fetch_all(
|
||||
"""SELECT c.*, (SELECT COUNT(*) FROM supermarkets s WHERE s.city ILIKE c.city) AS store_count
|
||||
FROM city_demographics c ORDER BY c.population DESC NULLS LAST LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
return {"items": [_row(r) for r in rows], "count": len(rows)}
|
||||
|
||||
|
||||
@router.post("/cities/sync")
|
||||
def sync_cities(limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:
|
||||
return retail_360.sync_city_demographics(limit)
|
||||
|
||||
|
||||
@router.get("/wholesalers")
|
||||
def list_wholesalers(limit: int = Query(500, ge=1, le=2000), q: Optional[str] = None) -> dict[str, Any]:
|
||||
clauses, params = [], []
|
||||
if q:
|
||||
clauses.append("(name ILIKE %s OR city ILIKE %s OR address ILIKE %s)")
|
||||
like = f"%{q}%"
|
||||
params.extend([like, like, like])
|
||||
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
rows = fetch_all(f"SELECT * FROM wholesalers{where} ORDER BY name LIMIT %s", tuple(params + [limit]))
|
||||
return {"items": [_row(r) for r in rows], "count": len(rows)}
|
||||
|
||||
|
||||
@router.post("/wholesalers/import")
|
||||
def import_wholesalers(background: bool = Query(False)) -> dict[str, Any]:
|
||||
log_agent_event(agent_name="wholesale_scraper", event_type="import", title="OSM wholesalers import")
|
||||
if background:
|
||||
import threading
|
||||
threading.Thread(target=wholesaler_scrapers.import_wholesalers, daemon=True).start()
|
||||
return {"status": "started", "message": "Wholesaler import running in background"}
|
||||
return wholesaler_scrapers.import_wholesalers()
|
||||
|
||||
|
||||
@router.get("/rss/live")
|
||||
def rss_live(limit: int = Query(30, ge=1, le=100), category: Optional[str] = None) -> dict[str, Any]:
|
||||
rows = rss_feeds.list_live_feed(limit, category)
|
||||
return {"items": [_row(r) for r in rows], "count": len(rows)}
|
||||
|
||||
|
||||
@router.post("/rss/refresh")
|
||||
def rss_refresh() -> dict[str, Any]:
|
||||
log_agent_event(agent_name="rss_feeds", event_type="refresh", title="RSS feeds refresh")
|
||||
return rss_feeds.refresh_all_feeds()
|
||||
|
||||
|
||||
@router.get("/market/stocks")
|
||||
def retail_market_stocks() -> dict[str, Any]:
|
||||
quotes = market_stocks.fetch_retail_quotes()
|
||||
return {
|
||||
"items": quotes,
|
||||
"summary": market_stocks.market_summary(quotes),
|
||||
"updated_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/regulations")
|
||||
def retail_regulations(limit: int = Query(30, ge=1, le=100)) -> dict[str, Any]:
|
||||
reg = rss_feeds.list_live_feed(limit, "regelgeving")
|
||||
cbs = rss_feeds.list_live_feed(limit, "cbs")
|
||||
markt = rss_feeds.list_live_feed(min(limit, 15), "markt")
|
||||
return {
|
||||
"regelgeving": [_row(r) for r in reg],
|
||||
"cbs": [_row(r) for r in cbs],
|
||||
"markt": [_row(r) for r in markt],
|
||||
"updated_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/live-dashboard")
|
||||
def live_dashboard() -> dict[str, Any]:
|
||||
trends = fetch_all(
|
||||
"SELECT * FROM market_trends ORDER BY updated_at DESC NULLS LAST LIMIT 8"
|
||||
)
|
||||
rss = rss_feeds.list_live_feed(12)
|
||||
opportunities = fetch_all(
|
||||
"""SELECT s.name, s.chain, s.city, ros.halal_opportunity_score
|
||||
FROM retail_opportunity_scores ros JOIN supermarkets s ON s.id = ros.supermarket_id
|
||||
ORDER BY ros.halal_opportunity_score DESC LIMIT 5"""
|
||||
)
|
||||
quotes = market_stocks.fetch_retail_quotes()
|
||||
return {
|
||||
"trends": [_row(t) for t in trends],
|
||||
"rss": [_row(r) for r in rss],
|
||||
"top_opportunities": [_row(o) for o in opportunities],
|
||||
"market_stocks": quotes,
|
||||
"market_summary": market_stocks.market_summary(quotes),
|
||||
"updated_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
"""RSS feed ingestion — filtered for kant-en-klaar & supermarkt only."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Optional
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from app.db import execute, execute_returning, fetch_all, fetch_one
|
||||
|
||||
USER_AGENT = "Foodlinkk-Intel/1.0"
|
||||
|
||||
INCLUDE_KEYWORDS = (
|
||||
"kant en klaar", "kant-en-klaar", "kant&klaa", "ready meal", "ready-to-eat",
|
||||
"maaltijd", "maaltijden", "supermarkt", "supermarket", "retail", "jumbo",
|
||||
"albert heijn", "ah ", " plus ", "lidl", "aldi", "dirk", "halal",
|
||||
"convenience", "schap", "filiaal", "foodservice", "vers", "meal",
|
||||
"grocery", "food retail", "kant-en-klaar",
|
||||
)
|
||||
|
||||
EXCLUDE_KEYWORDS = (
|
||||
"voetbal", "sport", "politiek", "verkiezing", "trump", "bbc", "oorlog",
|
||||
"crypto", "bitcoin", "aandelenbeurs", "beurs ", "weerbericht",
|
||||
)
|
||||
|
||||
|
||||
def _parse_date(raw: Optional[str]) -> Optional[datetime]:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return parsedate_to_datetime(raw).astimezone(timezone.utc)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
return re.sub(r"<[^>]+>", "", text or "").strip()[:2000]
|
||||
|
||||
|
||||
def is_relevant(title: str, summary: Optional[str] = None) -> bool:
|
||||
blob = f"{title} {summary or ''}".lower()
|
||||
for bad in EXCLUDE_KEYWORDS:
|
||||
if bad in blob:
|
||||
return False
|
||||
for good in INCLUDE_KEYWORDS:
|
||||
if good in blob:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _fetch_xml(url: str) -> ET.Element:
|
||||
req = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urlopen(req, timeout=25) as resp:
|
||||
data = resp.read()
|
||||
return ET.fromstring(data)
|
||||
|
||||
|
||||
def _skip_keyword_filter(category: Optional[str]) -> bool:
|
||||
return category in ("regelgeving", "cbs", "markt")
|
||||
|
||||
|
||||
def refresh_feed(feed_id: int) -> dict[str, Any]:
|
||||
feed = fetch_one("SELECT * FROM rss_feeds WHERE id = %s AND is_active = TRUE", (feed_id,))
|
||||
if not feed:
|
||||
return {"error": "feed not found"}
|
||||
skip_filter = _skip_keyword_filter(feed.get("category"))
|
||||
root = _fetch_xml(feed["url"])
|
||||
items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
|
||||
inserted = skipped = 0
|
||||
for item in items[:50]:
|
||||
title = (item.findtext("title") or item.findtext("{http://www.w3.org/2005/Atom}title") or "").strip()
|
||||
link = (item.findtext("link") or "").strip()
|
||||
if not link:
|
||||
link_el = item.find("{http://www.w3.org/2005/Atom}link")
|
||||
if link_el is not None:
|
||||
link = link_el.get("href") or ""
|
||||
summary = item.findtext("description") or item.findtext("summary") or item.findtext("{http://www.w3.org/2005/Atom}summary") or ""
|
||||
pub = item.findtext("pubDate") or item.findtext("published") or item.findtext("{http://www.w3.org/2005/Atom}published")
|
||||
if not title or not link:
|
||||
continue
|
||||
clean_summary = _strip_html(summary)
|
||||
cat = (feed.get("category") or "").lower()
|
||||
if cat not in ("regelgeving", "cbs", "markt") and not is_relevant(title, clean_summary):
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
execute_returning(
|
||||
"""INSERT INTO rss_items (feed_id, title, link, summary, published_at)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING id""",
|
||||
(feed_id, title[:500], link[:1000], clean_summary, _parse_date(pub)),
|
||||
)
|
||||
inserted += 1
|
||||
except Exception:
|
||||
pass
|
||||
execute(
|
||||
"UPDATE rss_feeds SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
|
||||
(feed_id,),
|
||||
)
|
||||
return {"feed": feed["name"], "inserted": inserted, "skipped": skipped}
|
||||
|
||||
|
||||
def refresh_all_feeds() -> dict[str, Any]:
|
||||
feeds = fetch_all("SELECT id, name FROM rss_feeds WHERE is_active = TRUE")
|
||||
results = []
|
||||
for f in feeds:
|
||||
try:
|
||||
results.append(refresh_feed(int(f["id"])))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
execute("UPDATE rss_feeds SET last_status = %s WHERE id = %s", (str(exc)[:32], f["id"]))
|
||||
results.append({"feed": f["name"], "error": str(exc)})
|
||||
return {"feeds": len(feeds), "results": results}
|
||||
|
||||
|
||||
def list_live_feed(limit: int = 40, category: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
params: list[Any] = []
|
||||
if category and category.lower() in ("regelgeving", "cbs", "markt"):
|
||||
base = """
|
||||
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
|
||||
FROM rss_items i
|
||||
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
|
||||
WHERE f.category = %s
|
||||
"""
|
||||
params.append(category.lower())
|
||||
else:
|
||||
like_clauses = " OR ".join(
|
||||
f"(i.title ILIKE %s OR COALESCE(i.summary,'') ILIKE %s)" for _ in INCLUDE_KEYWORDS[:12]
|
||||
)
|
||||
for kw in INCLUDE_KEYWORDS[:12]:
|
||||
p = f"%{kw}%"
|
||||
params.extend([p, p])
|
||||
base = f"""
|
||||
SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
|
||||
FROM rss_items i
|
||||
JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
|
||||
WHERE ({like_clauses})
|
||||
"""
|
||||
if category:
|
||||
base += " AND f.category = %s"
|
||||
params.append(category)
|
||||
base += " ORDER BY i.published_at DESC NULLS LAST, i.fetched_at DESC LIMIT %s"
|
||||
params.append(limit)
|
||||
rows = fetch_all(base, tuple(params))
|
||||
skip_filter = category and category.lower() in ("regelgeving", "cbs", "markt")
|
||||
if skip_filter:
|
||||
return [dict(r) for r in rows]
|
||||
return [dict(r) for r in rows if is_relevant(r.get("title") or "", r.get("summary"))]
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Market trends feed for kant-en-klaar / halal ready meals."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.db import execute_returning, fetch_all, json_param
|
||||
|
||||
|
||||
TREND_SEEDS = [
|
||||
{
|
||||
"category": "kant-en-klaar",
|
||||
"trend_name": "Halal ready-meals groei stedelijk",
|
||||
"description": "Stedelijke gebieden met hoge niet-westerse bevolking tonen vraag naar halal kant-en-klaar zonder voldoende schap-aanbod.",
|
||||
"source": "CBS + retail intelligence",
|
||||
"confidence_score": 0.82,
|
||||
"opportunity_score": 0.88,
|
||||
"related_products": ["halal maaltijden", "microwave meals", "salades"],
|
||||
"action_items": ["Target Plus/Jumbo regio's met halal-gap score >60", "Pilot schap bij 3 filialen"],
|
||||
},
|
||||
{
|
||||
"category": "halal",
|
||||
"trend_name": "Certificering als vertrouwen-driver",
|
||||
"description": "Filialen met halal-certificering maar beperkt ready-meal assortiment = upsell kans voor Foodlinkk.",
|
||||
"source": "halal_registry + CRM",
|
||||
"confidence_score": 0.75,
|
||||
"opportunity_score": 0.80,
|
||||
"related_products": ["HQC gecertificeerde maaltijden"],
|
||||
"action_items": ["Match HQC stores met CRM pipeline", "Cross-sell bestaande klanten"],
|
||||
},
|
||||
{
|
||||
"category": "kant-en-klaar",
|
||||
"trend_name": "Convenience trend post-COVID",
|
||||
"description": "Gemiddeld inkomen en eenpersoonshuishoudens correleren met groei kant-en-klaar segment.",
|
||||
"source": "CBS kerncijfers",
|
||||
"confidence_score": 0.70,
|
||||
"opportunity_score": 0.72,
|
||||
"related_products": ["single-serve", "meal kits"],
|
||||
"action_items": ["Filter winkels op huishoudens + inkomen >€35k"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def refresh_trends_from_social() -> dict[str, Any]:
|
||||
"""Derive trend signals from social mentions keywords."""
|
||||
mentions = fetch_all(
|
||||
"""SELECT platform, text, sentiment_score, created_at FROM social_mentions
|
||||
WHERE created_at > NOW() - interval '30 days'
|
||||
ORDER BY created_at DESC LIMIT 100"""
|
||||
)
|
||||
keywords = {
|
||||
"halal": 0, "kant-en-klaar": 0, "ready meal": 0, "meal prep": 0,
|
||||
"supermarkt": 0, "schap": 0, "afhalen": 0,
|
||||
}
|
||||
for m in mentions:
|
||||
text = (m.get("text") or "").lower()
|
||||
for kw in keywords:
|
||||
if kw in text:
|
||||
keywords[kw] += 1
|
||||
|
||||
created = 0
|
||||
for kw, count in keywords.items():
|
||||
if count < 1:
|
||||
continue
|
||||
execute_returning(
|
||||
"""INSERT INTO market_trends (category, trend_name, description, source,
|
||||
confidence_score, opportunity_score, related_products, data_source)
|
||||
VALUES (%s, %s, %s, 'social_mentions', %s, %s, %s, 'live_feed')
|
||||
RETURNING id""",
|
||||
(
|
||||
"kant-en-klaar" if "meal" in kw or "kant" in kw else "halal",
|
||||
f"Social buzz: {kw} ({count} mentions)",
|
||||
f"{count} vermeldingen afgelopen 30 dagen rond '{kw}'.",
|
||||
min(0.95, 0.5 + count * 0.05),
|
||||
min(0.95, 0.4 + count * 0.06),
|
||||
[kw],
|
||||
),
|
||||
)
|
||||
created += 1
|
||||
|
||||
for seed in TREND_SEEDS:
|
||||
exists = fetch_all(
|
||||
"SELECT id FROM market_trends WHERE trend_name = %s LIMIT 1",
|
||||
(seed["trend_name"],),
|
||||
)
|
||||
if not exists:
|
||||
execute_returning(
|
||||
"""INSERT INTO market_trends (category, trend_name, description, source,
|
||||
confidence_score, opportunity_score, related_products, action_items, data_source)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'seed') RETURNING id""",
|
||||
(
|
||||
seed["category"], seed["trend_name"], seed["description"], seed["source"],
|
||||
seed["confidence_score"], seed["opportunity_score"],
|
||||
seed["related_products"], seed["action_items"],
|
||||
),
|
||||
)
|
||||
created += 1
|
||||
return {"trends_created": created, "keyword_hits": keywords}
|
||||
|
||||
|
||||
def list_live_trends(limit: int = 20) -> list[dict[str, Any]]:
|
||||
return fetch_all(
|
||||
"""SELECT * FROM market_trends ORDER BY updated_at DESC NULLS LAST, created_at DESC LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
Reference in New Issue
Block a user