Platform bundle: marketing publish, IT ops, packaging, agents mesh.

Volledige Foodlinkk Command Center uitbreiding met social automatisering, reclamefolder filters, Proxmox monitoring en documentatie.
2026-06-09 00:41:27 +00:00
commit 5d60d33db1
212 changed files with 30044 additions and 0 deletions
@@ -0,0 +1,152 @@
+"""RSS feed ingestion — filtered for kant-en-klaar & supermarkt only."""
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Optional
+from urllib.request import Request, urlopen
+
+from app.db import execute, execute_returning, fetch_all, fetch_one
+
+USER_AGENT = "Foodlinkk-Intel/1.0"
+
+INCLUDE_KEYWORDS = (
+    "kant en klaar", "kant-en-klaar", "kant&klaa", "ready meal", "ready-to-eat",
+    "maaltijd", "maaltijden", "supermarkt", "supermarket", "retail", "jumbo",
+    "albert heijn", "ah ", " plus ", "lidl", "aldi", "dirk", "halal",
+    "convenience", "schap", "filiaal", "foodservice", "vers", "meal",
+    "grocery", "food retail", "kant-en-klaar",
+)
+
+EXCLUDE_KEYWORDS = (
+    "voetbal", "sport", "politiek", "verkiezing", "trump", "bbc", "oorlog",
+    "crypto", "bitcoin", "aandelenbeurs", "beurs ", "weerbericht",
+)
+
+
+def _parse_date(raw: Optional[str]) -> Optional[datetime]:
+    if not raw:
+        return None
+    try:
+        return parsedate_to_datetime(raw).astimezone(timezone.utc)
+    except Exception:
+        pass
+    try:
+        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except Exception:
+        return None
+
+
+def _strip_html(text: str) -> str:
+    return re.sub(r"<[^>]+>", "", text or "").strip()[:2000]
+
+
+def is_relevant(title: str, summary: Optional[str] = None) -> bool:
+    blob = f"{title} {summary or ''}".lower()
+    for bad in EXCLUDE_KEYWORDS:
+        if bad in blob:
+            return False
+    for good in INCLUDE_KEYWORDS:
+        if good in blob:
+            return True
+    return False
+
+
+def _fetch_xml(url: str) -> ET.Element:
+    req = Request(url, headers={"User-Agent": USER_AGENT})
+    with urlopen(req, timeout=25) as resp:
+        data = resp.read()
+    return ET.fromstring(data)
+
+
+def _skip_keyword_filter(category: Optional[str]) -> bool:
+    return category in ("regelgeving", "cbs", "markt")
+
+
+def refresh_feed(feed_id: int) -> dict[str, Any]:
+    feed = fetch_one("SELECT * FROM rss_feeds WHERE id = %s AND is_active = TRUE", (feed_id,))
+    if not feed:
+        return {"error": "feed not found"}
+    skip_filter = _skip_keyword_filter(feed.get("category"))
+    root = _fetch_xml(feed["url"])
+    items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
+    inserted = skipped = 0
+    for item in items[:50]:
+        title = (item.findtext("title") or item.findtext("{http://www.w3.org/2005/Atom}title") or "").strip()
+        link = (item.findtext("link") or "").strip()
+        if not link:
+            link_el = item.find("{http://www.w3.org/2005/Atom}link")
+            if link_el is not None:
+                link = link_el.get("href") or ""
+        summary = item.findtext("description") or item.findtext("summary") or item.findtext("{http://www.w3.org/2005/Atom}summary") or ""
+        pub = item.findtext("pubDate") or item.findtext("published") or item.findtext("{http://www.w3.org/2005/Atom}published")
+        if not title or not link:
+            continue
+        clean_summary = _strip_html(summary)
+        cat = (feed.get("category") or "").lower()
+        if cat not in ("regelgeving", "cbs", "markt") and not is_relevant(title, clean_summary):
+            skipped += 1
+            continue
+        try:
+            execute_returning(
+                """INSERT INTO rss_items (feed_id, title, link, summary, published_at)
+                   VALUES (%s, %s, %s, %s, %s) RETURNING id""",
+                (feed_id, title[:500], link[:1000], clean_summary, _parse_date(pub)),
+            )
+            inserted += 1
+        except Exception:
+            pass
+    execute(
+        "UPDATE rss_feeds SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
+        (feed_id,),
+    )
+    return {"feed": feed["name"], "inserted": inserted, "skipped": skipped}
+
+
+def refresh_all_feeds() -> dict[str, Any]:
+    feeds = fetch_all("SELECT id, name FROM rss_feeds WHERE is_active = TRUE")
+    results = []
+    for f in feeds:
+        try:
+            results.append(refresh_feed(int(f["id"])))
+        except Exception as exc:  # noqa: BLE001
+            execute("UPDATE rss_feeds SET last_status = %s WHERE id = %s", (str(exc)[:32], f["id"]))
+            results.append({"feed": f["name"], "error": str(exc)})
+    return {"feeds": len(feeds), "results": results}
+
+
+def list_live_feed(limit: int = 40, category: Optional[str] = None) -> list[dict[str, Any]]:
+    params: list[Any] = []
+    if category and category.lower() in ("regelgeving", "cbs", "markt"):
+        base = """
+            SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
+            FROM rss_items i
+            JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
+            WHERE f.category = %s
+        """
+        params.append(category.lower())
+    else:
+        like_clauses = " OR ".join(
+            f"(i.title ILIKE %s OR COALESCE(i.summary,'') ILIKE %s)" for _ in INCLUDE_KEYWORDS[:12]
+        )
+        for kw in INCLUDE_KEYWORDS[:12]:
+            p = f"%{kw}%"
+            params.extend([p, p])
+        base = f"""
+            SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
+            FROM rss_items i
+            JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
+            WHERE ({like_clauses})
+        """
+        if category:
+            base += " AND f.category = %s"
+            params.append(category)
+    base += " ORDER BY i.published_at DESC NULLS LAST, i.fetched_at DESC LIMIT %s"
+    params.append(limit)
+    rows = fetch_all(base, tuple(params))
+    skip_filter = category and category.lower() in ("regelgeving", "cbs", "markt")
+    if skip_filter:
+        return [dict(r) for r in rows]
+    return [dict(r) for r in rows if is_relevant(r.get("title") or "", r.get("summary"))]