foodlinkk-command-center/tools-api/app/connectors/rss_feeds.py

"""RSS feed ingestion — filtered for kant-en-klaar & supermarkt only."""
from __future__ import annotations

import re
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any, Optional
from urllib.request import Request, urlopen

from app.db import execute, execute_returning, fetch_all, fetch_one

USER_AGENT = "Foodlinkk-Intel/1.0"

INCLUDE_KEYWORDS = (
    "kant en klaar", "kant-en-klaar", "kant&klaa", "ready meal", "ready-to-eat",
    "maaltijd", "maaltijden", "supermarkt", "supermarket", "retail", "jumbo",
    "albert heijn", "ah ", " plus ", "lidl", "aldi", "dirk", "halal",
    "convenience", "schap", "filiaal", "foodservice", "vers", "meal",
    "grocery", "food retail", "kant-en-klaar",
)

EXCLUDE_KEYWORDS = (
    "voetbal", "sport", "politiek", "verkiezing", "trump", "bbc", "oorlog",
    "crypto", "bitcoin", "aandelenbeurs", "beurs ", "weerbericht",
)


def _parse_date(raw: Optional[str]) -> Optional[datetime]:
    if not raw:
        return None
    try:
        return parsedate_to_datetime(raw).astimezone(timezone.utc)
    except Exception:
        pass
    try:
        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except Exception:
        return None


def _strip_html(text: str) -> str:
    return re.sub(r"<[^>]+>", "", text or "").strip()[:2000]


def is_relevant(title: str, summary: Optional[str] = None) -> bool:
    blob = f"{title} {summary or ''}".lower()
    for bad in EXCLUDE_KEYWORDS:
        if bad in blob:
            return False
    for good in INCLUDE_KEYWORDS:
        if good in blob:
            return True
    return False


def _fetch_xml(url: str) -> ET.Element:
    req = Request(url, headers={"User-Agent": USER_AGENT})
    with urlopen(req, timeout=25) as resp:
        data = resp.read()
    return ET.fromstring(data)


def _skip_keyword_filter(category: Optional[str]) -> bool:
    return category in ("regelgeving", "cbs", "markt")


def refresh_feed(feed_id: int) -> dict[str, Any]:
    feed = fetch_one("SELECT * FROM rss_feeds WHERE id = %s AND is_active = TRUE", (feed_id,))
    if not feed:
        return {"error": "feed not found"}
    skip_filter = _skip_keyword_filter(feed.get("category"))
    root = _fetch_xml(feed["url"])
    items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
    inserted = skipped = 0
    for item in items[:50]:
        title = (item.findtext("title") or item.findtext("{http://www.w3.org/2005/Atom}title") or "").strip()
        link = (item.findtext("link") or "").strip()
        if not link:
            link_el = item.find("{http://www.w3.org/2005/Atom}link")
            if link_el is not None:
                link = link_el.get("href") or ""
        summary = item.findtext("description") or item.findtext("summary") or item.findtext("{http://www.w3.org/2005/Atom}summary") or ""
        pub = item.findtext("pubDate") or item.findtext("published") or item.findtext("{http://www.w3.org/2005/Atom}published")
        if not title or not link:
            continue
        clean_summary = _strip_html(summary)
        cat = (feed.get("category") or "").lower()
        if cat not in ("regelgeving", "cbs", "markt") and not is_relevant(title, clean_summary):
            skipped += 1
            continue
        try:
            execute_returning(
                """INSERT INTO rss_items (feed_id, title, link, summary, published_at)
                   VALUES (%s, %s, %s, %s, %s) RETURNING id""",
                (feed_id, title[:500], link[:1000], clean_summary, _parse_date(pub)),
            )
            inserted += 1
        except Exception:
            pass
    execute(
        "UPDATE rss_feeds SET last_fetch_at = NOW(), last_status = 'ok' WHERE id = %s",
        (feed_id,),
    )
    return {"feed": feed["name"], "inserted": inserted, "skipped": skipped}


def refresh_all_feeds() -> dict[str, Any]:
    feeds = fetch_all("SELECT id, name FROM rss_feeds WHERE is_active = TRUE")
    results = []
    for f in feeds:
        try:
            results.append(refresh_feed(int(f["id"])))
        except Exception as exc:  # noqa: BLE001
            execute("UPDATE rss_feeds SET last_status = %s WHERE id = %s", (str(exc)[:32], f["id"]))
            results.append({"feed": f["name"], "error": str(exc)})
    return {"feeds": len(feeds), "results": results}


def list_live_feed(limit: int = 40, category: Optional[str] = None) -> list[dict[str, Any]]:
    params: list[Any] = []
    if category and category.lower() in ("regelgeving", "cbs", "markt"):
        base = """
            SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
            FROM rss_items i
            JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
            WHERE f.category = %s
        """
        params.append(category.lower())
    else:
        like_clauses = " OR ".join(
            f"(i.title ILIKE %s OR COALESCE(i.summary,'') ILIKE %s)" for _ in INCLUDE_KEYWORDS[:12]
        )
        for kw in INCLUDE_KEYWORDS[:12]:
            p = f"%{kw}%"
            params.extend([p, p])
        base = f"""
            SELECT i.*, f.name AS feed_name, f.category, f.url AS feed_url
            FROM rss_items i
            JOIN rss_feeds f ON f.id = i.feed_id AND f.is_active = TRUE
            WHERE ({like_clauses})
        """
        if category:
            base += " AND f.category = %s"
            params.append(category)
    base += " ORDER BY i.published_at DESC NULLS LAST, i.fetched_at DESC LIMIT %s"
    params.append(limit)
    rows = fetch_all(base, tuple(params))
    skip_filter = category and category.lower() in ("regelgeving", "cbs", "markt")
    if skip_filter:
        return [dict(r) for r in rows]
    return [dict(r) for r in rows if is_relevant(r.get("title") or "", r.get("summary"))]