foodlinkk-command-center/cockpit/app/services/monitor.py

from __future__ import annotations

import hashlib
import re
import subprocess
from urllib.parse import urlparse

import httpx
from bs4 import BeautifulSoup

from app.db import execute, fetch_one, get_connection

USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 FoodlinkkBot/2.0"


def _validate_url(url: str) -> str:
    url = (url or "").strip()
    if not url.startswith(("http://", "https://")):
        url = "https://" + url.lstrip("/")
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https") or not parsed.netloc:
        raise ValueError("URL must start with http:// or https://")
    return url


def _fetch_page(url: str) -> tuple[str, str, str, str] | None:
    """Returns final_url, title, normalized_text, raw_html."""
    try:
        resp = httpx.get(
            url,
            timeout=25.0,
            follow_redirects=True,
            headers={"User-Agent": USER_AGENT, "Accept-Language": "nl-NL,nl;q=0.9"},
        )
        if resp.status_code >= 400:
            return None
        html = resp.text
        soup = BeautifulSoup(html, "html.parser")
        title = (soup.title.string or "").strip() if soup.title else ""
        for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
            tag.decompose()
        text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))
        return str(resp.url), title, text, html
    except Exception:
        return None


def get_page_hash(url: str) -> str | None:
    fetched = _fetch_page(url)
    if not fetched:
        return None
    _, _, text, _ = fetched
    return hashlib.md5(text.encode("utf-8")).hexdigest()


def _save_snapshot(site_id: int, url: str, final_url: str, title: str, text: str, html: str) -> int | None:
    import json

    try:
        with get_connection() as conn:
            with conn.cursor() as cur:
                cur.execute(
                    """
                    INSERT INTO crawled_pages (url, final_url, title, content, content_html, site_id, metadata, crawled_at)
                    VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, NOW())
                    ON CONFLICT (url) DO UPDATE SET
                        final_url=EXCLUDED.final_url, title=EXCLUDED.title,
                        content=EXCLUDED.content, content_html=EXCLUDED.content_html,
                        site_id=EXCLUDED.site_id, crawled_at=NOW()
                    RETURNING id
                    """,
                    (
                        url,
                        final_url,
                        title,
                        text[:50000],
                        html[:100000],
                        site_id,
                        json.dumps({"source": "monitor"}),
                    ),
                )
                page_id = cur.fetchone()[0]
                cur.execute(
                    """
                    INSERT INTO browser_sessions (url, final_url, title, status, content_text, site_id, completed_at)
                    VALUES (%s,%s,%s,'completed',%s,%s,NOW()) RETURNING id
                    """,
                    (url, final_url, title, text[:80000], site_id),
                )
                session_id = cur.fetchone()[0]
                cur.execute(
                    "UPDATE monitored_sites SET last_title=%s, last_snapshot_id=%s WHERE id=%s",
                    (title, session_id, site_id),
                )
        return session_id
    except Exception:
        return None


def add_site(url: str, name: str) -> dict:
    url = _validate_url(url)
    name = (name or url).strip()
    fetched = _fetch_page(url)
    if fetched:
        final_url, title, text, html = fetched
        h = hashlib.md5(text.encode("utf-8")).hexdigest()
    else:
        final_url, title, text, html = url, name, "", ""
        h = hashlib.md5(url.encode("utf-8")).hexdigest()
    with get_connection() as conn:
        with conn.cursor() as cur:
            cur.execute(
                """
                INSERT INTO monitored_sites (url, name, last_hash, last_crawled, last_title, is_active)
                VALUES (%s, %s, %s, NOW(), %s, TRUE)
                ON CONFLICT (url) DO UPDATE SET
                    name = EXCLUDED.name,
                    last_hash = EXCLUDED.last_hash,
                    last_crawled = NOW(),
                    last_title = EXCLUDED.last_title,
                    is_active = TRUE
                RETURNING id
                """,
                (url, name, h, title),
            )
            site_id = cur.fetchone()[0]
    if fetched:
        _save_snapshot(site_id, url, final_url, title, text, html)
    row = fetch_one(
        "SELECT id, url, name, last_hash, last_crawled, last_title, is_active, last_snapshot_id FROM monitored_sites WHERE id = %s",
        (site_id,),
    )
    return dict(row) if row else {"id": site_id, "url": url, "name": name}


def remove_site(site_id: int, soft: bool = True) -> None:
    if soft:
        execute("UPDATE monitored_sites SET is_active = FALSE WHERE id = %s", (site_id,))
    else:
        execute("DELETE FROM crawl_logs WHERE site_id = %s", (site_id,))
        execute("DELETE FROM page_changes WHERE site_id = %s", (site_id,))
        execute("DELETE FROM monitored_sites WHERE id = %s", (site_id,))


def trigger_crawl(site_id: int | None = None) -> dict:
    try:
        cmd = ["docker", "exec", "foodlinkk_worker", "python", "-c", "import trigger"]
        subprocess.run(cmd, capture_output=True, timeout=120, check=False)
        return {"ok": True, "method": "worker"}
    except Exception:
        pass

    from app.db import fetch_all

    if site_id:
        row = fetch_one(
            "SELECT id, url, last_hash, last_title FROM monitored_sites WHERE id = %s AND is_active = TRUE",
            (site_id,),
        )
        sites = [row] if row else []
    else:
        sites = fetch_all(
            "SELECT id, url, last_hash, last_title FROM monitored_sites WHERE is_active = TRUE"
        )

    changed = 0
    with get_connection() as conn:
        with conn.cursor() as cur:
            for site in sites:
                fetched = _fetch_page(site["url"])
                if not fetched:
                    cur.execute(
                        "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
                        (site["id"], "ERROR", f"Cannot reach {site['url']}"),
                    )
                    continue
                final_url, title, text, html = fetched
                new_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
                old_hash = site.get("last_hash")
                if old_hash and old_hash != new_hash:
                    cur.execute(
                        "INSERT INTO page_changes (site_id, old_hash, new_hash) VALUES (%s, %s, %s)",
                        (site["id"], old_hash, new_hash),
                    )
                    cur.execute(
                        "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
                        (site["id"], "CHANGE", f"Change detected on {site['url']} — {title}"),
                    )
                    changed += 1
                else:
                    cur.execute(
                        "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
                        (site["id"], "OK", f"Crawl OK — {title}"),
                    )
                cur.execute(
                    """
                    UPDATE monitored_sites SET last_hash=%s, last_crawled=NOW(), last_title=%s WHERE id=%s
                    """,
                    (new_hash, title, site["id"]),
                )
                _save_snapshot(site["id"], site["url"], final_url, title, text, html)
    return {"ok": True, "method": "inline", "changes": changed, "sites": len(sites)}