from __future__ import annotations import hashlib import re import subprocess from urllib.parse import urlparse import httpx from bs4 import BeautifulSoup from app.db import execute, fetch_one, get_connection USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 FoodlinkkBot/2.0" def _validate_url(url: str) -> str: url = (url or "").strip() if not url.startswith(("http://", "https://")): url = "https://" + url.lstrip("/") parsed = urlparse(url) if parsed.scheme not in ("http", "https") or not parsed.netloc: raise ValueError("URL must start with http:// or https://") return url def _fetch_page(url: str) -> tuple[str, str, str, str] | None: """Returns final_url, title, normalized_text, raw_html.""" try: resp = httpx.get( url, timeout=25.0, follow_redirects=True, headers={"User-Agent": USER_AGENT, "Accept-Language": "nl-NL,nl;q=0.9"}, ) if resp.status_code >= 400: return None html = resp.text soup = BeautifulSoup(html, "html.parser") title = (soup.title.string or "").strip() if soup.title else "" for tag in soup(["script", "style", "noscript", "svg", "iframe"]): tag.decompose() text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True)) return str(resp.url), title, text, html except Exception: return None def get_page_hash(url: str) -> str | None: fetched = _fetch_page(url) if not fetched: return None _, _, text, _ = fetched return hashlib.md5(text.encode("utf-8")).hexdigest() def _save_snapshot(site_id: int, url: str, final_url: str, title: str, text: str, html: str) -> int | None: import json try: with get_connection() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO crawled_pages (url, final_url, title, content, content_html, site_id, metadata, crawled_at) VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, NOW()) ON CONFLICT (url) DO UPDATE SET final_url=EXCLUDED.final_url, title=EXCLUDED.title, content=EXCLUDED.content, content_html=EXCLUDED.content_html, site_id=EXCLUDED.site_id, crawled_at=NOW() RETURNING id """, ( url, final_url, title, text[:50000], html[:100000], site_id, json.dumps({"source": "monitor"}), ), ) page_id = cur.fetchone()[0] cur.execute( """ INSERT INTO browser_sessions (url, final_url, title, status, content_text, site_id, completed_at) VALUES (%s,%s,%s,'completed',%s,%s,NOW()) RETURNING id """, (url, final_url, title, text[:80000], site_id), ) session_id = cur.fetchone()[0] cur.execute( "UPDATE monitored_sites SET last_title=%s, last_snapshot_id=%s WHERE id=%s", (title, session_id, site_id), ) return session_id except Exception: return None def add_site(url: str, name: str) -> dict: url = _validate_url(url) name = (name or url).strip() fetched = _fetch_page(url) if fetched: final_url, title, text, html = fetched h = hashlib.md5(text.encode("utf-8")).hexdigest() else: final_url, title, text, html = url, name, "", "" h = hashlib.md5(url.encode("utf-8")).hexdigest() with get_connection() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO monitored_sites (url, name, last_hash, last_crawled, last_title, is_active) VALUES (%s, %s, %s, NOW(), %s, TRUE) ON CONFLICT (url) DO UPDATE SET name = EXCLUDED.name, last_hash = EXCLUDED.last_hash, last_crawled = NOW(), last_title = EXCLUDED.last_title, is_active = TRUE RETURNING id """, (url, name, h, title), ) site_id = cur.fetchone()[0] if fetched: _save_snapshot(site_id, url, final_url, title, text, html) row = fetch_one( "SELECT id, url, name, last_hash, last_crawled, last_title, is_active, last_snapshot_id FROM monitored_sites WHERE id = %s", (site_id,), ) return dict(row) if row else {"id": site_id, "url": url, "name": name} def remove_site(site_id: int, soft: bool = True) -> None: if soft: execute("UPDATE monitored_sites SET is_active = FALSE WHERE id = %s", (site_id,)) else: execute("DELETE FROM crawl_logs WHERE site_id = %s", (site_id,)) execute("DELETE FROM page_changes WHERE site_id = %s", (site_id,)) execute("DELETE FROM monitored_sites WHERE id = %s", (site_id,)) def trigger_crawl(site_id: int | None = None) -> dict: try: cmd = ["docker", "exec", "foodlinkk_worker", "python", "-c", "import trigger"] subprocess.run(cmd, capture_output=True, timeout=120, check=False) return {"ok": True, "method": "worker"} except Exception: pass from app.db import fetch_all if site_id: row = fetch_one( "SELECT id, url, last_hash, last_title FROM monitored_sites WHERE id = %s AND is_active = TRUE", (site_id,), ) sites = [row] if row else [] else: sites = fetch_all( "SELECT id, url, last_hash, last_title FROM monitored_sites WHERE is_active = TRUE" ) changed = 0 with get_connection() as conn: with conn.cursor() as cur: for site in sites: fetched = _fetch_page(site["url"]) if not fetched: cur.execute( "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)", (site["id"], "ERROR", f"Cannot reach {site['url']}"), ) continue final_url, title, text, html = fetched new_hash = hashlib.md5(text.encode("utf-8")).hexdigest() old_hash = site.get("last_hash") if old_hash and old_hash != new_hash: cur.execute( "INSERT INTO page_changes (site_id, old_hash, new_hash) VALUES (%s, %s, %s)", (site["id"], old_hash, new_hash), ) cur.execute( "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)", (site["id"], "CHANGE", f"Change detected on {site['url']} — {title}"), ) changed += 1 else: cur.execute( "INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)", (site["id"], "OK", f"Crawl OK — {title}"), ) cur.execute( """ UPDATE monitored_sites SET last_hash=%s, last_crawled=NOW(), last_title=%s WHERE id=%s """, (new_hash, title, site["id"]), ) _save_snapshot(site["id"], site["url"], final_url, title, text, html) return {"ok": True, "method": "inline", "changes": changed, "sites": len(sites)}