Platform bundle: marketing publish, IT ops, packaging, agents mesh.
Volledige Foodlinkk Command Center uitbreiding met social automatisering, reclamefolder filters, Proxmox monitoring en documentatie.
This commit is contained in:
Aissa
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.db import execute, fetch_one, get_connection
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 FoodlinkkBot/2.0"
|
||||
|
||||
|
||||
def _validate_url(url: str) -> str:
|
||||
url = (url or "").strip()
|
||||
if not url.startswith(("http://", "https://")):
|
||||
url = "https://" + url.lstrip("/")
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
||||
raise ValueError("URL must start with http:// or https://")
|
||||
return url
|
||||
|
||||
|
||||
def _fetch_page(url: str) -> tuple[str, str, str, str] | None:
|
||||
"""Returns final_url, title, normalized_text, raw_html."""
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url,
|
||||
timeout=25.0,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT, "Accept-Language": "nl-NL,nl;q=0.9"},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
html = resp.text
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
title = (soup.title.string or "").strip() if soup.title else ""
|
||||
for tag in soup(["script", "style", "noscript", "svg", "iframe"]):
|
||||
tag.decompose()
|
||||
text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True))
|
||||
return str(resp.url), title, text, html
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_page_hash(url: str) -> str | None:
|
||||
fetched = _fetch_page(url)
|
||||
if not fetched:
|
||||
return None
|
||||
_, _, text, _ = fetched
|
||||
return hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _save_snapshot(site_id: int, url: str, final_url: str, title: str, text: str, html: str) -> int | None:
|
||||
import json
|
||||
|
||||
try:
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO crawled_pages (url, final_url, title, content, content_html, site_id, metadata, crawled_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, NOW())
|
||||
ON CONFLICT (url) DO UPDATE SET
|
||||
final_url=EXCLUDED.final_url, title=EXCLUDED.title,
|
||||
content=EXCLUDED.content, content_html=EXCLUDED.content_html,
|
||||
site_id=EXCLUDED.site_id, crawled_at=NOW()
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
url,
|
||||
final_url,
|
||||
title,
|
||||
text[:50000],
|
||||
html[:100000],
|
||||
site_id,
|
||||
json.dumps({"source": "monitor"}),
|
||||
),
|
||||
)
|
||||
page_id = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO browser_sessions (url, final_url, title, status, content_text, site_id, completed_at)
|
||||
VALUES (%s,%s,%s,'completed',%s,%s,NOW()) RETURNING id
|
||||
""",
|
||||
(url, final_url, title, text[:80000], site_id),
|
||||
)
|
||||
session_id = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"UPDATE monitored_sites SET last_title=%s, last_snapshot_id=%s WHERE id=%s",
|
||||
(title, session_id, site_id),
|
||||
)
|
||||
return session_id
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def add_site(url: str, name: str) -> dict:
|
||||
url = _validate_url(url)
|
||||
name = (name or url).strip()
|
||||
fetched = _fetch_page(url)
|
||||
if fetched:
|
||||
final_url, title, text, html = fetched
|
||||
h = hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
final_url, title, text, html = url, name, "", ""
|
||||
h = hashlib.md5(url.encode("utf-8")).hexdigest()
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO monitored_sites (url, name, last_hash, last_crawled, last_title, is_active)
|
||||
VALUES (%s, %s, %s, NOW(), %s, TRUE)
|
||||
ON CONFLICT (url) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
last_hash = EXCLUDED.last_hash,
|
||||
last_crawled = NOW(),
|
||||
last_title = EXCLUDED.last_title,
|
||||
is_active = TRUE
|
||||
RETURNING id
|
||||
""",
|
||||
(url, name, h, title),
|
||||
)
|
||||
site_id = cur.fetchone()[0]
|
||||
if fetched:
|
||||
_save_snapshot(site_id, url, final_url, title, text, html)
|
||||
row = fetch_one(
|
||||
"SELECT id, url, name, last_hash, last_crawled, last_title, is_active, last_snapshot_id FROM monitored_sites WHERE id = %s",
|
||||
(site_id,),
|
||||
)
|
||||
return dict(row) if row else {"id": site_id, "url": url, "name": name}
|
||||
|
||||
|
||||
def remove_site(site_id: int, soft: bool = True) -> None:
|
||||
if soft:
|
||||
execute("UPDATE monitored_sites SET is_active = FALSE WHERE id = %s", (site_id,))
|
||||
else:
|
||||
execute("DELETE FROM crawl_logs WHERE site_id = %s", (site_id,))
|
||||
execute("DELETE FROM page_changes WHERE site_id = %s", (site_id,))
|
||||
execute("DELETE FROM monitored_sites WHERE id = %s", (site_id,))
|
||||
|
||||
|
||||
def trigger_crawl(site_id: int | None = None) -> dict:
|
||||
try:
|
||||
cmd = ["docker", "exec", "foodlinkk_worker", "python", "-c", "import trigger"]
|
||||
subprocess.run(cmd, capture_output=True, timeout=120, check=False)
|
||||
return {"ok": True, "method": "worker"}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from app.db import fetch_all
|
||||
|
||||
if site_id:
|
||||
row = fetch_one(
|
||||
"SELECT id, url, last_hash, last_title FROM monitored_sites WHERE id = %s AND is_active = TRUE",
|
||||
(site_id,),
|
||||
)
|
||||
sites = [row] if row else []
|
||||
else:
|
||||
sites = fetch_all(
|
||||
"SELECT id, url, last_hash, last_title FROM monitored_sites WHERE is_active = TRUE"
|
||||
)
|
||||
|
||||
changed = 0
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
for site in sites:
|
||||
fetched = _fetch_page(site["url"])
|
||||
if not fetched:
|
||||
cur.execute(
|
||||
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
|
||||
(site["id"], "ERROR", f"Cannot reach {site['url']}"),
|
||||
)
|
||||
continue
|
||||
final_url, title, text, html = fetched
|
||||
new_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
old_hash = site.get("last_hash")
|
||||
if old_hash and old_hash != new_hash:
|
||||
cur.execute(
|
||||
"INSERT INTO page_changes (site_id, old_hash, new_hash) VALUES (%s, %s, %s)",
|
||||
(site["id"], old_hash, new_hash),
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
|
||||
(site["id"], "CHANGE", f"Change detected on {site['url']} — {title}"),
|
||||
)
|
||||
changed += 1
|
||||
else:
|
||||
cur.execute(
|
||||
"INSERT INTO crawl_logs (site_id, status, message) VALUES (%s, %s, %s)",
|
||||
(site["id"], "OK", f"Crawl OK — {title}"),
|
||||
)
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE monitored_sites SET last_hash=%s, last_crawled=NOW(), last_title=%s WHERE id=%s
|
||||
""",
|
||||
(new_hash, title, site["id"]),
|
||||
)
|
||||
_save_snapshot(site["id"], site["url"], final_url, title, text, html)
|
||||
return {"ok": True, "method": "inline", "changes": changed, "sites": len(sites)}
|
||||
Reference in New Issue
Block a user