"""Automatische NAS + second-brain synchronisatie.""" from __future__ import annotations import hashlib import json import logging import os from typing import Any import httpx from app.db import execute, fetch_all, fetch_one log = logging.getLogger("cockpit.auto_ingest") DOC_INGEST_URL = os.getenv("DOC_INGEST_URL", "http://10.4.7.19:8750") TOOLS_API_URL = os.getenv("TOOLS_API_URL", "http://tools-api:8700") def _log_run(source: str, status: str, details: dict[str, Any]) -> None: try: execute( "INSERT INTO ingest_automation_log (source, status, details) VALUES (%s, %s, %s::jsonb)", (source, status, json.dumps(details)), ) except Exception as exc: log.warning("ingest log failed: %s", exc) def _brain_chat_id_for_client(client_id: int) -> int: return -int(client_id) async def sync_nas_to_brain(limit: int = 40) -> dict[str, Any]: """Indexeer geanalyseerde documenten in second brain (pgvector) per klant.""" rows = fetch_all( """ SELECT da.storage_path, da.filename, da.doc_type, da.sentiment_label, da.word_count, dl.client_id, c.name AS client_name FROM document_analytics da LEFT JOIN document_links dl ON ( da.storage_path = dl.storage_path OR da.storage_path LIKE dl.storage_path || '/%' ) AND dl.is_folder = TRUE LEFT JOIN document_links dl2 ON da.storage_path = dl2.storage_path AND dl2.is_folder = FALSE LEFT JOIN clients c ON c.id = COALESCE(dl2.client_id, dl.client_id) WHERE da.storage_path IS NOT NULL ORDER BY da.analyzed_at DESC NULLS LAST LIMIT %s """, (max(1, min(limit, 200)),), ) synced = 0 errors = 0 async with httpx.AsyncClient(timeout=120.0) as client: for row in rows: path = row.get("storage_path") or "" if not path: continue client_id = row.get("client_id") chat_id = _brain_chat_id_for_client(client_id) if client_id else 888_000_001 text = ( f"NAS document: {row.get('filename') or path}\n" f"Pad: {path}\nType: {row.get('doc_type')}\n" f"Sentiment: {row.get('sentiment_label')}\nWoorden: {row.get('word_count')}" ) msg_hash = int(hashlib.md5(path.encode()).hexdigest()[:8], 16) try: r = await client.post( f"{TOOLS_API_URL.rstrip('/')}/brain/messages", json={ "chat_id": chat_id, "direction": "inbound", "role": "system", "content_type": "document", "content_text": text[:4000], "agent_name": "auto-ingest", "embed": True, "chat_type": "client_brain" if client_id else "nas_corpus", "user_name": row.get("client_name") or "NAS", "content_json": {"storage_path": path, "client_id": client_id}, "telegram_message_id": msg_hash, }, ) if r.status_code < 400: synced += 1 else: errors += 1 except Exception as exc: log.warning("brain sync %s: %s", path, exc) errors += 1 return {"synced": synced, "errors": errors, "candidates": len(rows)} async def run_full_auto_sync(force_scan: bool = False) -> dict[str, Any]: """Volledige pipeline: NAS scan → Chroma RAG → brain embeddings.""" out: dict[str, Any] = {"ok": True, "steps": {}} try: async with httpx.AsyncClient(timeout=300.0) as client: r = await client.post( f"{DOC_INGEST_URL.rstrip('/')}/ingest/scan", params={"force": "true" if force_scan else "false"}, ) r.raise_for_status() out["steps"]["nas_scan"] = r.json() except Exception as exc: out["steps"]["nas_scan"] = {"error": str(exc)} brain = await sync_nas_to_brain() out["steps"]["brain_sync"] = brain status = "ok" if not out["steps"].get("nas_scan", {}).get("error") else "partial" _log_run("auto_sync", status, out) out["last_logged"] = True return out def last_auto_sync() -> dict[str, Any] | None: row = fetch_one( "SELECT source, status, details, created_at FROM ingest_automation_log ORDER BY created_at DESC LIMIT 1" ) if not row: return None d = dict(row) if hasattr(d.get("created_at"), "isoformat"): d["created_at"] = d["created_at"].isoformat() if isinstance(d.get("details"), str): try: d["details"] = json.loads(d["details"]) except Exception: pass return d