Platform bundle: marketing publish, IT ops, packaging, agents mesh.

Volledige Foodlinkk Command Center uitbreiding met social automatisering,
reclamefolder filters, Proxmox monitoring en documentatie.

tools-api/app/brain.py

@@ -0,0 +1,425 @@
+"""PostgreSQL second brain — Telegram messages, graph edges, pgvector."""
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Optional
+
+import httpx
+
+from app.db import execute, execute_returning, fetch_all, fetch_one, json_param
+
+log = logging.getLogger("tools.brain")
+
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.4.7.19:11434").rstrip("/")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "nomic-embed-text")
+EMBED_DIM = 768
+
+
+async def embed_text(text: str) -> list[float]:
+    text = (text or "").strip()[:4000]
+    if not text:
+        return []
+    async with httpx.AsyncClient(timeout=120.0) as client:
+        r = await client.post(
+            f"{OLLAMA_URL}/api/embeddings",
+            json={"model": EMBED_MODEL, "prompt": text},
+        )
+        r.raise_for_status()
+        vec = r.json().get("embedding") or []
+        if len(vec) != EMBED_DIM:
+            raise ValueError(f"embedding dim {len(vec)} != {EMBED_DIM}")
+        return vec
+
+
+def _vec_param(vec: list[float]) -> str:
+    return "[" + ",".join(f"{x:.8f}" for x in vec) + "]"
+
+
+def upsert_conversation(
+    chat_id: int,
+    *,
+    chat_type: str = "private",
+    user_name: str | None = None,
+    user_role: str | None = None,
+    metadata: dict | None = None,
+) -> dict[str, Any]:
+    row = execute_returning(
+        """
+        INSERT INTO telegram_conversations (chat_id, chat_type, user_name, user_role, metadata, updated_at)
+        VALUES (%s, %s, %s, %s, %s, NOW())
+        ON CONFLICT (chat_id) DO UPDATE SET
+            user_name = COALESCE(EXCLUDED.user_name, telegram_conversations.user_name),
+            user_role = COALESCE(EXCLUDED.user_role, telegram_conversations.user_role),
+            metadata = telegram_conversations.metadata || EXCLUDED.metadata,
+            updated_at = NOW()
+        RETURNING *
+        """,
+        (chat_id, chat_type, user_name, user_role, json_param(metadata or {})),
+    )
+    return row or {}
+
+
+def store_message(
+    chat_id: int,
+    *,
+    direction: str,
+    content_text: str | None = None,
+    content_type: str = "text",
+    role: str = "user",
+    telegram_message_id: int | None = None,
+    reply_to_db_id: int | None = None,
+    agent_name: str | None = None,
+    content_json: dict | None = None,
+    user_name: str | None = None,
+    user_role: str | None = None,
+    chat_type: str = "private",
+) -> dict[str, Any]:
+    conv = upsert_conversation(
+        chat_id, chat_type=chat_type, user_name=user_name, user_role=user_role
+    )
+    row = execute_returning(
+        """
+        INSERT INTO telegram_messages (
+            conversation_id, telegram_message_id, direction, role, content_type,
+            content_text, content_json, reply_to_message_id, agent_name
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+        RETURNING *
+        """,
+        (
+            conv["id"],
+            telegram_message_id,
+            direction,
+            role,
+            content_type,
+            (content_text or "")[:8000] or None,
+            json_param(content_json or {}),
+            reply_to_db_id,
+            agent_name,
+        ),
+    )
+    return row or {}
+
+
+def store_edge(
+    source_message_id: int,
+    edge_type: str,
+    *,
+    target_message_id: int | None = None,
+    target_entity_type: str | None = None,
+    target_entity_id: int | None = None,
+    weight: float = 1.0,
+    metadata: dict | None = None,
+) -> dict[str, Any]:
+    row = execute_returning(
+        """
+        INSERT INTO telegram_message_edges (
+            source_message_id, target_message_id, target_entity_type,
+            target_entity_id, edge_type, weight, metadata
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s)
+        RETURNING *
+        """,
+        (
+            source_message_id,
+            target_message_id,
+            target_entity_type,
+            target_entity_id,
+            edge_type,
+            weight,
+            json_param(metadata or {}),
+        ),
+    )
+    return row or {}
+
+
+def store_embedding(message_id: int, vec: list[float], chunk_index: int = 0) -> None:
+    execute(
+        """
+        INSERT INTO telegram_message_embeddings (message_id, chunk_index, embedding, model)
+        VALUES (%s, %s, %s::vector, %s)
+        ON CONFLICT (message_id, chunk_index) DO UPDATE SET
+            embedding = EXCLUDED.embedding,
+            model = EXCLUDED.model
+        """,
+        (message_id, chunk_index, _vec_param(vec), EMBED_MODEL),
+    )
+
+
+async def store_message_with_embedding(
+    chat_id: int,
+    *,
+    direction: str,
+    content_text: str | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    msg = store_message(chat_id, direction=direction, content_text=content_text, **kwargs)
+    text = (content_text or "").strip()
+    if text and len(text) >= 8:
+        try:
+            vec = await embed_text(text)
+            if vec:
+                store_embedding(msg["id"], vec)
+        except Exception as exc:
+            log.warning("embed failed msg=%s: %s", msg.get("id"), exc)
+    return msg
+
+
+def get_graph(chat_id: int, limit: int = 50) -> dict[str, Any]:
+    conv = fetch_one("SELECT * FROM telegram_conversations WHERE chat_id = %s", (chat_id,))
+    if not conv:
+        return {"chat_id": chat_id, "nodes": [], "edges": []}
+
+    messages = fetch_all(
+        """
+        SELECT id, direction, role, content_type, content_text, agent_name,
+               reply_to_message_id, created_at
+        FROM telegram_messages
+        WHERE conversation_id = %s
+        ORDER BY created_at DESC
+        LIMIT %s
+        """,
+        (conv["id"], limit),
+    )
+    msg_ids = [m["id"] for m in messages]
+    edges: list[dict] = []
+    if msg_ids:
+        edges = fetch_all(
+            """
+            SELECT e.*, sm.content_text AS source_preview, tm.content_text AS target_preview
+            FROM telegram_message_edges e
+            JOIN telegram_messages sm ON sm.id = e.source_message_id
+            LEFT JOIN telegram_messages tm ON tm.id = e.target_message_id
+            WHERE e.source_message_id = ANY(%s) OR e.target_message_id = ANY(%s)
+            ORDER BY e.created_at DESC
+            LIMIT %s
+            """,
+            (msg_ids, msg_ids, limit * 2),
+        )
+
+    nodes = [
+        {
+            "id": m["id"],
+            "label": (m.get("content_text") or "")[:80],
+            "direction": m.get("direction"),
+            "role": m.get("role"),
+            "agent": m.get("agent_name"),
+            "type": m.get("content_type"),
+            "created_at": m.get("created_at"),
+        }
+        for m in reversed(messages)
+    ]
+    edge_list = [
+        {
+            "id": e["id"],
+            "from": e["source_message_id"],
+            "to": e.get("target_message_id"),
+            "type": e["edge_type"],
+            "entity_type": e.get("target_entity_type"),
+            "entity_id": e.get("target_entity_id"),
+            "weight": e.get("weight"),
+        }
+        for e in edges
+    ]
+    return {
+        "chat_id": chat_id,
+        "conversation_id": conv["id"],
+        "nodes": nodes,
+        "edges": edge_list,
+        "stats": {"messages": len(nodes), "edges": len(edge_list)},
+    }
+
+
+async def search_memory(
+    query: str,
+    *,
+    chat_id: int | None = None,
+    limit: int = 8,
+) -> list[dict[str, Any]]:
+    query = query.strip()
+    if not query:
+        return []
+
+    results: list[dict[str, Any]] = []
+
+    try:
+        vec = await embed_text(query)
+        if vec:
+            v = _vec_param(vec)
+            if chat_id is not None:
+                rows = fetch_all(
+                    """
+                    SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at,
+                           c.chat_id,
+                           1 - (e.embedding <=> %s::vector) AS score
+                    FROM telegram_message_embeddings e
+                    JOIN telegram_messages m ON m.id = e.message_id
+                    JOIN telegram_conversations c ON c.id = m.conversation_id
+                    WHERE m.content_text IS NOT NULL AND c.chat_id = %s
+                    ORDER BY e.embedding <=> %s::vector
+                    LIMIT %s
+                    """,
+                    (v, chat_id, v, limit),
+                )
+            else:
+                rows = fetch_all(
+                    """
+                    SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at,
+                           c.chat_id,
+                           1 - (e.embedding <=> %s::vector) AS score
+                    FROM telegram_message_embeddings e
+                    JOIN telegram_messages m ON m.id = e.message_id
+                    JOIN telegram_conversations c ON c.id = m.conversation_id
+                    WHERE m.content_text IS NOT NULL
+                    ORDER BY e.embedding <=> %s::vector
+                    LIMIT %s
+                    """,
+                    (v, v, limit),
+                )
+            results.extend(rows)
+    except Exception as exc:
+        log.warning("vector search failed: %s", exc)
+
+    if len(results) < limit:
+        params2: list[Any] = [query, limit - len(results)]
+        chat_clause = ""
+        if chat_id is not None:
+            chat_clause = "AND c.chat_id = %s"
+            params2.append(chat_id)
+        fts = fetch_all(
+            f"""
+            SELECT m.id, m.content_text, m.direction, m.agent_name, m.created_at, c.chat_id,
+                   ts_rank(to_tsvector('simple', coalesce(m.content_text, '')),
+                           plainto_tsquery('simple', %s)) AS score
+            FROM telegram_messages m
+            JOIN telegram_conversations c ON c.id = m.conversation_id
+            WHERE to_tsvector('simple', coalesce(m.content_text, '')) @@ plainto_tsquery('simple', %s)
+            {chat_clause}
+            ORDER BY score DESC
+            LIMIT %s
+            """,
+            tuple([query, query, *([chat_id] if chat_id else []), limit - len(results)]),
+        )
+        seen = {r["id"] for r in results}
+        for row in fts:
+            if row["id"] not in seen:
+                results.append(row)
+
+    return results[:limit]
+
+# Append to tools-api/app/brain.py
+
+def list_conversations(limit: int = 50) -> list[dict[str, Any]]:
+    return fetch_all(
+        """
+        SELECT c.*,
+               (SELECT COUNT(*) FROM telegram_messages m WHERE m.conversation_id = c.id) AS message_count,
+               (SELECT MAX(m2.created_at) FROM telegram_messages m2 WHERE m2.conversation_id = c.id) AS last_message_at
+        FROM telegram_conversations c
+        ORDER BY (SELECT MAX(m3.created_at) FROM telegram_messages m3 WHERE m3.conversation_id = c.id) DESC NULLS LAST
+        LIMIT %s
+        """,
+        (limit,),
+    )
+
+
+def list_feed(*, chat_id: int | None = None, limit: int = 80, offset: int = 0) -> list[dict[str, Any]]:
+    if chat_id is not None:
+        return fetch_all(
+            """
+            SELECT m.*, c.chat_id, c.user_name, c.user_role,
+                   EXISTS(SELECT 1 FROM telegram_message_embeddings e WHERE e.message_id = m.id) AS has_embedding
+            FROM telegram_messages m
+            JOIN telegram_conversations c ON c.id = m.conversation_id
+            WHERE c.chat_id = %s
+            ORDER BY m.created_at DESC
+            LIMIT %s OFFSET %s
+            """,
+            (chat_id, limit, offset),
+        )
+    return fetch_all(
+        """
+        SELECT m.*, c.chat_id, c.user_name, c.user_role,
+               EXISTS(SELECT 1 FROM telegram_message_embeddings e WHERE e.message_id = m.id) AS has_embedding
+        FROM telegram_messages m
+        JOIN telegram_conversations c ON c.id = m.conversation_id
+        ORDER BY m.created_at DESC
+        LIMIT %s OFFSET %s
+        """,
+        (limit, offset),
+    )
+
+
+def get_dashboard_stats() -> dict[str, Any]:
+    stats = fetch_one(
+        """
+        SELECT
+            (SELECT COUNT(*) FROM telegram_conversations) AS conversations,
+            (SELECT COUNT(*) FROM telegram_messages) AS messages,
+            (SELECT COUNT(*) FROM telegram_messages WHERE direction = 'in') AS inbound,
+            (SELECT COUNT(*) FROM telegram_messages WHERE direction = 'out') AS outbound,
+            (SELECT COUNT(*) FROM telegram_message_edges) AS edges,
+            (SELECT COUNT(*) FROM telegram_message_embeddings) AS embeddings
+        """
+    ) or {}
+    recent = list_feed(limit=15)
+    events = fetch_all(
+        """
+        SELECT id, agent_name, event_type, title, body, status, channel, created_at
+        FROM agent_events
+        WHERE channel = 'telegram' OR agent_name IN ('hermes', 'herman', 'personal_pa')
+        ORDER BY created_at DESC
+        LIMIT 20
+        """
+    )
+    return {"stats": stats, "recent_messages": recent, "agent_events": events}
+
+
+def get_global_graph(limit: int = 100) -> dict[str, Any]:
+    messages = fetch_all(
+        """
+        SELECT m.id, m.direction, m.role, m.content_type, m.content_text, m.agent_name,
+               m.created_at, c.chat_id, c.user_name
+        FROM telegram_messages m
+        JOIN telegram_conversations c ON c.id = m.conversation_id
+        ORDER BY m.created_at DESC
+        LIMIT %s
+        """,
+        (limit,),
+    )
+    if not messages:
+        return {"nodes": [], "edges": []}
+    ids = [m["id"] for m in messages]
+    edges = fetch_all(
+        """
+        SELECT e.*
+        FROM telegram_message_edges e
+        WHERE e.source_message_id = ANY(%s) OR e.target_message_id = ANY(%s)
+        """,
+        (ids, ids),
+    )
+    nodes = [
+        {
+            "id": m["id"],
+            "label": (m.get("content_text") or "")[:100],
+            "direction": m.get("direction"),
+            "role": m.get("role"),
+            "agent": m.get("agent_name"),
+            "chat_id": m.get("chat_id"),
+            "user_name": m.get("user_name"),
+            "type": m.get("content_type"),
+            "created_at": m.get("created_at"),
+        }
+        for m in reversed(messages)
+    ]
+    edge_list = [
+        {
+            "id": e["id"],
+            "from": e["source_message_id"],
+            "to": e.get("target_message_id"),
+            "type": e["edge_type"],
+            "entity_type": e.get("target_entity_type"),
+            "entity_id": e.get("target_entity_id"),
+        }
+        for e in edges
+    ]
+    return {"nodes": nodes, "edges": edge_list, "stats": {"nodes": len(nodes), "edges": len(edge_list)}}