SysOps: deploy-all — 2026-06-09 11:27 UTC

cockpit/app/routes/api.py

@@ -1,12 +1,19 @@
+import json
 from datetime import date, datetime
 from typing import Any, Optional
 
 import httpx
 from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
 
 from app.config import settings
 from app.db import execute, fetch_all, fetch_one
-from app.services.briefing import collect_briefing_data, generate_daily_briefing, serialize_stats
+from app.services.briefing import (
+    collect_briefing_data,
+    generate_daily_briefing,
+    serialize_stats,
+    stream_daily_briefing,
+)
 
 router = APIRouter(prefix="/api", tags=["api"])
 
@@ -91,6 +98,25 @@ async def herman_briefing():
     return {"ok": True, "content": content, "stats": stats, "generated_at": datetime.utcnow().isoformat()}
 
 
+@router.post("/herman/briefing/stream")
+async def herman_briefing_stream():
+    """SSE stream — live stappen tijdens dagrapport generatie."""
+
+    async def event_gen():
+        try:
+            async for event in stream_daily_briefing():
+                yield f"data: {json.dumps(event, default=str)}\n\n"
+        except Exception as exc:
+            err = {"type": "error", "message": str(exc)}
+            yield f"data: {json.dumps(err)}\n\n"
+
+    return StreamingResponse(
+        event_gen(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+    )
+
+
 @router.get("/herman/briefing/latest")
 async def herman_briefing_latest():
     try:

cockpit/app/services/briefing.py

@@ -42,11 +42,27 @@ def serialize_stats(data: dict[str, Any]) -> dict[str, Any]:
     return json.loads(json.dumps(data, default=_default))
 
 
-def collect_briefing_data() -> dict[str, Any]:
-    data: dict[str, Any] = {
-        "date": date.today().isoformat(),
-        "generated_at": datetime.now(timezone.utc).isoformat(),
+def _briefing_step(
+    phase: str,
+    source: str,
+    agent: str,
+    status: str,
+    message: str,
+    detail: str | None = None,
+) -> dict[str, Any]:
+    return {
+        "type": "step",
+        "phase": phase,
+        "source": source,
+        "agent": agent,
+        "status": status,
+        "message": message,
+        "detail": detail,
+        "at": datetime.now(timezone.utc).isoformat(),
     }
+
+
+def _collect_crm_core(data: dict[str, Any]) -> None:
     data["clients"] = _safe_count("clients")
     data["deals"] = _safe_count("deals")
     data["products"] = _safe_count("products")
@@ -54,6 +70,8 @@ def collect_briefing_data() -> dict[str, Any]:
     data["pipeline_eur"] = _safe_sum("deals", "value", "stage NOT IN ('won', 'lost')")
     data["pending_approvals"] = _safe_count("agent_events", "status = 'needs_approval'")
 
+
+def _collect_agent_queue(data: dict[str, Any]) -> None:
     try:
         data["pending_approval_requests"] = fetch_all(
             """SELECT id, agent_key, action_type, title, query_payload, created_at
@@ -74,6 +92,8 @@ def collect_briefing_data() -> dict[str, Any]:
     except Exception:
         data["recent_executed_actions"] = []
 
+
+def _collect_projects_ops(data: dict[str, Any]) -> None:
     try:
         data["project_assets_recent"] = fetch_all(
             """SELECT pa.title, pa.asset_type, pa.source_agent, pa.created_at, cp.name AS project_name
@@ -122,6 +142,8 @@ def collect_briefing_data() -> dict[str, Any]:
     except Exception:
         data["sysops_events_24h"] = []
 
+
+def _collect_pipeline_events(data: dict[str, Any]) -> None:
     try:
         data["deals_by_stage"] = fetch_all(
             "SELECT stage, COUNT(*) AS cnt, COALESCE(SUM(value), 0) AS total FROM deals GROUP BY stage ORDER BY cnt DESC"
@@ -153,6 +175,8 @@ def collect_briefing_data() -> dict[str, Any]:
     except Exception:
         data["pending_items"] = []
 
+
+def _collect_nas_analytics(data: dict[str, Any]) -> None:
     try:
         row = fetch_one(
             """SELECT COUNT(*) AS docs, COALESCE(SUM(word_count), 0) AS words,
@@ -194,7 +218,8 @@ def collect_briefing_data() -> dict[str, Any]:
     except Exception:
         data["calendar_events"] = []
 
-    # Retail intelligence
+
+def _collect_retail_intel(data: dict[str, Any]) -> None:
     data["supermarkets"] = _safe_count("supermarkets")
     data["clients_active"] = _safe_count("clients", "stage = 'active'")
     data["clients_total"] = _safe_count("clients")
@@ -237,6 +262,8 @@ def collect_briefing_data() -> dict[str, Any]:
     except Exception:
         data["milestones_recent"] = []
 
+
+def _collect_rss_market(data: dict[str, Any]) -> None:
     try:
         data["rss_highlights"] = fetch_all(
             """SELECT i.id, i.title, i.link, i.summary, f.name AS feed_name, f.url AS feed_url
@@ -324,10 +351,172 @@ def collect_briefing_data() -> dict[str, Any]:
 
     data["rss_items"] = _safe_count("rss_items")
 
+
+def collect_briefing_data() -> dict[str, Any]:
+    data: dict[str, Any] = {
+        "date": date.today().isoformat(),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+    }
+    _collect_crm_core(data)
+    _collect_agent_queue(data)
+    _collect_projects_ops(data)
+    _collect_pipeline_events(data)
+    _collect_nas_analytics(data)
+    _collect_retail_intel(data)
+    _collect_rss_market(data)
     data["activity_log"] = _build_activity_log(data)
     return data
 
 
+async def stream_daily_briefing():
+    """Yield SSE step events while building the CEO daily report."""
+    data: dict[str, Any] = {
+        "date": date.today().isoformat(),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    yield _briefing_step("init", "dashboard", "herman", "running", "Herman start CEO dagrapport")
+
+    yield _briefing_step("crm", "PostgreSQL", "crm", "running", "Ophalen klanten, deals & pipeline uit CRM database…")
+    _collect_crm_core(data)
+    yield _briefing_step(
+        "crm", "PostgreSQL", "crm", "ok",
+        f"{data['clients']} klanten · {data['deals']} deals · pipeline €{data['pipeline_eur']:,.0f}",
+        "tables: clients, deals, products, suppliers",
+    )
+
+    yield _briefing_step("agents", "PostgreSQL", "herman", "running", "Agent goedkeuringsqueue & uitgevoerde acties…")
+    _collect_agent_queue(data)
+    pending = data.get("pending_approval_requests") or []
+    yield _briefing_step(
+        "agents", "PostgreSQL", "herman", "ok",
+        f"{len(pending)} open goedkeuringen · {len(data.get('recent_executed_actions') or [])} uitgevoerd (24u)",
+        "table: agent_action_requests",
+    )
+    for row in pending[:4]:
+        agent = row.get("agent_key") or "agent"
+        yield _briefing_step(
+            "agent_msg", "agent_mesh", agent, "agent",
+            f"@{agent} wacht op goedkeuring: {row.get('title') or row.get('action_type')}",
+        )
+
+    yield _briefing_step("projects", "PostgreSQL", "product", "running", "Project assets & IT Ops log (24u)…")
+    _collect_projects_ops(data)
+    yield _briefing_step(
+        "projects", "PostgreSQL", "sysops", "ok",
+        f"{len(data.get('project_assets_recent') or [])} project assets · "
+        f"{len(data.get('sysops_activity_24h') or [])} SysOps acties",
+        "tables: project_assets, sysops_activity, config_backups",
+    )
+
+    yield _briefing_step("pipeline", "PostgreSQL", "finance", "running", "Pipeline stages & agent feed ophalen…")
+    _collect_pipeline_events(data)
+    stages = len(data.get("deals_by_stage") or [])
+    events = data.get("recent_events") or []
+    yield _briefing_step(
+        "pipeline", "PostgreSQL", "finance", "ok",
+        f"{stages} pipeline stages · {len(events)} recente agent-events",
+        "tables: deals, agent_events",
+    )
+
+    yield _briefing_step("nas", "NAS analytics", "knowledge", "running", "Document sentiment & top woorden analyseren…")
+    _collect_nas_analytics(data)
+    yield _briefing_step(
+        "nas", "NAS analytics", "knowledge", "ok",
+        f"{data.get('nas_docs', 0)} documenten · sentiment {data.get('nas_sentiment', 0):.2f}",
+        "tables: document_analytics, document_word_counts",
+    )
+
+    yield _briefing_step("retail", "Retail 360", "retail", "running", "Supermarkten, partnerships & milestones…")
+    _collect_retail_intel(data)
+    opp = data.get("top_opportunities") or []
+    yield _briefing_step(
+        "retail", "Retail 360", "retail", "ok",
+        f"{data.get('supermarkets', 0)} supermarkten · {data.get('crm_partnerships', 0)} partnerships · "
+        f"{len(data.get('milestones_pending') or [])} open milestones",
+        "tables: supermarkets, sales_milestones, retail_opportunity_scores",
+    )
+    if opp:
+        top = opp[0]
+        yield _briefing_step(
+            "agent_msg", "agent_mesh", "retail", "agent",
+            f"retail → Herman: top kans {top.get('chain')} {top.get('name')} ({top.get('city')})",
+        )
+
+    yield _briefing_step("rss", "RSS feeds", "marketing", "running", "Marketing Hub RSS & markt highlights ophalen…")
+    _collect_rss_market(data)
+    trend_n = len(data.get("trending_food") or [])
+    yield _briefing_step(
+        "rss", "RSS feeds", "marketing", "ok",
+        f"{data.get('rss_items', 0)} RSS items · {trend_n} food trends · {data.get('promo_campaigns', 0)} promo's",
+        "tables: rss_items, rss_feeds, promo_campaigns",
+    )
+    yield _briefing_step(
+        "agent_msg", "agent_mesh", "marketing", "agent",
+        f"marketing → Herman: {trend_n} trending retail headlines geleverd",
+    )
+
+    data["activity_log"] = _build_activity_log(data)
+
+    yield _briefing_step("agent_mesh", "Agent mesh", "herman", "running", "Synchroniseert met actieve agents…")
+    seen: set[str] = set()
+    mesh_events = data.get("recent_events") or []
+    for ev in mesh_events[:10]:
+        agent = (ev.get("agent_name") or "agent").lower()
+        if agent in seen:
+            continue
+        seen.add(agent)
+        yield _briefing_step(
+            "agent_msg", "agent_mesh", agent, "agent",
+            f"@{agent}: {ev.get('title') or ev.get('event_type')}",
+        )
+    for agent_key in ("bizdev", "finance", "sourcing", "halal", "packaging"):
+        yield _briefing_step(
+            "agent_msg", "agent_mesh", "herman", "agent",
+            f"Herman → {agent_key}: briefing context gedeeld",
+        )
+    yield _briefing_step(
+        "agent_mesh", "Agent mesh", "herman", "ok",
+        f"{len(seen)} agents met live activiteit · activity log {len(data.get('activity_log') or [])} regels",
+    )
+
+    yield _briefing_step(
+        "ai", f"Ollama ({settings.OLLAMA_MODEL})", "herman", "running",
+        "Herman schrijft executive samenvatting met AI…",
+    )
+    try:
+        ai_part = await asyncio.wait_for(_ai_executive_summary(data), timeout=25.0)
+    except (asyncio.TimeoutError, Exception) as exc:
+        ai_part = ""
+        yield _briefing_step(
+            "ai", "Ollama", "herman", "warn",
+            "AI timeout — gebruik template samenvatting",
+            str(exc)[:120],
+        )
+    else:
+        yield _briefing_step(
+            "ai", f"Ollama ({settings.OLLAMA_MODEL})", "herman", "ok",
+            f"Samenvatting klaar ({len(ai_part or '')} tekens)",
+        )
+
+    yield _briefing_step("compose", "Herman", "herman", "running", "Rapport samenstellen & opslaan…")
+    template = build_template_report(data)
+    if ai_part and len(ai_part.strip()) > 80:
+        content = ai_part.strip() + "\n\n---\n\n" + template
+    else:
+        content = _fallback_summary(data) + "\n\n---\n\n" + template
+    _save_briefing(content, data)
+    stats = serialize_stats(data)
+    yield _briefing_step("compose", "PostgreSQL", "herman", "ok", "Dagrapport opgeslagen in daily_briefings")
+
+    yield {
+        "type": "done",
+        "content": content,
+        "stats": stats,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+
 def _build_activity_log(data: dict[str, Any]) -> list[str]:
     lines: list[str] = []
     for row in data.get("pending_approval_requests") or []:

cockpit/static/css/herman-dashboard.css

@@ -201,18 +201,64 @@
 .page-viz-panel.viz-mode-alt .viz-eq { flex: 1; min-height: 140px; max-height: 220px; margin-top: 0.75rem; }
 .page-viz-panel.viz-mode-alt .viz-eq-row { font-size: 0.82rem; padding: 0.15rem 0; }
 
-.briefing-loading-overlay {
-  position: absolute; inset: 0; background: rgba(10, 14, 20, 0.8);
-  backdrop-filter: blur(4px); display: flex; flex-direction: column;
-  align-items: center; justify-content: center; gap: 1rem; z-index: 20; border-radius: 12px;
+.hm-live-bar {
+  position: sticky;
+  top: 0;
+  z-index: 40;
+  margin-bottom: 1rem;
+  padding: 0.75rem 1rem;
+  background: rgba(8, 12, 18, 0.97);
+  border: 1px solid rgba(0, 229, 255, 0.35);
+  border-radius: 12px;
+  box-shadow: 0 8px 32px rgba(0, 0, 0, 0.45), 0 0 20px rgba(0, 229, 255, 0.08);
+  backdrop-filter: blur(8px);
 }
-.briefing-loading-overlay .loading-dots { display: flex; gap: 0.5rem; }
-.briefing-loading-overlay .loading-dots span {
-  width: 12px; height: 12px; border-radius: 50%; background: var(--hm-cyan);
-  animation: hmEq 0.8s ease-in-out infinite alternate;
+.hm-live-bar-active { border-color: var(--hm-cyan); animation: hmLivePulse 2s ease-in-out infinite; }
+.hm-live-bar-done { border-color: rgba(34, 197, 94, 0.45); }
+@keyframes hmLivePulse {
+  0%, 100% { box-shadow: 0 8px 32px rgba(0,0,0,0.45), 0 0 12px rgba(0,229,255,0.1); }
+  50% { box-shadow: 0 8px 32px rgba(0,0,0,0.45), 0 0 22px rgba(0,229,255,0.22); }
+}
+.hm-live-bar-head {
+  display: flex; align-items: center; gap: 0.65rem; flex-wrap: wrap;
+  margin-bottom: 0.5rem;
+}
+.hm-live-title { font-size: 0.9rem; color: #f1f5f9; flex: 1; min-width: 180px; }
+.hm-live-progress { font-size: 0.72rem; color: var(--hm-cyan); }
+.hm-live-close { margin-left: auto; opacity: 0.7; }
+.hm-live-log {
+  list-style: none; margin: 0; padding: 0;
+  max-height: 200px; overflow-y: auto;
+  display: flex; flex-direction: column; gap: 0.3rem;
+  font-size: 0.78rem;
+}
+.hm-live-entry {
+  display: grid;
+  grid-template-columns: 110px 1fr;
+  gap: 0.35rem 0.65rem;
+  padding: 0.35rem 0.5rem;
+  border-radius: 8px;
+  background: rgba(0, 0, 0, 0.25);
+  border-left: 3px solid #64748b;
+}
+.hm-live-entry.hm-live-running { border-left-color: var(--hm-cyan); }
+.hm-live-entry.hm-live-ok { border-left-color: #22c55e; }
+.hm-live-entry.hm-live-warn { border-left-color: #f59e0b; }
+.hm-live-entry.hm-live-agent {
+  border-left-color: #a855f7;
+  background: rgba(168, 85, 247, 0.08);
+  grid-template-columns: 90px 1fr;
+}
+.hm-live-src {
+  font-size: 0.65rem; text-transform: uppercase; letter-spacing: 0.04em;
+  color: #94a3b8; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
+}
+.hm-live-msg { color: #e2e8f0; }
+.hm-live-detail {
+  grid-column: 2;
+  color: #64748b;
+  font-size: 0.68rem;
 }
-.briefing-loading-overlay .loading-dots span:nth-child(2) { animation-delay: 0.15s; }
-.briefing-loading-overlay .loading-dots span:nth-child(3) { animation-delay: 0.3s; }
 
 .briefing-typewriter { color: #f1f5f9; line-height: 1.75; font-size: 1rem; min-height: 3rem; }
 .horizon-card ul { color: #dce6f0; line-height: 1.65; }

cockpit/templates/dashboard.html

@@ -2,7 +2,7 @@
 {% block extra_head %}
 <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
 <link rel="stylesheet" href="/static/css/hermes.css" />
-<link rel="stylesheet" href="/static/css/herman-dashboard.css?v=14" />
+<link rel="stylesheet" href="/static/css/herman-dashboard.css?v=15" />
 {% endblock %}
 {% block content %}
 <div class="herman-shell hm-neo">
@@ -24,11 +24,6 @@
 </div>
 
 <section class="panel herman-briefing" x-data="dashboardBriefing()" x-init="init()">
-  <div class="briefing-loading-overlay" x-show="loading" x-cloak>
-    <div class="loading-dots"><span></span><span></span><span></span></div>
-    <p x-text="loadingMsg"></p>
-  </div>
-
   <header class="herman-briefing-header">
     <div>
       <h2>Dagelijkse briefing</h2>
@@ -44,6 +39,24 @@
     </div>
   </header>
 
+  <div id="hm-live-bar" class="hm-live-bar" x-show="loading || liveLog.length" x-cloak :class="loading ? 'hm-live-bar-active' : 'hm-live-bar-done'">
+    <div class="hm-live-bar-head">
+      <span class="hm-live-dot"></span>
+      <strong class="hm-live-title" x-text="loading ? loadingMsg : 'Dagrapport voltooid'"></strong>
+      <span class="hm-live-progress" x-show="loading" x-text="liveProgress"></span>
+      <button type="button" class="btn btn-sm hm-live-close" x-show="!loading && liveLog.length" @click="liveLog = []">✕</button>
+    </div>
+    <ol id="hm-live-log" class="hm-live-log">
+      <template x-for="(entry, idx) in liveLog" :key="idx">
+        <li :class="'hm-live-entry hm-live-' + (entry.status || 'ok') + (entry.type === 'agent_msg' ? ' hm-live-agent' : '')">
+          <span class="hm-live-src" x-text="entry.source || entry.agent"></span>
+          <span class="hm-live-msg" x-text="entry.message"></span>
+          <small class="hm-live-detail" x-show="entry.detail" x-text="entry.detail"></small>
+        </li>
+      </template>
+    </ol>
+  </div>
+
   <div id="viz-toolbar" class="viz-toolbar"></div>
   <div id="dashboard-customize" class="hm-customize-wrap" x-show="editLayout" x-cloak></div>
 
@@ -190,7 +203,8 @@
 const INITIAL_BRIEFING = {{ briefing_payload | tojson }};
 function dashboardBriefing() {
   return {
-    loading: false, loadingMsg: 'Herman werkt…', editLayout: false,
+    loading: false, loadingMsg: 'Herman werkt…', liveLog: [], liveProgress: '',
+    editLayout: false,
     content: INITIAL_BRIEFING.content || '', createdAt: INITIAL_BRIEFING.created_at || '',
     _pollStop: null, _wsStop: null, _vizMode: 'neo-bars', _stats: null,
     _prefs: null, _setDragMode: null, _saveTimer: null,
@@ -359,20 +373,70 @@ function dashboardBriefing() {
         if (toast !== false) Cockpit.toast('Live data bijgewerkt', 'success');
       } catch (e) { if (toast !== false) Cockpit.toast(e.message, 'error'); }
     },
+    pushLiveEntry(entry) {
+      this.liveLog.push(entry);
+      this.loadingMsg = entry.message || this.loadingMsg;
+      if (entry.type === 'step') {
+        const done = this.liveLog.filter(e => e.type === 'step' && e.status === 'ok').length;
+        this.liveProgress = done + ' bronnen geladen';
+      }
+      this.$nextTick(() => {
+        const log = document.getElementById('hm-live-log');
+        if (log) log.scrollTop = log.scrollHeight;
+        const bar = document.getElementById('hm-live-bar');
+        if (bar && this.loading) bar.scrollIntoView({ behavior: 'smooth', block: 'start' });
+      });
+    },
     async generate() {
       this.loading = true;
-      this.loadingMsg = 'Live data laden…';
-      await this.refreshLive(false);
-      this.loadingMsg = 'Herman schrijft dagrapport…';
+      this.liveLog = [];
+      this.liveProgress = '';
+      this.loadingMsg = 'Herman start dagrapport…';
+      window.scrollTo({ top: 0, behavior: 'smooth' });
+      const bar = document.getElementById('hm-live-bar');
+      if (bar) bar.scrollIntoView({ behavior: 'smooth', block: 'start' });
       try {
-        const r = await Cockpit.api('/api/herman/briefing', { method: 'POST' });
-        this.content = r.content || '';
-        this.createdAt = r.generated_at || new Date().toISOString();
-        BriefingCharts.render(document.getElementById('briefing-dashboard'), r.stats, r.content, this.createdAt, this._vizMode);
-        this.renderClientsMini(r.stats || {});
+        const resp = await fetch('/api/herman/briefing/stream', { method: 'POST' });
+        if (!resp.ok) throw new Error('Stream mislukt (' + resp.status + ')');
+        const reader = resp.body.getReader();
+        const decoder = new TextDecoder();
+        let buf = '';
+        let result = null;
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          buf += decoder.decode(value, { stream: true });
+          const parts = buf.split('\n\n');
+          buf = parts.pop() || '';
+          for (const chunk of parts) {
+            const line = chunk.trim();
+            if (!line.startsWith('data:')) continue;
+            const evt = JSON.parse(line.replace(/^data:\s*/, ''));
+            if (evt.type === 'step' || evt.type === 'agent_msg') {
+              this.pushLiveEntry(evt);
+            } else if (evt.type === 'done') {
+              result = evt;
+              this.pushLiveEntry({ type: 'step', status: 'ok', source: 'Herman', message: '✓ Dagrapport klaar!' });
+            } else if (evt.type === 'error') {
+              throw new Error(evt.message || 'Onbekende fout');
+            }
+          }
+        }
+        if (!result) throw new Error('Geen resultaat van Herman');
+        this.content = result.content || '';
+        this.createdAt = result.generated_at || new Date().toISOString();
+        this._stats = result.stats;
+        BriefingCharts.render(document.getElementById('briefing-dashboard'), result.stats, result.content, this.createdAt, this._vizMode);
+        this.renderClientsMini(result.stats || {});
+        this.renderActivityLog((result.stats || {}).activity_log || []);
+        document.getElementById('briefing-meta').textContent = 'Rapport: ' + this.createdAt.substring(0, 19).replace('T', ' ') + ' UTC';
         Cockpit.toast('Dagrapport klaar!', 'success');
-      } catch (e) { Cockpit.toast(e.message, 'error'); }
+      } catch (e) {
+        this.pushLiveEntry({ type: 'step', status: 'warn', source: 'Fout', message: e.message || String(e) });
+        Cockpit.toast(e.message || 'Fout bij genereren', 'error');
+      }
       this.loading = false;
+      this.loadingMsg = 'Klaar';
     }
   };
 }