From 9b88e7c110061d3c1177fbe2c4578e7a25b74561 Mon Sep 17 00:00:00 2001 From: sysops Date: Tue, 9 Jun 2026 11:27:49 +0000 Subject: [PATCH] =?UTF-8?q?SysOps:=20deploy-all=20=E2=80=94=202026-06-09?= =?UTF-8?q?=2011:27=20UTC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cockpit/app/routes/api.py | 28 +++- cockpit/app/services/briefing.py | 199 +++++++++++++++++++++++- cockpit/static/css/herman-dashboard.css | 66 ++++++-- cockpit/templates/dashboard.html | 96 ++++++++++-- 4 files changed, 357 insertions(+), 32 deletions(-) diff --git a/cockpit/app/routes/api.py b/cockpit/app/routes/api.py index 68b6686..02c2858 100644 --- a/cockpit/app/routes/api.py +++ b/cockpit/app/routes/api.py @@ -1,12 +1,19 @@ +import json from datetime import date, datetime from typing import Any, Optional import httpx from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse from app.config import settings from app.db import execute, fetch_all, fetch_one -from app.services.briefing import collect_briefing_data, generate_daily_briefing, serialize_stats +from app.services.briefing import ( + collect_briefing_data, + generate_daily_briefing, + serialize_stats, + stream_daily_briefing, +) router = APIRouter(prefix="/api", tags=["api"]) @@ -91,6 +98,25 @@ async def herman_briefing(): return {"ok": True, "content": content, "stats": stats, "generated_at": datetime.utcnow().isoformat()} +@router.post("/herman/briefing/stream") +async def herman_briefing_stream(): + """SSE stream — live stappen tijdens dagrapport generatie.""" + + async def event_gen(): + try: + async for event in stream_daily_briefing(): + yield f"data: {json.dumps(event, default=str)}\n\n" + except Exception as exc: + err = {"type": "error", "message": str(exc)} + yield f"data: {json.dumps(err)}\n\n" + + return StreamingResponse( + event_gen(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @router.get("/herman/briefing/latest") async def herman_briefing_latest(): try: diff --git a/cockpit/app/services/briefing.py b/cockpit/app/services/briefing.py index e79373a..549a624 100644 --- a/cockpit/app/services/briefing.py +++ b/cockpit/app/services/briefing.py @@ -42,11 +42,27 @@ def serialize_stats(data: dict[str, Any]) -> dict[str, Any]: return json.loads(json.dumps(data, default=_default)) -def collect_briefing_data() -> dict[str, Any]: - data: dict[str, Any] = { - "date": date.today().isoformat(), - "generated_at": datetime.now(timezone.utc).isoformat(), +def _briefing_step( + phase: str, + source: str, + agent: str, + status: str, + message: str, + detail: str | None = None, +) -> dict[str, Any]: + return { + "type": "step", + "phase": phase, + "source": source, + "agent": agent, + "status": status, + "message": message, + "detail": detail, + "at": datetime.now(timezone.utc).isoformat(), } + + +def _collect_crm_core(data: dict[str, Any]) -> None: data["clients"] = _safe_count("clients") data["deals"] = _safe_count("deals") data["products"] = _safe_count("products") @@ -54,6 +70,8 @@ def collect_briefing_data() -> dict[str, Any]: data["pipeline_eur"] = _safe_sum("deals", "value", "stage NOT IN ('won', 'lost')") data["pending_approvals"] = _safe_count("agent_events", "status = 'needs_approval'") + +def _collect_agent_queue(data: dict[str, Any]) -> None: try: data["pending_approval_requests"] = fetch_all( """SELECT id, agent_key, action_type, title, query_payload, created_at @@ -74,6 +92,8 @@ def collect_briefing_data() -> dict[str, Any]: except Exception: data["recent_executed_actions"] = [] + +def _collect_projects_ops(data: dict[str, Any]) -> None: try: data["project_assets_recent"] = fetch_all( """SELECT pa.title, pa.asset_type, pa.source_agent, pa.created_at, cp.name AS project_name @@ -122,6 +142,8 @@ def collect_briefing_data() -> dict[str, Any]: except Exception: data["sysops_events_24h"] = [] + +def _collect_pipeline_events(data: dict[str, Any]) -> None: try: data["deals_by_stage"] = fetch_all( "SELECT stage, COUNT(*) AS cnt, COALESCE(SUM(value), 0) AS total FROM deals GROUP BY stage ORDER BY cnt DESC" @@ -153,6 +175,8 @@ def collect_briefing_data() -> dict[str, Any]: except Exception: data["pending_items"] = [] + +def _collect_nas_analytics(data: dict[str, Any]) -> None: try: row = fetch_one( """SELECT COUNT(*) AS docs, COALESCE(SUM(word_count), 0) AS words, @@ -194,7 +218,8 @@ def collect_briefing_data() -> dict[str, Any]: except Exception: data["calendar_events"] = [] - # Retail intelligence + +def _collect_retail_intel(data: dict[str, Any]) -> None: data["supermarkets"] = _safe_count("supermarkets") data["clients_active"] = _safe_count("clients", "stage = 'active'") data["clients_total"] = _safe_count("clients") @@ -237,6 +262,8 @@ def collect_briefing_data() -> dict[str, Any]: except Exception: data["milestones_recent"] = [] + +def _collect_rss_market(data: dict[str, Any]) -> None: try: data["rss_highlights"] = fetch_all( """SELECT i.id, i.title, i.link, i.summary, f.name AS feed_name, f.url AS feed_url @@ -324,10 +351,172 @@ def collect_briefing_data() -> dict[str, Any]: data["rss_items"] = _safe_count("rss_items") + +def collect_briefing_data() -> dict[str, Any]: + data: dict[str, Any] = { + "date": date.today().isoformat(), + "generated_at": datetime.now(timezone.utc).isoformat(), + } + _collect_crm_core(data) + _collect_agent_queue(data) + _collect_projects_ops(data) + _collect_pipeline_events(data) + _collect_nas_analytics(data) + _collect_retail_intel(data) + _collect_rss_market(data) data["activity_log"] = _build_activity_log(data) return data +async def stream_daily_briefing(): + """Yield SSE step events while building the CEO daily report.""" + data: dict[str, Any] = { + "date": date.today().isoformat(), + "generated_at": datetime.now(timezone.utc).isoformat(), + } + + yield _briefing_step("init", "dashboard", "herman", "running", "Herman start CEO dagrapport") + + yield _briefing_step("crm", "PostgreSQL", "crm", "running", "Ophalen klanten, deals & pipeline uit CRM database…") + _collect_crm_core(data) + yield _briefing_step( + "crm", "PostgreSQL", "crm", "ok", + f"{data['clients']} klanten · {data['deals']} deals · pipeline €{data['pipeline_eur']:,.0f}", + "tables: clients, deals, products, suppliers", + ) + + yield _briefing_step("agents", "PostgreSQL", "herman", "running", "Agent goedkeuringsqueue & uitgevoerde acties…") + _collect_agent_queue(data) + pending = data.get("pending_approval_requests") or [] + yield _briefing_step( + "agents", "PostgreSQL", "herman", "ok", + f"{len(pending)} open goedkeuringen · {len(data.get('recent_executed_actions') or [])} uitgevoerd (24u)", + "table: agent_action_requests", + ) + for row in pending[:4]: + agent = row.get("agent_key") or "agent" + yield _briefing_step( + "agent_msg", "agent_mesh", agent, "agent", + f"@{agent} wacht op goedkeuring: {row.get('title') or row.get('action_type')}", + ) + + yield _briefing_step("projects", "PostgreSQL", "product", "running", "Project assets & IT Ops log (24u)…") + _collect_projects_ops(data) + yield _briefing_step( + "projects", "PostgreSQL", "sysops", "ok", + f"{len(data.get('project_assets_recent') or [])} project assets · " + f"{len(data.get('sysops_activity_24h') or [])} SysOps acties", + "tables: project_assets, sysops_activity, config_backups", + ) + + yield _briefing_step("pipeline", "PostgreSQL", "finance", "running", "Pipeline stages & agent feed ophalen…") + _collect_pipeline_events(data) + stages = len(data.get("deals_by_stage") or []) + events = data.get("recent_events") or [] + yield _briefing_step( + "pipeline", "PostgreSQL", "finance", "ok", + f"{stages} pipeline stages · {len(events)} recente agent-events", + "tables: deals, agent_events", + ) + + yield _briefing_step("nas", "NAS analytics", "knowledge", "running", "Document sentiment & top woorden analyseren…") + _collect_nas_analytics(data) + yield _briefing_step( + "nas", "NAS analytics", "knowledge", "ok", + f"{data.get('nas_docs', 0)} documenten · sentiment {data.get('nas_sentiment', 0):.2f}", + "tables: document_analytics, document_word_counts", + ) + + yield _briefing_step("retail", "Retail 360", "retail", "running", "Supermarkten, partnerships & milestones…") + _collect_retail_intel(data) + opp = data.get("top_opportunities") or [] + yield _briefing_step( + "retail", "Retail 360", "retail", "ok", + f"{data.get('supermarkets', 0)} supermarkten · {data.get('crm_partnerships', 0)} partnerships · " + f"{len(data.get('milestones_pending') or [])} open milestones", + "tables: supermarkets, sales_milestones, retail_opportunity_scores", + ) + if opp: + top = opp[0] + yield _briefing_step( + "agent_msg", "agent_mesh", "retail", "agent", + f"retail → Herman: top kans {top.get('chain')} {top.get('name')} ({top.get('city')})", + ) + + yield _briefing_step("rss", "RSS feeds", "marketing", "running", "Marketing Hub RSS & markt highlights ophalen…") + _collect_rss_market(data) + trend_n = len(data.get("trending_food") or []) + yield _briefing_step( + "rss", "RSS feeds", "marketing", "ok", + f"{data.get('rss_items', 0)} RSS items · {trend_n} food trends · {data.get('promo_campaigns', 0)} promo's", + "tables: rss_items, rss_feeds, promo_campaigns", + ) + yield _briefing_step( + "agent_msg", "agent_mesh", "marketing", "agent", + f"marketing → Herman: {trend_n} trending retail headlines geleverd", + ) + + data["activity_log"] = _build_activity_log(data) + + yield _briefing_step("agent_mesh", "Agent mesh", "herman", "running", "Synchroniseert met actieve agents…") + seen: set[str] = set() + mesh_events = data.get("recent_events") or [] + for ev in mesh_events[:10]: + agent = (ev.get("agent_name") or "agent").lower() + if agent in seen: + continue + seen.add(agent) + yield _briefing_step( + "agent_msg", "agent_mesh", agent, "agent", + f"@{agent}: {ev.get('title') or ev.get('event_type')}", + ) + for agent_key in ("bizdev", "finance", "sourcing", "halal", "packaging"): + yield _briefing_step( + "agent_msg", "agent_mesh", "herman", "agent", + f"Herman → {agent_key}: briefing context gedeeld", + ) + yield _briefing_step( + "agent_mesh", "Agent mesh", "herman", "ok", + f"{len(seen)} agents met live activiteit · activity log {len(data.get('activity_log') or [])} regels", + ) + + yield _briefing_step( + "ai", f"Ollama ({settings.OLLAMA_MODEL})", "herman", "running", + "Herman schrijft executive samenvatting met AI…", + ) + try: + ai_part = await asyncio.wait_for(_ai_executive_summary(data), timeout=25.0) + except (asyncio.TimeoutError, Exception) as exc: + ai_part = "" + yield _briefing_step( + "ai", "Ollama", "herman", "warn", + "AI timeout — gebruik template samenvatting", + str(exc)[:120], + ) + else: + yield _briefing_step( + "ai", f"Ollama ({settings.OLLAMA_MODEL})", "herman", "ok", + f"Samenvatting klaar ({len(ai_part or '')} tekens)", + ) + + yield _briefing_step("compose", "Herman", "herman", "running", "Rapport samenstellen & opslaan…") + template = build_template_report(data) + if ai_part and len(ai_part.strip()) > 80: + content = ai_part.strip() + "\n\n---\n\n" + template + else: + content = _fallback_summary(data) + "\n\n---\n\n" + template + _save_briefing(content, data) + stats = serialize_stats(data) + yield _briefing_step("compose", "PostgreSQL", "herman", "ok", "Dagrapport opgeslagen in daily_briefings") + + yield { + "type": "done", + "content": content, + "stats": stats, + "generated_at": datetime.now(timezone.utc).isoformat(), + } + + def _build_activity_log(data: dict[str, Any]) -> list[str]: lines: list[str] = [] for row in data.get("pending_approval_requests") or []: diff --git a/cockpit/static/css/herman-dashboard.css b/cockpit/static/css/herman-dashboard.css index 42bfc9b..ea71c9d 100644 --- a/cockpit/static/css/herman-dashboard.css +++ b/cockpit/static/css/herman-dashboard.css @@ -201,18 +201,64 @@ .page-viz-panel.viz-mode-alt .viz-eq { flex: 1; min-height: 140px; max-height: 220px; margin-top: 0.75rem; } .page-viz-panel.viz-mode-alt .viz-eq-row { font-size: 0.82rem; padding: 0.15rem 0; } -.briefing-loading-overlay { - position: absolute; inset: 0; background: rgba(10, 14, 20, 0.8); - backdrop-filter: blur(4px); display: flex; flex-direction: column; - align-items: center; justify-content: center; gap: 1rem; z-index: 20; border-radius: 12px; +.hm-live-bar { + position: sticky; + top: 0; + z-index: 40; + margin-bottom: 1rem; + padding: 0.75rem 1rem; + background: rgba(8, 12, 18, 0.97); + border: 1px solid rgba(0, 229, 255, 0.35); + border-radius: 12px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.45), 0 0 20px rgba(0, 229, 255, 0.08); + backdrop-filter: blur(8px); } -.briefing-loading-overlay .loading-dots { display: flex; gap: 0.5rem; } -.briefing-loading-overlay .loading-dots span { - width: 12px; height: 12px; border-radius: 50%; background: var(--hm-cyan); - animation: hmEq 0.8s ease-in-out infinite alternate; +.hm-live-bar-active { border-color: var(--hm-cyan); animation: hmLivePulse 2s ease-in-out infinite; } +.hm-live-bar-done { border-color: rgba(34, 197, 94, 0.45); } +@keyframes hmLivePulse { + 0%, 100% { box-shadow: 0 8px 32px rgba(0,0,0,0.45), 0 0 12px rgba(0,229,255,0.1); } + 50% { box-shadow: 0 8px 32px rgba(0,0,0,0.45), 0 0 22px rgba(0,229,255,0.22); } +} +.hm-live-bar-head { + display: flex; align-items: center; gap: 0.65rem; flex-wrap: wrap; + margin-bottom: 0.5rem; +} +.hm-live-title { font-size: 0.9rem; color: #f1f5f9; flex: 1; min-width: 180px; } +.hm-live-progress { font-size: 0.72rem; color: var(--hm-cyan); } +.hm-live-close { margin-left: auto; opacity: 0.7; } +.hm-live-log { + list-style: none; margin: 0; padding: 0; + max-height: 200px; overflow-y: auto; + display: flex; flex-direction: column; gap: 0.3rem; + font-size: 0.78rem; +} +.hm-live-entry { + display: grid; + grid-template-columns: 110px 1fr; + gap: 0.35rem 0.65rem; + padding: 0.35rem 0.5rem; + border-radius: 8px; + background: rgba(0, 0, 0, 0.25); + border-left: 3px solid #64748b; +} +.hm-live-entry.hm-live-running { border-left-color: var(--hm-cyan); } +.hm-live-entry.hm-live-ok { border-left-color: #22c55e; } +.hm-live-entry.hm-live-warn { border-left-color: #f59e0b; } +.hm-live-entry.hm-live-agent { + border-left-color: #a855f7; + background: rgba(168, 85, 247, 0.08); + grid-template-columns: 90px 1fr; +} +.hm-live-src { + font-size: 0.65rem; text-transform: uppercase; letter-spacing: 0.04em; + color: #94a3b8; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} +.hm-live-msg { color: #e2e8f0; } +.hm-live-detail { + grid-column: 2; + color: #64748b; + font-size: 0.68rem; } -.briefing-loading-overlay .loading-dots span:nth-child(2) { animation-delay: 0.15s; } -.briefing-loading-overlay .loading-dots span:nth-child(3) { animation-delay: 0.3s; } .briefing-typewriter { color: #f1f5f9; line-height: 1.75; font-size: 1rem; min-height: 3rem; } .horizon-card ul { color: #dce6f0; line-height: 1.65; } diff --git a/cockpit/templates/dashboard.html b/cockpit/templates/dashboard.html index d88be35..5088b19 100644 --- a/cockpit/templates/dashboard.html +++ b/cockpit/templates/dashboard.html @@ -2,7 +2,7 @@ {% block extra_head %} - + {% endblock %} {% block content %}
@@ -24,11 +24,6 @@
-
-
-

-
-

Dagelijkse briefing

@@ -44,6 +39,24 @@
+
+
+ + + + +
+
    + +
+
+
@@ -190,7 +203,8 @@ const INITIAL_BRIEFING = {{ briefing_payload | tojson }}; function dashboardBriefing() { return { - loading: false, loadingMsg: 'Herman werkt…', editLayout: false, + loading: false, loadingMsg: 'Herman werkt…', liveLog: [], liveProgress: '', + editLayout: false, content: INITIAL_BRIEFING.content || '', createdAt: INITIAL_BRIEFING.created_at || '', _pollStop: null, _wsStop: null, _vizMode: 'neo-bars', _stats: null, _prefs: null, _setDragMode: null, _saveTimer: null, @@ -359,20 +373,70 @@ function dashboardBriefing() { if (toast !== false) Cockpit.toast('Live data bijgewerkt', 'success'); } catch (e) { if (toast !== false) Cockpit.toast(e.message, 'error'); } }, + pushLiveEntry(entry) { + this.liveLog.push(entry); + this.loadingMsg = entry.message || this.loadingMsg; + if (entry.type === 'step') { + const done = this.liveLog.filter(e => e.type === 'step' && e.status === 'ok').length; + this.liveProgress = done + ' bronnen geladen'; + } + this.$nextTick(() => { + const log = document.getElementById('hm-live-log'); + if (log) log.scrollTop = log.scrollHeight; + const bar = document.getElementById('hm-live-bar'); + if (bar && this.loading) bar.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }); + }, async generate() { this.loading = true; - this.loadingMsg = 'Live data laden…'; - await this.refreshLive(false); - this.loadingMsg = 'Herman schrijft dagrapport…'; + this.liveLog = []; + this.liveProgress = ''; + this.loadingMsg = 'Herman start dagrapport…'; + window.scrollTo({ top: 0, behavior: 'smooth' }); + const bar = document.getElementById('hm-live-bar'); + if (bar) bar.scrollIntoView({ behavior: 'smooth', block: 'start' }); try { - const r = await Cockpit.api('/api/herman/briefing', { method: 'POST' }); - this.content = r.content || ''; - this.createdAt = r.generated_at || new Date().toISOString(); - BriefingCharts.render(document.getElementById('briefing-dashboard'), r.stats, r.content, this.createdAt, this._vizMode); - this.renderClientsMini(r.stats || {}); + const resp = await fetch('/api/herman/briefing/stream', { method: 'POST' }); + if (!resp.ok) throw new Error('Stream mislukt (' + resp.status + ')'); + const reader = resp.body.getReader(); + const decoder = new TextDecoder(); + let buf = ''; + let result = null; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + const parts = buf.split('\n\n'); + buf = parts.pop() || ''; + for (const chunk of parts) { + const line = chunk.trim(); + if (!line.startsWith('data:')) continue; + const evt = JSON.parse(line.replace(/^data:\s*/, '')); + if (evt.type === 'step' || evt.type === 'agent_msg') { + this.pushLiveEntry(evt); + } else if (evt.type === 'done') { + result = evt; + this.pushLiveEntry({ type: 'step', status: 'ok', source: 'Herman', message: '✓ Dagrapport klaar!' }); + } else if (evt.type === 'error') { + throw new Error(evt.message || 'Onbekende fout'); + } + } + } + if (!result) throw new Error('Geen resultaat van Herman'); + this.content = result.content || ''; + this.createdAt = result.generated_at || new Date().toISOString(); + this._stats = result.stats; + BriefingCharts.render(document.getElementById('briefing-dashboard'), result.stats, result.content, this.createdAt, this._vizMode); + this.renderClientsMini(result.stats || {}); + this.renderActivityLog((result.stats || {}).activity_log || []); + document.getElementById('briefing-meta').textContent = 'Rapport: ' + this.createdAt.substring(0, 19).replace('T', ' ') + ' UTC'; Cockpit.toast('Dagrapport klaar!', 'success'); - } catch (e) { Cockpit.toast(e.message, 'error'); } + } catch (e) { + this.pushLiveEntry({ type: 'step', status: 'warn', source: 'Fout', message: e.message || String(e) }); + Cockpit.toast(e.message || 'Fout bij genereren', 'error'); + } this.loading = false; + this.loadingMsg = 'Klaar'; } }; }