feat(cdc): add Re-sync sources button + auto-heal for Debezium connectors

- streaming_ops: add connector status discovery, manual resync endpoint
  (POST /api/pipeline/streaming/resync), connectors status endpoint, and a
  background connector_autoheal_loop that restarts FAILED tasks automatically
- main: wire connector_autoheal_loop into app lifespan
- api.ts: add resyncSources() helper
- ChangesView: add "Re-sync sources" header button with live status
This commit is contained in:
mo
2026-06-29 13:31:11 +00:00
parent 1e2cfe80f2
commit d066def8b4
4 changed files with 173 additions and 2 deletions
+35 -2
View File
@@ -1,6 +1,6 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { Activity, Radio, RefreshCw, PlusCircle, Pencil, Trash2, Database, Layers, TrendingUp } from 'lucide-react'
import { fetchChanges, fetchChangeStats } from '../../lib/api'
import { Activity, Radio, RefreshCw, PlusCircle, Pencil, Trash2, Database, Layers, TrendingUp, Cable } from 'lucide-react'
import { fetchChanges, fetchChangeStats, resyncSources } from '../../lib/api'
import type { CdcChange, CdcStats } from '../../types'
import { cn } from '../../lib/utils'
@@ -224,6 +224,8 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
const [expanded, setExpanded] = useState<string | null>(null)
const [connected, setConnected] = useState(false)
const [flash, setFlash] = useState(false)
const [resyncing, setResyncing] = useState(false)
const [resyncMsg, setResyncMsg] = useState<string | null>(null)
// Live overlay: CDC events counted straight off the WebSocket stream since the
// last server stats snapshot. The top KPIs/charts = authoritative server stats
// (refreshed every 2.5s) + this overlay, so they move in lock-step with the
@@ -246,6 +248,25 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
applyStats(s)
}, [applyStats])
const doResync = useCallback(async () => {
setResyncing(true)
setResyncMsg('Restarting Debezium connectors…')
try {
const res = await resyncSources(true)
if (res) {
setResyncMsg(`Re-synced · ${res.healthy ?? 0}/${res.total ?? 0} connectors healthy`)
setTimeout(() => load(), 2500)
} else {
setResyncMsg('Re-sync failed — check ETL Guardian terminal')
}
} catch {
setResyncMsg('Re-sync failed — check ETL Guardian terminal')
} finally {
setResyncing(false)
setTimeout(() => setResyncMsg(null), 7000)
}
}, [load])
useEffect(() => {
load()
const iv = setInterval(() => fetchChangeStats(15).then((s) => applyStats(s)), 2500)
@@ -346,6 +367,18 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
connected ? 'border-emerald-500/40 bg-emerald-500/10 text-emerald-300' : 'border-rose-500/40 bg-rose-500/10 text-rose-300')}>
<Radio className={cn('h-3 w-3', connected && 'animate-pulse')} /> {connected ? 'STREAMING' : 'OFFLINE'}
</span>
{resyncMsg && (
<span className="hidden text-[10px] text-amber-300/90 md:inline">{resyncMsg}</span>
)}
<button
type="button"
onClick={doResync}
disabled={resyncing}
title="Restart all Debezium source connectors so CDC catches up after a database outage"
className="flex items-center gap-1 rounded border border-amber-500/40 bg-amber-500/10 px-2 py-1 text-[10px] font-medium text-amber-300 hover:bg-amber-500/20 disabled:opacity-60"
>
<Cable className={cn('h-3 w-3', resyncing && 'animate-spin')} /> {resyncing ? 'Re-syncing…' : 'Re-sync sources'}
</button>
<button type="button" onClick={load} className="flex items-center gap-1 rounded border border-border/60 px-2 py-1 text-[10px] text-foreground-muted hover:text-docker">
<RefreshCw className="h-3 w-3" /> Refresh
</button>
+12
View File
@@ -115,6 +115,18 @@ export async function fetchChangeStats(minutes = 15): Promise<CdcStats | null> {
return fetchJson<CdcStats>(`/api/changes/stats?minutes=${minutes}`, 8000)
}
export type ConnectorState = { name: string; state?: string; failed?: number[] }
export type ResyncResult = { ok: boolean; restarted?: string[]; healthy?: number; total?: number; after?: ConnectorState[] }
export async function resyncSources(force = true): Promise<ResyncResult | null> {
const r = await fetch('/api/pipeline/streaming/resync', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ force }),
})
return r.ok ? ((await r.json()) as ResyncResult) : null
}
export async function fetchAgentOpsStatus() {
return fetchJson<Record<string, unknown>>('/api/agent-ops/status', 8000)
}