feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability
Adds native Command Center features (no new containers) integrated as sub-tabs in the existing Data Explorer and Data Quality views: - Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/ freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring". - Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix, orphan detection, business glossary; local store best-effort synced to OpenMetadata (owner PATCH) → Data Explorer "Ownership". - Access & policy posture: per-dataset compliance combining PII masking, ownership, live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies". - Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with live row counts and column-level PII/masking tracing → Data Explorer "Lineage". - Observability (observability.py): volume/freshness/schema-drift monitoring with alerts → Data Explorer "Observability". - Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count and PK-indexed freshness so monitors stay cheap on 25-54M-row tables. - LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
mo
@@ -0,0 +1,141 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { Activity, RefreshCw, Loader2, AlertTriangle, Clock, Database, TrendingUp, TrendingDown } from 'lucide-react'
|
||||
import { cn } from '../../lib/utils'
|
||||
|
||||
type Point = { t: string; rows: number; delta: number }
|
||||
type DsMetric = {
|
||||
key: string; label: string; engine: string; color: string; table: string
|
||||
rows: number | null; delta: number | null; freshness_age_min: number | null
|
||||
columns: number | null; stalled_cycles: number; error: string | null; ts: string | null
|
||||
series: Point[]
|
||||
}
|
||||
type Alert = { id: string; dataset: string; type: string; severity: string; message: string; count: number; ts: string; last_ts: string }
|
||||
type Metrics = {
|
||||
ok: boolean; enabled: boolean; cycles: number; freshness_min: number
|
||||
alert_counts: { critical: number; warning: number; info: number; total: number }
|
||||
datasets: DsMetric[]
|
||||
}
|
||||
|
||||
function Spark({ data, color }: { data: Point[]; color: string }) {
|
||||
const pts = data.slice(-40)
|
||||
if (pts.length < 2) return <div className="h-10 w-full" />
|
||||
const w = 240
|
||||
const h = 40
|
||||
const vals = pts.map((p) => p.rows)
|
||||
const max = Math.max(...vals)
|
||||
const min = Math.min(...vals)
|
||||
const range = max - min || 1
|
||||
const step = w / (pts.length - 1)
|
||||
const line = pts.map((p, i) => `${i === 0 ? 'M' : 'L'}${(i * step).toFixed(1)},${(h - ((p.rows - min) / range) * (h - 6) - 3).toFixed(1)}`).join(' ')
|
||||
return (
|
||||
<svg viewBox={`0 0 ${w} ${h}`} className="h-10 w-full" preserveAspectRatio="none">
|
||||
<path d={`${line} L${w},${h} L0,${h} Z`} fill={color} fillOpacity="0.12" />
|
||||
<path d={line} fill="none" stroke={color} strokeWidth="1.5" />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
export function ObservabilityView() {
|
||||
const [metrics, setMetrics] = useState<Metrics | null>(null)
|
||||
const [alerts, setAlerts] = useState<Alert[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const [m, a] = await Promise.all([
|
||||
fetch('/api/observability/metrics').then((r) => r.json()),
|
||||
fetch('/api/observability/alerts').then((r) => r.json()),
|
||||
])
|
||||
setMetrics(m)
|
||||
setAlerts(a.active || [])
|
||||
} catch { /* */ } finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => { load() }, [load])
|
||||
useEffect(() => { const t = setInterval(load, 8000); return () => clearInterval(t) }, [load])
|
||||
|
||||
const ac = metrics?.alert_counts
|
||||
const sevBorder = (s: string) => (s === 'critical' ? 'border-rose-500/40 bg-rose-500/5' : s === 'warning' ? 'border-amber-500/40 bg-amber-500/5' : 'border-sky-500/40 bg-sky-500/5')
|
||||
const sevText = (s: string) => (s === 'critical' ? 'text-rose-400' : s === 'warning' ? 'text-amber-400' : 'text-sky-400')
|
||||
|
||||
return (
|
||||
<div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto">
|
||||
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
|
||||
<Kpi icon={AlertTriangle} label="Critical" value={String(ac?.critical ?? 0)} accent={ac?.critical ? '#f87171' : '#34d399'} />
|
||||
<Kpi icon={AlertTriangle} label="Warnings" value={String(ac?.warning ?? 0)} accent={ac?.warning ? '#fbbf24' : '#34d399'} />
|
||||
<Kpi icon={Activity} label="Sweeps" value={String(metrics?.cycles ?? 0)} accent="#60a5fa" />
|
||||
<Kpi icon={Clock} label="Freshness SLA" value={`${metrics?.freshness_min ?? '—'}m`} accent="#a78bfa" />
|
||||
</div>
|
||||
|
||||
<div className="flex shrink-0 items-center px-1">
|
||||
<span className="text-[10px] text-foreground-faint">Tracking volume, freshness & schema drift across every business table</span>
|
||||
<button type="button" onClick={load} className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-border px-2.5 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
|
||||
{loading ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} Refresh
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* active alerts */}
|
||||
{alerts.length > 0 && (
|
||||
<div className="shrink-0 space-y-1.5">
|
||||
{alerts.map((a) => (
|
||||
<div key={a.id} className={cn('flex items-center gap-2 rounded-md border px-3 py-2 text-[10px]', sevBorder(a.severity))}>
|
||||
<AlertTriangle className={cn('h-3.5 w-3.5 shrink-0', sevText(a.severity))} />
|
||||
<span className="font-semibold text-foreground">{a.dataset}</span>
|
||||
<span className="text-foreground-muted">{a.message}</span>
|
||||
<span className={cn('ml-auto rounded px-1.5 py-0.5 text-[8px] uppercase', sevText(a.severity))}>{a.type}</span>
|
||||
{a.count > 1 && <span className="rounded bg-surface-overlay px-1.5 py-0.5 text-[8px] text-foreground-faint">×{a.count}</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* per-dataset volume + freshness */}
|
||||
<div className="grid gap-2 pb-2 lg:grid-cols-2 xl:grid-cols-3">
|
||||
{(metrics?.datasets || []).map((d) => {
|
||||
const stale = d.freshness_age_min != null && d.freshness_age_min > (metrics?.freshness_min ?? 30)
|
||||
return (
|
||||
<div key={d.key} className="panel p-3">
|
||||
<div className="mb-1 flex items-center gap-1.5">
|
||||
<Database className="h-3.5 w-3.5" style={{ color: d.color }} />
|
||||
<span className="text-[11px] font-semibold text-foreground">{d.label}</span>
|
||||
{d.delta != null && d.delta !== 0 && (
|
||||
<span className={cn('ml-auto flex items-center gap-0.5 text-[9px]', d.delta > 0 ? 'text-emerald-400' : 'text-rose-400')}>
|
||||
{d.delta > 0 ? <TrendingUp className="h-3 w-3" /> : <TrendingDown className="h-3 w-3" />}
|
||||
{d.delta > 0 ? '+' : ''}{d.delta.toLocaleString()}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<Spark data={d.series} color={d.color} />
|
||||
<div className="mt-1.5 flex flex-wrap items-center gap-x-3 gap-y-1 text-[9px] text-foreground-muted">
|
||||
<span>Rows: <span className="font-mono text-foreground">{d.rows?.toLocaleString() ?? '—'}</span></span>
|
||||
<span className={cn('flex items-center gap-1', stale && 'text-amber-400')}>
|
||||
<Clock className="h-3 w-3" /> {d.freshness_age_min != null ? `${d.freshness_age_min.toFixed(0)}m old` : 'n/a'}
|
||||
</span>
|
||||
{d.columns != null && <span>{d.columns} cols</span>}
|
||||
{d.stalled_cycles > 0 && <span className="text-amber-400">stalled ×{d.stalled_cycles}</span>}
|
||||
{d.error && <span className="text-rose-400">err</span>}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Kpi({ icon: Icon, label, value, accent }: { icon: typeof Activity; label: string; value: string; accent: string }) {
|
||||
return (
|
||||
<div className="panel flex items-center gap-3 px-3 py-2.5">
|
||||
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
|
||||
<Icon className="h-4 w-4" />
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
|
||||
<p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user