feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability

Adds native Command Center features (no new containers) integrated as sub-tabs
in the existing Data Explorer and Data Quality views:

- Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/
  freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring".
- Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix,
  orphan detection, business glossary; local store best-effort synced to
  OpenMetadata (owner PATCH) → Data Explorer "Ownership".
- Access & policy posture: per-dataset compliance combining PII masking, ownership,
  live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies".
- Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with
  live row counts and column-level PII/masking tracing → Data Explorer "Lineage".
- Observability (observability.py): volume/freshness/schema-drift monitoring with
  alerts → Data Explorer "Observability".
- Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count
  and PK-indexed freshness so monitors stay cheap on 25-54M-row tables.
- LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
mo
2026-06-29 17:43:37 +00:00
parent d066def8b4
commit f36c8906bc
15 changed files with 2479 additions and 4 deletions
@@ -11,9 +11,11 @@ import {
Table2,
Upload,
XCircle,
Gauge,
} from 'lucide-react'
import { cn } from '../../lib/utils'
import { subTabActive, subTabIdle } from '../../lib/tabActive'
import { DqMonitoringPanel } from './DqMonitoringPanel'
type Dimension = {
id: string
@@ -167,13 +169,13 @@ type ReportSummary = {
columns: number
}
type Tab = 'assess' | 'docling' | 'reports'
type Tab = 'assess' | 'docling' | 'reports' | 'monitoring'
const SCORE_COLOR = (s: number) => (s >= 80 ? 'text-success' : s >= 60 ? 'text-warning' : 'text-danger')
const BAR_COLOR = (s: number) => (s >= 80 ? 'bg-success' : s >= 60 ? 'bg-warning' : 'bg-danger')
export function DataQualityView() {
const [tab, setTab] = useState<Tab>('assess')
const [tab, setTab] = useState<Tab>('monitoring')
const [caps, setCaps] = useState<Capabilities | null>(null)
const [loading, setLoading] = useState(false)
const [assess, setAssess] = useState<AssessResult | null>(null)
@@ -252,6 +254,7 @@ export function DataQualityView() {
}
const tabs: { id: Tab; label: string; icon: typeof FileSearch }[] = [
{ id: 'monitoring', label: 'Live Monitoring', icon: Gauge },
{ id: 'assess', label: 'Maturity Assessment', icon: FileSearch },
{ id: 'docling', label: 'Docling Parser', icon: FileText },
{ id: 'reports', label: 'Reports', icon: CheckCircle2 },
@@ -312,6 +315,8 @@ export function DataQualityView() {
</div>
)}
{tab === 'monitoring' && <DqMonitoringPanel />}
{tab === 'assess' && (
<div className="space-y-5">
<UploadZone