feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability

Adds native Command Center features (no new containers) integrated as sub-tabs
in the existing Data Explorer and Data Quality views:

- Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/
  freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring".
- Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix,
  orphan detection, business glossary; local store best-effort synced to
  OpenMetadata (owner PATCH) → Data Explorer "Ownership".
- Access & policy posture: per-dataset compliance combining PII masking, ownership,
  live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies".
- Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with
  live row counts and column-level PII/masking tracing → Data Explorer "Lineage".
- Observability (observability.py): volume/freshness/schema-drift monitoring with
  alerts → Data Explorer "Observability".
- Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count
  and PK-indexed freshness so monitors stay cheap on 25-54M-row tables.
- LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
mo
2026-06-29 17:43:37 +00:00
parent d066def8b4
commit f36c8906bc
15 changed files with 2479 additions and 4 deletions
@@ -16,12 +16,18 @@ import {
HardDrive,
ShieldCheck,
Radio,
GitBranch,
Lock,
} from 'lucide-react'
import { cn } from '../../lib/utils'
import { TrinoFederationView, type SubTab } from './TrinoFederationView'
import { LiveDashboard } from './LiveDashboard'
import { LineageView } from './LineageView'
import { GovernanceOwnershipView } from './GovernanceOwnershipView'
import { GovernanceAccessView } from './GovernanceAccessView'
import { ObservabilityView } from './ObservabilityView'
type ExplorerTab = 'business' | 'live' | SubTab
type ExplorerTab = 'business' | 'live' | SubTab | 'lineage' | 'ownership' | 'access' | 'observability'
const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string; live?: boolean }[] = [
{ id: 'business', label: 'Business Overview', icon: BarChart3, hint: 'Customers, orders, workforce, supply chain & telemetry across every source' },
@@ -29,6 +35,10 @@ const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string;
{ id: 'federated', label: 'Federated (Trino)', icon: Network, hint: 'One SQL across all 5 databases + region scorecard joined live' },
{ id: 'lake', label: 'Hadoop Lake', icon: HardDrive, hint: 'All business data mirrored as external Iceberg tables on HDFS' },
{ id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck, hint: 'Every table & column with PII / masking status — exactly what the assistant sees' },
{ id: 'lineage', label: 'Lineage', icon: GitBranch, hint: 'End-to-end data lineage with column-level PII tracing from source to curated layer' },
{ id: 'ownership', label: 'Ownership', icon: UserCog, hint: 'Data owners, stewards, tiers & business glossary — accountability per dataset' },
{ id: 'access', label: 'Access & Policies', icon: Lock, hint: 'Governance posture: PII masking, ownership, live DQ & alerts vs each data contract' },
{ id: 'observability', label: 'Observability', icon: Activity, hint: 'Volume, freshness & schema-drift monitoring with live alerts across every table' },
]
type Bucket = { key: string; count: number; value?: number }
@@ -295,6 +305,12 @@ export function DataExplorerView() {
{/* Trino federation / lake / dictionary tabs */}
{(view === 'federated' || view === 'lake' || view === 'dictionary') && <TrinoFederationView embedded activeTab={view} />}
{/* Governance / lineage / observability sub-tabs */}
{view === 'lineage' && <div className="flex min-h-0 flex-1 flex-col"><LineageView /></div>}
{view === 'ownership' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceOwnershipView /></div>}
{view === 'access' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceAccessView /></div>}
{view === 'observability' && <div className="flex min-h-0 flex-1 flex-col"><ObservabilityView /></div>}
{/* ───────── BUSINESS OVERVIEW ───────── */}
{view === 'business' && (
<>