feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability
Adds native Command Center features (no new containers) integrated as sub-tabs in the existing Data Explorer and Data Quality views: - Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/ freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring". - Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix, orphan detection, business glossary; local store best-effort synced to OpenMetadata (owner PATCH) → Data Explorer "Ownership". - Access & policy posture: per-dataset compliance combining PII masking, ownership, live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies". - Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with live row counts and column-level PII/masking tracing → Data Explorer "Lineage". - Observability (observability.py): volume/freshness/schema-drift monitoring with alerts → Data Explorer "Observability". - Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count and PK-indexed freshness so monitors stay cheap on 25-54M-row tables. - LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
@@ -16,12 +16,18 @@ import {
|
||||
HardDrive,
|
||||
ShieldCheck,
|
||||
Radio,
|
||||
GitBranch,
|
||||
Lock,
|
||||
} from 'lucide-react'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { TrinoFederationView, type SubTab } from './TrinoFederationView'
|
||||
import { LiveDashboard } from './LiveDashboard'
|
||||
import { LineageView } from './LineageView'
|
||||
import { GovernanceOwnershipView } from './GovernanceOwnershipView'
|
||||
import { GovernanceAccessView } from './GovernanceAccessView'
|
||||
import { ObservabilityView } from './ObservabilityView'
|
||||
|
||||
type ExplorerTab = 'business' | 'live' | SubTab
|
||||
type ExplorerTab = 'business' | 'live' | SubTab | 'lineage' | 'ownership' | 'access' | 'observability'
|
||||
|
||||
const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string; live?: boolean }[] = [
|
||||
{ id: 'business', label: 'Business Overview', icon: BarChart3, hint: 'Customers, orders, workforce, supply chain & telemetry across every source' },
|
||||
@@ -29,6 +35,10 @@ const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string;
|
||||
{ id: 'federated', label: 'Federated (Trino)', icon: Network, hint: 'One SQL across all 5 databases + region scorecard joined live' },
|
||||
{ id: 'lake', label: 'Hadoop Lake', icon: HardDrive, hint: 'All business data mirrored as external Iceberg tables on HDFS' },
|
||||
{ id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck, hint: 'Every table & column with PII / masking status — exactly what the assistant sees' },
|
||||
{ id: 'lineage', label: 'Lineage', icon: GitBranch, hint: 'End-to-end data lineage with column-level PII tracing from source to curated layer' },
|
||||
{ id: 'ownership', label: 'Ownership', icon: UserCog, hint: 'Data owners, stewards, tiers & business glossary — accountability per dataset' },
|
||||
{ id: 'access', label: 'Access & Policies', icon: Lock, hint: 'Governance posture: PII masking, ownership, live DQ & alerts vs each data contract' },
|
||||
{ id: 'observability', label: 'Observability', icon: Activity, hint: 'Volume, freshness & schema-drift monitoring with live alerts across every table' },
|
||||
]
|
||||
|
||||
type Bucket = { key: string; count: number; value?: number }
|
||||
@@ -295,6 +305,12 @@ export function DataExplorerView() {
|
||||
{/* Trino federation / lake / dictionary tabs */}
|
||||
{(view === 'federated' || view === 'lake' || view === 'dictionary') && <TrinoFederationView embedded activeTab={view} />}
|
||||
|
||||
{/* Governance / lineage / observability sub-tabs */}
|
||||
{view === 'lineage' && <div className="flex min-h-0 flex-1 flex-col"><LineageView /></div>}
|
||||
{view === 'ownership' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceOwnershipView /></div>}
|
||||
{view === 'access' && <div className="flex min-h-0 flex-1 flex-col"><GovernanceAccessView /></div>}
|
||||
{view === 'observability' && <div className="flex min-h-0 flex-1 flex-col"><ObservabilityView /></div>}
|
||||
|
||||
{/* ───────── BUSINESS OVERVIEW ───────── */}
|
||||
{view === 'business' && (
|
||||
<>
|
||||
|
||||
Reference in New Issue
Block a user