feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability

Adds native Command Center features (no new containers) integrated as sub-tabs
in the existing Data Explorer and Data Quality views:

- Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/
  freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring".
- Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix,
  orphan detection, business glossary; local store best-effort synced to
  OpenMetadata (owner PATCH) → Data Explorer "Ownership".
- Access & policy posture: per-dataset compliance combining PII masking, ownership,
  live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies".
- Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with
  live row counts and column-level PII/masking tracing → Data Explorer "Lineage".
- Observability (observability.py): volume/freshness/schema-drift monitoring with
  alerts → Data Explorer "Observability".
- Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count
  and PK-indexed freshness so monitors stay cheap on 25-54M-row tables.
- LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
mo
2026-06-29 17:43:37 +00:00
parent d066def8b4
commit f36c8906bc
15 changed files with 2479 additions and 4 deletions
+36
View File
@@ -671,6 +671,21 @@ def collect_governance(log: TerminalLogFn | None = None) -> dict[str, Any]:
"mysql_hr.hr.employee_events -> iceberg.curated_masked.employee_events_masked (PII masked)",
"hdfs:/data/historical/sales_orders -> iceberg.hadoop.historical_sales_hdfs",
]
try:
from dq_monitor import summary_for_llm as _dq
out["data_quality"] = _dq()
except Exception as exc:
out["data_quality"] = {"error": str(exc)}
try:
from observability import summary_for_llm as _obs
out["observability"] = _obs()
except Exception as exc:
out["observability"] = {"error": str(exc)}
try:
from catalog_governance import summary_for_llm as _own
out["ownership"] = _own()
except Exception as exc:
out["ownership"] = {"error": str(exc)}
return out
@@ -700,6 +715,27 @@ def _section_governance(g: dict[str, Any]) -> list[str]:
lines.append(" Lineage:")
for ln in g.get("lineage") or []:
lines.append(f" - {ln}")
dq = g.get("data_quality") or {}
if isinstance(dq, dict) and "error" not in dq:
lines.append(f" Data quality (continuous, live tables): platform score {dq.get('platform_dq_score')}")
for d in dq.get("datasets") or []:
iss = f" issues: {', '.join(d['issues'][:3])}" if d.get("issues") else ""
lines.append(f" - {d['key']}: score {d.get('score')}{iss}")
own = g.get("ownership") or {}
if isinstance(own, dict) and "error" not in own:
orph = own.get("orphan_datasets") or []
lines.append(f" Ownership: {len(own.get('owners', {}))} assigned"
+ (f", orphans (no owner): {', '.join(orph)}" if orph else ", no orphans"))
for k, v in (own.get("owners") or {}).items():
if v.get("owner"):
lines.append(f" - {k}: owner={v.get('owner')} steward={v.get('steward') or ''} tier={v.get('tier') or ''}")
obs = g.get("observability") or {}
if isinstance(obs, dict) and "error" not in obs:
ac = obs.get("active_alerts") or {}
lines.append(f" Observability alerts: {ac.get('total', 0)} active "
f"(critical={ac.get('critical', 0)}, warning={ac.get('warning', 0)})")
for a in (obs.get("alerts") or [])[:5]:
lines.append(f" - [{a['severity']}] {a['dataset']}: {a['message']}")
return lines