feat(governance): close the 6 data-disease gaps — DQ monitoring, ownership, access posture, lineage & observability
Adds native Command Center features (no new containers) integrated as sub-tabs in the existing Data Explorer and Data Quality views: - Continuous Data Quality (dq_monitor.py): live completeness/uniqueness/validity/ freshness scorecards via Trino with rolling trends → DataQuality "Live Monitoring". - Ownership & stewardship (catalog_governance.py): owner/steward/tier matrix, orphan detection, business glossary; local store best-effort synced to OpenMetadata (owner PATCH) → Data Explorer "Ownership". - Access & policy posture: per-dataset compliance combining PII masking, ownership, live DQ and observability alerts vs data contracts → Data Explorer "Access & Policies". - Lineage (lineage.py): staged source→CDC→Spark→S3→Iceberg→Trino→serving graph with live row counts and column-level PII/masking tracing → Data Explorer "Lineage". - Observability (observability.py): volume/freshness/schema-drift monitoring with alerts → Data Explorer "Observability". - Shared lake_meta.py dataset registry + bounded Trino client; fast native row-count and PK-indexed freshness so monitors stay cheap on 25-54M-row tables. - LLM context (lab_context.py) enriched with DQ scores, ownership and active alerts.
This commit is contained in:
@@ -671,6 +671,21 @@ def collect_governance(log: TerminalLogFn | None = None) -> dict[str, Any]:
|
||||
"mysql_hr.hr.employee_events -> iceberg.curated_masked.employee_events_masked (PII masked)",
|
||||
"hdfs:/data/historical/sales_orders -> iceberg.hadoop.historical_sales_hdfs",
|
||||
]
|
||||
try:
|
||||
from dq_monitor import summary_for_llm as _dq
|
||||
out["data_quality"] = _dq()
|
||||
except Exception as exc:
|
||||
out["data_quality"] = {"error": str(exc)}
|
||||
try:
|
||||
from observability import summary_for_llm as _obs
|
||||
out["observability"] = _obs()
|
||||
except Exception as exc:
|
||||
out["observability"] = {"error": str(exc)}
|
||||
try:
|
||||
from catalog_governance import summary_for_llm as _own
|
||||
out["ownership"] = _own()
|
||||
except Exception as exc:
|
||||
out["ownership"] = {"error": str(exc)}
|
||||
return out
|
||||
|
||||
|
||||
@@ -700,6 +715,27 @@ def _section_governance(g: dict[str, Any]) -> list[str]:
|
||||
lines.append(" Lineage:")
|
||||
for ln in g.get("lineage") or []:
|
||||
lines.append(f" - {ln}")
|
||||
dq = g.get("data_quality") or {}
|
||||
if isinstance(dq, dict) and "error" not in dq:
|
||||
lines.append(f" Data quality (continuous, live tables): platform score {dq.get('platform_dq_score')}")
|
||||
for d in dq.get("datasets") or []:
|
||||
iss = f" issues: {', '.join(d['issues'][:3])}" if d.get("issues") else ""
|
||||
lines.append(f" - {d['key']}: score {d.get('score')}{iss}")
|
||||
own = g.get("ownership") or {}
|
||||
if isinstance(own, dict) and "error" not in own:
|
||||
orph = own.get("orphan_datasets") or []
|
||||
lines.append(f" Ownership: {len(own.get('owners', {}))} assigned"
|
||||
+ (f", orphans (no owner): {', '.join(orph)}" if orph else ", no orphans"))
|
||||
for k, v in (own.get("owners") or {}).items():
|
||||
if v.get("owner"):
|
||||
lines.append(f" - {k}: owner={v.get('owner')} steward={v.get('steward') or '—'} tier={v.get('tier') or '—'}")
|
||||
obs = g.get("observability") or {}
|
||||
if isinstance(obs, dict) and "error" not in obs:
|
||||
ac = obs.get("active_alerts") or {}
|
||||
lines.append(f" Observability alerts: {ac.get('total', 0)} active "
|
||||
f"(critical={ac.get('critical', 0)}, warning={ac.get('warning', 0)})")
|
||||
for a in (obs.get("alerts") or [])[:5]:
|
||||
lines.append(f" - [{a['severity']}] {a['dataset']}: {a['message']}")
|
||||
return lines
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user