feat: Authentik login + switchable GPU prod target
Add OIDC auth for Command Center and runtime GPU endpoint selection pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
This commit is contained in:
mo
+252
-5
@@ -34,6 +34,7 @@ POLICY_PATH = Path(os.getenv("MASKING_POLICY_PATH", "/data/masking_policy.json")
|
||||
DEFAULT_MASKED = True
|
||||
MASK_TOKEN = "🔒 MASKED (masking policy ON)"
|
||||
_policy_cache: dict[str, bool] | None = None
|
||||
_policy_mtime: float = -1.0
|
||||
|
||||
# Datasets we surface in the PII overlay. node_id matches dataflow.py node ids.
|
||||
# om_fqn = OpenMetadata table FQN (service.database.schema.table) for tag lookup.
|
||||
@@ -77,21 +78,42 @@ DATASET_BY_KEY = {ds["key"]: ds for ds in DATASETS}
|
||||
|
||||
|
||||
def _load_policy() -> dict[str, bool]:
|
||||
global _policy_cache
|
||||
if _policy_cache is None:
|
||||
"""Load policy from disk; re-read when the file mtime changes (Data Flow toggles)."""
|
||||
global _policy_cache, _policy_mtime
|
||||
try:
|
||||
mtime = POLICY_PATH.stat().st_mtime
|
||||
except Exception:
|
||||
mtime = 0.0
|
||||
if _policy_cache is None or mtime != _policy_mtime:
|
||||
try:
|
||||
_policy_cache = {k: bool(v) for k, v in json.loads(POLICY_PATH.read_text()).items()}
|
||||
except Exception:
|
||||
_policy_cache = {}
|
||||
if _policy_cache is None:
|
||||
_policy_cache = {}
|
||||
_policy_mtime = mtime
|
||||
return _policy_cache
|
||||
|
||||
|
||||
def _save_policy(p: dict[str, bool]) -> None:
|
||||
global _policy_cache
|
||||
global _policy_cache, _policy_mtime
|
||||
_policy_cache = p
|
||||
try:
|
||||
POLICY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
POLICY_PATH.write_text(json.dumps(p, indent=2))
|
||||
_policy_mtime = POLICY_PATH.stat().st_mtime
|
||||
except Exception:
|
||||
_policy_mtime = time.time()
|
||||
|
||||
|
||||
def invalidate_pii_caches() -> None:
|
||||
"""Force catalog rebuild so chat/Data Flow see the same mask flags immediately."""
|
||||
_cache["data"] = None
|
||||
_cache["ts"] = 0.0
|
||||
try:
|
||||
import trino_federated as tf
|
||||
if hasattr(tf, "_dict_cache"):
|
||||
tf._dict_cache["data"] = None
|
||||
tf._dict_cache["at"] = 0.0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -325,10 +347,235 @@ async def set_policy(body: MaskPolicyRequest) -> JSONResponse:
|
||||
for col in cols:
|
||||
p[f"{key}.{col}"] = bool(body.masked)
|
||||
_save_policy(p)
|
||||
_cache["data"] = None # force rebuild so masked flags reflect the new policy
|
||||
invalidate_pii_caches() # chat + Data Flow must reflect the toggle immediately
|
||||
return JSONResponse({"ok": True, "key": key, "columns": cols, "masked": bool(body.masked)})
|
||||
|
||||
|
||||
def policy_column_lists(*, use_cache: bool = False) -> dict[str, Any]:
|
||||
"""Live masked vs visible PII columns — same source as the Data Flow tab."""
|
||||
catalog = get_pii(use_cache=use_cache)
|
||||
masked: list[dict[str, str]] = []
|
||||
visible: list[dict[str, str]] = []
|
||||
for d in catalog.get("datasets", []):
|
||||
key = d.get("key") or ""
|
||||
label = d.get("label") or key
|
||||
locked = bool(d.get("masked_layer") or d.get("policy_locked"))
|
||||
for c in d.get("pii_columns", []):
|
||||
entry = {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"column": c.get("name") or "",
|
||||
"category": c.get("category") or "PII",
|
||||
"locked": locked,
|
||||
}
|
||||
(masked if c.get("masked") else visible).append(entry)
|
||||
summ = catalog.get("summary") or {}
|
||||
return {
|
||||
"catalog": catalog,
|
||||
"masked": masked,
|
||||
"visible": visible,
|
||||
"summary": summ,
|
||||
"mask_token": MASK_TOKEN,
|
||||
}
|
||||
|
||||
|
||||
def _interest_categories(message: str) -> list[str]:
|
||||
lower = (message or "").lower()
|
||||
cat_map = [
|
||||
(("email", "e-mail", "mail"), "EMAIL"),
|
||||
(("phone", "telefoon", "mobile"), "PHONE"),
|
||||
(("name", "naam"), "NAME"),
|
||||
(("iban", "bank", "card"), "FINANCIAL"),
|
||||
(("ssn", "bsn", "national", "passport"), "NATIONAL_ID"),
|
||||
(("address", "adres"), "ADDRESS"),
|
||||
(("birth", "dob", "geboorte"), "DOB"),
|
||||
(("ip",), "IP"),
|
||||
]
|
||||
out: list[str] = []
|
||||
for words, cat in cat_map:
|
||||
if any(w in lower for w in words):
|
||||
out.append(cat)
|
||||
return out
|
||||
|
||||
|
||||
def _sample_rows_for_chat(
|
||||
*,
|
||||
categories: list[str] | None = None,
|
||||
max_datasets: int = 2,
|
||||
rows_per: int = 2,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Fetch a few live rows; values already policy-masked."""
|
||||
catalog = get_pii(use_cache=False)
|
||||
prefer = ["mysql", "postgres", "mongodb", "cassandra", "neo4j", "curated"]
|
||||
samples: list[dict[str, Any]] = []
|
||||
for key in prefer:
|
||||
dset = next((d for d in catalog.get("datasets", []) if d.get("key") == key), None)
|
||||
if not dset or not dset.get("pii_columns"):
|
||||
continue
|
||||
ds = DATASET_BY_KEY.get(key)
|
||||
if not ds:
|
||||
continue
|
||||
pii_cols = dset["pii_columns"]
|
||||
if categories:
|
||||
focus = [c for c in pii_cols if c.get("category") in categories]
|
||||
# Always keep one id-like visible column for context when focusing
|
||||
ids = [c for c in pii_cols if c.get("category") == "IDENTIFIER" and not c.get("masked")]
|
||||
pick = (ids[:1] + focus) if focus else pii_cols
|
||||
else:
|
||||
pick = pii_cols
|
||||
if not pick:
|
||||
continue
|
||||
# de-dupe preserving order
|
||||
seen: set[str] = set()
|
||||
select_cols: list[str] = []
|
||||
masked_map: dict[str, bool] = {}
|
||||
for c in pick:
|
||||
name = c["name"]
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
select_cols.append(name)
|
||||
masked_map[name] = bool(c.get("masked"))
|
||||
if len(select_cols) >= 6:
|
||||
break
|
||||
name_col = next((c["name"] for c in pii_cols if c.get("category") == "NAME"), None)
|
||||
try:
|
||||
cols, rows = _lookup_rows(ds, select_cols, name_col, None, rows_per)
|
||||
except Exception:
|
||||
continue
|
||||
if not rows:
|
||||
continue
|
||||
rendered = []
|
||||
for row in rows[:rows_per]:
|
||||
rendered.append({
|
||||
cname: (MASK_TOKEN if masked_map.get(cname) else val)
|
||||
for cname, val in zip(cols, row)
|
||||
})
|
||||
samples.append({
|
||||
"key": key,
|
||||
"label": dset.get("label", key),
|
||||
"table": ds.get("table"),
|
||||
"rows": rendered,
|
||||
"masked_cols": [c for c, m in masked_map.items() if m],
|
||||
"visible_cols": [c for c, m in masked_map.items() if not m],
|
||||
})
|
||||
if len(samples) >= max_datasets:
|
||||
break
|
||||
return samples
|
||||
|
||||
|
||||
def build_policy_evidence(*, max_datasets: int = 2, rows_per: int = 2) -> str:
|
||||
"""Compact internal evidence for LLM context (not shown raw to users)."""
|
||||
snap = policy_column_lists(use_cache=False)
|
||||
masked = snap["masked"]
|
||||
visible = snap["visible"]
|
||||
summ = snap["summary"]
|
||||
lines = [
|
||||
"=== PII POLICY (Data Flow synced) ===",
|
||||
f"Masked {summ.get('masked_columns', len(masked))}/{summ.get('pii_columns', 0)} · "
|
||||
f"visible {summ.get('unmasked_columns', len(visible))}. Token: {MASK_TOKEN}",
|
||||
"Masked: " + ", ".join(f"{m['key']}.{m['column']}" for m in masked[:25]) + (
|
||||
f" …(+{len(masked)-25})" if len(masked) > 25 else ""
|
||||
),
|
||||
"Visible: " + (", ".join(f"{v['key']}.{v['column']}" for v in visible[:25]) or "(none)"),
|
||||
]
|
||||
for s in _sample_rows_for_chat(max_datasets=max_datasets, rows_per=rows_per):
|
||||
lines.append(f"Sample {s['label']}:")
|
||||
for row in s["rows"]:
|
||||
lines.append(" " + " | ".join(f"{k}={v}" for k, v in row.items()))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_pii_chat_answer(message: str = "") -> str:
|
||||
"""Short personal reply: masked → say masked; visible → show values. Never list field names."""
|
||||
snap = policy_column_lists(use_cache=False)
|
||||
masked = snap["masked"]
|
||||
visible = snap["visible"]
|
||||
lower = (message or "").lower()
|
||||
interest = _interest_categories(message)
|
||||
greeting = any(w in lower for w in ("hi", "hello", "hey", "hallo", "goedemorgen", "goedemiddag"))
|
||||
hi = "Hi! " if greeting else ""
|
||||
|
||||
label = {
|
||||
"EMAIL": "email",
|
||||
"PHONE": "phone number",
|
||||
"NAME": "name",
|
||||
"FINANCIAL": "bank / IBAN details",
|
||||
"NATIONAL_ID": "national ID",
|
||||
"ADDRESS": "address",
|
||||
"DOB": "date of birth",
|
||||
"IP": "IP address",
|
||||
}
|
||||
|
||||
# No specific PII type asked — keep it vague, never enumerate columns
|
||||
if not interest:
|
||||
if any(w in lower for w in ("mask", "pii", "sensitive", "privacy", "personal")):
|
||||
return (
|
||||
f"{hi}Personal data is protected by the masking policy. "
|
||||
f"Ask for something specific (an email, a phone number, a name…) and I'll tell you "
|
||||
f"whether I can share it — or only `{MASK_TOKEN}`."
|
||||
)
|
||||
return (
|
||||
f"{hi}I can't share personal data that's masked. "
|
||||
f"Ask me for an email, phone number, or name if you want to check."
|
||||
)
|
||||
|
||||
topic = ", ".join(label.get(c, c.lower()) for c in interest)
|
||||
interested_masked = [c for c in masked if c["category"] in interest]
|
||||
interested_visible = [c for c in visible if c["category"] in interest]
|
||||
|
||||
# Collect visible sample *values* only (no column names in the reply)
|
||||
values: list[str] = []
|
||||
if interested_visible:
|
||||
samples = _sample_rows_for_chat(categories=interest, max_datasets=2, rows_per=2)
|
||||
for s in samples:
|
||||
for row in s["rows"]:
|
||||
for col in interested_visible:
|
||||
if col["column"] in row:
|
||||
val = row[col["column"]]
|
||||
if val is None or val == "" or val == MASK_TOKEN:
|
||||
continue
|
||||
values.append(str(val))
|
||||
# unique, preserve order
|
||||
seen: set[str] = set()
|
||||
uniq: list[str] = []
|
||||
for v in values:
|
||||
if v not in seen:
|
||||
seen.add(v)
|
||||
uniq.append(v)
|
||||
values = uniq[:5]
|
||||
|
||||
# Fully masked for this ask
|
||||
if interested_masked and not interested_visible:
|
||||
return (
|
||||
f"{hi}No — that {topic} is masked (`{MASK_TOKEN}`). "
|
||||
"I can't share it."
|
||||
)
|
||||
|
||||
# Fully visible
|
||||
if interested_visible and not interested_masked:
|
||||
if values:
|
||||
listed = ", ".join(values)
|
||||
return f"{hi}Sure — here's what I can share: {listed}."
|
||||
return f"{hi}That {topic} isn't masked, but I don't have a sample value right now."
|
||||
|
||||
# Mixed: some sources masked, some visible — still don't name columns
|
||||
if interested_visible and interested_masked:
|
||||
if values:
|
||||
listed = ", ".join(values)
|
||||
return (
|
||||
f"{hi}Some of that is masked (`{MASK_TOKEN}`); "
|
||||
f"what I can share: {listed}."
|
||||
)
|
||||
return (
|
||||
f"{hi}Some of that {topic} is masked (`{MASK_TOKEN}`). "
|
||||
"I can't share the protected parts."
|
||||
)
|
||||
|
||||
return f"{hi}I don't have that personal data available."
|
||||
|
||||
|
||||
|
||||
def _lookup_rows(ds: dict[str, Any], select: list[str], name_col: str | None,
|
||||
search: str | None, limit: int) -> tuple[list[str], list[list[Any]]]:
|
||||
"""Fetch rows from the source. Native DB queries (fast, early LIMIT) for
|
||||
|
||||
Reference in New Issue
Block a user