feat(pii): add masking toggle for Hadoop nodes + entity-identifier (quasi-PII) classification

This commit is contained in:
mo
2026-06-27 21:04:30 +00:00
parent 0bdc10ae78
commit e2ac114863
+10 -1
View File
@@ -54,6 +54,14 @@ DATASETS = [
"table": "iceberg.curated_masked.sales_orders_masked", "table_name": "sales_orders_masked", "catalog": "iceberg",
"schema": "curated_masked", "masked_layer": True,
"om_fqn": "atc_trino.iceberg.curated_masked.sales_orders_masked"},
{"key": "hadoop", "node_id": "iceberg_hadoop", "label": "Iceberg hadoop historical_sales_hdfs",
"table": "iceberg.hadoop.historical_sales_hdfs", "table_name": "historical_sales_hdfs", "catalog": "iceberg",
"schema": "hadoop",
"om_fqn": "atc_trino.iceberg.hadoop.historical_sales_hdfs"},
{"key": "hdfs", "node_id": "hdfs", "label": "Hadoop HDFS historical_sales_hdfs",
"table": "iceberg.hadoop.historical_sales_hdfs", "table_name": "historical_sales_hdfs", "catalog": "iceberg",
"schema": "hadoop",
"om_fqn": "atc_trino.iceberg.hadoop.historical_sales_hdfs"},
]
KEY_BY_NODE = {ds["node_id"]: ds["key"] for ds in DATASETS}
@@ -103,7 +111,8 @@ PII_RULES: list[tuple[str, str]] = [
(r"first_name|last_name|full_name|customer_name|employee_name|contact_name|^name$", "NAME"),
(r"address|street|city|zip|postal|postcode", "ADDRESS"),
(r"dob|birth|date_of_birth", "DOB"),
(r"ip_addr|ip_address", "IP"),
(r"ip_addr|ip_address|_ip$|customer_ip|client_ip", "IP"),
(r"customer_id|client_id|user_id|account_id|member_id|device_id|subscriber_id|employee_id|guest_id|person_id", "IDENTIFIER"),
]
_cache: dict[str, Any] = {"ts": 0.0, "data": None}