feat(masking): mask_to_curated DAG + OpenMetadata lineage + versioned OM artifacts

- deploy/airflow/mask_to_curated_dag.py: Trino-SQL masking (hash/redact/generalize)
  of PII from postgres/mysql sources into iceberg.curated_masked.* (verified:
  14k+11k masked rows, all PII masked).
- OM source->masked lineage edges created; curated tables cataloged.
- Version OM compose + ingestion configs + om_api helper under deploy/openmetadata.
This commit is contained in:
mo
2026-06-27 03:13:16 +02:00
parent 944635ffe1
commit c3cacc141d
11 changed files with 987 additions and 20 deletions
@@ -0,0 +1,27 @@
source:
type: mongodb
serviceName: atc_mongodb
serviceConnection:
config:
type: MongoDB
scheme: mongodb
hostPort: 10.0.21.51:27017
databaseName: supplychain
connectionOptions:
replicaSet: rs0
directConnection: "true"
sourceConfig:
config:
type: DatabaseMetadata
databaseFilterPattern:
includes:
- supplychain
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
@@ -0,0 +1,32 @@
source:
type: mysql
serviceName: atc_mysql
serviceConnection:
config:
type: Mysql
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:3306
databaseSchema: hr
sourceConfig:
config:
type: AutoClassification
enableAutoClassification: true
storeSampleData: true
confidence: 80
tableFilterPattern:
includes:
- employee_events
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
@@ -0,0 +1,23 @@
source:
type: mysql
serviceName: atc_mysql
serviceConnection:
config:
type: Mysql
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:3306
databaseSchema: hr
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
@@ -0,0 +1,35 @@
source:
type: postgres
serviceName: atc_postgres
serviceConnection:
config:
type: Postgres
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:5432
database: postgres
sourceConfig:
config:
type: AutoClassification
enableAutoClassification: true
storeSampleData: true
confidence: 80
schemaFilterPattern:
includes:
- public
tableFilterPattern:
includes:
- sales_orders
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
+26
View File
@@ -0,0 +1,26 @@
source:
type: postgres
serviceName: atc_postgres
serviceConnection:
config:
type: Postgres
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:5432
database: postgres
sourceConfig:
config:
type: DatabaseMetadata
schemaFilterPattern:
includes:
- public
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Usage: run.sh <JWT> <yaml-basename without .yaml>
# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion
# container, and executes the metadata ingestion workflow.
set -uo pipefail
JWT="$1"
NAME="$2"
CMD="${3:-ingest}"
DIR=/opt/openmetadata/ingest
# Use the stored stable token if no/placeholder JWT is passed (avoids rotating
# the bot token, which would invalidate the Command Center's stored token).
if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then
JWT="$(cat "$DIR/.token" 2>/dev/null)"
fi
SRC="$DIR/${NAME}.yaml"
OUT="$DIR/.${NAME}.run.yaml"
if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi
sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT"
docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null
echo "=== metadata ${CMD}: ${NAME} ==="
docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45
rm -f "$OUT"
@@ -0,0 +1,21 @@
source:
type: trino
serviceName: atc_trino
serviceConnection:
config:
type: Trino
username: mo
hostPort: 10.0.21.50:8089
catalog: iceberg
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"