feat(masking): mask_to_curated DAG + OpenMetadata lineage + versioned OM artifacts
- deploy/airflow/mask_to_curated_dag.py: Trino-SQL masking (hash/redact/generalize) of PII from postgres/mysql sources into iceberg.curated_masked.* (verified: 14k+11k masked rows, all PII masked). - OM source->masked lineage edges created; curated tables cataloged. - Version OM compose + ingestion configs + om_api helper under deploy/openmetadata.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
source:
|
||||
type: mongodb
|
||||
serviceName: atc_mongodb
|
||||
serviceConnection:
|
||||
config:
|
||||
type: MongoDB
|
||||
scheme: mongodb
|
||||
hostPort: 10.0.21.51:27017
|
||||
databaseName: supplychain
|
||||
connectionOptions:
|
||||
replicaSet: rs0
|
||||
directConnection: "true"
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DatabaseMetadata
|
||||
databaseFilterPattern:
|
||||
includes:
|
||||
- supplychain
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
@@ -0,0 +1,32 @@
|
||||
source:
|
||||
type: mysql
|
||||
serviceName: atc_mysql
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Mysql
|
||||
username: mo
|
||||
authType:
|
||||
password: Dell2026!
|
||||
hostPort: 10.0.21.51:3306
|
||||
databaseSchema: hr
|
||||
sourceConfig:
|
||||
config:
|
||||
type: AutoClassification
|
||||
enableAutoClassification: true
|
||||
storeSampleData: true
|
||||
confidence: 80
|
||||
tableFilterPattern:
|
||||
includes:
|
||||
- employee_events
|
||||
processor:
|
||||
type: orm-profiler
|
||||
config: {}
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
@@ -0,0 +1,23 @@
|
||||
source:
|
||||
type: mysql
|
||||
serviceName: atc_mysql
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Mysql
|
||||
username: mo
|
||||
authType:
|
||||
password: Dell2026!
|
||||
hostPort: 10.0.21.51:3306
|
||||
databaseSchema: hr
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DatabaseMetadata
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
@@ -0,0 +1,35 @@
|
||||
source:
|
||||
type: postgres
|
||||
serviceName: atc_postgres
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Postgres
|
||||
username: mo
|
||||
authType:
|
||||
password: Dell2026!
|
||||
hostPort: 10.0.21.51:5432
|
||||
database: postgres
|
||||
sourceConfig:
|
||||
config:
|
||||
type: AutoClassification
|
||||
enableAutoClassification: true
|
||||
storeSampleData: true
|
||||
confidence: 80
|
||||
schemaFilterPattern:
|
||||
includes:
|
||||
- public
|
||||
tableFilterPattern:
|
||||
includes:
|
||||
- sales_orders
|
||||
processor:
|
||||
type: orm-profiler
|
||||
config: {}
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
@@ -0,0 +1,26 @@
|
||||
source:
|
||||
type: postgres
|
||||
serviceName: atc_postgres
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Postgres
|
||||
username: mo
|
||||
authType:
|
||||
password: Dell2026!
|
||||
hostPort: 10.0.21.51:5432
|
||||
database: postgres
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DatabaseMetadata
|
||||
schemaFilterPattern:
|
||||
includes:
|
||||
- public
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Usage: run.sh <JWT> <yaml-basename without .yaml>
|
||||
# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion
|
||||
# container, and executes the metadata ingestion workflow.
|
||||
set -uo pipefail
|
||||
JWT="$1"
|
||||
NAME="$2"
|
||||
CMD="${3:-ingest}"
|
||||
DIR=/opt/openmetadata/ingest
|
||||
# Use the stored stable token if no/placeholder JWT is passed (avoids rotating
|
||||
# the bot token, which would invalidate the Command Center's stored token).
|
||||
if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then
|
||||
JWT="$(cat "$DIR/.token" 2>/dev/null)"
|
||||
fi
|
||||
SRC="$DIR/${NAME}.yaml"
|
||||
OUT="$DIR/.${NAME}.run.yaml"
|
||||
|
||||
if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi
|
||||
sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT"
|
||||
docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null
|
||||
echo "=== metadata ${CMD}: ${NAME} ==="
|
||||
docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45
|
||||
rm -f "$OUT"
|
||||
@@ -0,0 +1,21 @@
|
||||
source:
|
||||
type: trino
|
||||
serviceName: atc_trino
|
||||
serviceConnection:
|
||||
config:
|
||||
type: Trino
|
||||
username: mo
|
||||
hostPort: 10.0.21.50:8089
|
||||
catalog: iceberg
|
||||
sourceConfig:
|
||||
config:
|
||||
type: DatabaseMetadata
|
||||
sink:
|
||||
type: metadata-rest
|
||||
config: {}
|
||||
workflowConfig:
|
||||
openMetadataServerConfig:
|
||||
hostPort: http://openmetadata-server:8585/api
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "__JWT__"
|
||||
Reference in New Issue
Block a user