feat(masking): mask_to_curated DAG + OpenMetadata lineage + versioned OM artifacts
- deploy/airflow/mask_to_curated_dag.py: Trino-SQL masking (hash/redact/generalize) of PII from postgres/mysql sources into iceberg.curated_masked.* (verified: 14k+11k masked rows, all PII masked). - OM source->masked lineage edges created; curated tables cataloged. - Version OM compose + ingestion configs + om_api helper under deploy/openmetadata.
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Usage: run.sh <JWT> <yaml-basename without .yaml>
|
||||
# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion
|
||||
# container, and executes the metadata ingestion workflow.
|
||||
set -uo pipefail
|
||||
JWT="$1"
|
||||
NAME="$2"
|
||||
CMD="${3:-ingest}"
|
||||
DIR=/opt/openmetadata/ingest
|
||||
# Use the stored stable token if no/placeholder JWT is passed (avoids rotating
|
||||
# the bot token, which would invalidate the Command Center's stored token).
|
||||
if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then
|
||||
JWT="$(cat "$DIR/.token" 2>/dev/null)"
|
||||
fi
|
||||
SRC="$DIR/${NAME}.yaml"
|
||||
OUT="$DIR/.${NAME}.run.yaml"
|
||||
|
||||
if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi
|
||||
sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT"
|
||||
docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null
|
||||
echo "=== metadata ${CMD}: ${NAME} ==="
|
||||
docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45
|
||||
rm -f "$OUT"
|
||||
Reference in New Issue
Block a user