mo
c3cacc141d
- deploy/airflow/mask_to_curated_dag.py: Trino-SQL masking (hash/redact/generalize) of PII from postgres/mysql sources into iceberg.curated_masked.* (verified: 14k+11k masked rows, all PII masked). - OM source->masked lineage edges created; curated tables cataloged. - Version OM compose + ingestion configs + om_api helper under deploy/openmetadata.
24 lines
888 B
Bash
24 lines
888 B
Bash
#!/bin/bash
|
|
# Usage: run.sh <JWT> <yaml-basename without .yaml>
|
|
# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion
|
|
# container, and executes the metadata ingestion workflow.
|
|
set -uo pipefail
|
|
JWT="$1"
|
|
NAME="$2"
|
|
CMD="${3:-ingest}"
|
|
DIR=/opt/openmetadata/ingest
|
|
# Use the stored stable token if no/placeholder JWT is passed (avoids rotating
|
|
# the bot token, which would invalidate the Command Center's stored token).
|
|
if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then
|
|
JWT="$(cat "$DIR/.token" 2>/dev/null)"
|
|
fi
|
|
SRC="$DIR/${NAME}.yaml"
|
|
OUT="$DIR/.${NAME}.run.yaml"
|
|
|
|
if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi
|
|
sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT"
|
|
docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null
|
|
echo "=== metadata ${CMD}: ${NAME} ==="
|
|
docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45
|
|
rm -f "$OUT"
|