ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(masking): mask_to_curated DAG + OpenMetadata lineage + versioned OM artifacts

Browse Source 
- deploy/airflow/mask_to_curated_dag.py: Trino-SQL masking (hash/redact/generalize)
  of PII from postgres/mysql sources into iceberg.curated_masked.* (verified:
  14k+11k masked rows, all PII masked).
- OM source->masked lineage edges created; curated tables cataloged.
- Version OM compose + ingestion configs + om_api helper under deploy/openmetadata.
This commit is contained in:
mo
2026-06-27 03:13:16 +02:00
parent 944635ffe1
commit c3cacc141d
11 changed files with 987 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/airflow/mask_to_curated_dag.py
+102
View File
@@ -0,0 +1,102 @@
"""mask_to_curated — ETL with PII masking into the curated Iceberg layer.
Reads PII-bearing rows from the source databases via Trino (postgres_sales,
mysql_hr catalogs) and writes MASKED copies into iceberg.curated_masked.*.
Masking is expressed as Trino SQL (hash / partial-redact / generalize) so no PII
ever lands in the curated layer. Triggered by the Command Center (movement
'mask_to_curated') or autonomously by the ETL agent.
"""
from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime
import json
import os
import urllib.request
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
TRINO_USER = os.getenv("TRINO_USER", "mo")
LIMIT = int(os.getenv("MASK_LIMIT", "20000"))
default_args = {"owner": "airflow", "retries": 0}
def trino(sql: str):
"""Run a Trino statement, following nextUri pages. Returns all data rows."""
req = urllib.request.Request(
f"{TRINO_URL}/v1/statement", data=sql.encode(),
headers={"X-Trino-User": TRINO_USER, "Content-Type": "text/plain"},
)
rows = []
with urllib.request.urlopen(req, timeout=30) as resp:
d = json.loads(resp.read().decode())
while True:
rows += d.get("data") or []
err = d.get("error")
if err:
raise Exception(f"Trino error: {err.get('message')}")
nxt = d.get("nextUri")
if not nxt:
break
with urllib.request.urlopen(nxt, timeout=30) as resp:
d = json.loads(resp.read().decode())
return rows
# Masked projections. PII columns are hashed / partially redacted / generalized.
SALES_SQL = f"""
CREATE TABLE iceberg.curated_masked.sales_orders_masked
WITH (format = 'PARQUET') AS
SELECT
order_id, customer_id, product_id, region, sales_channel,
CAST(order_ts AS timestamp(6)) AS order_ts, amount, currency, order_status,
to_hex(md5(to_utf8(customer_name))) AS customer_name_hash,
regexp_replace(customer_email, '(^.)[^@]*(@.*)$', '$1***$2') AS customer_email_masked,
concat('***', substr(customer_phone, -4)) AS customer_phone_masked,
concat(substr(billing_iban, 1, 6), '****') AS billing_iban_masked,
regexp_replace(customer_ip, '\\.\\d+$', '.0') AS customer_ip_masked,
element_at(split(shipping_address, ', '), 2) AS shipping_area
FROM postgres_sales.public.sales_orders
WHERE customer_email IS NOT NULL
LIMIT {LIMIT}
"""
EMP_SQL = f"""
CREATE TABLE iceberg.curated_masked.employee_events_masked
WITH (format = 'PARQUET') AS
SELECT
event_id, employee_id, department, role_name, region, event_type, salary_change,
CAST(event_ts AS timestamp(6)) AS event_ts,
to_hex(md5(to_utf8(employee_name))) AS employee_name_hash,
regexp_replace(employee_email, '(^.)[^@]*(@.*)$', '$1***$2') AS employee_email_masked,
concat('***', substr(employee_phone, -4)) AS employee_phone_masked,
concat('***', substr(national_id, -3)) AS national_id_masked,
element_at(split(home_address, ', '), 2) AS home_area,
CAST(year(CAST(date_of_birth AS date)) AS varchar) AS birth_year
FROM mysql_hr.hr.employee_events
WHERE employee_email IS NOT NULL
LIMIT {LIMIT}
"""
def _mask(**context):
trino("CREATE SCHEMA IF NOT EXISTS iceberg.curated_masked")
for name, sql in [("sales_orders_masked", SALES_SQL), ("employee_events_masked", EMP_SQL)]:
print(f"Refreshing iceberg.curated_masked.{name} ...")
trino(f"DROP TABLE IF EXISTS iceberg.curated_masked.{name}")
trino(sql)
cnt = trino(f"SELECT count(*) FROM iceberg.curated_masked.{name}")
print(f" {name}: {cnt[0][0] if cnt else '?'} masked rows")
print("mask_to_curated complete")
dag = DAG(
dag_id="mask_to_curated",
default_args=default_args,
description="Mask PII from sources into iceberg.curated_masked (Trino SQL masking)",
schedule=None,
start_date=datetime(2025, 1, 1),
catchup=False,
tags=["etl", "masking", "pii", "curated"],
)
PythonOperator(task_id="mask_to_curated", python_callable=_mask, dag=dag)
deploy/mask_pii_setup.py
+33 -20
View File
@@ -17,6 +17,13 @@ DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51")
DB_USER = os.getenv("SRC_DB_USER", "mo")
DB_PASS = os.getenv("SRC_DB_PASSWORD", "Dell2026!")
N = int(os.getenv("PII_BACKFILL_ROWS", "4000"))
# asc: backfill lowest ids (so OpenMetadata's first-rows sample sees PII);
# desc: newest rows; both: do both ends.
ORDER = os.getenv("PII_BACKFILL_ORDER", "both").lower()
def _orders():
return (["ASC", "DESC"] if ORDER == "both" else [ORDER.upper()])
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
@@ -70,16 +77,19 @@ def do_postgres():
("customer_ip", "text"), ("billing_iban", "text"), ("shipping_address", "text")]:
cur.execute(f"ALTER TABLE public.sales_orders ADD COLUMN IF NOT EXISTS {col} {typ}")
print(" columns ensured")
cur.execute("SELECT order_id FROM public.sales_orders ORDER BY order_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for oid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid))
cur.executemany(
"UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, "
"customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
total = 0
for direction in _orders():
cur.execute(f"SELECT order_id FROM public.sales_orders ORDER BY order_id {direction} LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for oid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid))
cur.executemany(
"UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, "
"customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows)
total += len(rows)
print(f" backfilled {total} rows with PII ({'+'.join(_orders())})")
cur.close()
conn.close()
@@ -98,16 +108,19 @@ def do_mysql():
if "Duplicate column" not in str(e):
raise
print(" columns ensured")
cur.execute("SELECT event_id FROM employee_events ORDER BY event_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for eid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid))
cur.executemany(
"UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, "
"national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
total = 0
for direction in _orders():
cur.execute(f"SELECT event_id FROM employee_events ORDER BY event_id {direction} LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for eid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid))
cur.executemany(
"UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, "
"national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows)
total += len(rows)
print(f" backfilled {total} rows with PII ({'+'.join(_orders())})")
cur.close()
conn.close()
deploy/openmetadata/docker-compose.yml
+555
View File
@@ -0,0 +1,555 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: "3.9"
volumes:
ingestion-volume-dag-airflow:
ingestion-volume-dags:
ingestion-volume-tmp:
es-data:
services:
mysql:
container_name: openmetadata_mysql
image: docker.getcollate.io/openmetadata/db:1.13.0
command: "--sort_buffer_size=10M"
restart: always
environment:
MYSQL_ROOT_PASSWORD: password
expose:
- 3306
ports:
- "3306:3306"
volumes:
- ./docker-volume/db-data:/var/lib/mysql
networks:
- app_net
healthcheck:
test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
interval: 15s
timeout: 10s
retries: 10
elasticsearch:
container_name: openmetadata_elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
environment:
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1024m -Xmx1024m
- xpack.security.enabled=false
networks:
- app_net
ports:
- "9200:9200"
- "9300:9300"
healthcheck:
test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
interval: 15s
timeout: 10s
retries: 10
volumes:
- es-data:/usr/share/elasticsearch/data
execute-migrate-all:
container_name: execute_migrate_all
image: docker.getcollate.io/openmetadata/server:1.13.0
command: "./bootstrap/openmetadata-ops.sh migrate"
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
#For OIDC Authentication, when client is confidential
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
OIDC_TENANT: ${OIDC_TENANT:-""}
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
# For SAML Authentication
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
# For LDAP Authentication
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-db}
# AWS:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
# Azure:
OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
#Referrer-Policy
WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
#Permission-Policy
WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
#Cache
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
depends_on:
elasticsearch:
condition: service_healthy
mysql:
condition: service_healthy
networks:
- app_net
openmetadata-server:
container_name: openmetadata_server
restart: always
image: docker.getcollate.io/openmetadata/server:1.13.0
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
#For OIDC Authentication, when client is confidential
OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"}
OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"}
OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"}
OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
OIDC_TENANT: ${OIDC_TENANT:-""}
OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}
OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"}
OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"}
OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"}
# For SAML Authentication
# SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false}
# SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""}
# SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""}
# SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""}
# SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"}
# SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"}
# SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"}
# SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"}
# SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""}
# SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"}
# SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false}
# SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"}
# SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false}
# SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false}
# SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false}
# SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false}
# SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false}
# SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false}
# SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false}
# SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""}
# SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""}
# SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""}
# For LDAP Authentication
# AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-}
# AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-}
# AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""}
# AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""}
# AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""}
# AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-}
# AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3}
# AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-}
# AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll}
# AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-}
# AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-}
# AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-}
# AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-}
# AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]}
# AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-}
# AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100}
ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
# AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set)
# Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html
AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""}
SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true}
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-db}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
#Cache
WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
expose:
- 8585
- 8586
ports:
- "8585:8585"
- "8586:8586"
depends_on:
elasticsearch:
condition: service_healthy
mysql:
condition: service_healthy
execute-migrate-all:
condition: service_completed_successfully
networks:
- app_net
healthcheck:
test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ]
ingestion:
container_name: openmetadata_ingestion
image: docker.getcollate.io/openmetadata/ingestion:1.13.0
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
openmetadata-server:
condition: service_started
environment:
AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
DB_PORT: ${AIRFLOW_DB_PORT:-3306}
AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
# extra connection-string properties for the database
# EXAMPLE
# require SSL (only for Postgres)
# properties: "?sslmode=require"
DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-}
# To test the lineage backend
# AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
# AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
# AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
# AIRFLOW__LINEAGE__JWT_TOKEN: ...
entrypoint: /bin/bash
command:
- "/opt/airflow/ingestion_dependency.sh"
expose:
- 8080
ports:
- "8080:8080"
networks:
- app_net
volumes:
- ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
- ingestion-volume-dags:/opt/airflow/dags
- ingestion-volume-tmp:/tmp
networks:
app_net:
ipam:
driver: default
config:
- subnet: "172.16.240.0/24"
deploy/openmetadata/ingest/mongo_meta.yaml
+27
View File
@@ -0,0 +1,27 @@
source:
type: mongodb
serviceName: atc_mongodb
serviceConnection:
config:
type: MongoDB
scheme: mongodb
hostPort: 10.0.21.51:27017
databaseName: supplychain
connectionOptions:
replicaSet: rs0
directConnection: "true"
sourceConfig:
config:
type: DatabaseMetadata
databaseFilterPattern:
includes:
- supplychain
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/mysql_classify.yaml
+32
View File
@@ -0,0 +1,32 @@
source:
type: mysql
serviceName: atc_mysql
serviceConnection:
config:
type: Mysql
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:3306
databaseSchema: hr
sourceConfig:
config:
type: AutoClassification
enableAutoClassification: true
storeSampleData: true
confidence: 80
tableFilterPattern:
includes:
- employee_events
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/mysql_meta.yaml
+23
View File
@@ -0,0 +1,23 @@
source:
type: mysql
serviceName: atc_mysql
serviceConnection:
config:
type: Mysql
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:3306
databaseSchema: hr
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/pg_classify.yaml
+35
View File
@@ -0,0 +1,35 @@
source:
type: postgres
serviceName: atc_postgres
serviceConnection:
config:
type: Postgres
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:5432
database: postgres
sourceConfig:
config:
type: AutoClassification
enableAutoClassification: true
storeSampleData: true
confidence: 80
schemaFilterPattern:
includes:
- public
tableFilterPattern:
includes:
- sales_orders
processor:
type: orm-profiler
config: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/pg_meta.yaml
+26
View File
@@ -0,0 +1,26 @@
source:
type: postgres
serviceName: atc_postgres
serviceConnection:
config:
type: Postgres
username: mo
authType:
password: Dell2026!
hostPort: 10.0.21.51:5432
database: postgres
sourceConfig:
config:
type: DatabaseMetadata
schemaFilterPattern:
includes:
- public
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/ingest/run.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Usage: run.sh <JWT> <yaml-basename without .yaml>
# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion
# container, and executes the metadata ingestion workflow.
set -uo pipefail
JWT="$1"
NAME="$2"
CMD="${3:-ingest}"
DIR=/opt/openmetadata/ingest
# Use the stored stable token if no/placeholder JWT is passed (avoids rotating
# the bot token, which would invalidate the Command Center's stored token).
if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then
JWT="$(cat "$DIR/.token" 2>/dev/null)"
fi
SRC="$DIR/${NAME}.yaml"
OUT="$DIR/.${NAME}.run.yaml"
if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi
sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT"
docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null
echo "=== metadata ${CMD}: ${NAME} ==="
docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45
rm -f "$OUT"
deploy/openmetadata/ingest/trino_meta.yaml
+21
View File
@@ -0,0 +1,21 @@
source:
type: trino
serviceName: atc_trino
serviceConnection:
config:
type: Trino
username: mo
hostPort: 10.0.21.50:8089
catalog: iceberg
sourceConfig:
config:
type: DatabaseMetadata
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "__JWT__"
deploy/openmetadata/om_api.py
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""OpenMetadata API helper: login, fetch ingestion-bot JWT, create admin users."""
import base64
import json
import sys
import urllib.request
import urllib.error
OM = "http://10.0.21.47:8585"
def req(method, path, token=None, body=None):
url = OM + path
data = json.dumps(body).encode() if body is not None else None
r = urllib.request.Request(url, data=data, method=method)
r.add_header("Content-Type", "application/json")
if token:
r.add_header("Authorization", "Bearer " + token)
try:
with urllib.request.urlopen(r, timeout=20) as resp:
raw = resp.read().decode()
return resp.status, (json.loads(raw) if raw else {})
except urllib.error.HTTPError as e:
raw = e.read().decode()
try:
return e.code, json.loads(raw)
except Exception:
return e.code, {"raw": raw}
def login(email, password):
st, d = req("POST", "/api/v1/users/login", body={
"email": email,
"password": base64.b64encode(password.encode()).decode(),
})
return st, d
def main():
action = sys.argv[1] if len(sys.argv) > 1 else "bootstrap"
st, d = login("admin@open-metadata.org", "admin")
if st != 200 or "accessToken" not in d:
print("LOGIN_FAILED", st, json.dumps(d)[:300])
sys.exit(1)
token = d["accessToken"]
print("ADMIN_LOGIN_OK")
if action in ("bootstrap", "bottoken"):
# Reuse the ingestion-bot's existing JWT (do NOT regenerate to avoid breaking internals).
st, b = req("GET", "/api/v1/bots/name/ingestion-bot", token=token)
uid = (b.get("botUser") or {}).get("id")
jwt = None
if uid:
st2, u = req("GET", f"/api/v1/users/{uid}?include=all", token=token)
am = u.get("authenticationMechanism") or {}
jwt = (am.get("config") or {}).get("JWTToken")
if not jwt and uid and "--regen" in sys.argv:
st3, t = req("PUT", f"/api/v1/users/generateToken/{uid}",
token=token, body={"JWTTokenExpiry": "Unlimited"})
jwt = t.get("JWTToken")
if jwt:
print("INGESTION_BOT_JWT=" + jwt)
else:
print("BOT_JWT_NOT_FOUND", json.dumps(b)[:200])
if action == "lineage":
pairs = [
("atc_postgres.postgres.public.sales_orders", "atc_trino.iceberg.curated_masked.sales_orders_masked"),
("atc_mysql.default.hr.employee_events", "atc_trino.iceberg.curated_masked.employee_events_masked"),
]
for src_fqn, dst_fqn in pairs:
s1, src = req("GET", f"/api/v1/tables/name/{src_fqn}", token=token)
s2, dst = req("GET", f"/api/v1/tables/name/{dst_fqn}", token=token)
sid, did = src.get("id"), dst.get("id")
if not sid or not did:
print(f"SKIP {src_fqn}->{dst_fqn} (missing id s={s1} d={s2})")
continue
st, r = req("PUT", "/api/v1/lineage", token=token, body={
"edge": {
"fromEntity": {"id": sid, "type": "table"},
"toEntity": {"id": did, "type": "table"},
"lineageDetails": {"description": "PII masking via mask_to_curated (Trino SQL hash/redact/generalize)"},
},
})
print(f"LINEAGE {src_fqn} -> {dst_fqn}: {st}")
return
if action == "tags":
for fqn in sys.argv[2:]:
st, t = req("GET", f"/api/v1/tables/name/{fqn}?fields=columns,tags", token=token)
cols = t.get("columns", [])
print(f"== {fqn} ({st}) cols={len(cols)} ==")
for c in cols:
tg = [x.get("tagFQN") for x in (c.get("tags") or [])]
if tg:
print(f" {c['name']}: {tg}")
return
if action == "bootstrap":
for name in ("mo", "bart"):
st, d2 = req("POST", "/api/v1/users/signup", body={
"firstName": name.capitalize(), "lastName": "ATC",
"email": f"{name}@open-metadata.org",
"password": "Dell2026!",
})
print(f"USER_{name}: {st} {json.dumps(d2)[:140]}")
if __name__ == "__main__":
main()