diff --git a/deploy/airflow/mask_to_curated_dag.py b/deploy/airflow/mask_to_curated_dag.py new file mode 100644 index 0000000..c5cb5e5 --- /dev/null +++ b/deploy/airflow/mask_to_curated_dag.py @@ -0,0 +1,102 @@ +"""mask_to_curated — ETL with PII masking into the curated Iceberg layer. + +Reads PII-bearing rows from the source databases via Trino (postgres_sales, +mysql_hr catalogs) and writes MASKED copies into iceberg.curated_masked.*. +Masking is expressed as Trino SQL (hash / partial-redact / generalize) so no PII +ever lands in the curated layer. Triggered by the Command Center (movement +'mask_to_curated') or autonomously by the ETL agent. +""" +from airflow import DAG +from airflow.operators.python import PythonOperator +from datetime import datetime +import json +import os +import urllib.request + +TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/") +TRINO_USER = os.getenv("TRINO_USER", "mo") +LIMIT = int(os.getenv("MASK_LIMIT", "20000")) + +default_args = {"owner": "airflow", "retries": 0} + + +def trino(sql: str): + """Run a Trino statement, following nextUri pages. Returns all data rows.""" + req = urllib.request.Request( + f"{TRINO_URL}/v1/statement", data=sql.encode(), + headers={"X-Trino-User": TRINO_USER, "Content-Type": "text/plain"}, + ) + rows = [] + with urllib.request.urlopen(req, timeout=30) as resp: + d = json.loads(resp.read().decode()) + while True: + rows += d.get("data") or [] + err = d.get("error") + if err: + raise Exception(f"Trino error: {err.get('message')}") + nxt = d.get("nextUri") + if not nxt: + break + with urllib.request.urlopen(nxt, timeout=30) as resp: + d = json.loads(resp.read().decode()) + return rows + + +# Masked projections. PII columns are hashed / partially redacted / generalized. +SALES_SQL = f""" +CREATE TABLE iceberg.curated_masked.sales_orders_masked +WITH (format = 'PARQUET') AS +SELECT + order_id, customer_id, product_id, region, sales_channel, + CAST(order_ts AS timestamp(6)) AS order_ts, amount, currency, order_status, + to_hex(md5(to_utf8(customer_name))) AS customer_name_hash, + regexp_replace(customer_email, '(^.)[^@]*(@.*)$', '$1***$2') AS customer_email_masked, + concat('***', substr(customer_phone, -4)) AS customer_phone_masked, + concat(substr(billing_iban, 1, 6), '****') AS billing_iban_masked, + regexp_replace(customer_ip, '\\.\\d+$', '.0') AS customer_ip_masked, + element_at(split(shipping_address, ', '), 2) AS shipping_area +FROM postgres_sales.public.sales_orders +WHERE customer_email IS NOT NULL +LIMIT {LIMIT} +""" + +EMP_SQL = f""" +CREATE TABLE iceberg.curated_masked.employee_events_masked +WITH (format = 'PARQUET') AS +SELECT + event_id, employee_id, department, role_name, region, event_type, salary_change, + CAST(event_ts AS timestamp(6)) AS event_ts, + to_hex(md5(to_utf8(employee_name))) AS employee_name_hash, + regexp_replace(employee_email, '(^.)[^@]*(@.*)$', '$1***$2') AS employee_email_masked, + concat('***', substr(employee_phone, -4)) AS employee_phone_masked, + concat('***', substr(national_id, -3)) AS national_id_masked, + element_at(split(home_address, ', '), 2) AS home_area, + CAST(year(CAST(date_of_birth AS date)) AS varchar) AS birth_year +FROM mysql_hr.hr.employee_events +WHERE employee_email IS NOT NULL +LIMIT {LIMIT} +""" + + +def _mask(**context): + trino("CREATE SCHEMA IF NOT EXISTS iceberg.curated_masked") + for name, sql in [("sales_orders_masked", SALES_SQL), ("employee_events_masked", EMP_SQL)]: + print(f"Refreshing iceberg.curated_masked.{name} ...") + trino(f"DROP TABLE IF EXISTS iceberg.curated_masked.{name}") + trino(sql) + cnt = trino(f"SELECT count(*) FROM iceberg.curated_masked.{name}") + print(f" {name}: {cnt[0][0] if cnt else '?'} masked rows") + print("mask_to_curated complete") + + +dag = DAG( + dag_id="mask_to_curated", + default_args=default_args, + description="Mask PII from sources into iceberg.curated_masked (Trino SQL masking)", + schedule=None, + start_date=datetime(2025, 1, 1), + catchup=False, + tags=["etl", "masking", "pii", "curated"], +) + +PythonOperator(task_id="mask_to_curated", python_callable=_mask, dag=dag) diff --git a/deploy/mask_pii_setup.py b/deploy/mask_pii_setup.py index 9cb0b6e..6c7fd5d 100644 --- a/deploy/mask_pii_setup.py +++ b/deploy/mask_pii_setup.py @@ -17,6 +17,13 @@ DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51") DB_USER = os.getenv("SRC_DB_USER", "mo") DB_PASS = os.getenv("SRC_DB_PASSWORD", "Dell2026!") N = int(os.getenv("PII_BACKFILL_ROWS", "4000")) +# asc: backfill lowest ids (so OpenMetadata's first-rows sample sees PII); +# desc: newest rows; both: do both ends. +ORDER = os.getenv("PII_BACKFILL_ORDER", "both").lower() + + +def _orders(): + return (["ASC", "DESC"] if ORDER == "both" else [ORDER.upper()]) FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem", "Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"] @@ -70,16 +77,19 @@ def do_postgres(): ("customer_ip", "text"), ("billing_iban", "text"), ("shipping_address", "text")]: cur.execute(f"ALTER TABLE public.sales_orders ADD COLUMN IF NOT EXISTS {col} {typ}") print(" columns ensured") - cur.execute("SELECT order_id FROM public.sales_orders ORDER BY order_id DESC LIMIT %s", (N,)) - ids = [r[0] for r in cur.fetchall()] - rows = [] - for oid in ids: - nm = full_name() - rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid)) - cur.executemany( - "UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, " - "customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows) - print(f" backfilled {len(rows)} rows with PII") + total = 0 + for direction in _orders(): + cur.execute(f"SELECT order_id FROM public.sales_orders ORDER BY order_id {direction} LIMIT %s", (N,)) + ids = [r[0] for r in cur.fetchall()] + rows = [] + for oid in ids: + nm = full_name() + rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid)) + cur.executemany( + "UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, " + "customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows) + total += len(rows) + print(f" backfilled {total} rows with PII ({'+'.join(_orders())})") cur.close() conn.close() @@ -98,16 +108,19 @@ def do_mysql(): if "Duplicate column" not in str(e): raise print(" columns ensured") - cur.execute("SELECT event_id FROM employee_events ORDER BY event_id DESC LIMIT %s", (N,)) - ids = [r[0] for r in cur.fetchall()] - rows = [] - for eid in ids: - nm = full_name() - rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid)) - cur.executemany( - "UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, " - "national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows) - print(f" backfilled {len(rows)} rows with PII") + total = 0 + for direction in _orders(): + cur.execute(f"SELECT event_id FROM employee_events ORDER BY event_id {direction} LIMIT %s", (N,)) + ids = [r[0] for r in cur.fetchall()] + rows = [] + for eid in ids: + nm = full_name() + rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid)) + cur.executemany( + "UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, " + "national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows) + total += len(rows) + print(f" backfilled {total} rows with PII ({'+'.join(_orders())})") cur.close() conn.close() diff --git a/deploy/openmetadata/docker-compose.yml b/deploy/openmetadata/docker-compose.yml new file mode 100644 index 0000000..174bf5e --- /dev/null +++ b/deploy/openmetadata/docker-compose.yml @@ -0,0 +1,555 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.9" +volumes: + ingestion-volume-dag-airflow: + ingestion-volume-dags: + ingestion-volume-tmp: + es-data: +services: + mysql: + container_name: openmetadata_mysql + image: docker.getcollate.io/openmetadata/db:1.13.0 + command: "--sort_buffer_size=10M" + restart: always + environment: + MYSQL_ROOT_PASSWORD: password + expose: + - 3306 + ports: + - "3306:3306" + volumes: + - ./docker-volume/db-data:/var/lib/mysql + networks: + - app_net + healthcheck: + test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db" + interval: 15s + timeout: 10s + retries: 10 + + elasticsearch: + container_name: openmetadata_elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0 + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms1024m -Xmx1024m + - xpack.security.enabled=false + networks: + - app_net + ports: + - "9200:9200" + - "9300:9300" + healthcheck: + test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1" + interval: 15s + timeout: 10s + retries: 10 + volumes: + - es-data:/usr/share/elasticsearch/data + + execute-migrate-all: + container_name: execute_migrate_all + image: docker.getcollate.io/openmetadata/server:1.13.0 + command: "./bootstrap/openmetadata-ops.sh migrate" + environment: + OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} + SERVER_PORT: ${SERVER_PORT:-8585} + SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + + # Migration + MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200} + + # OpenMetadata Server Authentication Configuration + AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer} + AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter} + AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]} + AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]} + AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]} + AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"} + AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]} + AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false} + AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false} + AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic} + AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token} + CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} + AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]} + AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} + AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""} + AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} + AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} + AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]} + AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} + OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"} + OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"} + OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"} + # For SAML Authentication + # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} + # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} + # SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""} + # SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""} + # SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"} + # SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"} + # SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"} + # SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"} + # SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""} + # SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"} + # SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false} + # SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"} + # SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false} + # SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false} + # SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false} + # SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false} + # SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false} + # SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false} + # SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false} + # SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""} + # SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""} + # SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""} + # For LDAP Authentication + # AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-} + # AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-} + # AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""} + # AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""} + # AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""} + # AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-} + # AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3} + # AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-} + # AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll} + # AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-} + # AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-} + # AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-} + # AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-} + # AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]} + # AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-} + # AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true} + + # JWT Configuration + RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"} + RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"} + JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"} + JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"} + # OpenMetadata Server Pipeline Service Client Configuration + PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080} + PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300} + SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api} + PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} + PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} + # Database configuration for MySQL + DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver} + DB_SCHEME: ${DB_SCHEME:-mysql} + DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC} + DB_USER: ${DB_USER:-openmetadata_user} + DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password} + DB_HOST: ${DB_HOST:-mysql} + DB_PORT: ${DB_PORT:-3306} + OM_DATABASE: ${OM_DATABASE:-openmetadata_db} + # ElasticSearch Configurations + ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} + ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} + ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} + ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} + SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"} + ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} + ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} + ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} + ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} + ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} + # AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set) + # Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""} + AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""} + SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"} + + #eventMonitoringConfiguration + EVENT_MONITOR: ${EVENT_MONITOR:-prometheus} + EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10} + EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]} + EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]} + + #pipelineServiceClientConfiguration + PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true} + PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"} + PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false} + PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""} + PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"} + #airflow parameters + AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin} + AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin} + AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10} + AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""} + AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""} + FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=} + + #secretsManagerConfiguration + SECRET_MANAGER: ${SECRET_MANAGER:-db} + # AWS: + OM_SM_REGION: ${OM_SM_REGION:-""} + OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""} + OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""} + # Azure: + OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""} + OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""} + OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""} + OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""} + + #email configuration: + OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"} + OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"} + AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false} + OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""} + OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""} + SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""} + SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""} + SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""} + SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""} + SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"} + + # Heap OPTS Configurations + OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G} + # Mask passwords values in UI + MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false} + + #OpenMetadata Web Configuration + WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"} + #HSTS + WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false} + WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"} + WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"} + WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"} + #Frame Options + WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false} + WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"} + WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""} + #Content Type + WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false} + #XSS-Protection + WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false} + WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true} + WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true} + #CSP + WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false} + WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"} + WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""} + #Referrer-Policy + WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false} + WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"} + #Permission-Policy + WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false} + WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""} + #Cache + WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""} + WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""} + depends_on: + elasticsearch: + condition: service_healthy + mysql: + condition: service_healthy + networks: + - app_net + + openmetadata-server: + container_name: openmetadata_server + restart: always + image: docker.getcollate.io/openmetadata/server:1.13.0 + environment: + OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} + SERVER_PORT: ${SERVER_PORT:-8585} + SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + + # OpenMetadata Server Authentication Configuration + AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer} + AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter} + AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]} + AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]} + AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]} + AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"} + AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]} + AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false} + AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false} + AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic} + AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token} + CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""} + AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]} + AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com} + AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""} + AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""} + AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]} + AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]} + AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true} + AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public} + #For OIDC Authentication, when client is confidential + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""} + OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc. + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""} + OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"} + OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""} + OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true} + OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-"RS256"} + OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"} + OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true} + OIDC_CALLBACK: ${OIDC_CALLBACK:-"http://localhost:8585/callback"} + OIDC_SERVER_URL: ${OIDC_SERVER_URL:-"http://localhost:8585"} + OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"} + OIDC_TENANT: ${OIDC_TENANT:-""} + OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""} + OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}} + OIDC_MAX_AGE: ${OIDC_MAX_AGE:-"0"} + OIDC_PROMPT_TYPE: ${OIDC_PROMPT_TYPE:-"consent"} + OIDC_SESSION_EXPIRY: ${OIDC_SESSION_EXPIRY:-"604800"} + # For SAML Authentication + # SAML_DEBUG_MODE: ${SAML_DEBUG_MODE:-false} + # SAML_IDP_ENTITY_ID: ${SAML_IDP_ENTITY_ID:-""} + # SAML_IDP_SSO_LOGIN_URL: ${SAML_IDP_SSO_LOGIN_URL:-""} + # SAML_IDP_CERTIFICATE: ${SAML_IDP_CERTIFICATE:-""} + # SAML_AUTHORITY_URL: ${SAML_AUTHORITY_URL:-"http://localhost:8585/api/v1/saml/login"} + # SAML_IDP_NAME_ID: ${SAML_IDP_NAME_ID:-"urn:oasis:names:tc:SAML:2.0:nameid-format:emailAddress"} + # SAML_SP_ENTITY_ID: ${SAML_SP_ENTITY_ID:-"http://localhost:8585/api/v1/saml/metadata"} + # SAML_SP_ACS: ${SAML_SP_ACS:-"http://localhost:8585/api/v1/saml/acs"} + # SAML_SP_CERTIFICATE: ${SAML_SP_CERTIFICATE:-""} + # SAML_SP_CALLBACK: ${SAML_SP_CALLBACK:-"http://localhost:8585/saml/callback"} + # SAML_STRICT_MODE: ${SAML_STRICT_MODE:-false} + # SAML_SP_TOKEN_VALIDITY: ${SAML_SP_TOKEN_VALIDITY:-"3600"} + # SAML_SEND_ENCRYPTED_NAME_ID: ${SAML_SEND_ENCRYPTED_NAME_ID:-false} + # SAML_SEND_SIGNED_AUTH_REQUEST: ${SAML_SEND_SIGNED_AUTH_REQUEST:-false} + # SAML_SIGNED_SP_METADATA: ${SAML_SIGNED_SP_METADATA:-false} + # SAML_WANT_MESSAGE_SIGNED: ${SAML_WANT_MESSAGE_SIGNED:-false} + # SAML_WANT_ASSERTION_SIGNED: ${SAML_WANT_ASSERTION_SIGNED:-false} + # SAML_WANT_ASSERTION_ENCRYPTED: ${SAML_WANT_ASSERTION_ENCRYPTED:-false} + # SAML_WANT_NAME_ID_ENCRYPTED: ${SAML_WANT_NAME_ID_ENCRYPTED:-false} + # SAML_KEYSTORE_FILE_PATH: ${SAML_KEYSTORE_FILE_PATH:-""} + # SAML_KEYSTORE_ALIAS: ${SAML_KEYSTORE_ALIAS:-""} + # SAML_KEYSTORE_PASSWORD: ${SAML_KEYSTORE_PASSWORD:-""} + # For LDAP Authentication + # AUTHENTICATION_LDAP_HOST: ${AUTHENTICATION_LDAP_HOST:-} + # AUTHENTICATION_LDAP_PORT: ${AUTHENTICATION_LDAP_PORT:-} + # AUTHENTICATION_LOOKUP_ADMIN_DN: ${AUTHENTICATION_LOOKUP_ADMIN_DN:-""} + # AUTHENTICATION_LOOKUP_ADMIN_PWD: ${AUTHENTICATION_LOOKUP_ADMIN_PWD:-""} + # AUTHENTICATION_USER_LOOKUP_BASEDN: ${AUTHENTICATION_USER_LOOKUP_BASEDN:-""} + # AUTHENTICATION_USER_MAIL_ATTR: ${AUTHENTICATION_USER_MAIL_ATTR:-} + # AUTHENTICATION_LDAP_POOL_SIZE: ${AUTHENTICATION_LDAP_POOL_SIZE:-3} + # AUTHENTICATION_LDAP_SSL_ENABLED: ${AUTHENTICATION_LDAP_SSL_ENABLED:-} + # AUTHENTICATION_LDAP_TRUSTSTORE_TYPE: ${AUTHENTICATION_LDAP_TRUSTSTORE_TYPE:-TrustAll} + # AUTHENTICATION_LDAP_TRUSTSTORE_PATH: ${AUTHENTICATION_LDAP_TRUSTSTORE_PATH:-} + # AUTHENTICATION_LDAP_KEYSTORE_PASSWORD: ${AUTHENTICATION_LDAP_KEYSTORE_PASSWORD:-} + # AUTHENTICATION_LDAP_SSL_KEY_FORMAT: ${AUTHENTICATION_LDAP_SSL_KEY_FORMAT:-} + # AUTHENTICATION_LDAP_ALLOW_WILDCARDS: ${AUTHENTICATION_LDAP_ALLOW_WILDCARDS:-} + # AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES: ${AUTHENTICATION_LDAP_ALLOWED_HOSTNAMES:-[]} + # AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST: ${AUTHENTICATION_LDAP_SSL_VERIFY_CERT_HOST:-} + # AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES: ${AUTHENTICATION_LDAP_EXAMINE_VALIDITY_DATES:-true} + + # JWT Configuration + RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"} + RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"} + JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"} + JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"} + # OpenMetadata Server Pipeline Service Client Configuration + PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080} + PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300} + SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api} + PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} + PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""} + # Database configuration for MySQL + DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver} + DB_SCHEME: ${DB_SCHEME:-mysql} + DB_PARAMS: ${DB_PARAMS:-allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC} + DB_USER: ${DB_USER:-openmetadata_user} + DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password} + DB_HOST: ${DB_HOST:-mysql} + DB_PORT: ${DB_PORT:-3306} + OM_DATABASE: ${OM_DATABASE:-openmetadata_db} + # ElasticSearch Configurations + ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} + ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} + ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} + ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} + SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"} + ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} + ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} + ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5} + ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60} + ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600} + ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-100} + ELASTICSEARCH_PAYLOAD_BYTES_SIZE: ${ELASTICSEARCH_PAYLOAD_BYTES_SIZE:-10485760} #max payLoadSize in Bytes + ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN} + # AWS IAM Authentication for OpenSearch (auto-enabled when AWS_DEFAULT_REGION is set) + # Uses standard AWS env vars: https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-envvars.html + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-""} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""} + AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-""} + SEARCH_AWS_SERVICE_NAME: ${SEARCH_AWS_SERVICE_NAME:-"es"} + + #eventMonitoringConfiguration + EVENT_MONITOR: ${EVENT_MONITOR:-prometheus} + EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10} + EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]} + EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]} + + #pipelineServiceClientConfiguration + PIPELINE_SERVICE_CLIENT_ENABLED: ${PIPELINE_SERVICE_CLIENT_ENABLED:-true} + PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"} + PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false} + PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""} + PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"} + #airflow parameters + AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin} + AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin} + AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10} + AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""} + AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""} + FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=} + + #secretsManagerConfiguration + SECRET_MANAGER: ${SECRET_MANAGER:-db} + #parameters: + OM_SM_REGION: ${OM_SM_REGION:-""} + OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""} + OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""} + + #email configuration: + OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"} + OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"} + AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false} + OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""} + OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""} + SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""} + SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""} + SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""} + SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""} + SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"} + + # Heap OPTS Configurations + OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G} + # Mask passwords values in UI + MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false} + + #OpenMetadata Web Configuration + WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"} + #HSTS + WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false} + WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"} + WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"} + WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"} + #Frame Options + WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false} + WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"} + WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""} + #Content Type + WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false} + #XSS-Protection + WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false} + WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true} + WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true} + #CSP + WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false} + WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"} + WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""} + #Cache + WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""} + WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""} + + expose: + - 8585 + - 8586 + ports: + - "8585:8585" + - "8586:8586" + depends_on: + elasticsearch: + condition: service_healthy + mysql: + condition: service_healthy + execute-migrate-all: + condition: service_completed_successfully + networks: + - app_net + healthcheck: + test: [ "CMD", "wget", "-q", "--spider", "http://localhost:8586/healthcheck" ] + + ingestion: + container_name: openmetadata_ingestion + image: docker.getcollate.io/openmetadata/ingestion:1.13.0 + depends_on: + elasticsearch: + condition: service_started + mysql: + condition: service_healthy + openmetadata-server: + condition: service_started + environment: + AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session" + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs" + DB_HOST: ${AIRFLOW_DB_HOST:-mysql} + DB_PORT: ${AIRFLOW_DB_PORT:-3306} + AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} + DB_USER: ${AIRFLOW_DB_USER:-airflow_user} + DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} + # extra connection-string properties for the database + # EXAMPLE + # require SSL (only for Postgres) + # properties: "?sslmode=require" + DB_PROPERTIES: ${AIRFLOW_DB_PROPERTIES:-} + # To test the lineage backend + # AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend + # AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow + # AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api + # AIRFLOW__LINEAGE__JWT_TOKEN: ... + entrypoint: /bin/bash + command: + - "/opt/airflow/ingestion_dependency.sh" + expose: + - 8080 + ports: + - "8080:8080" + networks: + - app_net + volumes: + - ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs + - ingestion-volume-dags:/opt/airflow/dags + - ingestion-volume-tmp:/tmp + + +networks: + app_net: + ipam: + driver: default + config: + - subnet: "172.16.240.0/24" diff --git a/deploy/openmetadata/ingest/mongo_meta.yaml b/deploy/openmetadata/ingest/mongo_meta.yaml new file mode 100644 index 0000000..928c7b5 --- /dev/null +++ b/deploy/openmetadata/ingest/mongo_meta.yaml @@ -0,0 +1,27 @@ +source: + type: mongodb + serviceName: atc_mongodb + serviceConnection: + config: + type: MongoDB + scheme: mongodb + hostPort: 10.0.21.51:27017 + databaseName: supplychain + connectionOptions: + replicaSet: rs0 + directConnection: "true" + sourceConfig: + config: + type: DatabaseMetadata + databaseFilterPattern: + includes: + - supplychain +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/mysql_classify.yaml b/deploy/openmetadata/ingest/mysql_classify.yaml new file mode 100644 index 0000000..27d3206 --- /dev/null +++ b/deploy/openmetadata/ingest/mysql_classify.yaml @@ -0,0 +1,32 @@ +source: + type: mysql + serviceName: atc_mysql + serviceConnection: + config: + type: Mysql + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:3306 + databaseSchema: hr + sourceConfig: + config: + type: AutoClassification + enableAutoClassification: true + storeSampleData: true + confidence: 80 + tableFilterPattern: + includes: + - employee_events +processor: + type: orm-profiler + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/mysql_meta.yaml b/deploy/openmetadata/ingest/mysql_meta.yaml new file mode 100644 index 0000000..fef8426 --- /dev/null +++ b/deploy/openmetadata/ingest/mysql_meta.yaml @@ -0,0 +1,23 @@ +source: + type: mysql + serviceName: atc_mysql + serviceConnection: + config: + type: Mysql + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:3306 + databaseSchema: hr + sourceConfig: + config: + type: DatabaseMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/pg_classify.yaml b/deploy/openmetadata/ingest/pg_classify.yaml new file mode 100644 index 0000000..4fbc1d0 --- /dev/null +++ b/deploy/openmetadata/ingest/pg_classify.yaml @@ -0,0 +1,35 @@ +source: + type: postgres + serviceName: atc_postgres + serviceConnection: + config: + type: Postgres + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:5432 + database: postgres + sourceConfig: + config: + type: AutoClassification + enableAutoClassification: true + storeSampleData: true + confidence: 80 + schemaFilterPattern: + includes: + - public + tableFilterPattern: + includes: + - sales_orders +processor: + type: orm-profiler + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/pg_meta.yaml b/deploy/openmetadata/ingest/pg_meta.yaml new file mode 100644 index 0000000..a643529 --- /dev/null +++ b/deploy/openmetadata/ingest/pg_meta.yaml @@ -0,0 +1,26 @@ +source: + type: postgres + serviceName: atc_postgres + serviceConnection: + config: + type: Postgres + username: mo + authType: + password: Dell2026! + hostPort: 10.0.21.51:5432 + database: postgres + sourceConfig: + config: + type: DatabaseMetadata + schemaFilterPattern: + includes: + - public +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/run.sh b/deploy/openmetadata/ingest/run.sh new file mode 100644 index 0000000..79e8503 --- /dev/null +++ b/deploy/openmetadata/ingest/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Usage: run.sh +# Runs on docker02; substitutes JWT into the YAML, copies into the ingestion +# container, and executes the metadata ingestion workflow. +set -uo pipefail +JWT="$1" +NAME="$2" +CMD="${3:-ingest}" +DIR=/opt/openmetadata/ingest +# Use the stored stable token if no/placeholder JWT is passed (avoids rotating +# the bot token, which would invalidate the Command Center's stored token). +if [ -z "$JWT" ] || [ "$JWT" = "-" ]; then + JWT="$(cat "$DIR/.token" 2>/dev/null)" +fi +SRC="$DIR/${NAME}.yaml" +OUT="$DIR/.${NAME}.run.yaml" + +if [ ! -f "$SRC" ]; then echo "MISSING $SRC"; exit 2; fi +sed "s|__JWT__|${JWT}|g" "$SRC" > "$OUT" +docker cp "$OUT" openmetadata_ingestion:/tmp/${NAME}.yaml >/dev/null +echo "=== metadata ${CMD}: ${NAME} ===" +docker exec openmetadata_ingestion metadata ${CMD} -c /tmp/${NAME}.yaml 2>&1 | tail -45 +rm -f "$OUT" diff --git a/deploy/openmetadata/ingest/trino_meta.yaml b/deploy/openmetadata/ingest/trino_meta.yaml new file mode 100644 index 0000000..b1a2889 --- /dev/null +++ b/deploy/openmetadata/ingest/trino_meta.yaml @@ -0,0 +1,21 @@ +source: + type: trino + serviceName: atc_trino + serviceConnection: + config: + type: Trino + username: mo + hostPort: 10.0.21.50:8089 + catalog: iceberg + sourceConfig: + config: + type: DatabaseMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/om_api.py b/deploy/openmetadata/om_api.py new file mode 100644 index 0000000..50240ff --- /dev/null +++ b/deploy/openmetadata/om_api.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""OpenMetadata API helper: login, fetch ingestion-bot JWT, create admin users.""" +import base64 +import json +import sys +import urllib.request +import urllib.error + +OM = "http://10.0.21.47:8585" + + +def req(method, path, token=None, body=None): + url = OM + path + data = json.dumps(body).encode() if body is not None else None + r = urllib.request.Request(url, data=data, method=method) + r.add_header("Content-Type", "application/json") + if token: + r.add_header("Authorization", "Bearer " + token) + try: + with urllib.request.urlopen(r, timeout=20) as resp: + raw = resp.read().decode() + return resp.status, (json.loads(raw) if raw else {}) + except urllib.error.HTTPError as e: + raw = e.read().decode() + try: + return e.code, json.loads(raw) + except Exception: + return e.code, {"raw": raw} + + +def login(email, password): + st, d = req("POST", "/api/v1/users/login", body={ + "email": email, + "password": base64.b64encode(password.encode()).decode(), + }) + return st, d + + +def main(): + action = sys.argv[1] if len(sys.argv) > 1 else "bootstrap" + st, d = login("admin@open-metadata.org", "admin") + if st != 200 or "accessToken" not in d: + print("LOGIN_FAILED", st, json.dumps(d)[:300]) + sys.exit(1) + token = d["accessToken"] + print("ADMIN_LOGIN_OK") + + if action in ("bootstrap", "bottoken"): + # Reuse the ingestion-bot's existing JWT (do NOT regenerate to avoid breaking internals). + st, b = req("GET", "/api/v1/bots/name/ingestion-bot", token=token) + uid = (b.get("botUser") or {}).get("id") + jwt = None + if uid: + st2, u = req("GET", f"/api/v1/users/{uid}?include=all", token=token) + am = u.get("authenticationMechanism") or {} + jwt = (am.get("config") or {}).get("JWTToken") + if not jwt and uid and "--regen" in sys.argv: + st3, t = req("PUT", f"/api/v1/users/generateToken/{uid}", + token=token, body={"JWTTokenExpiry": "Unlimited"}) + jwt = t.get("JWTToken") + if jwt: + print("INGESTION_BOT_JWT=" + jwt) + else: + print("BOT_JWT_NOT_FOUND", json.dumps(b)[:200]) + + if action == "lineage": + pairs = [ + ("atc_postgres.postgres.public.sales_orders", "atc_trino.iceberg.curated_masked.sales_orders_masked"), + ("atc_mysql.default.hr.employee_events", "atc_trino.iceberg.curated_masked.employee_events_masked"), + ] + for src_fqn, dst_fqn in pairs: + s1, src = req("GET", f"/api/v1/tables/name/{src_fqn}", token=token) + s2, dst = req("GET", f"/api/v1/tables/name/{dst_fqn}", token=token) + sid, did = src.get("id"), dst.get("id") + if not sid or not did: + print(f"SKIP {src_fqn}->{dst_fqn} (missing id s={s1} d={s2})") + continue + st, r = req("PUT", "/api/v1/lineage", token=token, body={ + "edge": { + "fromEntity": {"id": sid, "type": "table"}, + "toEntity": {"id": did, "type": "table"}, + "lineageDetails": {"description": "PII masking via mask_to_curated (Trino SQL hash/redact/generalize)"}, + }, + }) + print(f"LINEAGE {src_fqn} -> {dst_fqn}: {st}") + return + + if action == "tags": + for fqn in sys.argv[2:]: + st, t = req("GET", f"/api/v1/tables/name/{fqn}?fields=columns,tags", token=token) + cols = t.get("columns", []) + print(f"== {fqn} ({st}) cols={len(cols)} ==") + for c in cols: + tg = [x.get("tagFQN") for x in (c.get("tags") or [])] + if tg: + print(f" {c['name']}: {tg}") + return + + if action == "bootstrap": + for name in ("mo", "bart"): + st, d2 = req("POST", "/api/v1/users/signup", body={ + "firstName": name.capitalize(), "lastName": "ATC", + "email": f"{name}@open-metadata.org", + "password": "Dell2026!", + }) + print(f"USER_{name}: {st} {json.dumps(d2)[:140]}") + + +if __name__ == "__main__": + main()