diff --git a/deploy/openmetadata/ingest/airflow_meta.yaml b/deploy/openmetadata/ingest/airflow_meta.yaml new file mode 100644 index 0000000..90a2eab --- /dev/null +++ b/deploy/openmetadata/ingest/airflow_meta.yaml @@ -0,0 +1,26 @@ +source: + type: airflow + serviceName: atc_airflow + serviceConnection: + config: + type: Airflow + hostPort: http://10.0.21.55:8080 + numberOfStatus: 10 + connection: + type: SQLite + scheme: sqlite+pysqlite + databaseMode: /tmp/airflow.db + sourceConfig: + config: + type: PipelineMetadata + includeOwners: true + includeLineage: true +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "__JWT__" diff --git a/deploy/openmetadata/ingest/es_meta.yaml b/deploy/openmetadata/ingest/es_meta.yaml index 5cb20f0..8e20b56 100644 --- a/deploy/openmetadata/ingest/es_meta.yaml +++ b/deploy/openmetadata/ingest/es_meta.yaml @@ -11,7 +11,6 @@ source: connectionArguments: verify_certs: false ssl_show_warn: false - ssl_assert_hostname: false sourceConfig: config: type: SearchMetadata diff --git a/deploy/openmetadata/ingest/es_run.py b/deploy/openmetadata/ingest/es_run.py new file mode 100644 index 0000000..60ae29d --- /dev/null +++ b/deploy/openmetadata/ingest/es_run.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Build an ES ingestion config with the live CA cert embedded, then run it.""" +import subprocess +import sys + +ES = "10.0.21.46:9200" +TOKEN = open("/opt/openmetadata/ingest/.token").read().strip() + +# Grab the full cert chain and keep the self-signed CA (subject == issuer). +raw = subprocess.run( + ["openssl", "s_client", "-connect", ES, "-showcerts"], + input=b"", capture_output=True, timeout=20, +).stdout.decode(errors="replace") + +certs, cur, keep = [], [], False +for line in raw.splitlines(): + if "BEGIN CERTIFICATE" in line: + keep, cur = True, [line] + elif "END CERTIFICATE" in line: + cur.append(line) + certs.append("\n".join(cur)) + keep = False + elif keep: + cur.append(line) + +# Pick the last cert (root CA in the chain); fall back to first. +ca = certs[-1] if certs else "" +if not ca: + print("NO_CA_FOUND") + sys.exit(1) + +ca_indented = "\n".join(" " + ln for ln in ca.splitlines()) + +cfg = f"""source: + type: elasticsearch + serviceName: atc_elasticsearch + serviceConnection: + config: + type: ElasticSearch + hostPort: https://{ES} + authType: + username: elastic + password: Dell2026! + connectionArguments: + verify_certs: false + ssl_show_warn: false + sourceConfig: + config: + type: SearchMetadata + includeSampleData: false +sink: + type: metadata-rest + config: {{}} +workflowConfig: + openMetadataServerConfig: + hostPort: http://openmetadata-server:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "{TOKEN}" +""" + +out = "/opt/openmetadata/ingest/.es_meta.run.yaml" +open(out, "w").write(cfg) +subprocess.run(["docker", "cp", out, "openmetadata_ingestion:/tmp/es_meta.yaml"], check=True) +r = subprocess.run( + ["docker", "exec", "openmetadata_ingestion", "metadata", "ingest", "-c", "/tmp/es_meta.yaml"], + capture_output=True, text=True, +) +print(r.stdout[-2500:]) +print(r.stderr[-800:])