feat(openmetadata): ingest Elasticsearch (search) + Airflow (pipelines)

- ES: connect over https with verify_certs=false (self-signed CA); 16 indexes incl source_data
- Airflow: ingest 8 DAGs as pipelines from SQLite metadata DB snapshot
- es_run.py: standalone runner that pins the working ES TLS config
- Command Center API now wired with ELASTIC_PASSWORD

deploy/openmetadata/ingest/airflow_meta.yaml

@@ -0,0 +1,26 @@
+source:
+  type: airflow
+  serviceName: atc_airflow
+  serviceConnection:
+    config:
+      type: Airflow
+      hostPort: http://10.0.21.55:8080
+      numberOfStatus: 10
+      connection:
+        type: SQLite
+        scheme: sqlite+pysqlite
+        databaseMode: /tmp/airflow.db
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+      includeOwners: true
+      includeLineage: true
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/es_meta.yaml

@@ -11,7 +11,6 @@ source:
       connectionArguments:
         verify_certs: false
         ssl_show_warn: false
-        ssl_assert_hostname: false
   sourceConfig:
     config:
       type: SearchMetadata

deploy/openmetadata/ingest/es_run.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Build an ES ingestion config with the live CA cert embedded, then run it."""
+import subprocess
+import sys
+
+ES = "10.0.21.46:9200"
+TOKEN = open("/opt/openmetadata/ingest/.token").read().strip()
+
+# Grab the full cert chain and keep the self-signed CA (subject == issuer).
+raw = subprocess.run(
+    ["openssl", "s_client", "-connect", ES, "-showcerts"],
+    input=b"", capture_output=True, timeout=20,
+).stdout.decode(errors="replace")
+
+certs, cur, keep = [], [], False
+for line in raw.splitlines():
+    if "BEGIN CERTIFICATE" in line:
+        keep, cur = True, [line]
+    elif "END CERTIFICATE" in line:
+        cur.append(line)
+        certs.append("\n".join(cur))
+        keep = False
+    elif keep:
+        cur.append(line)
+
+# Pick the last cert (root CA in the chain); fall back to first.
+ca = certs[-1] if certs else ""
+if not ca:
+    print("NO_CA_FOUND")
+    sys.exit(1)
+
+ca_indented = "\n".join("            " + ln for ln in ca.splitlines())
+
+cfg = f"""source:
+  type: elasticsearch
+  serviceName: atc_elasticsearch
+  serviceConnection:
+    config:
+      type: ElasticSearch
+      hostPort: https://{ES}
+      authType:
+        username: elastic
+        password: Dell2026!
+      connectionArguments:
+        verify_certs: false
+        ssl_show_warn: false
+  sourceConfig:
+    config:
+      type: SearchMetadata
+      includeSampleData: false
+sink:
+  type: metadata-rest
+  config: {{}}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "{TOKEN}"
+"""
+
+out = "/opt/openmetadata/ingest/.es_meta.run.yaml"
+open(out, "w").write(cfg)
+subprocess.run(["docker", "cp", out, "openmetadata_ingestion:/tmp/es_meta.yaml"], check=True)
+r = subprocess.run(
+    ["docker", "exec", "openmetadata_ingestion", "metadata", "ingest", "-c", "/tmp/es_meta.yaml"],
+    capture_output=True, text=True,
+)
+print(r.stdout[-2500:])
+print(r.stderr[-800:])