feat(openmetadata): ingest full stack - Trino all catalogs + Kafka, S3, Superset, ES

Widen Trino ingestion to every catalog (postgres_sales/mysql_hr/mongodb/iceberg)
and add messaging (Kafka), storage (S3 object store), dashboards (Superset) and
search (Elasticsearch) service ingestion configs so OpenMetadata reflects the
whole running platform. Secrets are injected at run time, not committed.

deploy/openmetadata/ingest/es_meta.yaml

@@ -0,0 +1,26 @@
+source:
+  type: elasticsearch
+  serviceName: atc_elasticsearch
+  serviceConnection:
+    config:
+      type: ElasticSearch
+      hostPort: https://10.0.21.46:9200
+      authType:
+        username: elastic
+        password: __ES_PASS__
+      connectionArguments:
+        verify_certs: "false"
+        ssl_show_warn: "false"
+  sourceConfig:
+    config:
+      type: SearchMetadata
+      includeSampleData: false
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/kafka_meta.yaml

@@ -0,0 +1,24 @@
+source:
+  type: kafka
+  serviceName: atc_kafka
+  serviceConnection:
+    config:
+      type: Kafka
+      bootstrapServers: 10.0.21.36:9092
+  sourceConfig:
+    config:
+      type: MessagingMetadata
+      generateSampleData: false
+      topicFilterPattern:
+        excludes:
+          - "^__.*"
+          - "^_confluent.*"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/s3_meta.yaml

@@ -0,0 +1,23 @@
+source:
+  type: s3
+  serviceName: atc_s3
+  serviceConnection:
+    config:
+      type: S3
+      awsConfig:
+        awsAccessKeyId: __S3_KEY__
+        awsSecretAccessKey: __S3_SECRET__
+        awsRegion: us-east-1
+        endPointURL: http://10.0.20.111:9020
+  sourceConfig:
+    config:
+      type: StorageMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/superset_meta.yaml

@@ -0,0 +1,23 @@
+source:
+  type: superset
+  serviceName: atc_superset
+  serviceConnection:
+    config:
+      type: Superset
+      hostPort: http://10.0.21.45:8088
+      connection:
+        provider: db
+        username: mo
+        password: Dell2026!
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"

deploy/openmetadata/ingest/trino_meta.yaml

@@ -6,10 +6,17 @@ source:
       type: Trino
       username: mo
       hostPort: 10.0.21.50:8089
-      catalog: iceberg
   sourceConfig:
     config:
       type: DatabaseMetadata
+      markDeletedTables: true
+      databaseFilterPattern:
+        excludes:
+          - system
+          - jmx
+          - tpch
+          - tpcds
+          - memory
 sink:
   type: metadata-rest
   config: {}