From b7dab19590eaf5ea0d6fa4d8cd3e9a69b8762997 Mon Sep 17 00:00:00 2001
From: mo <mo@dell-atc.lan>
Date: Sat, 27 Jun 2026 12:25:51 +0200
Subject: [PATCH] feat(openmetadata): ingest full stack - Trino all catalogs +
 Kafka, S3, Superset, ES

Widen Trino ingestion to every catalog (postgres_sales/mysql_hr/mongodb/iceberg)
and add messaging (Kafka), storage (S3 object store), dashboards (Superset) and
search (Elasticsearch) service ingestion configs so OpenMetadata reflects the
whole running platform. Secrets are injected at run time, not committed.
---
 deploy/openmetadata/ingest/es_meta.yaml       | 26 +++++++++++++++++++
 deploy/openmetadata/ingest/kafka_meta.yaml    | 24 +++++++++++++++++
 deploy/openmetadata/ingest/s3_meta.yaml       | 23 ++++++++++++++++
 deploy/openmetadata/ingest/superset_meta.yaml | 23 ++++++++++++++++
 deploy/openmetadata/ingest/trino_meta.yaml    |  9 ++++++-
 5 files changed, 104 insertions(+), 1 deletion(-)
 create mode 100644 deploy/openmetadata/ingest/es_meta.yaml
 create mode 100644 deploy/openmetadata/ingest/kafka_meta.yaml
 create mode 100644 deploy/openmetadata/ingest/s3_meta.yaml
 create mode 100644 deploy/openmetadata/ingest/superset_meta.yaml

diff --git a/deploy/openmetadata/ingest/es_meta.yaml b/deploy/openmetadata/ingest/es_meta.yaml
new file mode 100644
index 0000000..544cb86
--- /dev/null
+++ b/deploy/openmetadata/ingest/es_meta.yaml
@@ -0,0 +1,26 @@
+source:
+  type: elasticsearch
+  serviceName: atc_elasticsearch
+  serviceConnection:
+    config:
+      type: ElasticSearch
+      hostPort: https://10.0.21.46:9200
+      authType:
+        username: elastic
+        password: __ES_PASS__
+      connectionArguments:
+        verify_certs: "false"
+        ssl_show_warn: "false"
+  sourceConfig:
+    config:
+      type: SearchMetadata
+      includeSampleData: false
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"
diff --git a/deploy/openmetadata/ingest/kafka_meta.yaml b/deploy/openmetadata/ingest/kafka_meta.yaml
new file mode 100644
index 0000000..6774d1b
--- /dev/null
+++ b/deploy/openmetadata/ingest/kafka_meta.yaml
@@ -0,0 +1,24 @@
+source:
+  type: kafka
+  serviceName: atc_kafka
+  serviceConnection:
+    config:
+      type: Kafka
+      bootstrapServers: 10.0.21.36:9092
+  sourceConfig:
+    config:
+      type: MessagingMetadata
+      generateSampleData: false
+      topicFilterPattern:
+        excludes:
+          - "^__.*"
+          - "^_confluent.*"
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"
diff --git a/deploy/openmetadata/ingest/s3_meta.yaml b/deploy/openmetadata/ingest/s3_meta.yaml
new file mode 100644
index 0000000..3646a07
--- /dev/null
+++ b/deploy/openmetadata/ingest/s3_meta.yaml
@@ -0,0 +1,23 @@
+source:
+  type: s3
+  serviceName: atc_s3
+  serviceConnection:
+    config:
+      type: S3
+      awsConfig:
+        awsAccessKeyId: __S3_KEY__
+        awsSecretAccessKey: __S3_SECRET__
+        awsRegion: us-east-1
+        endPointURL: http://10.0.20.111:9020
+  sourceConfig:
+    config:
+      type: StorageMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"
diff --git a/deploy/openmetadata/ingest/superset_meta.yaml b/deploy/openmetadata/ingest/superset_meta.yaml
new file mode 100644
index 0000000..3c6ab1a
--- /dev/null
+++ b/deploy/openmetadata/ingest/superset_meta.yaml
@@ -0,0 +1,23 @@
+source:
+  type: superset
+  serviceName: atc_superset
+  serviceConnection:
+    config:
+      type: Superset
+      hostPort: http://10.0.21.45:8088
+      connection:
+        provider: db
+        username: mo
+        password: Dell2026!
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://openmetadata-server:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "__JWT__"
diff --git a/deploy/openmetadata/ingest/trino_meta.yaml b/deploy/openmetadata/ingest/trino_meta.yaml
index b1a2889..bac6b6b 100644
--- a/deploy/openmetadata/ingest/trino_meta.yaml
+++ b/deploy/openmetadata/ingest/trino_meta.yaml
@@ -6,10 +6,17 @@ source:
       type: Trino
       username: mo
       hostPort: 10.0.21.50:8089
-      catalog: iceberg
   sourceConfig:
     config:
       type: DatabaseMetadata
+      markDeletedTables: true
+      databaseFilterPattern:
+        excludes:
+          - system
+          - jmx
+          - tpch
+          - tpcds
+          - memory
 sink:
   type: metadata-rest
   config: {}