Document lakehouse Debezium connectors and updated docker inventory.

Add atc-lake01 compose with s3-kafka-consumer; redact DB passwords in connector JSON.
2026-06-04 02:27:23 +02:00
parent 8f2bb06348
commit 8eb64381e3
6 changed files with 136 additions and 2 deletions
@@ -0,0 +1,13 @@
+# Debezium connector definitions
+
+Register on **atc-lake01** via `POST http://10.0.21.50:8083/connectors` with JSON body `{ "name": "...", "config": { ... } }`.
+
+Passwords use `${DEBEZIUM_DB_PASSWORD}` — set on the host, not in git.
+
+| Connector | Source DB | Topic prefix |
+|-----------|-----------|--------------|
+| postgres-sales-connector | postgres @ 10.0.21.51 `public.sales_orders` | postgres_sales |
+| mysql-hr-connector | mysql hr.employee_events | mysql_hr |
+| mongodb-supplychain-connector | mongodb supplychain.events | mongodb_supplychain |
+
+Cassandra CDC skipped (requires table CDC + agent).
@@ -0,0 +1,12 @@
+{
+  "name": "mongodb-supplychain-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mongodb.MongoDbConnector",
+    "mongodb.hosts": "rs0/10.0.21.51:27017",
+    "mongodb.user": "",
+    "mongodb.password": "${DEBEZIUM_DB_PASSWORD}",
+    "topic.prefix": "mongodb_supplychain",
+    "database.include.list": "supplychain",
+    "collection.include.list": "supplychain.events"
+  }
+}
@@ -0,0 +1,16 @@
+{
+  "name": "mysql-hr-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "3306",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.server.id": "184054",
+    "topic.prefix": "mysql_hr",
+    "database.include.list": "hr",
+    "table.include.list": "hr.employee_events",
+    "schema.history.internal.kafka.bootstrap.servers": "10.0.21.36:9092",
+    "schema.history.internal.kafka.topic": "schema-changes.hr"
+  }
+}
@@ -0,0 +1,16 @@
+{
+  "name": "postgres-sales-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "5432",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.dbname": "postgres",
+    "topic.prefix": "postgres_sales",
+    "schema.include.list": "public",
+    "table.include.list": "public.sales_orders",
+    "plugin.name": "pgoutput",
+    "publication.name": "dbz_publication"
+  }
+}