Document lakehouse Debezium connectors and updated docker inventory.

Add atc-lake01 compose with s3-kafka-consumer; redact DB passwords in connector JSON.
2026-06-04 02:27:23 +02:00
parent 8f2bb06348
commit 8eb64381e3
6 changed files with 136 additions and 2 deletions
@@ -0,0 +1,66 @@
+name: lakehouse
+services:
+  trino:
+    image: trinodb/trino:405
+    container_name: trino
+    ports:
+      - "8089:8080"
+    volumes:
+      - /home/lakehouse/trino/catalog:/etc/trino/catalog
+      - /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
+    deploy:
+      resources:
+        limits:
+          memory: 16G
+    environment:
+      - TRINO_USERNAME=mo
+
+  spark-master:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-master
+    ports:
+      - "8081:8080"
+      - "7077:7077"
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+    environment:
+      - SPARK_MODE=master
+      - SPARK_RPC_AUTHENTICATION_ENABLED=no
+
+  spark-worker:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-worker
+    environment:
+      - SPARK_MODE=worker
+      - SPARK_MASTER_URL=spark://spark-master:7077
+      - SPARK_WORKER_MEMORY=10G
+      - SPARK_RPC_AUTHENTICATION_ENABLED=no
+    deploy:
+      resources:
+        limits:
+          memory: 12G
+
+  kafka-connect:
+    image: debezium/connect:2.5.4.Final
+    container_name: kafka-connect
+    ports:
+      - "8083:8083"
+    environment:
+      BOOTSTRAP_SERVERS: 10.0.21.36:9092
+      GROUP_ID: lakehouse-connect
+      CONFIG_STORAGE_TOPIC: connect-configs
+      OFFSET_STORAGE_TOPIC: connect-offsets
+      STATUS_STORAGE_TOPIC: connect-status
+      CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
+      CONNECT_REST_PORT: 8083
+    restart: unless-stopped
+
+  s3-kafka-consumer:
+    image: python:3.11-slim
+    container_name: s3-kafka-consumer
+    restart: unless-stopped
+    volumes:
+      - /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
+    command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]
@@ -0,0 +1,13 @@
+# Debezium connector definitions
+
+Register on **atc-lake01** via `POST http://10.0.21.50:8083/connectors` with JSON body `{ "name": "...", "config": { ... } }`.
+
+Passwords use `${DEBEZIUM_DB_PASSWORD}` — set on the host, not in git.
+
+| Connector | Source DB | Topic prefix |
+|-----------|-----------|--------------|
+| postgres-sales-connector | postgres @ 10.0.21.51 `public.sales_orders` | postgres_sales |
+| mysql-hr-connector | mysql hr.employee_events | mysql_hr |
+| mongodb-supplychain-connector | mongodb supplychain.events | mongodb_supplychain |
+
+Cassandra CDC skipped (requires table CDC + agent).
@@ -0,0 +1,12 @@
+{
+  "name": "mongodb-supplychain-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mongodb.MongoDbConnector",
+    "mongodb.hosts": "rs0/10.0.21.51:27017",
+    "mongodb.user": "",
+    "mongodb.password": "${DEBEZIUM_DB_PASSWORD}",
+    "topic.prefix": "mongodb_supplychain",
+    "database.include.list": "supplychain",
+    "collection.include.list": "supplychain.events"
+  }
+}
@@ -0,0 +1,16 @@
+{
+  "name": "mysql-hr-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "3306",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.server.id": "184054",
+    "topic.prefix": "mysql_hr",
+    "database.include.list": "hr",
+    "table.include.list": "hr.employee_events",
+    "schema.history.internal.kafka.bootstrap.servers": "10.0.21.36:9092",
+    "schema.history.internal.kafka.topic": "schema-changes.hr"
+  }
+}
@@ -0,0 +1,16 @@
+{
+  "name": "postgres-sales-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "5432",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.dbname": "postgres",
+    "topic.prefix": "postgres_sales",
+    "schema.include.list": "public",
+    "table.include.list": "public.sales_orders",
+    "plugin.name": "pgoutput",
+    "publication.name": "dbz_publication"
+  }
+}
@@ -34,12 +34,23 @@ Compose: `config/docker/atc-db02/docker-compose.yml`
 | Container | Image | Ports |
 |-----------|-------|-------|
 | kafka-connect | debezium/connect:2.5.4 | 8083 |
-| spark-master | bitnami/spark | 7077, 8080 |
+| s3-kafka-consumer | python:3.11-slim | (internal) |
+| spark-master | bitnami/spark | 7077, 8081→8080 |
 | spark-worker | bitnami/spark | (internal) |
 | trino | trinodb/trino:405 | 8089→8080 |
 | spark-temp | apache/spark:3.4.0 | — |

-Compose: `config/docker/atc-lake01/docker-compose.yml`
+Debezium connectors: `postgres-sales-connector`, `mysql-hr-connector`, `mongodb-supplychain-connector` (see `config/debezium/connectors/`).
+
+Compose: `compose/atc-lake01/docker-compose.yml`
+
+## atc-airflow01 (10.0.21.55)
+
+| Container | Image | Ports |
+|-----------|-------|-------|
+| airflow | atc-airflow:3.0.6 | 8080 |
+
+DAG `generate_data_all_databases` — data generators for atc-db02. Compose: `/opt/airflow/docker-compose.yml` on host.

 ## atc-kafka01 (10.0.21.36)