Document lakehouse Debezium connectors and updated docker inventory.

Add atc-lake01 compose with s3-kafka-consumer; redact DB passwords in connector JSON.
This commit is contained in:
mo
2026-06-04 02:27:23 +02:00
parent 8f2bb06348
commit 8eb64381e3
6 changed files with 136 additions and 2 deletions
+66
View File
@@ -0,0 +1,66 @@
name: lakehouse
services:
trino:
image: trinodb/trino:405
container_name: trino
ports:
- "8089:8080"
volumes:
- /home/lakehouse/trino/catalog:/etc/trino/catalog
- /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
deploy:
resources:
limits:
memory: 16G
environment:
- TRINO_USERNAME=mo
spark-master:
image: bitnamilegacy/spark:latest
container_name: spark-master
ports:
- "8081:8080"
- "7077:7077"
deploy:
resources:
limits:
memory: 2G
environment:
- SPARK_MODE=master
- SPARK_RPC_AUTHENTICATION_ENABLED=no
spark-worker:
image: bitnamilegacy/spark:latest
container_name: spark-worker
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark-master:7077
- SPARK_WORKER_MEMORY=10G
- SPARK_RPC_AUTHENTICATION_ENABLED=no
deploy:
resources:
limits:
memory: 12G
kafka-connect:
image: debezium/connect:2.5.4.Final
container_name: kafka-connect
ports:
- "8083:8083"
environment:
BOOTSTRAP_SERVERS: 10.0.21.36:9092
GROUP_ID: lakehouse-connect
CONFIG_STORAGE_TOPIC: connect-configs
OFFSET_STORAGE_TOPIC: connect-offsets
STATUS_STORAGE_TOPIC: connect-status
CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
CONNECT_REST_PORT: 8083
restart: unless-stopped
s3-kafka-consumer:
image: python:3.11-slim
container_name: s3-kafka-consumer
restart: unless-stopped
volumes:
- /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]