Document lakehouse Debezium connectors and updated docker inventory.
Add atc-lake01 compose with s3-kafka-consumer; redact DB passwords in connector JSON.
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
name: lakehouse
|
||||
services:
|
||||
trino:
|
||||
image: trinodb/trino:405
|
||||
container_name: trino
|
||||
ports:
|
||||
- "8089:8080"
|
||||
volumes:
|
||||
- /home/lakehouse/trino/catalog:/etc/trino/catalog
|
||||
- /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 16G
|
||||
environment:
|
||||
- TRINO_USERNAME=mo
|
||||
|
||||
spark-master:
|
||||
image: bitnamilegacy/spark:latest
|
||||
container_name: spark-master
|
||||
ports:
|
||||
- "8081:8080"
|
||||
- "7077:7077"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
environment:
|
||||
- SPARK_MODE=master
|
||||
- SPARK_RPC_AUTHENTICATION_ENABLED=no
|
||||
|
||||
spark-worker:
|
||||
image: bitnamilegacy/spark:latest
|
||||
container_name: spark-worker
|
||||
environment:
|
||||
- SPARK_MODE=worker
|
||||
- SPARK_MASTER_URL=spark://spark-master:7077
|
||||
- SPARK_WORKER_MEMORY=10G
|
||||
- SPARK_RPC_AUTHENTICATION_ENABLED=no
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 12G
|
||||
|
||||
kafka-connect:
|
||||
image: debezium/connect:2.5.4.Final
|
||||
container_name: kafka-connect
|
||||
ports:
|
||||
- "8083:8083"
|
||||
environment:
|
||||
BOOTSTRAP_SERVERS: 10.0.21.36:9092
|
||||
GROUP_ID: lakehouse-connect
|
||||
CONFIG_STORAGE_TOPIC: connect-configs
|
||||
OFFSET_STORAGE_TOPIC: connect-offsets
|
||||
STATUS_STORAGE_TOPIC: connect-status
|
||||
CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
|
||||
CONNECT_REST_PORT: 8083
|
||||
restart: unless-stopped
|
||||
|
||||
s3-kafka-consumer:
|
||||
image: python:3.11-slim
|
||||
container_name: s3-kafka-consumer
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
|
||||
command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]
|
||||
Reference in New Issue
Block a user