67 lines
1.7 KiB
YAML
67 lines
1.7 KiB
YAML
|
name: lakehouse
|
||
|
|
services:
|
||
|
|
trino:
|
||
|
|
image: trinodb/trino:405
|
||
|
|
container_name: trino
|
||
|
|
ports:
|
||
|
|
- "8089:8080"
|
||
|
|
volumes:
|
||
|
|
- /home/lakehouse/trino/catalog:/etc/trino/catalog
|
||
|
|
- /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
limits:
|
||
|
|
memory: 16G
|
||
|
|
environment:
|
||
|
|
- TRINO_USERNAME=mo
|
||
|
|
|
||
|
|
spark-master:
|
||
|
|
image: bitnamilegacy/spark:latest
|
||
|
|
container_name: spark-master
|
||
|
|
ports:
|
||
|
|
- "8081:8080"
|
||
|
|
- "7077:7077"
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
limits:
|
||
|
|
memory: 2G
|
||
|
|
environment:
|
||
|
|
- SPARK_MODE=master
|
||
|
|
- SPARK_RPC_AUTHENTICATION_ENABLED=no
|
||
|
|
|
||
|
|
spark-worker:
|
||
|
|
image: bitnamilegacy/spark:latest
|
||
|
|
container_name: spark-worker
|
||
|
|
environment:
|
||
|
|
- SPARK_MODE=worker
|
||
|
|
- SPARK_MASTER_URL=spark://spark-master:7077
|
||
|
|
- SPARK_WORKER_MEMORY=10G
|
||
|
|
- SPARK_RPC_AUTHENTICATION_ENABLED=no
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
limits:
|
||
|
|
memory: 12G
|
||
|
|
|
||
|
|
kafka-connect:
|
||
|
|
image: debezium/connect:2.5.4.Final
|
||
|
|
container_name: kafka-connect
|
||
|
|
ports:
|
||
|
|
- "8083:8083"
|
||
|
|
environment:
|
||
|
|
BOOTSTRAP_SERVERS: 10.0.21.36:9092
|
||
|
|
GROUP_ID: lakehouse-connect
|
||
|
|
CONFIG_STORAGE_TOPIC: connect-configs
|
||
|
|
OFFSET_STORAGE_TOPIC: connect-offsets
|
||
|
|
STATUS_STORAGE_TOPIC: connect-status
|
||
|
|
CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
|
||
|
|
CONNECT_REST_PORT: 8083
|
||
|
|
restart: unless-stopped
|
||
|
|
|
||
|
|
s3-kafka-consumer:
|
||
|
|
image: python:3.11-slim
|
||
|
|
container_name: s3-kafka-consumer
|
||
|
|
restart: unless-stopped
|
||
|
|
volumes:
|
||
|
|
- /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
|
||
|
|
command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]
|