54 lines
1.4 KiB
YAML
54 lines
1.4 KiB
YAML
|
# Lakehouse compute — atc-lake01 (10.0.21.50)
|
||
|
|
# Reconstructed from running containers; use docker network for spark worker DNS.
|
||
|
|
version: "3.9"
|
||
|
|
|
||
|
|
services:
|
||
|
|
kafka-connect:
|
||
|
|
image: debezium/connect:2.5.4.Final
|
||
|
|
container_name: kafka-connect
|
||
|
|
restart: unless-stopped
|
||
|
|
ports:
|
||
|
|
- "8083:8083"
|
||
|
|
environment:
|
||
|
|
BOOTSTRAP_SERVERS: 10.0.21.36:9092
|
||
|
|
GROUP_ID: "1"
|
||
|
|
CONFIG_STORAGE_TOPIC: connect-configs
|
||
|
|
OFFSET_STORAGE_TOPIC: connect-offsets
|
||
|
|
STATUS_STORAGE_TOPIC: connect-status
|
||
|
|
|
||
|
|
spark-master:
|
||
|
|
image: bitnamilegacy/spark:latest
|
||
|
|
container_name: spark-master
|
||
|
|
restart: unless-stopped
|
||
|
|
environment:
|
||
|
|
SPARK_MODE: master
|
||
|
|
SPARK_RPC_AUTHENTICATION_ENABLED: "no"
|
||
|
|
ports:
|
||
|
|
- "7077:7077"
|
||
|
|
- "8080:8080"
|
||
|
|
|
||
|
|
spark-worker:
|
||
|
|
image: bitnamilegacy/spark:latest
|
||
|
|
container_name: spark-worker
|
||
|
|
restart: unless-stopped
|
||
|
|
depends_on:
|
||
|
|
- spark-master
|
||
|
|
environment:
|
||
|
|
SPARK_MODE: worker
|
||
|
|
SPARK_MASTER_URL: spark://spark-master:7077
|
||
|
|
SPARK_WORKER_MEMORY: 10G
|
||
|
|
SPARK_RPC_AUTHENTICATION_ENABLED: "no"
|
||
|
|
|
||
|
|
trino:
|
||
|
|
image: trinodb/trino:405
|
||
|
|
container_name: trino
|
||
|
|
restart: unless-stopped
|
||
|
|
ports:
|
||
|
|
- "8089:8080"
|
||
|
|
|
||
|
|
# Ad-hoc Spark shell image (optional)
|
||
|
|
spark-temp:
|
||
|
|
image: apache/spark:3.4.0-scala2.12-java11-python3-ubuntu
|
||
|
|
container_name: spark-temp
|
||
|
|
profiles: ["tools"]
|