name: lakehouse
services:
  trino:
    image: trinodb/trino:405
    container_name: trino
    ports:
      - "8089:8080"
    volumes:
      - /home/lakehouse/trino/catalog:/etc/trino/catalog
      - /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
    deploy:
      resources:
        limits:
          memory: 16G
    environment:
      - TRINO_USERNAME=mo

  spark-master:
    image: bitnamilegacy/spark:latest
    container_name: spark-master
    ports:
      - "8081:8080"
      - "7077:7077"
    deploy:
      resources:
        limits:
          memory: 2G
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no

  spark-worker:
    image: bitnamilegacy/spark:latest
    container_name: spark-worker
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=10G
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
    deploy:
      resources:
        limits:
          memory: 12G

  kafka-connect:
    image: debezium/connect:2.5.4.Final
    container_name: kafka-connect
    ports:
      - "8083:8083"
    environment:
      BOOTSTRAP_SERVERS: 10.0.21.36:9092
      GROUP_ID: lakehouse-connect
      CONFIG_STORAGE_TOPIC: connect-configs
      OFFSET_STORAGE_TOPIC: connect-offsets
      STATUS_STORAGE_TOPIC: connect-status
      CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
      CONNECT_REST_PORT: 8083
    restart: unless-stopped

  s3-kafka-consumer:
    image: python:3.11-slim
    container_name: s3-kafka-consumer
    restart: unless-stopped
    volumes:
      - /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
    command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]