diff --git a/compose/atc-lake01/docker-compose.yml b/compose/atc-lake01/docker-compose.yml new file mode 100644 index 0000000..4060121 --- /dev/null +++ b/compose/atc-lake01/docker-compose.yml @@ -0,0 +1,66 @@ +name: lakehouse +services: + trino: + image: trinodb/trino:405 + container_name: trino + ports: + - "8089:8080" + volumes: + - /home/lakehouse/trino/catalog:/etc/trino/catalog + - /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties + deploy: + resources: + limits: + memory: 16G + environment: + - TRINO_USERNAME=mo + + spark-master: + image: bitnamilegacy/spark:latest + container_name: spark-master + ports: + - "8081:8080" + - "7077:7077" + deploy: + resources: + limits: + memory: 2G + environment: + - SPARK_MODE=master + - SPARK_RPC_AUTHENTICATION_ENABLED=no + + spark-worker: + image: bitnamilegacy/spark:latest + container_name: spark-worker + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + - SPARK_WORKER_MEMORY=10G + - SPARK_RPC_AUTHENTICATION_ENABLED=no + deploy: + resources: + limits: + memory: 12G + + kafka-connect: + image: debezium/connect:2.5.4.Final + container_name: kafka-connect + ports: + - "8083:8083" + environment: + BOOTSTRAP_SERVERS: 10.0.21.36:9092 + GROUP_ID: lakehouse-connect + CONFIG_STORAGE_TOPIC: connect-configs + OFFSET_STORAGE_TOPIC: connect-offsets + STATUS_STORAGE_TOPIC: connect-status + CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50 + CONNECT_REST_PORT: 8083 + restart: unless-stopped + + s3-kafka-consumer: + image: python:3.11-slim + container_name: s3-kafka-consumer + restart: unless-stopped + volumes: + - /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro + command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"] diff --git a/config/debezium/README.md b/config/debezium/README.md new file mode 100644 index 0000000..cc440b8 --- /dev/null +++ b/config/debezium/README.md @@ -0,0 +1,13 @@ +# Debezium connector definitions + +Register on **atc-lake01** via `POST http://10.0.21.50:8083/connectors` with JSON body `{ "name": "...", "config": { ... } }`. + +Passwords use `${DEBEZIUM_DB_PASSWORD}` — set on the host, not in git. + +| Connector | Source DB | Topic prefix | +|-----------|-----------|--------------| +| postgres-sales-connector | postgres @ 10.0.21.51 `public.sales_orders` | postgres_sales | +| mysql-hr-connector | mysql hr.employee_events | mysql_hr | +| mongodb-supplychain-connector | mongodb supplychain.events | mongodb_supplychain | + +Cassandra CDC skipped (requires table CDC + agent). diff --git a/config/debezium/connectors/mongodb-supplychain-connector.json b/config/debezium/connectors/mongodb-supplychain-connector.json new file mode 100644 index 0000000..110c8b7 --- /dev/null +++ b/config/debezium/connectors/mongodb-supplychain-connector.json @@ -0,0 +1,12 @@ +{ + "name": "mongodb-supplychain-connector", + "config": { + "connector.class": "io.debezium.connector.mongodb.MongoDbConnector", + "mongodb.hosts": "rs0/10.0.21.51:27017", + "mongodb.user": "", + "mongodb.password": "${DEBEZIUM_DB_PASSWORD}", + "topic.prefix": "mongodb_supplychain", + "database.include.list": "supplychain", + "collection.include.list": "supplychain.events" + } +} diff --git a/config/debezium/connectors/mysql-hr-connector.json b/config/debezium/connectors/mysql-hr-connector.json new file mode 100644 index 0000000..02a0520 --- /dev/null +++ b/config/debezium/connectors/mysql-hr-connector.json @@ -0,0 +1,16 @@ +{ + "name": "mysql-hr-connector", + "config": { + "connector.class": "io.debezium.connector.mysql.MySqlConnector", + "database.hostname": "10.0.21.51", + "database.port": "3306", + "database.user": "mo", + "database.password": "${DEBEZIUM_DB_PASSWORD}", + "database.server.id": "184054", + "topic.prefix": "mysql_hr", + "database.include.list": "hr", + "table.include.list": "hr.employee_events", + "schema.history.internal.kafka.bootstrap.servers": "10.0.21.36:9092", + "schema.history.internal.kafka.topic": "schema-changes.hr" + } +} diff --git a/config/debezium/connectors/postgres-sales-connector.json b/config/debezium/connectors/postgres-sales-connector.json new file mode 100644 index 0000000..638a5bb --- /dev/null +++ b/config/debezium/connectors/postgres-sales-connector.json @@ -0,0 +1,16 @@ +{ + "name": "postgres-sales-connector", + "config": { + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "database.hostname": "10.0.21.51", + "database.port": "5432", + "database.user": "mo", + "database.password": "${DEBEZIUM_DB_PASSWORD}", + "database.dbname": "postgres", + "topic.prefix": "postgres_sales", + "schema.include.list": "public", + "table.include.list": "public.sales_orders", + "plugin.name": "pgoutput", + "publication.name": "dbz_publication" + } +} diff --git a/docs/docker-inventory.md b/docs/docker-inventory.md index e7abedc..e6d9471 100644 --- a/docs/docker-inventory.md +++ b/docs/docker-inventory.md @@ -34,12 +34,23 @@ Compose: `config/docker/atc-db02/docker-compose.yml` | Container | Image | Ports | |-----------|-------|-------| | kafka-connect | debezium/connect:2.5.4 | 8083 | -| spark-master | bitnami/spark | 7077, 8080 | +| s3-kafka-consumer | python:3.11-slim | (internal) | +| spark-master | bitnami/spark | 7077, 8081→8080 | | spark-worker | bitnami/spark | (internal) | | trino | trinodb/trino:405 | 8089→8080 | | spark-temp | apache/spark:3.4.0 | — | -Compose: `config/docker/atc-lake01/docker-compose.yml` +Debezium connectors: `postgres-sales-connector`, `mysql-hr-connector`, `mongodb-supplychain-connector` (see `config/debezium/connectors/`). + +Compose: `compose/atc-lake01/docker-compose.yml` + +## atc-airflow01 (10.0.21.55) + +| Container | Image | Ports | +|-----------|-------|-------| +| airflow | atc-airflow:3.0.6 | 8080 | + +DAG `generate_data_all_databases` — data generators for atc-db02. Compose: `/opt/airflow/docker-compose.yml` on host. ## atc-kafka01 (10.0.21.36)