Document full application landscape and collect Docker configs

- Add docs/landscape.md and docker-inventory.md with Mermaid architecture - Add config/docker per host (db02, lake01, kafka01) and objectscale/elastic refs - Add live architecture page at icons/docs/architecture.html - Homepage OPS tab: Lakehouse Architecture group with diagram link - Add scripts/collect/collect-fleet-config.sh to refresh configs from fleet
2026-05-19 23:01:47 +02:00
parent e1180e14c9
commit 007f6cedc5
15 changed files with 881 additions and 42 deletions
@@ -0,0 +1,17 @@
+# Docker configurations
+
+Per-host Compose files and READMEs. These mirror (or document) live deployments.
+
+| Host | Path | Live location |
+|------|------|---------------|
+| atc-docker01 | `../compose/`, `../../deploy/` | `/root/lakehouse/compose/` |
+| atc-db02 | `atc-db02/docker-compose.yml` | `/opt/sources/docker-compose.yml` |
+| atc-lake01 | `atc-lake01/docker-compose.yml` | Docker CLI (no single compose file on host) |
+| atc-kafka01 | `atc-kafka01/docker-compose.yml` | Docker CLI |
+| atc-mgt01 | `../../compose/forgejo/` | Docker volumes |
+
+Refresh from fleet:
+
+```bash
+./scripts/collect/collect-fleet-config.sh
+```
@@ -0,0 +1,65 @@
+version: "3.9"
+
+services:
+
+  postgres-sales:
+    image: postgres:15
+    container_name: postgres_sales
+    environment:
+      POSTGRES_DB: sales
+      POSTGRES_USER: sales_user
+      POSTGRES_PASSWORD: sales_pwd
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_sales_data:/var/lib/postgresql/data
+
+  mysql-hr:
+    image: mysql:8.0
+    container_name: mysql_hr
+    environment:
+      MYSQL_DATABASE: hr
+      MYSQL_USER: hr_user
+      MYSQL_PASSWORD: hr_pwd
+      MYSQL_ROOT_PASSWORD: root_pwd
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql_hr_data:/var/lib/mysql
+
+  mongodb-supplychain:
+    image: mongo:4.4
+    container_name: mongodb_supplychain
+    command: ["mongod", "--auth"]
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongodb_supplychain_data:/data/db
+
+  cassandra-telemetry:
+    image: cassandra:4.1
+    container_name: cassandra-telemetry
+    ports:
+      - "9042:9042"
+    volumes:
+      - cassandra_telemetry_data:/var/lib/cassandra
+    environment:
+      CASSANDRA_CLUSTER_NAME: TelemetryCluster
+
+  neo4j-graph:
+    image: neo4j:4.4
+    container_name: neo4j_graph
+    environment:
+      NEO4J_AUTH: neo4j/testpwd
+    ports:
+      - "7474:7474"
+      - "7687:7687"
+    volumes:
+      - neo4j_graph_data:/data
+
+volumes:
+  postgres_sales_data:
+  mysql_hr_data:
+  mongodb_supplychain_data:
+  cassandra_telemetry_data:
+  neo4j_graph_data:
@@ -0,0 +1,7 @@
+# atc-kafka01
+
+- **Kafka broker:** `10.0.21.36:9092` (native install under `/opt/kafka`)
+- **Kafka UI:** Docker — port `9000`
+- **Kafka Connect plugins:** `/opt/kafka-connect` on host
+
+Debezium Connect runs on **atc-lake01:8083**, not on this host.
@@ -0,0 +1,14 @@
+# Kafka UI — atc-kafka01 (10.0.21.36)
+# Kafka broker runs natively on :9092 (see /opt/kafka on host).
+version: "3.9"
+
+services:
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: kafka-ui
+    restart: unless-stopped
+    ports:
+      - "9000:8080"
+    environment:
+      KAFKA_CLUSTERS_0_NAME: atc-kafka
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: localhost:9092
@@ -0,0 +1,53 @@
+# Lakehouse compute — atc-lake01 (10.0.21.50)
+# Reconstructed from running containers; use docker network for spark worker DNS.
+version: "3.9"
+
+services:
+  kafka-connect:
+    image: debezium/connect:2.5.4.Final
+    container_name: kafka-connect
+    restart: unless-stopped
+    ports:
+      - "8083:8083"
+    environment:
+      BOOTSTRAP_SERVERS: 10.0.21.36:9092
+      GROUP_ID: "1"
+      CONFIG_STORAGE_TOPIC: connect-configs
+      OFFSET_STORAGE_TOPIC: connect-offsets
+      STATUS_STORAGE_TOPIC: connect-status
+
+  spark-master:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-master
+    restart: unless-stopped
+    environment:
+      SPARK_MODE: master
+      SPARK_RPC_AUTHENTICATION_ENABLED: "no"
+    ports:
+      - "7077:7077"
+      - "8080:8080"
+
+  spark-worker:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-worker
+    restart: unless-stopped
+    depends_on:
+      - spark-master
+    environment:
+      SPARK_MODE: worker
+      SPARK_MASTER_URL: spark://spark-master:7077
+      SPARK_WORKER_MEMORY: 10G
+      SPARK_RPC_AUTHENTICATION_ENABLED: "no"
+
+  trino:
+    image: trinodb/trino:405
+    container_name: trino
+    restart: unless-stopped
+    ports:
+      - "8089:8080"
+
+  # Ad-hoc Spark shell image (optional)
+  spark-temp:
+    image: apache/spark:3.4.0-scala2.12-java11-python3-ubuntu
+    container_name: spark-temp
+    profiles: ["tools"]