diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e5fe2fc --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Kopieer naar .env op atc-docker01: cp .env.example .env + +# Superset (compose/superset) +SUPERSET_SECRET_KEY=change-me-to-a-long-random-string + +# Homepage (optioneel) +HOMEPAGE_ALLOWED_HOSTS=atc-docker01.dell-atc.lan,localhost + +# Proxmox API — ook in config/homepage/proxmox.yaml +# PROXMOX_URL=https://10.0.10.65:8006 +# PROXMOX_TOKEN=root@pam!homepage +# PROXMOX_SECRET= + +# Kafka connectors — vervang in config/kafka/*.json bij deploy +# DATABASE_PASSWORD= diff --git a/.gitignore b/.gitignore index c324aef..ae68cc5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,27 @@ -# Runtime logs (not config) +# Runtime logs config/homepage/logs/ *.log +# Secrets (lokaal op host, niet committen indien je .env gebruikt) +.env +.env.local +.env.*.local + # OS / editor .DS_Store *~ +.idea/ +.vscode/ + +# Python +__pycache__/ +*.pyc +.venv/ + +# Docker build cache +**/.build/ + +# Backup dumps (bewust buiten git) +backup/ +*.tgz +*.tar.gz diff --git a/README.md b/README.md new file mode 100644 index 0000000..d859eeb --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# ATC Lakehouse + +Infrastructure-as-code en documentatie voor het Dell ATC Lakehouse lab — Forward Deployed Engineering (Bart & Mo). + +## Repository-structuur + +``` +├── config/ # Applicatie-configuratie (homepage, kafka, trino, spark, airflow, superset) +├── compose/ # Docker Compose per stack (bron van waarheid voor containers) +├── deploy/ # Gecombineerde compose-bestanden per host +├── docs/ # Documentatie als code +├── inventory/ # Export van draaiende containers (referentie) +├── rss-proxy/ # RSS-feed proxy voor homepage widgets +└── scripts/ # Deploy- en hulpscripts +``` + +## Hosts (overzicht) + +| Host | Rol | Belangrijkste services | +|------|-----|------------------------| +| `atc-docker01` | Docker hub | Homepage, Superset, Forgejo (lokaal), LAM, RSS-proxy, icons | +| `atc-mgt01` | Management | Forgejo (Git), LDAP/LAM | +| `atc-kafka01` | Streaming | Kafka, Kafka UI | +| `atc-lake01` | Lakehouse | Spark, Trino, Debezium, Airflow | +| `atc-elastic01` | Search | Elasticsearch, Kibana | +| `atc-db01` / `atc-db02` | Databases | PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j | +| `atc-grafana` | Monitoring | Grafana | +| `pve01` (`10.0.10.65`) | Hypervisor | Proxmox VE | + +Zie [docs/hosts.md](docs/hosts.md) voor poorten en IP-adressen. + +## Snel starten — atc-docker01 + +```bash +cd /root/lakehouse +cp .env.example .env # vul secrets in +./scripts/deploy/deploy-atc-docker01.sh +``` + +## Documentatie + +- [Architectuur](docs/architecture.md) +- [Hosts & poorten](docs/hosts.md) +- [Disaster recovery](docs/disaster-recovery.md) +- [Dagelijkse operatie](docs/operations.md) +- [Homepage dashboard](config/homepage/README.md) + +## Git remote + +Forgejo: `http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse` + +```bash +git clone http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git /root/lakehouse +``` + +## Geheimen + +Bestanden met credentials (`proxmox.yaml`, `.env`, kafka connectors) staan in git voor **lab-restore**. Voor productie: gebruik `.env` + `*.example` en houd secrets buiten een publieke remote. diff --git a/compose/cadvisor/compose.yaml b/compose/cadvisor/compose.yaml new file mode 100644 index 0000000..dcbf117 --- /dev/null +++ b/compose/cadvisor/compose.yaml @@ -0,0 +1,14 @@ +# cAdvisor — container metrics (optioneel; poort 8080 botst met icons-server indien beide op 8080) +services: + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + restart: unless-stopped + privileged: true + ports: + - "8081:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro diff --git a/compose/forgejo/compose.yaml b/compose/forgejo/compose.yaml new file mode 100644 index 0000000..58d1b38 --- /dev/null +++ b/compose/forgejo/compose.yaml @@ -0,0 +1,19 @@ +# Forgejo (lokaal op atc-docker01) — primaire Git staat op atc-mgt01:3001 +services: + forgejo: + image: codeberg.org/forgejo/forgejo:14 + container_name: forgejo + restart: always + environment: + USER_UID: "1000" + USER_GID: "1000" + ports: + - "4002:3000" + - "222:22" + volumes: + - forgejo_data:/data + - /etc/localtime:/etc/localtime:ro + +volumes: + forgejo_data: + name: forgejo_forgejo diff --git a/compose/lam/compose.yml b/compose/lam/compose.yml new file mode 100644 index 0000000..dd02c75 --- /dev/null +++ b/compose/lam/compose.yml @@ -0,0 +1,17 @@ +# LDAP Account Manager (lokaal op atc-docker01) +# Productie LDAP/LAM: http://atc-mgt01.dell-atc.lan/lam/ +services: + lam: + image: ghcr.io/ldapaccountmanager/lam:stable + container_name: lam-app-1 + restart: unless-stopped + ports: + - "4001:80" + environment: + DEBIAN_FRONTEND: noninteractive + volumes: + - lam_data:/var/lib/ldap-account-manager + +volumes: + lam_data: + name: lam_lam diff --git a/compose/superset/Dockerfile b/compose/superset/Dockerfile new file mode 100644 index 0000000..521c1a4 --- /dev/null +++ b/compose/superset/Dockerfile @@ -0,0 +1,3 @@ +FROM apache/superset:latest + +RUN pip install psycopg2-binary diff --git a/compose/superset/docker-compose.yaml b/compose/superset/docker-compose.yaml new file mode 100644 index 0000000..0dec98f --- /dev/null +++ b/compose/superset/docker-compose.yaml @@ -0,0 +1,27 @@ +services: + superset: + build: . + container_name: superset + restart: unless-stopped + ports: + - "8088:8088" + env_file: + - ../../.env + environment: + - SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-change-me} + - SUPERSET_LOAD_EXAMPLES=no + volumes: + - ./superset_config.py:/app/pythonpath/superset_config.py + - superset_home:/app/superset_home + depends_on: + - redis + + redis: + image: redis:7 + container_name: superset_redis + restart: unless-stopped + ports: + - "6379:6379" + +volumes: + superset_home: diff --git a/compose/superset/superset_config.py b/compose/superset/superset_config.py new file mode 100644 index 0000000..0ebd1d2 --- /dev/null +++ b/compose/superset/superset_config.py @@ -0,0 +1,39 @@ +import os + +# Secret key for session signing +SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY', 'your-secret-key-here') + +# Database configuration - use SQLite to avoid psycopg2 issues +SQLALCHEMY_DATABASE_URI = 'sqlite:////app/superset_home/superset.db' + +# Redis cache configuration +CACHE_CONFIG = { + 'CACHE_TYPE': 'redis', + 'CACHE_REDIS_URL': 'redis://redis:6379/0', + 'CACHE_DEFAULT_TIMEOUT': 300 +} + +# Enable CSRF protection +ENABLE_PROXY_FIX = True + +# Feature flags +FEATURE_FLAGS = { + 'ENABLE_TEMPLATE_PROCESSING': True, + 'ALERT_REPORTS': True, +} + +# Row limit +ROW_LIMIT = 50000 + +# Viz types +VIZ_TYPE_DICT = { + 'table': {}, + 'dist_bar': {}, + 'line': {}, + 'area': {}, + 'pie': {}, + 'number': {}, +} + +# Timezone +TIMEZONE = 'Europe/Amsterdam' diff --git a/config/airflow/README.md b/config/airflow/README.md new file mode 100644 index 0000000..5c3f30d --- /dev/null +++ b/config/airflow/README.md @@ -0,0 +1,14 @@ +# Airflow + +| Bestand | Beschrijving | +|---------|--------------| +| `airflow.cfg` | Referentie-configuratie (deploy naar Airflow home op lake01) | +| `generate_data_dag.py` | DAG voor demo data-generatie | + +## URL + +http://atc-lake01.dell-atc.lan:8080/ (of `10.0.21.55:8080` volgens netwerk) + +## Deploy + +Kopieer `generate_data_dag.py` naar de Airflow `dags/` folder op de lake-host en herstart de scheduler/webserver. diff --git a/config/kafka/README.md b/config/kafka/README.md new file mode 100644 index 0000000..a464c5b --- /dev/null +++ b/config/kafka/README.md @@ -0,0 +1,25 @@ +# Kafka Connect — Debezium connectors + +JSON-definities voor CDC van databases naar Kafka. + +## Connectors + +| Bestand | Bron | Topic prefix | +|---------|------|--------------| +| `postgres-connector.json` | PostgreSQL op db02 | `postgres-sales` | +| `mysql-connector.json` | MySQL | — | +| `mongodb-connector.json` | MongoDB | — | + +## Deploy + +Vervang `PASSWORD_PLACEHOLDER` en pas host/poort aan. Registreer via Kafka Connect REST API op `atc-lake01` (Debezium `:8083`). + +```bash +curl -X POST -H "Content-Type: application/json" \ + --data @postgres-connector.json \ + http://atc-lake01.dell-atc.lan:8083/connectors/ +``` + +## Referentie + +- Kafka UI: http://atc-kafka01.dell-atc.lan:9000/ diff --git a/config/superset/docker-compose.yaml b/config/superset/docker-compose.yaml index db3878c..0dec98f 100644 --- a/config/superset/docker-compose.yaml +++ b/config/superset/docker-compose.yaml @@ -5,8 +5,10 @@ services: restart: unless-stopped ports: - "8088:8088" + env_file: + - ../../.env environment: - - SUPERSET_SECRET_KEY=your-secret-key-here + - SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-change-me} - SUPERSET_LOAD_EXAMPLES=no volumes: - ./superset_config.py:/app/pythonpath/superset_config.py diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..292d4e3 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,20 @@ +# Deploy + +| Bestand | Doel | +|---------|------| +| `docker-compose.homepage.yml` | Homepage + RSS-proxy + icon nginx | +| `docker-compose.atc-docker01.yml` | Alle stacks via Compose `include` | +| `../scripts/deploy/deploy-atc-docker01.sh` | Aanbevolen: stapsgewijs deployen | + +## Commando's + +```bash +# Alleen dashboard +docker compose -f deploy/docker-compose.homepage.yml up -d --build + +# Volledige docker01 stack +docker compose -f deploy/docker-compose.atc-docker01.yml up -d --build + +# Of script +./scripts/deploy/deploy-atc-docker01.sh +``` diff --git a/deploy/docker-compose.atc-docker01.yml b/deploy/docker-compose.atc-docker01.yml new file mode 100644 index 0000000..fdd0a29 --- /dev/null +++ b/deploy/docker-compose.atc-docker01.yml @@ -0,0 +1,14 @@ +# Volledige stack voor atc-docker01 (Docker Compose v2.20+ include) +# Gebruik: docker compose -f deploy/docker-compose.atc-docker01.yml up -d --build +# Of: ./scripts/deploy/deploy-atc-docker01.sh + +include: + - path: docker-compose.homepage.yml + - path: ../compose/superset/docker-compose.yaml + project_directory: ../compose/superset + - path: ../compose/forgejo/compose.yaml + project_directory: ../compose/forgejo + - path: ../compose/lam/compose.yml + project_directory: ../compose/lam + # - path: ../compose/cadvisor/compose.yaml + # project_directory: ../compose/cadvisor diff --git a/deploy/docker-compose.homepage.yml b/deploy/docker-compose.homepage.yml index 30d0f80..bef1092 100644 --- a/deploy/docker-compose.homepage.yml +++ b/deploy/docker-compose.homepage.yml @@ -12,7 +12,7 @@ services: - ../config/homepage:/app/config - /var/run/docker.sock:/var/run/docker.sock:ro environment: - HOMEPAGE_ALLOWED_HOSTS: "*" + HOMEPAGE_ALLOWED_HOSTS: ${HOMEPAGE_ALLOWED_HOSTS:-atc-docker01.dell-atc.lan,localhost} rss-proxy: build: ../rss-proxy diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..91b1f25 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,67 @@ +# Architectuur — ATC Lakehouse + +## Dataflow (conceptueel) + +```mermaid +flowchart LR + subgraph sources [Bronnen] + DB1[(atc-db01)] + DB2[(atc-db02)] + end + + subgraph ingest [Ingest] + DEB[Debezium] + KAFKA[Kafka] + end + + subgraph lake [Lake] + SPARK[Spark] + TRINO[Trino] + ICE[Iceberg tables] + end + + subgraph consume [Consumptie] + SS[Superset] + KB[Kibana] + GF[Grafana] + end + + DB1 --> DEB + DB2 --> DEB + DEB --> KAFKA + KAFKA --> SPARK + SPARK --> ICE + TRINO --> ICE + TRINO --> SS + KAFKA --> SS +``` + +## Lagen + +| Laag | Componenten | Doel | +|------|-------------|------| +| **Bron** | PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j | OLTP / events / graph demo data | +| **Ingest** | Debezium, Kafka Connect | CDC naar Kafka topics | +| **Processing** | Spark, Airflow | ETL, batch, orchestratie | +| **Query** | Trino | Federated SQL over lake + bronnen | +| **Storage** | ObjectScale / MinIO (indien actief) | Object storage | +| **Analytics** | Superset, Kibana | BI en search UI | +| **Platform** | Proxmox, Docker hosts | VMs en containers | +| **Ops** | Homepage dashboard, Grafana | Zichtbaarheid en links | + +## atc-docker01 — centrale dashboard-host + +Draait de **gethomepage**-dashboard (poort 80), **RSS-proxy** (8090), **icon server** (8080), **Apache Superset** (8088), en optioneel lokale **Forgejo** (4002) / **LAM** (4001). + +Configuratiepad op de host: `/root/lakehouse/config/homepage` → gemount in container `/app/config`. + +## Configuratie in git + +| Pad | Beschrijving | +|-----|--------------| +| `config/kafka/*.json` | Debezium/Kafka Connect connector definities | +| `config/trino/config.properties` | Trino coordinator (single-node) | +| `config/spark/spark-defaults.conf` | Spark defaults | +| `config/airflow/` | Airflow DAG + `airflow.cfg` | +| `config/superset/` | Superset Python config + compose | +| `config/homepage/` | Volledige dashboardconfiguratie | diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md new file mode 100644 index 0000000..6182852 --- /dev/null +++ b/docs/disaster-recovery.md @@ -0,0 +1,84 @@ +# Disaster recovery + +Doel: volledige **atc-docker01** dashboard-stack en configuratie terugzetten vanuit git. + +## Vereisten + +- VM `atc-docker01` met Docker en git +- Netwerk naar interne `.dell-atc.lan` hosts +- Secrets: `.env` of bestaande `config/homepage/proxmox.yaml` + +## Stap 1 — Repository + +```bash +dnf install -y git docker-compose-plugin # of equivalent +git clone http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git /root/lakehouse +cd /root/lakehouse +cp .env.example .env +# Bewerk .env: SUPERSET_SECRET_KEY, eventueel Proxmox token +``` + +## Stap 2 — Homepage + RSS + icons + +```bash +docker compose -f deploy/docker-compose.homepage.yml up -d --build +docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate +``` + +## Stap 3 — Superset + +```bash +docker compose -f compose/superset/docker-compose.yaml up -d --build +``` + +## Stap 4 — Optioneel: Forgejo, LAM, cAdvisor + +```bash +docker compose -f compose/forgejo/compose.yaml up -d +docker compose -f compose/lam/compose.yml up -d +# docker compose -f compose/cadvisor/compose.yaml up -d +``` + +Of alles tegelijk: + +```bash +./scripts/deploy/deploy-atc-docker01.sh +``` + +## Stap 5 — Andere hosts (handmatig) + +Deze repo bevat **config-referenties**, geen volledige VM-images: + +| Host | Herstel uit git | +|------|-----------------| +| Kafka connectors | `config/kafka/*.json` → deploy op Connect API | +| Trino | `config/trino/config.properties` → `/etc/trino/` op lake01 | +| Spark | `config/spark/spark-defaults.conf` | +| Airflow | `config/airflow/` | +| Data scripts | `config/scripts/data-generation/` | + +Zie `inventory/containers-atc-docker01.json` voor laatst geëxporteerde containerstate. + +## Volumes (niet in git) + +Back-up apart: + +- `forgejo_forgejo` — Git data +- `superset_superset_home` — Superset metadata +- `lam_lam` — LAM configuratie + +```bash +docker run --rm -v forgejo_forgejo:/data -v /backup:/backup alpine \ + tar czf /backup/forgejo_forgejo.tgz -C /data . +``` + +## Verificatie + +1. http://atc-docker01.dell-atc.lan/ — homepage laadt, 4 tabs +2. http://atc-docker01.dell-atc.lan:8090/health — RSS proxy +3. http://atc-docker01.dell-atc.lan:8088/health — Superset +4. FEEDS-tab toont RSS-widgets + +## Proxmox-widget + +Token staat in `config/homepage/proxmox.yaml`. Na restore: Proxmox → Permissions → API Tokens → controleer `root@pam!homepage`. diff --git a/docs/hosts.md b/docs/hosts.md new file mode 100644 index 0000000..eb00c7f --- /dev/null +++ b/docs/hosts.md @@ -0,0 +1,38 @@ +# Hosts & poorten + +## VM-overzicht + +| Hostname | IP (indien bekend) | Rol | +|----------|-------------------|-----| +| `atc-docker01.dell-atc.lan` | `10.0.21.45` | Docker: homepage, superset, rss-proxy, icons | +| `atc-mgt01.dell-atc.lan` | — | Forgejo Git `:3001`, LDAP/LAM | +| `atc-kafka01.dell-atc.lan` | — | Kafka UI `:9000` | +| `atc-lake01.dell-atc.lan` | — | Spark `:8080`, Trino `:8089`, Debezium `:8083` | +| `atc-elastic01.dell-atc.lan` | — | ES `:9200`, Kibana `:5601` | +| `atc-db01.dell-atc.lan` | `10.0.20.112` | PostgreSQL `5432`, MySQL `3306` | +| `atc-db02.dell-atc.lan` | `10.0.21.51` | PG, MySQL, Mongo `27017`, Cassandra `9042`, Neo4j `7474`/`7687` | +| `atc-grafana.dell-atc.lan` | `10.0.20.103` | Grafana `:3000` | +| `pve01` / Proxmox | `10.0.10.65` | Hypervisor `:8006` | +| iDRAC | `10.0.41.102` | Out-of-band management | +| ObjectScale | `10.0.20.111` | Object storage HTTPS | + +## atc-docker01 — Docker services + +| Service | Poort | URL | +|---------|-------|-----| +| Homepage | 80 | http://atc-docker01.dell-atc.lan/ | +| Icons (nginx) | 8080 | http://atc-docker01.dell-atc.lan:8080/ | +| RSS proxy | 8090 | http://atc-docker01.dell-atc.lan:8090/health | +| Superset | 8088 | http://atc-docker01.dell-atc.lan:8088/ | +| Superset Redis | 6379 | intern | +| Forgejo (lokaal) | 4002, 222 | http://atc-docker01.dell-atc.lan:4002/ | +| LAM (lokaal) | 4001 | http://atc-docker01.dell-atc.lan:4001/ | +| cAdvisor | — | optioneel, niet standaard gestart | + +## Paden op atc-docker01 + +| Pad | Inhoud | +|-----|--------| +| `/root/lakehouse` | Deze git clone | +| `/root/compose/superset` | Legacy symlink-werkdir (sync met `compose/superset`) | +| `/var/lib/docker/volumes/` | Persistente data (niet in git) | diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..0595a51 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,43 @@ +# Operaties + +## Config wijzigen en deployen + +```bash +cd /root/lakehouse +# Bewerk config/homepage/*.yaml of rss-proxy/server.py +git add -A && git commit -m "beschrijving" && git push origin master + +# Homepage herladen +docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate + +# RSS proxy rebuild +docker compose -f deploy/docker-compose.homepage.yml build rss-proxy +docker compose -f deploy/docker-compose.homepage.yml up -d rss-proxy +``` + +## Logs + +```bash +docker logs -f homepage +docker logs -f rss-proxy +docker logs -f superset +``` + +Homepage applicatielog staat buiten git: `config/homepage/logs/` (genegeerd door `.gitignore`). + +## Container inventory bijwerken + +```bash +./scripts/deploy/export-inventory.sh +git add inventory/ && git commit -m "chore: refresh container inventory" +``` + +## Stack herstarten + +```bash +./scripts/deploy/deploy-atc-docker01.sh +``` + +## Git op de server + +Remote staat op `atc-docker01` in `/root/lakehouse`. Gebruik je eigen credentials voor push naar Forgejo op `atc-mgt01`. diff --git a/inventory/containers-atc-docker01.json b/inventory/containers-atc-docker01.json new file mode 100644 index 0000000..f705177 --- /dev/null +++ b/inventory/containers-atc-docker01.json @@ -0,0 +1,223 @@ +[ + { + "name": "rss-proxy", + "image": "rss-proxy:latest", + "ports": { + "8090/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "8090" + }, + { + "HostIp": "::", + "HostPort": "8090" + } + ] + }, + "restart": "unless-stopped", + "binds": [], + "volumes": [], + "env": [] + }, + { + "name": "cadvisor", + "image": "gcr.io/cadvisor/cadvisor:latest", + "ports": {}, + "restart": "no", + "binds": [ + "/:/rootfs:ro", + "/var/run:/var/run:ro", + "/sys:/sys:ro", + "/var/lib/docker/:/var/lib/docker:ro" + ], + "volumes": [], + "env": [] + }, + { + "name": "homepage", + "image": "ghcr.io/gethomepage/homepage:latest", + "ports": { + "3000/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "80" + }, + { + "HostIp": "::", + "HostPort": "80" + } + ] + }, + "restart": "unless-stopped", + "binds": [ + "/root/lakehouse/config/homepage:/app/config", + "/var/run/docker.sock:/var/run/docker.sock" + ], + "volumes": [], + "env": [ + "HOMEPAGE_ALLOWED_HOSTS=atc-docker01.dell-atc.lan,localhost", + "YARN_VERSION=1.22.22", + "PORT=3000" + ] + }, + { + "name": "icons-server", + "image": "nginx:alpine", + "ports": { + "80/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "8080" + }, + { + "HostIp": "::", + "HostPort": "8080" + } + ] + }, + "restart": "no", + "binds": [ + "/root/lakehouse/config/homepage/icons:/usr/share/nginx/html:ro" + ], + "volumes": [], + "env": [] + }, + { + "name": "superset", + "image": "superset-superset", + "ports": { + "8088/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "8088" + }, + { + "HostIp": "::", + "HostPort": "8088" + } + ] + }, + "restart": "unless-stopped", + "binds": [ + "/root/compose/superset/superset_config.py:/app/pythonpath/superset_config.py:rw", + "superset_superset_home:/app/superset_home:rw" + ], + "volumes": [ + { + "name": "superset_superset_home", + "dest": "/app/superset_home" + } + ], + "env": [ + "SUPERSET_SECRET_KEY=your-secret-key-here", + "SUPERSET_LOAD_EXAMPLES=no", + "SUPERSET_HOME=/app/superset_home", + "HOME=/app/superset_home", + "SUPERSET_ENV=production", + "FLASK_APP=superset.app:create_app()", + "PYTHONPATH=/app/pythonpath", + "SUPERSET_PORT=8088", + "PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright-browsers" + ] + }, + { + "name": "superset_redis", + "image": "redis:7", + "ports": { + "6379/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "6379" + }, + { + "HostIp": "::", + "HostPort": "6379" + } + ] + }, + "restart": "unless-stopped", + "binds": [], + "volumes": [ + { + "name": "75b6d34e34dd318d9c3bb25d37250f53eaeadff8334cf6301ef87151139457e2", + "dest": "/data" + } + ], + "env": [ + "REDIS_VERSION=7.4.9" + ] + }, + { + "name": "forgejo", + "image": "codeberg.org/forgejo/forgejo:14", + "ports": { + "22/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "222" + }, + { + "HostIp": "::", + "HostPort": "222" + } + ], + "3000/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "4002" + }, + { + "HostIp": "::", + "HostPort": "4002" + } + ] + }, + "restart": "always", + "binds": [ + "forgejo_forgejo:/data:rw", + "/etc/localtime:/etc/localtime:ro" + ], + "volumes": [ + { + "name": "forgejo_forgejo", + "dest": "/data" + } + ], + "env": [ + "USER_GID=1000", + "USER_UID=1000", + "USER=git", + "GITEA_CUSTOM=/data/gitea" + ] + }, + { + "name": "lam-app-1", + "image": "ghcr.io/ldapaccountmanager/lam:stable", + "ports": { + "80/tcp": [ + { + "HostIp": "0.0.0.0", + "HostPort": "4001" + }, + { + "HostIp": "::", + "HostPort": "4001" + } + ] + }, + "restart": "unless-stopped", + "binds": [ + "lam_lam:/var/lib/ldap-account-manager:rw" + ], + "volumes": [ + { + "name": "lam_lam", + "dest": "/var/lib/ldap-account-manager" + } + ], + "env": [ + "DEBIAN_FRONTEND=noninteractive", + "DEBUG=" + ] + } +] \ No newline at end of file diff --git a/scripts/atclab/README.md b/scripts/atclab/README.md new file mode 100644 index 0000000..1eb0eca --- /dev/null +++ b/scripts/atclab/README.md @@ -0,0 +1,12 @@ +# atclab scripts (referentie) + +Kopie van `/opt/atclab/bin/` op lab hosts — handige wrappers voor yum/git/syslog. + +| Script | Functie | +|--------|---------| +| `atclab-update` | yum update van atclab repo | +| `gitcommit` | `git commit -m "$@"` shortcut | +| `gittag` | git tagging helper | +| `syslog` / `syslogf` | syslog viewers | + +Deze scripts zijn host-specifiek; pas aan per omgeving. diff --git a/scripts/atclab/atclab-update b/scripts/atclab/atclab-update new file mode 100755 index 0000000..fcb1fb8 --- /dev/null +++ b/scripts/atclab/atclab-update @@ -0,0 +1,4 @@ +#!/usr/bin/bash +#yum makecache + +yum --disablerepo="*" --enablerepo="atclab" clean all && yum --disablerepo="*" --enablerepo="atclab" -y update diff --git a/scripts/atclab/gitcommit b/scripts/atclab/gitcommit new file mode 100755 index 0000000..f160bff --- /dev/null +++ b/scripts/atclab/gitcommit @@ -0,0 +1,5 @@ +#!/usr/bin/bash + +desc="$@" + +git commit -m "$desc" diff --git a/scripts/atclab/gittag b/scripts/atclab/gittag new file mode 100755 index 0000000..88acc24 --- /dev/null +++ b/scripts/atclab/gittag @@ -0,0 +1,18 @@ +#!/usr/bin/bash + +DIR=$(git rev-parse --show-toplevel) || exit 10 +NAME=$(rpm -q --qf '%{name}\n' --specfile ${DIR}/*.spec) +VERSION=$(rpm -q --qf '%{version}\n' --specfile ${DIR}/*.spec) + + +while getopts ":h:" OPT; do + case "$OPT" in + \?|h) usage ; exit 0 ;; + esac +done +shift $((OPTIND-1)) + +case $1 in + apply) git tag -a -mversion v${VERSION} ;; + *) echo git tag -a -mversion v${VERSION} ;; +esac diff --git a/scripts/atclab/r b/scripts/atclab/r new file mode 100755 index 0000000..26c0ad7 --- /dev/null +++ b/scripts/atclab/r @@ -0,0 +1,23 @@ +#!/usr/bin/bash + +DEL="" + +deletehost() { + local host ip + host=$(echo $1 | awk -F@ '{print $NF}') + ip=$(host $host | awk '/has address/ {print $NF}' | head -1) + ssh-keygen -R $host + test -n "$ip" && ssh-keygen -R $ip +} + +while getopts ":dh" OPT; do + case "$OPT" in + d) DEL=Y ;; + \?|h) usage >&2 ; exit 0 ;; + esac +done +shift $(expr $OPTIND - 1) + +test -n "$DEL" && deletehost $@ + +ssh root@${1:-none} diff --git a/scripts/atclab/syslog b/scripts/atclab/syslog new file mode 100755 index 0000000..a6c8576 --- /dev/null +++ b/scripts/atclab/syslog @@ -0,0 +1,4 @@ +#!/usr/bin/bash + +test -f /var/log/messages && tail -n100 /var/log/messages +test -f /var/log/syslog && tail -n100 /var/log/syslog diff --git a/scripts/atclab/syslogf b/scripts/atclab/syslogf new file mode 100755 index 0000000..c71d322 --- /dev/null +++ b/scripts/atclab/syslogf @@ -0,0 +1,4 @@ +#!/usr/bin/bash + +test -f /var/log/messages && tail -n100 -f /var/log/messages +test -f /var/log/syslog && tail -n100 -f /var/log/syslog diff --git a/scripts/deploy/deploy-atc-docker01.sh b/scripts/deploy/deploy-atc-docker01.sh new file mode 100755 index 0000000..0f54633 --- /dev/null +++ b/scripts/deploy/deploy-atc-docker01.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +if [[ -f .env ]]; then + set -a + # shellcheck source=/dev/null + source .env + set +a +fi + +echo "==> Homepage + RSS + icons" +docker compose -f deploy/docker-compose.homepage.yml up -d --build + +echo "==> Superset + Redis" +docker compose -f compose/superset/docker-compose.yaml up -d --build + +echo "==> Forgejo (lokaal)" +docker compose -f compose/forgejo/compose.yaml up -d + +echo "==> LAM (lokaal)" +docker compose -f compose/lam/compose.yml up -d + +echo "==> Revalidate homepage" +docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate 2>/dev/null || true + +echo "Done. Open http://atc-docker01.dell-atc.lan/" diff --git a/scripts/deploy/export-inventory.py b/scripts/deploy/export-inventory.py new file mode 100755 index 0000000..0832587 --- /dev/null +++ b/scripts/deploy/export-inventory.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import json +import subprocess + +SKIP_ENV_PREFIXES = ( + "PATH=", "NODE_", "PYTHON_", "GPG_", "LANG=", "YARN=", "HOSTNAME=", + "NGINX_", "NJS_", "ACME_", "DYNPKG_", "PKG_", "CADVISOR_", +) + +ids = subprocess.check_output(["docker", "ps", "-aq"], text=True).split() +if not ids or ids == [""]: + raise SystemExit("no containers") + +raw = subprocess.check_output(["docker", "inspect", *ids]) +data = json.loads(raw) +out = [] +for c in data: + env = [ + e for e in (c["Config"].get("Env") or []) + if not any(e.startswith(p) for p in SKIP_ENV_PREFIXES) + ] + out.append({ + "name": c["Name"].lstrip("/"), + "image": c["Config"]["Image"], + "ports": c["NetworkSettings"]["Ports"], + "restart": c["HostConfig"]["RestartPolicy"]["Name"], + "binds": c["HostConfig"].get("Binds") or [], + "volumes": [ + {"name": m["Name"], "dest": m["Destination"]} + for m in c["Mounts"] + if m.get("Type") == "volume" + ], + "env": env, + }) + +with open("/root/lakehouse/inventory/containers-atc-docker01.json", "w") as f: + json.dump(out, f, indent=2) +print("wrote", len(out), "containers") diff --git a/scripts/deploy/export-inventory.sh b/scripts/deploy/export-inventory.sh new file mode 100755 index 0000000..1b1daea --- /dev/null +++ b/scripts/deploy/export-inventory.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +OUT="$ROOT/inventory/containers-atc-docker01.json" + +python3 "$ROOT/scripts/deploy/export-inventory.py" "$OUT" +echo "Wrote $OUT"