From f3d24634e46aed8859138a70597841aa8d718fbc Mon Sep 17 00:00:00 2001 From: Lakehouse Admin Date: Tue, 19 May 2026 22:46:40 +0200 Subject: [PATCH] English docs and passwordless SSH mesh for lab fleet - Translate all documentation to English - Add docs/ssh-mesh.md and scripts/setup/setup-ssh-mesh.sh - Add config/hosts/atc-lab.hosts for /etc/hosts on all VMs - Ignore atc_cluster private key in git; mesh deployed on 12 hosts --- .gitignore | 4 +- README.md | 46 ++++++++-------- config/hosts/atc-lab.hosts | 17 ++++++ deploy/README.md | 16 +++--- docs/architecture.md | 48 ++++++++-------- docs/disaster-recovery.md | 56 +++++++++---------- docs/hosts.md | 47 +++++++++------- docs/operations.md | 30 ++++++---- docs/ssh-mesh.md | 69 +++++++++++++++++++++++ scripts/atclab/README.md | 10 ++-- scripts/setup/README.md | 7 +++ scripts/setup/setup-ssh-mesh.sh | 98 +++++++++++++++++++++++++++++++++ 12 files changed, 330 insertions(+), 118 deletions(-) create mode 100644 config/hosts/atc-lab.hosts create mode 100644 docs/ssh-mesh.md create mode 100644 scripts/setup/README.md create mode 100755 scripts/setup/setup-ssh-mesh.sh diff --git a/.gitignore b/.gitignore index ae68cc5..e74e808 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,10 @@ config/homepage/logs/ *.log -# Secrets (lokaal op host, niet committen indien je .env gebruikt) +# Secrets (keep on host; use .env.example in git) .env +**/atc_cluster +!**/atc_cluster.pub .env.local .env.*.local diff --git a/README.md b/README.md index d859eeb..51b3f2a 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,24 @@ # ATC Lakehouse -Infrastructure-as-code en documentatie voor het Dell ATC Lakehouse lab — Forward Deployed Engineering (Bart & Mo). +Infrastructure-as-code and documentation for the Dell ATC Lakehouse lab — Forward Deployed Engineering (Bart & Mo). -## Repository-structuur +## Repository layout ``` -├── config/ # Applicatie-configuratie (homepage, kafka, trino, spark, airflow, superset) -├── compose/ # Docker Compose per stack (bron van waarheid voor containers) -├── deploy/ # Gecombineerde compose-bestanden per host -├── docs/ # Documentatie als code -├── inventory/ # Export van draaiende containers (referentie) -├── rss-proxy/ # RSS-feed proxy voor homepage widgets -└── scripts/ # Deploy- en hulpscripts +├── config/ # Application config (homepage, kafka, trino, spark, airflow, superset) +├── compose/ # Docker Compose per stack (source of truth for containers) +├── deploy/ # Combined compose files per host +├── docs/ # Documentation as code +├── inventory/ # Exported container state (reference) +├── rss-proxy/ # RSS feed proxy for homepage widgets +└── scripts/ # Deploy and setup scripts ``` -## Hosts (overzicht) +## Hosts (summary) -| Host | Rol | Belangrijkste services | -|------|-----|------------------------| -| `atc-docker01` | Docker hub | Homepage, Superset, Forgejo (lokaal), LAM, RSS-proxy, icons | +| Host | Role | Key services | +|------|------|----------------| +| `atc-docker01` | Docker hub | Homepage, Superset, Forgejo (local), LAM, RSS-proxy, icons | | `atc-mgt01` | Management | Forgejo (Git), LDAP/LAM | | `atc-kafka01` | Streaming | Kafka, Kafka UI | | `atc-lake01` | Lakehouse | Spark, Trino, Debezium, Airflow | @@ -27,22 +27,24 @@ Infrastructure-as-code en documentatie voor het Dell ATC Lakehouse lab — Forwa | `atc-grafana` | Monitoring | Grafana | | `pve01` (`10.0.10.65`) | Hypervisor | Proxmox VE | -Zie [docs/hosts.md](docs/hosts.md) voor poorten en IP-adressen. +See [docs/hosts.md](docs/hosts.md) for ports and IPs. +See [docs/ssh-mesh.md](docs/ssh-mesh.md) for passwordless SSH between VMs. -## Snel starten — atc-docker01 +## Quick start — atc-docker01 ```bash cd /root/lakehouse -cp .env.example .env # vul secrets in +cp .env.example .env # fill in secrets ./scripts/deploy/deploy-atc-docker01.sh ``` -## Documentatie +## Documentation -- [Architectuur](docs/architecture.md) -- [Hosts & poorten](docs/hosts.md) +- [Architecture](docs/architecture.md) +- [Hosts & ports](docs/hosts.md) +- [SSH mesh (passwordless fleet)](docs/ssh-mesh.md) - [Disaster recovery](docs/disaster-recovery.md) -- [Dagelijkse operatie](docs/operations.md) +- [Day-to-day operations](docs/operations.md) - [Homepage dashboard](config/homepage/README.md) ## Git remote @@ -53,6 +55,6 @@ Forgejo: `http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse` git clone http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git /root/lakehouse ``` -## Geheimen +## Secrets -Bestanden met credentials (`proxmox.yaml`, `.env`, kafka connectors) staan in git voor **lab-restore**. Voor productie: gebruik `.env` + `*.example` en houd secrets buiten een publieke remote. +Files with credentials (`proxmox.yaml`, `.env`, kafka connectors) are in git for **lab disaster recovery**. For production, use `.env` + `*.example` and keep secrets off public remotes. diff --git a/config/hosts/atc-lab.hosts b/config/hosts/atc-lab.hosts new file mode 100644 index 0000000..a6e60c8 --- /dev/null +++ b/config/hosts/atc-lab.hosts @@ -0,0 +1,17 @@ +# ATC Lakehouse lab — append to /etc/hosts on each VM +# Managed by scripts/setup/setup-ssh-mesh.sh + +10.0.21.45 atc-docker01 atc-docker01.dell-atc.lan +10.0.21.47 atc-docker02 atc-docker02.dell-atc.lan +10.0.20.104 atc-mgt01 atc-mgt01.dell-atc.lan +10.0.21.36 atc-kafka01 atc-kafka01.dell-atc.lan atc-kafka +10.0.21.50 atc-lake01 atc-lake01.dell-atc.lan +10.0.21.46 atc-elastic01 atc-elastic01.dell-atc.lan atc-elastic +10.0.20.112 atc-db01 atc-db01.dell-atc.lan +10.0.21.51 atc-db02 atc-db02.dell-atc.lan +10.0.20.103 atc-grafana atc-grafana.dell-atc.lan +10.0.21.55 atc-airflow01 airflow01 atc-airflow01.dell-atc.lan +10.0.21.49 atc-portal01 atc-portal01.dell-atc.lan portal01 +10.0.21.39 atc-lama01 atc-lama atc-lama01.dell-atc.lan +10.0.10.65 pve01 proxmox pve.dell-atc.lan +10.0.20.111 atc-objectscale objectscale diff --git a/deploy/README.md b/deploy/README.md index 292d4e3..8792b99 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,20 +1,20 @@ # Deploy -| Bestand | Doel | -|---------|------| +| File | Purpose | +|------|---------| | `docker-compose.homepage.yml` | Homepage + RSS-proxy + icon nginx | -| `docker-compose.atc-docker01.yml` | Alle stacks via Compose `include` | -| `../scripts/deploy/deploy-atc-docker01.sh` | Aanbevolen: stapsgewijs deployen | +| `docker-compose.atc-docker01.yml` | All stacks via Compose `include` | +| `../scripts/deploy/deploy-atc-docker01.sh` | Recommended: step-by-step deploy | -## Commando's +## Commands ```bash -# Alleen dashboard +# Dashboard only docker compose -f deploy/docker-compose.homepage.yml up -d --build -# Volledige docker01 stack +# Full docker01 stack docker compose -f deploy/docker-compose.atc-docker01.yml up -d --build -# Of script +# Or use the script ./scripts/deploy/deploy-atc-docker01.sh ``` diff --git a/docs/architecture.md b/docs/architecture.md index 91b1f25..78da31b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,10 +1,10 @@ -# Architectuur — ATC Lakehouse +# Architecture — ATC Lakehouse -## Dataflow (conceptueel) +## Data flow (conceptual) ```mermaid flowchart LR - subgraph sources [Bronnen] + subgraph sources [Sources] DB1[(atc-db01)] DB2[(atc-db02)] end @@ -20,7 +20,7 @@ flowchart LR ICE[Iceberg tables] end - subgraph consume [Consumptie] + subgraph consume [Consumption] SS[Superset] KB[Kibana] GF[Grafana] @@ -36,32 +36,36 @@ flowchart LR KAFKA --> SS ``` -## Lagen +## Layers -| Laag | Componenten | Doel | -|------|-------------|------| -| **Bron** | PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j | OLTP / events / graph demo data | -| **Ingest** | Debezium, Kafka Connect | CDC naar Kafka topics | -| **Processing** | Spark, Airflow | ETL, batch, orchestratie | -| **Query** | Trino | Federated SQL over lake + bronnen | -| **Storage** | ObjectScale / MinIO (indien actief) | Object storage | -| **Analytics** | Superset, Kibana | BI en search UI | -| **Platform** | Proxmox, Docker hosts | VMs en containers | -| **Ops** | Homepage dashboard, Grafana | Zichtbaarheid en links | +| Layer | Components | Purpose | +|-------|------------|---------| +| **Source** | PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j | OLTP / events / graph demo data | +| **Ingest** | Debezium, Kafka Connect | CDC to Kafka topics | +| **Processing** | Spark, Airflow | ETL, batch, orchestration | +| **Query** | Trino | Federated SQL over lake + sources | +| **Storage** | ObjectScale / MinIO (when active) | Object storage | +| **Analytics** | Superset, Kibana | BI and search UI | +| **Platform** | Proxmox, Docker hosts | VMs and containers | +| **Ops** | Homepage dashboard, Grafana | Visibility and links | -## atc-docker01 — centrale dashboard-host +## atc-docker01 — central dashboard host -Draait de **gethomepage**-dashboard (poort 80), **RSS-proxy** (8090), **icon server** (8080), **Apache Superset** (8088), en optioneel lokale **Forgejo** (4002) / **LAM** (4001). +Runs **gethomepage** (port 80), **RSS-proxy** (8090), **icon server** (8080), **Apache Superset** (8088), and optional local **Forgejo** (4002) / **LAM** (4001). -Configuratiepad op de host: `/root/lakehouse/config/homepage` → gemount in container `/app/config`. +Config path on host: `/root/lakehouse/config/homepage` → mounted at `/app/config` in the container. -## Configuratie in git +## SSH mesh -| Pad | Beschrijving | +All lab VMs use a shared cluster key for root-to-root access. See [ssh-mesh.md](ssh-mesh.md). + +## Configuration in git + +| Path | Description | |-----|--------------| -| `config/kafka/*.json` | Debezium/Kafka Connect connector definities | +| `config/kafka/*.json` | Debezium/Kafka Connect connector definitions | | `config/trino/config.properties` | Trino coordinator (single-node) | | `config/spark/spark-defaults.conf` | Spark defaults | | `config/airflow/` | Airflow DAG + `airflow.cfg` | | `config/superset/` | Superset Python config + compose | -| `config/homepage/` | Volledige dashboardconfiguratie | +| `config/homepage/` | Full dashboard configuration | diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md index 6182852..fa4e18d 100644 --- a/docs/disaster-recovery.md +++ b/docs/disaster-recovery.md @@ -1,84 +1,84 @@ # Disaster recovery -Doel: volledige **atc-docker01** dashboard-stack en configuratie terugzetten vanuit git. +Goal: fully restore the **atc-docker01** dashboard stack and configuration from git. -## Vereisten +## Requirements -- VM `atc-docker01` met Docker en git -- Netwerk naar interne `.dell-atc.lan` hosts -- Secrets: `.env` of bestaande `config/homepage/proxmox.yaml` +- VM `atc-docker01` with Docker and git +- Network to internal `.dell-atc.lan` hosts +- Secrets: `.env` or existing `config/homepage/proxmox.yaml` +- SSH mesh deployed ([ssh-mesh.md](ssh-mesh.md)) for multi-host config pull -## Stap 1 — Repository +## Step 1 — Repository ```bash -dnf install -y git docker-compose-plugin # of equivalent +dnf install -y git docker-compose-plugin git clone http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git /root/lakehouse cd /root/lakehouse cp .env.example .env -# Bewerk .env: SUPERSET_SECRET_KEY, eventueel Proxmox token +# Edit .env: SUPERSET_SECRET_KEY, etc. ``` -## Stap 2 — Homepage + RSS + icons +## Step 2 — Homepage + RSS + icons ```bash docker compose -f deploy/docker-compose.homepage.yml up -d --build docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate ``` -## Stap 3 — Superset +## Step 3 — Superset ```bash docker compose -f compose/superset/docker-compose.yaml up -d --build ``` -## Stap 4 — Optioneel: Forgejo, LAM, cAdvisor +## Step 4 — Optional: Forgejo, LAM, cAdvisor ```bash docker compose -f compose/forgejo/compose.yaml up -d docker compose -f compose/lam/compose.yml up -d -# docker compose -f compose/cadvisor/compose.yaml up -d ``` -Of alles tegelijk: +Or all at once: ```bash ./scripts/deploy/deploy-atc-docker01.sh ``` -## Stap 5 — Andere hosts (handmatig) +## Step 5 — Other hosts (manual) -Deze repo bevat **config-referenties**, geen volledige VM-images: +This repo holds **config references**, not full VM images: -| Host | Herstel uit git | -|------|-----------------| -| Kafka connectors | `config/kafka/*.json` → deploy op Connect API | -| Trino | `config/trino/config.properties` → `/etc/trino/` op lake01 | +| Host | Restore from git | +|------|------------------| +| Kafka connectors | `config/kafka/*.json` → Connect REST API | +| Trino | `config/trino/config.properties` → lake host | | Spark | `config/spark/spark-defaults.conf` | | Airflow | `config/airflow/` | | Data scripts | `config/scripts/data-generation/` | -Zie `inventory/containers-atc-docker01.json` voor laatst geëxporteerde containerstate. +See `inventory/containers-atc-docker01.json` for last exported container state. -## Volumes (niet in git) +## Volumes (not in git) -Back-up apart: +Back up separately: - `forgejo_forgejo` — Git data - `superset_superset_home` — Superset metadata -- `lam_lam` — LAM configuratie +- `lam_lam` — LAM configuration ```bash docker run --rm -v forgejo_forgejo:/data -v /backup:/backup alpine \ tar czf /backup/forgejo_forgejo.tgz -C /data . ``` -## Verificatie +## Verification -1. http://atc-docker01.dell-atc.lan/ — homepage laadt, 4 tabs +1. http://atc-docker01.dell-atc.lan/ — homepage loads, 4 tabs 2. http://atc-docker01.dell-atc.lan:8090/health — RSS proxy 3. http://atc-docker01.dell-atc.lan:8088/health — Superset -4. FEEDS-tab toont RSS-widgets +4. FEEDS tab shows RSS widgets -## Proxmox-widget +## Proxmox widget -Token staat in `config/homepage/proxmox.yaml`. Na restore: Proxmox → Permissions → API Tokens → controleer `root@pam!homepage`. +Token in `config/homepage/proxmox.yaml`. After restore, verify in Proxmox → Permissions → API Tokens → `root@pam!homepage`. diff --git a/docs/hosts.md b/docs/hosts.md index eb00c7f..411579b 100644 --- a/docs/hosts.md +++ b/docs/hosts.md @@ -1,38 +1,43 @@ -# Hosts & poorten +# Hosts & ports -## VM-overzicht +## VM overview -| Hostname | IP (indien bekend) | Rol | -|----------|-------------------|-----| +| Hostname | IP | Role | +|----------|-----|------| | `atc-docker01.dell-atc.lan` | `10.0.21.45` | Docker: homepage, superset, rss-proxy, icons | -| `atc-mgt01.dell-atc.lan` | — | Forgejo Git `:3001`, LDAP/LAM | -| `atc-kafka01.dell-atc.lan` | — | Kafka UI `:9000` | -| `atc-lake01.dell-atc.lan` | — | Spark `:8080`, Trino `:8089`, Debezium `:8083` | -| `atc-elastic01.dell-atc.lan` | — | ES `:9200`, Kibana `:5601` | +| `atc-docker02.dell-atc.lan` | `10.0.21.47` | Secondary Docker host | +| `atc-mgt01.dell-atc.lan` | `10.0.20.104` | Forgejo Git `:3001`, LDAP/LAM | +| `atc-kafka01.dell-atc.lan` | `10.0.21.36` | Kafka UI `:9000` | +| `atc-lake01.dell-atc.lan` | `10.0.21.50` | Spark `:8080`, Trino `:8089`, Debezium `:8083` | +| `atc-elastic01.dell-atc.lan` | `10.0.21.46` | ES `:9200`, Kibana `:5601` | | `atc-db01.dell-atc.lan` | `10.0.20.112` | PostgreSQL `5432`, MySQL `3306` | | `atc-db02.dell-atc.lan` | `10.0.21.51` | PG, MySQL, Mongo `27017`, Cassandra `9042`, Neo4j `7474`/`7687` | | `atc-grafana.dell-atc.lan` | `10.0.20.103` | Grafana `:3000` | +| `atc-airflow01` | `10.0.21.55` | Airflow `:8080` | +| `atc-portal01.dell-atc.lan` | `10.0.21.49` | Portal | +| `atc-lama01.dell-atc.lan` | `10.0.21.39` | LAMA | | `pve01` / Proxmox | `10.0.10.65` | Hypervisor `:8006` | -| iDRAC | `10.0.41.102` | Out-of-band management | | ObjectScale | `10.0.20.111` | Object storage HTTPS | +| iDRAC | `10.0.41.102` | Out-of-band management | + +SSH mesh: see [ssh-mesh.md](ssh-mesh.md). ## atc-docker01 — Docker services -| Service | Poort | URL | -|---------|-------|-----| +| Service | Port | URL | +|---------|------|-----| | Homepage | 80 | http://atc-docker01.dell-atc.lan/ | | Icons (nginx) | 8080 | http://atc-docker01.dell-atc.lan:8080/ | | RSS proxy | 8090 | http://atc-docker01.dell-atc.lan:8090/health | | Superset | 8088 | http://atc-docker01.dell-atc.lan:8088/ | -| Superset Redis | 6379 | intern | -| Forgejo (lokaal) | 4002, 222 | http://atc-docker01.dell-atc.lan:4002/ | -| LAM (lokaal) | 4001 | http://atc-docker01.dell-atc.lan:4001/ | -| cAdvisor | — | optioneel, niet standaard gestart | +| Superset Redis | 6379 | internal | +| Forgejo (local) | 4002, 222 | http://atc-docker01.dell-atc.lan:4002/ | +| LAM (local) | 4001 | http://atc-docker01.dell-atc.lan:4001/ | +| cAdvisor | 8081 | optional | -## Paden op atc-docker01 +## Paths on atc-docker01 -| Pad | Inhoud | -|-----|--------| -| `/root/lakehouse` | Deze git clone | -| `/root/compose/superset` | Legacy symlink-werkdir (sync met `compose/superset`) | -| `/var/lib/docker/volumes/` | Persistente data (niet in git) | +| Path | Content | +|------|---------| +| `/root/lakehouse` | Git clone | +| `/root/compose/` | Legacy compose dirs (mirror `compose/` in git) | diff --git a/docs/operations.md b/docs/operations.md index 0595a51..b57d699 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,16 +1,16 @@ -# Operaties +# Operations -## Config wijzigen en deployen +## Change config and deploy ```bash cd /root/lakehouse -# Bewerk config/homepage/*.yaml of rss-proxy/server.py -git add -A && git commit -m "beschrijving" && git push origin master +# Edit config/homepage/*.yaml or rss-proxy/server.py +git add -A && git commit -m "description" && git push origin master -# Homepage herladen +# Reload homepage docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate -# RSS proxy rebuild +# Rebuild RSS proxy docker compose -f deploy/docker-compose.homepage.yml build rss-proxy docker compose -f deploy/docker-compose.homepage.yml up -d rss-proxy ``` @@ -23,21 +23,29 @@ docker logs -f rss-proxy docker logs -f superset ``` -Homepage applicatielog staat buiten git: `config/homepage/logs/` (genegeerd door `.gitignore`). +Application logs are outside git: `config/homepage/logs/` (see `.gitignore`). -## Container inventory bijwerken +## Refresh container inventory ```bash ./scripts/deploy/export-inventory.sh git add inventory/ && git commit -m "chore: refresh container inventory" ``` -## Stack herstarten +## Restart full docker01 stack ```bash ./scripts/deploy/deploy-atc-docker01.sh ``` -## Git op de server +## SSH between hosts -Remote staat op `atc-docker01` in `/root/lakehouse`. Gebruik je eigen credentials voor push naar Forgejo op `atc-mgt01`. +```bash +ssh -i /root/.ssh/atc_cluster root@atc-lake01 +``` + +See [ssh-mesh.md](ssh-mesh.md). + +## Git on the server + +Clone lives at `/root/lakehouse` on `atc-docker01`. Use your Forgejo credentials to push to `atc-mgt01`. diff --git a/docs/ssh-mesh.md b/docs/ssh-mesh.md new file mode 100644 index 0000000..c2897b6 --- /dev/null +++ b/docs/ssh-mesh.md @@ -0,0 +1,69 @@ +# SSH mesh — passwordless access + +All ATC lab VMs share a cluster SSH key so root can hop between hosts without passwords. + +## Design + +| Item | Path / value | +|------|----------------| +| Shared private key | `/root/.ssh/atc_cluster` (on each VM, **not** in git) | +| Shared public key | `/root/.ssh/atc_cluster.pub` | +| SSH client snippet | `/root/.ssh/config.d/99-atc-lab.conf` | +| Host aliases | `/etc/hosts` block tagged `ATC Lakehouse lab` | + +Proxmox (`pve01`) also keeps its admin key in each VM's `authorized_keys` for break-glass access. + +## Fleet (mesh enabled) + +| Hostname | IP | +|----------|-----| +| atc-docker01 | 10.0.21.45 | +| atc-docker02 | 10.0.21.47 | +| atc-mgt01 | 10.0.20.104 | +| atc-kafka01 | 10.0.21.36 | +| atc-lake01 | 10.0.21.50 | +| atc-elastic01 | 10.0.21.46 | +| atc-db01 | 10.0.20.112 | +| atc-db02 | 10.0.21.51 | +| atc-grafana | 10.0.20.103 | +| atc-airflow01 | 10.0.21.55 | +| atc-portal01 | 10.0.21.49 | +| atc-lama01 | 10.0.21.39 | + +## Deploy / refresh + +From Proxmox (or any host with root SSH to the fleet): + +```bash +cd /root/lakehouse +./scripts/setup/setup-ssh-mesh.sh +``` + +## Test + +```bash +ssh -i ~/.ssh/atc_cluster root@atc-docker01 +ssh -i ~/.ssh/atc_cluster root@atc-lake01 hostname +# or from docker01: +for h in atc-db01 atc-kafka01 atc-mgt01; do + ssh -i /root/.ssh/atc_cluster root@$h hostname -f +done +``` + +## Hosts not yet in the mesh + +These VMs did not accept the hypervisor key (different credentials or SSH policy): + +- `10.0.21.52`, `10.0.21.37`, `10.0.21.38`, `10.0.21.41` +- `10.0.20.111` (ObjectScale), `10.0.20.31`, `10.0.21.44` + +Add the cluster public key manually after fixing root access: + +```bash +cat /root/.ssh/atc_cluster.pub # from any mesh host +# paste into target:/root/.ssh/authorized_keys +``` + +## Security note + +The cluster private key is powerful. Restrict Forgejo repo access and rotate keys if the lab is exposed outside your network. diff --git a/scripts/atclab/README.md b/scripts/atclab/README.md index 1eb0eca..49446b7 100644 --- a/scripts/atclab/README.md +++ b/scripts/atclab/README.md @@ -1,12 +1,12 @@ -# atclab scripts (referentie) +# atclab scripts (reference) -Kopie van `/opt/atclab/bin/` op lab hosts — handige wrappers voor yum/git/syslog. +Copy of `/opt/atclab/bin/` from lab hosts — yum/git/syslog helpers. -| Script | Functie | +| Script | Purpose | |--------|---------| -| `atclab-update` | yum update van atclab repo | +| `atclab-update` | yum update from atclab repo | | `gitcommit` | `git commit -m "$@"` shortcut | | `gittag` | git tagging helper | | `syslog` / `syslogf` | syslog viewers | -Deze scripts zijn host-specifiek; pas aan per omgeving. +Host-specific; adjust per environment. diff --git a/scripts/setup/README.md b/scripts/setup/README.md new file mode 100644 index 0000000..ae4616e --- /dev/null +++ b/scripts/setup/README.md @@ -0,0 +1,7 @@ +# Setup scripts + +| Script | Purpose | +|--------|---------| +| `setup-ssh-mesh.sh` | Deploy passwordless SSH cluster key to all lab VMs | + +Run from Proxmox as root after cloning this repo. Requires existing hypervisor SSH access to targets. diff --git a/scripts/setup/setup-ssh-mesh.sh b/scripts/setup/setup-ssh-mesh.sh new file mode 100755 index 0000000..0cccbaf --- /dev/null +++ b/scripts/setup/setup-ssh-mesh.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Deploy shared ATC cluster SSH key and /etc/hosts to all reachable lab VMs. +# Run from Proxmox host (root, with SSH access to the fleet). +set -euo pipefail + +CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}" +HOSTS=( + 10.0.21.45 + 10.0.21.47 + 10.0.20.104 + 10.0.21.36 + 10.0.21.50 + 10.0.21.46 + 10.0.20.112 + 10.0.21.51 + 10.0.20.103 + 10.0.21.55 + 10.0.21.49 + 10.0.21.39 +) + +SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=8) +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts" + +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT + +scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || { + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster" + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub" +} + +PVE_PUB="" +[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub) + +for ip in "${HOSTS[@]}"; do + echo "==> $ip" + scp "${SSH_OPTS[@]}" "$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" "root@${ip}:/root/.ssh/" || { + echo " FAIL scp $ip" + continue + } + + ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s <<'REMOTE' +set -e +chmod 600 /root/.ssh/atc_cluster /root/.ssh/atc_cluster.pub +touch /root/.ssh/authorized_keys +grep -qF 'atc-lakehouse-cluster' /root/.ssh/authorized_keys 2>/dev/null || \ + cat /root/.ssh/atc_cluster.pub >> /root/.ssh/authorized_keys +REMOTE + + if [[ -n "$PVE_PUB" ]]; then + ssh "${SSH_OPTS[@]}" "root@${ip}" \ + "grep -qF '${PVE_PUB%% *}' /root/.ssh/authorized_keys 2>/dev/null || echo '$PVE_PUB' >> /root/.ssh/authorized_keys" + fi + + ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s </dev/null && \ + sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true +echo '# ATC Lakehouse lab (managed by Lakehouse git)' >> /etc/hosts +cat >> /etc/hosts <<'HOSTS' +$(grep -v '^#' "$HOSTS_FILE" | grep -v '^$') +HOSTS +mkdir -p /root/.ssh/config.d +cat > /root/.ssh/config.d/99-atc-lab.conf <<'CFG' +Host atc-* *.dell-atc.lan pve01 proxmox + User root + IdentityFile ~/.ssh/atc_cluster + StrictHostKeyChecking accept-new + ConnectTimeout 5 +CFG +grep -q 'config.d' /root/.ssh/config 2>/dev/null || \ + printf '%s\n' 'Include config.d/*.conf' > /root/.ssh/config +chmod 600 /root/.ssh/config /root/.ssh/config.d/99-atc-lab.conf 2>/dev/null || true +grep -q '^PubkeyAuthentication yes' /etc/ssh/sshd_config 2>/dev/null || \ + echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config +systemctl reload sshd 2>/dev/null || service sshd reload 2>/dev/null || true +hostname -f +REMOTE +done + +echo "" +echo "==> Mesh test from atc-docker01" +ssh "${SSH_OPTS[@]}" -i "$TMPDIR/atc_cluster" root@10.0.21.45 ' + ok=0 fail=0 + for t in atc-db01 atc-db02 atc-kafka01 atc-lake01 atc-elastic01 atc-mgt01 atc-grafana; do + if ssh -i /root/.ssh/atc_cluster -o BatchMode=yes -o ConnectTimeout=4 root@${t} hostname -f 2>/dev/null; then + ok=$((ok+1)) + else + echo "FAIL ${t}" + fail=$((fail+1)) + fi + done + echo "OK=${ok} FAIL=${fail}" +' + +echo "Done. See docs/ssh-mesh.md"