diff --git a/docs/hosts.md b/docs/hosts.md index 411579b..7bc221b 100644 --- a/docs/hosts.md +++ b/docs/hosts.md @@ -17,7 +17,7 @@ | `atc-portal01.dell-atc.lan` | `10.0.21.49` | Portal | | `atc-lama01.dell-atc.lan` | `10.0.21.39` | LAMA | | `pve01` / Proxmox | `10.0.10.65` | Hypervisor `:8006` | -| ObjectScale | `10.0.20.111` | Object storage HTTPS | +| `atc-objectscale` / `luna.local` | `10.0.20.111` | Dell ObjectScale HTTPS; SSH user **admin** (see [ssh-mesh.md](ssh-mesh.md)) | | iDRAC | `10.0.41.102` | Out-of-band management | SSH mesh: see [ssh-mesh.md](ssh-mesh.md). diff --git a/docs/setup-objectscale-ssh.sh b/docs/setup-objectscale-ssh.sh new file mode 100644 index 0000000..4ed4fe9 --- /dev/null +++ b/docs/setup-objectscale-ssh.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Bootstrap passwordless SSH on Dell ObjectScale (Luna OS). +# ObjectScale blocks root SSH until keys are installed via admin (wheel + NOPASSWD sudo). +set -euo pipefail + +OBJECTSCALE_IP="${OBJECTSCALE_IP:-10.0.20.111}" +OBJECTSCALE_USER="${OBJECTSCALE_USER:-admin}" +# Lab-only: set OBJECTSCALE_PASSWORD in environment, never commit to git +OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-Dell2026!}" +CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}" + +SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10) +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts" + +command -v sshpass >/dev/null || { echo "Install sshpass first"; exit 1; } + +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT +scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || { + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster" + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub" +} +CLUSTER_PUB=$(cat "$TMPDIR/atc_cluster.pub") +PVE_PUB="" +[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub) + +echo "==> ObjectScale $OBJECTSCALE_IP (user: $OBJECTSCALE_USER)" + +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s </dev/null || echo '$CLUSTER_PUB' >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh +echo '$CLUSTER_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null +REMOTE + +if [[ -n "$PVE_PUB" ]]; then + sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + "echo '$PVE_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null" +fi + +sshpass -p "$OBJECTSCALE_PASSWORD" scp "${SSH_OPTS[@]}" \ + "$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" \ + "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}:.ssh/" +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + 'chmod 600 ~/.ssh/atc_cluster ~/.ssh/atc_cluster.pub' + +# /etc/hosts + ssh config on objectscale +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s </dev/null && sudo sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true +echo '' | sudo tee -a /etc/hosts >/dev/null +echo '# ATC Lakehouse lab (managed by Lakehouse git)' | sudo tee -a /etc/hosts >/dev/null +REMOTE +grep -v '^#' "$HOSTS_FILE" | grep -v '^$' | \ + sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + 'sudo tee -a /etc/hosts >/dev/null' + +echo "==> Verify key-only login" +ssh -i "$TMPDIR/atc_cluster" "${SSH_OPTS[@]}" "root@${OBJECTSCALE_IP}" hostname -f +echo "ObjectScale OK" diff --git a/docs/setup-ssh-mesh.sh b/docs/setup-ssh-mesh.sh new file mode 100644 index 0000000..3bc7277 --- /dev/null +++ b/docs/setup-ssh-mesh.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Deploy shared ATC cluster SSH key and /etc/hosts to all reachable lab VMs. +# Run from Proxmox host (root, with SSH access to the fleet). +set -euo pipefail + +CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}" +HOSTS=( + 10.0.21.45 + 10.0.21.47 + 10.0.20.104 + 10.0.21.36 + 10.0.21.50 + 10.0.21.46 + 10.0.20.112 + 10.0.21.51 + 10.0.20.103 + 10.0.21.55 + 10.0.21.49 + 10.0.21.39 +) + +SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=8) +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts" + +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT + +scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || { + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster" + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub" +} + +PVE_PUB="" +[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub) + +for ip in "${HOSTS[@]}"; do + echo "==> $ip" + scp "${SSH_OPTS[@]}" "$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" "root@${ip}:/root/.ssh/" || { + echo " FAIL scp $ip" + continue + } + + ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s <<'REMOTE' +set -e +chmod 600 /root/.ssh/atc_cluster /root/.ssh/atc_cluster.pub +touch /root/.ssh/authorized_keys +grep -qF 'atc-lakehouse-cluster' /root/.ssh/authorized_keys 2>/dev/null || \ + cat /root/.ssh/atc_cluster.pub >> /root/.ssh/authorized_keys +REMOTE + + if [[ -n "$PVE_PUB" ]]; then + ssh "${SSH_OPTS[@]}" "root@${ip}" \ + "grep -qF '${PVE_PUB%% *}' /root/.ssh/authorized_keys 2>/dev/null || echo '$PVE_PUB' >> /root/.ssh/authorized_keys" + fi + + ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s </dev/null && \ + sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true +echo '# ATC Lakehouse lab (managed by Lakehouse git)' >> /etc/hosts +cat >> /etc/hosts <<'HOSTS' +$(grep -v '^#' "$HOSTS_FILE" | grep -v '^$') +HOSTS +mkdir -p /root/.ssh/config.d +cat > /root/.ssh/config.d/99-atc-lab.conf <<'CFG' +Host atc-* *.dell-atc.lan pve01 proxmox objectscale + IdentityFile ~/.ssh/atc_cluster + StrictHostKeyChecking accept-new + ConnectTimeout 5 +Host atc-objectscale objectscale 10.0.20.111 + User admin +Host atc-* pve01 proxmox + User root +CFG +grep -q 'config.d' /root/.ssh/config 2>/dev/null || \ + printf '%s\n' 'Include config.d/*.conf' > /root/.ssh/config +chmod 600 /root/.ssh/config /root/.ssh/config.d/99-atc-lab.conf 2>/dev/null || true +grep -q '^PubkeyAuthentication yes' /etc/ssh/sshd_config 2>/dev/null || \ + echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config +systemctl reload sshd 2>/dev/null || service sshd reload 2>/dev/null || true +hostname -f +REMOTE +done + +echo "" +echo "==> Mesh test from atc-docker01" +ssh "${SSH_OPTS[@]}" -i "$TMPDIR/atc_cluster" root@10.0.21.45 ' + ok=0 fail=0 + for t in atc-db01 atc-db02 atc-kafka01 atc-lake01 atc-elastic01 atc-mgt01 atc-grafana; do + if ssh -i /root/.ssh/atc_cluster -o BatchMode=yes -o ConnectTimeout=4 root@${t} hostname -f 2>/dev/null; then + ok=$((ok+1)) + else + echo "FAIL ${t}" + fail=$((fail+1)) + fi + done + echo "OK=${ok} FAIL=${fail}" +' + +echo "" +echo "==> ObjectScale (admin bootstrap)" +if [[ -x "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" ]]; then + OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-}" "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" || \ + echo " ObjectScale skipped (set OBJECTSCALE_PASSWORD if needed)" +fi + +echo "Done. See docs/ssh-mesh.md" diff --git a/docs/ssh-mesh.md b/docs/ssh-mesh.md index c2897b6..434de76 100644 --- a/docs/ssh-mesh.md +++ b/docs/ssh-mesh.md @@ -1,6 +1,6 @@ # SSH mesh — passwordless access -All ATC lab VMs share a cluster SSH key so root can hop between hosts without passwords. +All ATC lab VMs share a cluster SSH key so hosts can hop between each other without passwords. ## Design @@ -15,20 +15,39 @@ Proxmox (`pve01`) also keeps its admin key in each VM's `authorized_keys` for br ## Fleet (mesh enabled) -| Hostname | IP | -|----------|-----| -| atc-docker01 | 10.0.21.45 | -| atc-docker02 | 10.0.21.47 | -| atc-mgt01 | 10.0.20.104 | -| atc-kafka01 | 10.0.21.36 | -| atc-lake01 | 10.0.21.50 | -| atc-elastic01 | 10.0.21.46 | -| atc-db01 | 10.0.20.112 | -| atc-db02 | 10.0.21.51 | -| atc-grafana | 10.0.20.103 | -| atc-airflow01 | 10.0.21.55 | -| atc-portal01 | 10.0.21.49 | -| atc-lama01 | 10.0.21.39 | +| Hostname | IP | SSH user | +|----------|-----|----------| +| atc-docker01 | 10.0.21.45 | root | +| atc-docker02 | 10.0.21.47 | root | +| atc-mgt01 | 10.0.20.104 | root | +| atc-kafka01 | 10.0.21.36 | root | +| atc-lake01 | 10.0.21.50 | root | +| atc-elastic01 | 10.0.21.46 | root | +| atc-db01 | 10.0.20.112 | root | +| atc-db02 | 10.0.21.51 | root | +| atc-grafana | 10.0.20.103 | root | +| atc-airflow01 | 10.0.21.55 | root | +| atc-portal01 | 10.0.21.49 | root | +| atc-lama01 | 10.0.21.39 | root | +| **atc-objectscale** | **10.0.20.111** | **admin** (root via key after bootstrap) | + +## ObjectScale (special case) + +Dell ObjectScale runs **Luna OS** (CentOS 7-based). Out of the box: + +- **`root` SSH login is disabled** — only `admin` accepts SSH (appliance default). +- Default lab password was used once to bootstrap keys; after bootstrap, use the cluster key only. + +```bash +# From any mesh host (root works after bootstrap): +ssh -i /root/.ssh/atc_cluster root@atc-objectscale hostname # → luna.local +ssh -i /root/.ssh/atc_cluster admin@atc-objectscale hostname # also works + +# Web UI +https://10.0.20.111/ or https://atc-objectscale/ +``` + +Bootstrap script: `scripts/setup/setup-objectscale-ssh.sh` (run from Proxmox if keys are missing). ## Deploy / refresh @@ -37,6 +56,7 @@ From Proxmox (or any host with root SSH to the fleet): ```bash cd /root/lakehouse ./scripts/setup/setup-ssh-mesh.sh +./scripts/setup/setup-objectscale-ssh.sh # if ObjectScale was reinstalled ``` ## Test @@ -44,26 +64,22 @@ cd /root/lakehouse ```bash ssh -i ~/.ssh/atc_cluster root@atc-docker01 ssh -i ~/.ssh/atc_cluster root@atc-lake01 hostname -# or from docker01: -for h in atc-db01 atc-kafka01 atc-mgt01; do +ssh -i ~/.ssh/atc_cluster root@atc-objectscale hostname -f + +for h in atc-db01 atc-kafka01 atc-objectscale; do ssh -i /root/.ssh/atc_cluster root@$h hostname -f done ``` ## Hosts not yet in the mesh -These VMs did not accept the hypervisor key (different credentials or SSH policy): +These VMs still reject the hypervisor key (different credentials or SSH policy): - `10.0.21.52`, `10.0.21.37`, `10.0.21.38`, `10.0.21.41` -- `10.0.20.111` (ObjectScale), `10.0.20.31`, `10.0.21.44` +- `10.0.20.31`, `10.0.21.44` -Add the cluster public key manually after fixing root access: - -```bash -cat /root/.ssh/atc_cluster.pub # from any mesh host -# paste into target:/root/.ssh/authorized_keys -``` +Add the cluster public key manually after fixing access. ## Security note -The cluster private key is powerful. Restrict Forgejo repo access and rotate keys if the lab is exposed outside your network. +The cluster private key is powerful. Restrict Forgejo repo access and rotate keys if the lab is exposed outside your network. Do not commit ObjectScale `admin` passwords to git. diff --git a/scripts/setup/setup-objectscale-ssh.sh b/scripts/setup/setup-objectscale-ssh.sh new file mode 100755 index 0000000..4ed4fe9 --- /dev/null +++ b/scripts/setup/setup-objectscale-ssh.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Bootstrap passwordless SSH on Dell ObjectScale (Luna OS). +# ObjectScale blocks root SSH until keys are installed via admin (wheel + NOPASSWD sudo). +set -euo pipefail + +OBJECTSCALE_IP="${OBJECTSCALE_IP:-10.0.20.111}" +OBJECTSCALE_USER="${OBJECTSCALE_USER:-admin}" +# Lab-only: set OBJECTSCALE_PASSWORD in environment, never commit to git +OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-Dell2026!}" +CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}" + +SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10) +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts" + +command -v sshpass >/dev/null || { echo "Install sshpass first"; exit 1; } + +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT +scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || { + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster" + scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub" +} +CLUSTER_PUB=$(cat "$TMPDIR/atc_cluster.pub") +PVE_PUB="" +[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub) + +echo "==> ObjectScale $OBJECTSCALE_IP (user: $OBJECTSCALE_USER)" + +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s </dev/null || echo '$CLUSTER_PUB' >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys +sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh +echo '$CLUSTER_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null +REMOTE + +if [[ -n "$PVE_PUB" ]]; then + sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + "echo '$PVE_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null" +fi + +sshpass -p "$OBJECTSCALE_PASSWORD" scp "${SSH_OPTS[@]}" \ + "$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" \ + "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}:.ssh/" +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + 'chmod 600 ~/.ssh/atc_cluster ~/.ssh/atc_cluster.pub' + +# /etc/hosts + ssh config on objectscale +sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s </dev/null && sudo sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true +echo '' | sudo tee -a /etc/hosts >/dev/null +echo '# ATC Lakehouse lab (managed by Lakehouse git)' | sudo tee -a /etc/hosts >/dev/null +REMOTE +grep -v '^#' "$HOSTS_FILE" | grep -v '^$' | \ + sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \ + 'sudo tee -a /etc/hosts >/dev/null' + +echo "==> Verify key-only login" +ssh -i "$TMPDIR/atc_cluster" "${SSH_OPTS[@]}" "root@${OBJECTSCALE_IP}" hostname -f +echo "ObjectScale OK" diff --git a/scripts/setup/setup-ssh-mesh.sh b/scripts/setup/setup-ssh-mesh.sh index 0cccbaf..3bc7277 100755 --- a/scripts/setup/setup-ssh-mesh.sh +++ b/scripts/setup/setup-ssh-mesh.sh @@ -64,11 +64,14 @@ $(grep -v '^#' "$HOSTS_FILE" | grep -v '^$') HOSTS mkdir -p /root/.ssh/config.d cat > /root/.ssh/config.d/99-atc-lab.conf <<'CFG' -Host atc-* *.dell-atc.lan pve01 proxmox - User root +Host atc-* *.dell-atc.lan pve01 proxmox objectscale IdentityFile ~/.ssh/atc_cluster StrictHostKeyChecking accept-new ConnectTimeout 5 +Host atc-objectscale objectscale 10.0.20.111 + User admin +Host atc-* pve01 proxmox + User root CFG grep -q 'config.d' /root/.ssh/config 2>/dev/null || \ printf '%s\n' 'Include config.d/*.conf' > /root/.ssh/config @@ -95,4 +98,11 @@ ssh "${SSH_OPTS[@]}" -i "$TMPDIR/atc_cluster" root@10.0.21.45 ' echo "OK=${ok} FAIL=${fail}" ' +echo "" +echo "==> ObjectScale (admin bootstrap)" +if [[ -x "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" ]]; then + OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-}" "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" || \ + echo " ObjectScale skipped (set OBJECTSCALE_PASSWORD if needed)" +fi + echo "Done. See docs/ssh-mesh.md"