ATC/Lakehouse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ObjectScale to SSH mesh (admin bootstrap, root via keys)

Browse Source 
ObjectScale blocks root SSH by default; only admin accepts login.
Bootstrap installs cluster key for admin and root; document in ssh-mesh.md.
This commit is contained in:
Lakehouse Admin
2026-05-19 22:50:32 +02:00
parent f3d24634e4
commit f0c56f137c
6 changed files with 285 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/hosts.md
+1 -1
View File
@@ -17,7 +17,7 @@
| `atc-portal01.dell-atc.lan` | `10.0.21.49` | Portal |
| `atc-lama01.dell-atc.lan` | `10.0.21.39` | LAMA |
| `pve01` / Proxmox | `10.0.10.65` | Hypervisor `:8006` |
| ObjectScale | `10.0.20.111` | Object storage HTTPS |
| `atc-objectscale` / `luna.local` | `10.0.20.111` | Dell ObjectScale HTTPS; SSH user **admin** (see [ssh-mesh.md](ssh-mesh.md)) |
| iDRAC | `10.0.41.102` | Out-of-band management |
SSH mesh: see [ssh-mesh.md](ssh-mesh.md).
docs/setup-objectscale-ssh.sh
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# Bootstrap passwordless SSH on Dell ObjectScale (Luna OS).
# ObjectScale blocks root SSH until keys are installed via admin (wheel + NOPASSWD sudo).
set -euo pipefail
OBJECTSCALE_IP="${OBJECTSCALE_IP:-10.0.20.111}"
OBJECTSCALE_USER="${OBJECTSCALE_USER:-admin}"
# Lab-only: set OBJECTSCALE_PASSWORD in environment, never commit to git
OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-Dell2026!}"
CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts"
command -v sshpass >/dev/null || { echo "Install sshpass first"; exit 1; }
TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || {
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster"
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub"
}
CLUSTER_PUB=$(cat "$TMPDIR/atc_cluster.pub")
PVE_PUB=""
[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub)
echo "==> ObjectScale $OBJECTSCALE_IP (user: $OBJECTSCALE_USER)"
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s <<REMOTE
set -e
grep -qF 'atc-lakehouse-cluster' ~/.ssh/authorized_keys 2>/dev/null || echo '$CLUSTER_PUB' >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh
echo '$CLUSTER_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null
REMOTE
if [[ -n "$PVE_PUB" ]]; then
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
"echo '$PVE_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null"
fi
sshpass -p "$OBJECTSCALE_PASSWORD" scp "${SSH_OPTS[@]}" \
"$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" \
"${OBJECTSCALE_USER}@${OBJECTSCALE_IP}:.ssh/"
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
'chmod 600 ~/.ssh/atc_cluster ~/.ssh/atc_cluster.pub'
# /etc/hosts + ssh config on objectscale
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s <<REMOTE
sudo grep -q 'ATC Lakehouse lab' /etc/hosts 2>/dev/null && sudo sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true
echo '' | sudo tee -a /etc/hosts >/dev/null
echo '# ATC Lakehouse lab (managed by Lakehouse git)' | sudo tee -a /etc/hosts >/dev/null
REMOTE
grep -v '^#' "$HOSTS_FILE" | grep -v '^$' | \
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
'sudo tee -a /etc/hosts >/dev/null'
echo "==> Verify key-only login"
ssh -i "$TMPDIR/atc_cluster" "${SSH_OPTS[@]}" "root@${OBJECTSCALE_IP}" hostname -f
echo "ObjectScale OK"
docs/setup-ssh-mesh.sh
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
# Deploy shared ATC cluster SSH key and /etc/hosts to all reachable lab VMs.
# Run from Proxmox host (root, with SSH access to the fleet).
set -euo pipefail
CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}"
HOSTS=(
10.0.21.45
10.0.21.47
10.0.20.104
10.0.21.36
10.0.21.50
10.0.21.46
10.0.20.112
10.0.21.51
10.0.20.103
10.0.21.55
10.0.21.49
10.0.21.39
)
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=8)
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts"
TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || {
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster"
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub"
}
PVE_PUB=""
[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub)
for ip in "${HOSTS[@]}"; do
echo "==> $ip"
scp "${SSH_OPTS[@]}" "$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" "root@${ip}:/root/.ssh/" || {
echo " FAIL scp $ip"
continue
}
ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s <<'REMOTE'
set -e
chmod 600 /root/.ssh/atc_cluster /root/.ssh/atc_cluster.pub
touch /root/.ssh/authorized_keys
grep -qF 'atc-lakehouse-cluster' /root/.ssh/authorized_keys 2>/dev/null || \
cat /root/.ssh/atc_cluster.pub >> /root/.ssh/authorized_keys
REMOTE
if [[ -n "$PVE_PUB" ]]; then
ssh "${SSH_OPTS[@]}" "root@${ip}" \
"grep -qF '${PVE_PUB%% *}' /root/.ssh/authorized_keys 2>/dev/null || echo '$PVE_PUB' >> /root/.ssh/authorized_keys"
fi
ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s <<REMOTE
set -e
grep -q 'ATC Lakehouse lab' /etc/hosts 2>/dev/null && \
sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true
echo '# ATC Lakehouse lab (managed by Lakehouse git)' >> /etc/hosts
cat >> /etc/hosts <<'HOSTS'
$(grep -v '^#' "$HOSTS_FILE" | grep -v '^$')
HOSTS
mkdir -p /root/.ssh/config.d
cat > /root/.ssh/config.d/99-atc-lab.conf <<'CFG'
Host atc-* *.dell-atc.lan pve01 proxmox objectscale
IdentityFile ~/.ssh/atc_cluster
StrictHostKeyChecking accept-new
ConnectTimeout 5
Host atc-objectscale objectscale 10.0.20.111
User admin
Host atc-* pve01 proxmox
User root
CFG
grep -q 'config.d' /root/.ssh/config 2>/dev/null || \
printf '%s\n' 'Include config.d/*.conf' > /root/.ssh/config
chmod 600 /root/.ssh/config /root/.ssh/config.d/99-atc-lab.conf 2>/dev/null || true
grep -q '^PubkeyAuthentication yes' /etc/ssh/sshd_config 2>/dev/null || \
echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config
systemctl reload sshd 2>/dev/null || service sshd reload 2>/dev/null || true
hostname -f
REMOTE
done
echo ""
echo "==> Mesh test from atc-docker01"
ssh "${SSH_OPTS[@]}" -i "$TMPDIR/atc_cluster" root@10.0.21.45 '
ok=0 fail=0
for t in atc-db01 atc-db02 atc-kafka01 atc-lake01 atc-elastic01 atc-mgt01 atc-grafana; do
if ssh -i /root/.ssh/atc_cluster -o BatchMode=yes -o ConnectTimeout=4 root@${t} hostname -f 2>/dev/null; then
ok=$((ok+1))
else
echo "FAIL ${t}"
fail=$((fail+1))
fi
done
echo "OK=${ok} FAIL=${fail}"
'
echo ""
echo "==> ObjectScale (admin bootstrap)"
if [[ -x "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" ]]; then
OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-}" "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" || \
echo " ObjectScale skipped (set OBJECTSCALE_PASSWORD if needed)"
fi
echo "Done. See docs/ssh-mesh.md"
docs/ssh-mesh.md
+42 -26
View File
@@ -1,6 +1,6 @@
# SSH mesh — passwordless access
All ATC lab VMs share a cluster SSH key so root can hop between hosts without passwords.
All ATC lab VMs share a cluster SSH key so hosts can hop between each other without passwords.
## Design
@@ -15,20 +15,39 @@ Proxmox (`pve01`) also keeps its admin key in each VM's `authorized_keys` for br
## Fleet (mesh enabled)
| Hostname | IP |
|----------|-----|
| atc-docker01 | 10.0.21.45 |
| atc-docker02 | 10.0.21.47 |
| atc-mgt01 | 10.0.20.104 |
| atc-kafka01 | 10.0.21.36 |
| atc-lake01 | 10.0.21.50 |
| atc-elastic01 | 10.0.21.46 |
| atc-db01 | 10.0.20.112 |
| atc-db02 | 10.0.21.51 |
| atc-grafana | 10.0.20.103 |
| atc-airflow01 | 10.0.21.55 |
| atc-portal01 | 10.0.21.49 |
| atc-lama01 | 10.0.21.39 |
| Hostname | IP | SSH user |
|----------|-----|----------|
| atc-docker01 | 10.0.21.45 | root |
| atc-docker02 | 10.0.21.47 | root |
| atc-mgt01 | 10.0.20.104 | root |
| atc-kafka01 | 10.0.21.36 | root |
| atc-lake01 | 10.0.21.50 | root |
| atc-elastic01 | 10.0.21.46 | root |
| atc-db01 | 10.0.20.112 | root |
| atc-db02 | 10.0.21.51 | root |
| atc-grafana | 10.0.20.103 | root |
| atc-airflow01 | 10.0.21.55 | root |
| atc-portal01 | 10.0.21.49 | root |
| atc-lama01 | 10.0.21.39 | root |
| **atc-objectscale** | **10.0.20.111** | **admin** (root via key after bootstrap) |
## ObjectScale (special case)
Dell ObjectScale runs **Luna OS** (CentOS 7-based). Out of the box:
- **`root` SSH login is disabled** — only `admin` accepts SSH (appliance default).
- Default lab password was used once to bootstrap keys; after bootstrap, use the cluster key only.
```bash
# From any mesh host (root works after bootstrap):
ssh -i /root/.ssh/atc_cluster root@atc-objectscale hostname # → luna.local
ssh -i /root/.ssh/atc_cluster admin@atc-objectscale hostname # also works
# Web UI
https://10.0.20.111/ or https://atc-objectscale/
```
Bootstrap script: `scripts/setup/setup-objectscale-ssh.sh` (run from Proxmox if keys are missing).
## Deploy / refresh
@@ -37,6 +56,7 @@ From Proxmox (or any host with root SSH to the fleet):
```bash
cd /root/lakehouse
./scripts/setup/setup-ssh-mesh.sh
./scripts/setup/setup-objectscale-ssh.sh # if ObjectScale was reinstalled
```
## Test
@@ -44,26 +64,22 @@ cd /root/lakehouse
```bash
ssh -i ~/.ssh/atc_cluster root@atc-docker01
ssh -i ~/.ssh/atc_cluster root@atc-lake01 hostname
# or from docker01:
for h in atc-db01 atc-kafka01 atc-mgt01; do
ssh -i ~/.ssh/atc_cluster root@atc-objectscale hostname -f
for h in atc-db01 atc-kafka01 atc-objectscale; do
ssh -i /root/.ssh/atc_cluster root@$h hostname -f
done
```
## Hosts not yet in the mesh
These VMs did not accept the hypervisor key (different credentials or SSH policy):
These VMs still reject the hypervisor key (different credentials or SSH policy):
- `10.0.21.52`, `10.0.21.37`, `10.0.21.38`, `10.0.21.41`
- `10.0.20.111` (ObjectScale), `10.0.20.31`, `10.0.21.44`
- `10.0.20.31`, `10.0.21.44`
Add the cluster public key manually after fixing root access:
```bash
cat /root/.ssh/atc_cluster.pub # from any mesh host
# paste into target:/root/.ssh/authorized_keys
```
Add the cluster public key manually after fixing access.
## Security note
The cluster private key is powerful. Restrict Forgejo repo access and rotate keys if the lab is exposed outside your network.
The cluster private key is powerful. Restrict Forgejo repo access and rotate keys if the lab is exposed outside your network. Do not commit ObjectScale `admin` passwords to git.
scripts/setup/setup-objectscale-ssh.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# Bootstrap passwordless SSH on Dell ObjectScale (Luna OS).
# ObjectScale blocks root SSH until keys are installed via admin (wheel + NOPASSWD sudo).
set -euo pipefail
OBJECTSCALE_IP="${OBJECTSCALE_IP:-10.0.20.111}"
OBJECTSCALE_USER="${OBJECTSCALE_USER:-admin}"
# Lab-only: set OBJECTSCALE_PASSWORD in environment, never commit to git
OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-Dell2026!}"
CLUSTER_KEY_SRC="${CLUSTER_KEY_SRC:-root@10.0.21.45:/root/.ssh/atc_cluster}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10)
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
HOSTS_FILE="${REPO_ROOT}/config/hosts/atc-lab.hosts"
command -v sshpass >/dev/null || { echo "Install sshpass first"; exit 1; }
TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/" 2>/dev/null || {
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}" "$TMPDIR/atc_cluster"
scp "${SSH_OPTS[@]}" "${CLUSTER_KEY_SRC}.pub" "$TMPDIR/atc_cluster.pub"
}
CLUSTER_PUB=$(cat "$TMPDIR/atc_cluster.pub")
PVE_PUB=""
[[ -f ~/.ssh/id_rsa.pub ]] && PVE_PUB=$(cat ~/.ssh/id_rsa.pub)
echo "==> ObjectScale $OBJECTSCALE_IP (user: $OBJECTSCALE_USER)"
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s <<REMOTE
set -e
grep -qF 'atc-lakehouse-cluster' ~/.ssh/authorized_keys 2>/dev/null || echo '$CLUSTER_PUB' >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
sudo mkdir -p /root/.ssh && sudo chmod 700 /root/.ssh
echo '$CLUSTER_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null
REMOTE
if [[ -n "$PVE_PUB" ]]; then
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
"echo '$PVE_PUB' | sudo tee -a /root/.ssh/authorized_keys >/dev/null"
fi
sshpass -p "$OBJECTSCALE_PASSWORD" scp "${SSH_OPTS[@]}" \
"$TMPDIR/atc_cluster" "$TMPDIR/atc_cluster.pub" \
"${OBJECTSCALE_USER}@${OBJECTSCALE_IP}:.ssh/"
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
'chmod 600 ~/.ssh/atc_cluster ~/.ssh/atc_cluster.pub'
# /etc/hosts + ssh config on objectscale
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" bash -s <<REMOTE
sudo grep -q 'ATC Lakehouse lab' /etc/hosts 2>/dev/null && sudo sed -i '/# ATC Lakehouse lab/,/^$/d' /etc/hosts || true
echo '' | sudo tee -a /etc/hosts >/dev/null
echo '# ATC Lakehouse lab (managed by Lakehouse git)' | sudo tee -a /etc/hosts >/dev/null
REMOTE
grep -v '^#' "$HOSTS_FILE" | grep -v '^$' | \
sshpass -p "$OBJECTSCALE_PASSWORD" ssh "${SSH_OPTS[@]}" "${OBJECTSCALE_USER}@${OBJECTSCALE_IP}" \
'sudo tee -a /etc/hosts >/dev/null'
echo "==> Verify key-only login"
ssh -i "$TMPDIR/atc_cluster" "${SSH_OPTS[@]}" "root@${OBJECTSCALE_IP}" hostname -f
echo "ObjectScale OK"
scripts/setup/setup-ssh-mesh.sh
+12 -2
View File
@@ -64,11 +64,14 @@ $(grep -v '^#' "$HOSTS_FILE" | grep -v '^$')
HOSTS
mkdir -p /root/.ssh/config.d
cat > /root/.ssh/config.d/99-atc-lab.conf <<'CFG'
Host atc-* *.dell-atc.lan pve01 proxmox
User root
Host atc-* *.dell-atc.lan pve01 proxmox objectscale
IdentityFile ~/.ssh/atc_cluster
StrictHostKeyChecking accept-new
ConnectTimeout 5
Host atc-objectscale objectscale 10.0.20.111
User admin
Host atc-* pve01 proxmox
User root
CFG
grep -q 'config.d' /root/.ssh/config 2>/dev/null || \
printf '%s\n' 'Include config.d/*.conf' > /root/.ssh/config
@@ -95,4 +98,11 @@ ssh "${SSH_OPTS[@]}" -i "$TMPDIR/atc_cluster" root@10.0.21.45 '
echo "OK=${ok} FAIL=${fail}"
'
echo ""
echo "==> ObjectScale (admin bootstrap)"
if [[ -x "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" ]]; then
OBJECTSCALE_PASSWORD="${OBJECTSCALE_PASSWORD:-}" "${REPO_ROOT}/scripts/setup/setup-objectscale-ssh.sh" || \
echo " ObjectScale skipped (set OBJECTSCALE_PASSWORD if needed)"
fi
echo "Done. See docs/ssh-mesh.md"