ATC/Lakehouse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full lab documentation and infrastructure as code

Browse Source 
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose
- Airflow DAG scripts, Proxmox VM inventory, network docs
- Ansible playbook, Gitea CI validate workflow
- Backup and health-check scripts, cron documentation
- Homepage DOCS tab with links to all documentation
- Extended collect-fleet-config.sh and populate-repo.py
This commit is contained in:
Lakehouse Admin
2026-05-19 23:12:41 +02:00
parent 3c8993a41e
commit df5ec93dc3
46 changed files with 1809 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/collect/collect-fleet-config.sh
+23 -29
View File
@@ -1,47 +1,41 @@
#!/usr/bin/env bash
# Pull live configs from ATC fleet into the Lakehouse git repo.
# Run on atc-docker01 as root (requires SSH mesh).
# Pull live configs from entire ATC fleet into git.
set -euo pipefail
REPO="${REPO:-/root/lakehouse}"
KEY="${KEY:-/root/.ssh/atc_cluster}"
SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10)
SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=15)
redact_airflow() {
redact() {
sed -E \
-e 's/^(fernet_key = ).*/\1REDACTED/' \
-e 's/^(internal_api_secret_key = ).*/\1REDACTED/' \
-e 's/^(secret_key = ).*/\1REDACTED/'
-e 's/^(fernet_key|internal_api_secret_key|admin_password) = .*/\1 = REDACTED/' \
-e 's/^(connection-password=).*/\1REDACTED/' \
-e 's/(hive\.s3\.aws-access-key=).*/\1REDACTED/' \
-e 's/(hive\.s3\.aws-secret-key=).*/\1REDACTED/' \
-e 's/(S3_ACCESS_KEY|S3_SECRET_KEY) = .*/\1 = "REDACTED"/' \
-e 's/(ssh_password|ansible_become_pass):.*/\1: REDACTED/'
}
echo "==> Python fleet collector (Trino, Grafana, Spark, LDAP, NPM, Proxmox)"
python3 "$REPO/scripts/collect/populate-repo.py"
echo "==> db02 docker-compose"
"${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml"
echo "==> objectscale deploy.yml (redacted)"
"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | \
sed -E 's/(ssh_password|ansible_become_pass):.*/\1: "REDACTED"/' \
> "$REPO/config/objectscale/deploy.yml"
echo "==> objectscale"
"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | redact > "$REPO/config/objectscale/deploy.yml"
echo "==> elasticsearch.yml"
echo "==> elasticsearch"
"${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml"
echo "==> kafka kraft-server.properties"
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" \
> "$REPO/config/kafka/kraft-server.properties"
echo "==> kafka"
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" > "$REPO/config/kafka/kraft-server.properties"
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" > "$REPO/config/kafka/server.properties"
echo "==> kafka server.properties (ZK template, reference)"
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" \
> "$REPO/config/kafka/server.properties"
echo "==> airflow"
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact > "$REPO/config/airflow/airflow.cfg"
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" > "$REPO/config/airflow/generate_data_dag.py"
echo "==> airflow.cfg (secrets redacted)"
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact_airflow \
> "$REPO/config/airflow/airflow.cfg"
echo "==> airflow generate_data_dag.py"
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" \
> "$REPO/config/airflow/generate_data_dag.py"
echo "==> container inventory (docker01)"
echo "==> docker01 inventory"
"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true
echo "Done. Review: cd $REPO && git diff"
echo "Done. git diff && git commit"