Add Airflow and Kafka broker configs to git
- config/airflow/airflow.cfg (fernet/API keys redacted) + live DAG sync - config/kafka/kraft-server.properties (live KRaft broker on kafka01) - Extend collect-fleet-config.sh for airflow/kafka collection - Add docs/recommendations.md with prioritized next steps
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pull live configs from ATC fleet into the Lakehouse git repo.
|
||||
# Run on atc-docker01 as root (requires SSH mesh).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${REPO:-/root/lakehouse}"
|
||||
KEY="${KEY:-/root/.ssh/atc_cluster}"
|
||||
SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10)
|
||||
|
||||
redact_airflow() {
|
||||
sed -E \
|
||||
-e 's/^(fernet_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(internal_api_secret_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(secret_key = ).*/\1REDACTED/'
|
||||
}
|
||||
|
||||
echo "==> db02 docker-compose"
|
||||
"${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml"
|
||||
|
||||
echo "==> objectscale deploy.yml (redacted)"
|
||||
"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | \
|
||||
sed -E 's/(ssh_password|ansible_become_pass):.*/\1: "REDACTED"/' \
|
||||
> "$REPO/config/objectscale/deploy.yml"
|
||||
|
||||
echo "==> elasticsearch.yml"
|
||||
"${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml"
|
||||
|
||||
echo "==> kafka kraft-server.properties"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" \
|
||||
> "$REPO/config/kafka/kraft-server.properties"
|
||||
|
||||
echo "==> kafka server.properties (ZK template, reference)"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" \
|
||||
> "$REPO/config/kafka/server.properties"
|
||||
|
||||
echo "==> airflow.cfg (secrets redacted)"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact_airflow \
|
||||
> "$REPO/config/airflow/airflow.cfg"
|
||||
|
||||
echo "==> airflow generate_data_dag.py"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" \
|
||||
> "$REPO/config/airflow/generate_data_dag.py"
|
||||
|
||||
echo "==> container inventory (docker01)"
|
||||
"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true
|
||||
|
||||
echo "Done. Review: cd $REPO && git diff"
|
||||
Reference in New Issue
Block a user