Add Airflow and Kafka broker configs to git
- config/airflow/airflow.cfg (fernet/API keys redacted) + live DAG sync - config/kafka/kraft-server.properties (live KRaft broker on kafka01) - Extend collect-fleet-config.sh for airflow/kafka collection - Add docs/recommendations.md with prioritized next steps
This commit is contained in:
Lakehouse Admin
@@ -1,14 +1,35 @@
|
||||
# Airflow
|
||||
# Airflow — atc-airflow01
|
||||
|
||||
| Bestand | Beschrijving |
|
||||
|---------|--------------|
|
||||
| `airflow.cfg` | Referentie-configuratie (deploy naar Airflow home op lake01) |
|
||||
| `generate_data_dag.py` | DAG voor demo data-generatie |
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Host | `atc-airflow01` / `10.0.21.55` |
|
||||
| URL | http://atc-airflow01:8080/ or http://10.0.21.55:8080/ |
|
||||
| Home | `/root/airflow/` |
|
||||
| Config | `/root/airflow/airflow.cfg` |
|
||||
| DAGs | `/root/airflow/dags/` |
|
||||
| DB | SQLite (`airflow.db` — not in git) |
|
||||
|
||||
## URL
|
||||
## Files in git
|
||||
|
||||
http://atc-lake01.dell-atc.lan:8080/ (of `10.0.21.55:8080` volgens netwerk)
|
||||
| File | Notes |
|
||||
|------|-------|
|
||||
| `airflow.cfg` | Reference copy; `fernet_key` and `internal_api_secret_key` redacted |
|
||||
| `generate_data_dag.py` | Synced from live DAGs folder |
|
||||
| `../airflow/airflow.cfg` | Older path (legacy); prefer this directory |
|
||||
|
||||
## Deploy
|
||||
## Restore
|
||||
|
||||
Kopieer `generate_data_dag.py` naar de Airflow `dags/` folder op de lake-host en herstart de scheduler/webserver.
|
||||
```bash
|
||||
# On atc-airflow01
|
||||
mkdir -p /root/airflow/dags
|
||||
cp airflow.cfg /root/airflow/
|
||||
cp generate_data_dag.py /root/airflow/dags/
|
||||
# Set real fernet_key in airflow.cfg or via env
|
||||
systemctl restart airflow-webserver airflow-scheduler # if using systemd
|
||||
```
|
||||
|
||||
## Refresh
|
||||
|
||||
```bash
|
||||
./scripts/collect/collect-fleet-config.sh
|
||||
```
|
||||
|
||||
@@ -169,7 +169,7 @@ execute_tasks_new_python_interpreter = False
|
||||
#
|
||||
# Variable: AIRFLOW__CORE__FERNET_KEY
|
||||
#
|
||||
fernet_key = f4n7uT66HPrl8yxqKHVtme_qINN9YlZ1xUQcZy761MY=
|
||||
fernet_key = REDACTED
|
||||
|
||||
# Whether to disable pickling dags
|
||||
#
|
||||
@@ -445,7 +445,7 @@ database_access_isolation = False
|
||||
#
|
||||
# Variable: AIRFLOW__CORE__INTERNAL_API_SECRET_KEY
|
||||
#
|
||||
internal_api_secret_key = 010RL08807/JBjH4cWzNaw==
|
||||
internal_api_secret_key = REDACTED
|
||||
|
||||
# The ability to allow testing connections across Airflow UI, API and CLI.
|
||||
# Supported options: ``Disabled``, ``Enabled``, ``Hidden``. Default: Disabled
|
||||
@@ -1193,7 +1193,7 @@ enable_swagger_ui = True
|
||||
#
|
||||
# Variable: AIRFLOW__API__SECRET_KEY
|
||||
#
|
||||
secret_key = 010RL08807/JBjH4cWzNaw==
|
||||
secret_key = REDACTED
|
||||
|
||||
# Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values
|
||||
# except those that have security implications. ``True`` shows all values. ``False`` hides the
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pull live configs from ATC fleet into the Lakehouse git repo.
|
||||
# Run on atc-docker01 as root (requires SSH mesh).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${REPO:-/root/lakehouse}"
|
||||
KEY="${KEY:-/root/.ssh/atc_cluster}"
|
||||
SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10)
|
||||
|
||||
redact_airflow() {
|
||||
sed -E \
|
||||
-e 's/^(fernet_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(internal_api_secret_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(secret_key = ).*/\1REDACTED/'
|
||||
}
|
||||
|
||||
echo "==> db02 docker-compose"
|
||||
"${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml"
|
||||
|
||||
echo "==> objectscale deploy.yml (redacted)"
|
||||
"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | \
|
||||
sed -E 's/(ssh_password|ansible_become_pass):.*/\1: "REDACTED"/' \
|
||||
> "$REPO/config/objectscale/deploy.yml"
|
||||
|
||||
echo "==> elasticsearch.yml"
|
||||
"${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml"
|
||||
|
||||
echo "==> kafka kraft-server.properties"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" \
|
||||
> "$REPO/config/kafka/kraft-server.properties"
|
||||
|
||||
echo "==> kafka server.properties (ZK template, reference)"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" \
|
||||
> "$REPO/config/kafka/server.properties"
|
||||
|
||||
echo "==> airflow.cfg (secrets redacted)"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact_airflow \
|
||||
> "$REPO/config/airflow/airflow.cfg"
|
||||
|
||||
echo "==> airflow generate_data_dag.py"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" \
|
||||
> "$REPO/config/airflow/generate_data_dag.py"
|
||||
|
||||
echo "==> container inventory (docker01)"
|
||||
"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true
|
||||
|
||||
echo "Done. Review: cd $REPO && git diff"
|
||||
+22
-17
@@ -1,25 +1,30 @@
|
||||
# Kafka Connect — Debezium connectors
|
||||
# Kafka — atc-kafka01
|
||||
|
||||
JSON-definities voor CDC van databases naar Kafka.
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Host | `atc-kafka01` / `10.0.21.36` |
|
||||
| Mode | **KRaft** (no ZooKeeper) |
|
||||
| Broker | `10.0.21.36:9092` |
|
||||
| Controller | `localhost:9093` |
|
||||
| UI | Kafka UI Docker — port `9000` |
|
||||
| Data dir | `/home/kafka/data` |
|
||||
| Systemd | `kafka.service` → `kraft/server.properties` |
|
||||
|
||||
## Connectors
|
||||
## Files
|
||||
|
||||
| Bestand | Bron | Topic prefix |
|
||||
|---------|------|--------------|
|
||||
| `postgres-connector.json` | PostgreSQL op db02 | `postgres-sales` |
|
||||
| `mysql-connector.json` | MySQL | — |
|
||||
| `mongodb-connector.json` | MongoDB | — |
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `kraft-server.properties` | **Live** broker config (systemd) |
|
||||
| `server.properties` | Legacy ZK template (not used by current service) |
|
||||
|
||||
## Deploy
|
||||
## Connect
|
||||
|
||||
Vervang `PASSWORD_PLACEHOLDER` en pas host/poort aan. Registreer via Kafka Connect REST API op `atc-lake01` (Debezium `:8083`).
|
||||
Debezium Connect runs on **atc-lake01:8083** (`BOOTSTRAP_SERVERS=10.0.21.36:9092`).
|
||||
|
||||
Connector JSON: `../kafka/*.json` (postgres, mysql, mongodb).
|
||||
|
||||
## Refresh
|
||||
|
||||
```bash
|
||||
curl -X POST -H "Content-Type: application/json" \
|
||||
--data @postgres-connector.json \
|
||||
http://atc-lake01.dell-atc.lan:8083/connectors/
|
||||
./scripts/collect/collect-fleet-config.sh
|
||||
```
|
||||
|
||||
## Referentie
|
||||
|
||||
- Kafka UI: http://atc-kafka01.dell-atc.lan:9000/
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# Required KRaft roles
|
||||
process.roles=broker,controller
|
||||
node.id=1
|
||||
controller.quorum.voters=1@localhost:9093
|
||||
|
||||
# Listeners
|
||||
listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
|
||||
controller.listener.names=CONTROLLER
|
||||
inter.broker.listener.name=PLAINTEXT
|
||||
advertised.listeners=PLAINTEXT://10.0.21.36:9092
|
||||
listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
||||
|
||||
# Storage - use a clean directory
|
||||
log.dirs=/home/kafka/data
|
||||
|
||||
# Topic defaults
|
||||
num.partitions=3
|
||||
default.replication.factor=1
|
||||
|
||||
# Retention (7 days)
|
||||
log.retention.hours=168
|
||||
log.retention.check.interval.ms=300000
|
||||
log.segment.bytes=1073741824
|
||||
|
||||
# Performance (8 vCPUs)
|
||||
num.network.threads=8
|
||||
num.io.threads=8
|
||||
socket.send.buffer.bytes=102400
|
||||
socket.receive.buffer.bytes=102400
|
||||
socket.request.max.bytes=104857600
|
||||
|
||||
# Transaction state (single node)
|
||||
offsets.topic.replication.factor=1
|
||||
transaction.state.log.replication.factor=1
|
||||
transaction.state.log.min.isr=1
|
||||
@@ -0,0 +1,138 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# This configuration file is intended for use in ZK-based mode, where Apache ZooKeeper is required.
|
||||
# See kafka.server.KafkaConfig for additional details and defaults
|
||||
#
|
||||
|
||||
############################# Server Basics #############################
|
||||
|
||||
# The id of the broker. This must be set to a unique integer for each broker.
|
||||
broker.id=0
|
||||
|
||||
############################# Socket Server Settings #############################
|
||||
|
||||
# The address the socket server listens on. If not configured, the host name will be equal to the value of
|
||||
# java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092.
|
||||
# FORMAT:
|
||||
# listeners = listener_name://host_name:port
|
||||
# EXAMPLE:
|
||||
# listeners = PLAINTEXT://your.host.name:9092
|
||||
#listeners=PLAINTEXT://:9092
|
||||
|
||||
# Listener name, hostname and port the broker will advertise to clients.
|
||||
# If not set, it uses the value for "listeners".
|
||||
#advertised.listeners=PLAINTEXT://your.host.name:9092
|
||||
|
||||
# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
|
||||
#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
|
||||
|
||||
# The number of threads that the server uses for receiving requests from the network and sending responses to the network
|
||||
num.network.threads=3
|
||||
|
||||
# The number of threads that the server uses for processing requests, which may include disk I/O
|
||||
num.io.threads=8
|
||||
|
||||
# The send buffer (SO_SNDBUF) used by the socket server
|
||||
socket.send.buffer.bytes=102400
|
||||
|
||||
# The receive buffer (SO_RCVBUF) used by the socket server
|
||||
socket.receive.buffer.bytes=102400
|
||||
|
||||
# The maximum size of a request that the socket server will accept (protection against OOM)
|
||||
socket.request.max.bytes=104857600
|
||||
|
||||
|
||||
############################# Log Basics #############################
|
||||
|
||||
# A comma separated list of directories under which to store log files
|
||||
log.dirs=/tmp/kafka-logs
|
||||
|
||||
# The default number of log partitions per topic. More partitions allow greater
|
||||
# parallelism for consumption, but this will also result in more files across
|
||||
# the brokers.
|
||||
num.partitions=1
|
||||
|
||||
# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
|
||||
# This value is recommended to be increased for installations with data dirs located in RAID array.
|
||||
num.recovery.threads.per.data.dir=1
|
||||
|
||||
############################# Internal Topic Settings #############################
|
||||
# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
|
||||
# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
|
||||
offsets.topic.replication.factor=1
|
||||
transaction.state.log.replication.factor=1
|
||||
transaction.state.log.min.isr=1
|
||||
|
||||
############################# Log Flush Policy #############################
|
||||
|
||||
# Messages are immediately written to the filesystem but by default we only fsync() to sync
|
||||
# the OS cache lazily. The following configurations control the flush of data to disk.
|
||||
# There are a few important trade-offs here:
|
||||
# 1. Durability: Unflushed data may be lost if you are not using replication.
|
||||
# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
|
||||
# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
|
||||
# The settings below allow one to configure the flush policy to flush data after a period of time or
|
||||
# every N messages (or both). This can be done globally and overridden on a per-topic basis.
|
||||
|
||||
# The number of messages to accept before forcing a flush of data to disk
|
||||
#log.flush.interval.messages=10000
|
||||
|
||||
# The maximum amount of time a message can sit in a log before we force a flush
|
||||
#log.flush.interval.ms=1000
|
||||
|
||||
############################# Log Retention Policy #############################
|
||||
|
||||
# The following configurations control the disposal of log segments. The policy can
|
||||
# be set to delete segments after a period of time, or after a given size has accumulated.
|
||||
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
|
||||
# from the end of the log.
|
||||
|
||||
# The minimum age of a log file to be eligible for deletion due to age
|
||||
log.retention.hours=168
|
||||
|
||||
# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
|
||||
# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
|
||||
#log.retention.bytes=1073741824
|
||||
|
||||
# The maximum size of a log segment file. When this size is reached a new log segment will be created.
|
||||
#log.segment.bytes=1073741824
|
||||
|
||||
# The interval at which log segments are checked to see if they can be deleted according
|
||||
# to the retention policies
|
||||
log.retention.check.interval.ms=300000
|
||||
|
||||
############################# Zookeeper #############################
|
||||
|
||||
# Zookeeper connection string (see zookeeper docs for details).
|
||||
# This is a comma separated host:port pairs, each corresponding to a zk
|
||||
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
|
||||
# You can also append an optional chroot string to the urls to specify the
|
||||
# root directory for all kafka znodes.
|
||||
zookeeper.connect=localhost:2181
|
||||
|
||||
# Timeout in ms for connecting to zookeeper
|
||||
zookeeper.connection.timeout.ms=18000
|
||||
|
||||
|
||||
############################# Group Coordinator Settings #############################
|
||||
|
||||
# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
|
||||
# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
|
||||
# The default value for this is 3 seconds.
|
||||
# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
|
||||
# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
|
||||
group.initial.rebalance.delay.ms=0
|
||||
@@ -47,7 +47,7 @@ Compose: `config/docker/atc-lake01/docker-compose.yml`
|
||||
|-----------|-------|-------|
|
||||
| kafka-ui | provectuslabs/kafka-ui | 9000→8080 |
|
||||
|
||||
Kafka broker: native on `:9092` (not containerized in current lab).
|
||||
Kafka broker: **KRaft** native on `:9092` — config: `config/kafka/kraft-server.properties`.
|
||||
|
||||
## atc-mgt01 (10.0.20.104)
|
||||
|
||||
|
||||
+2
-2
@@ -108,9 +108,9 @@ flowchart TB
|
||||
| Spark, Trino, Debezium | Docker (lake01) | `config/docker/atc-lake01/` |
|
||||
| Source DBs | Docker (db02) | `config/docker/atc-db02/` |
|
||||
| Kafka UI | Docker (kafka01) | `config/docker/atc-kafka01/` |
|
||||
| Kafka broker | Native/systemd (kafka01) | `config/kafka/` |
|
||||
| Kafka broker | Native KRaft (kafka01) | `config/kafka/kraft-server.properties` |
|
||||
| Elasticsearch | Native (elastic01) | `config/elastic/` |
|
||||
| Airflow | Native (airflow01) | `config/airflow/` |
|
||||
| Airflow | Native (airflow01) | `config/airflow/airflow.cfg`, DAGs |
|
||||
| ObjectScale | ECS appliance | `config/objectscale/` |
|
||||
| Forgejo | Docker (mgt01) | `compose/forgejo/` |
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# Recommendations — next steps for documentation & IaC
|
||||
|
||||
Prioritized ideas to make the lab fully reproducible and operable.
|
||||
|
||||
## High priority
|
||||
|
||||
| Item | Host | Why |
|
||||
|------|------|-----|
|
||||
| **Trino catalog properties** | atc-lake01 | `etc/catalog/*.properties` — documents federated queries |
|
||||
| **Grafana `grafana.ini` + datasources** | atc-grafana | Monitoring as code |
|
||||
| **Kibana / ES keystore note** | atc-elastic01 | Passwords in keystore — document enrollment, not files |
|
||||
| **Proxmox VM notes** | pve01 | VMID → hostname → IP table (API export script) |
|
||||
| **Backup script** | docker01 | Nightly `git pull` + volume tarballs to ObjectScale |
|
||||
|
||||
## Medium priority
|
||||
|
||||
| Item | Host | Why |
|
||||
|------|------|-----|
|
||||
| **Spark jobs** | atc-lake01 | `/opt/spark-jobs/` in git |
|
||||
| **Airflow `dags/scripts/`** | atc-airflow01 | Supporting Python for DAGs |
|
||||
| **Nginx Proxy Manager** | atc-mgt01 | Export NPM config (if used for TLS) |
|
||||
| **LDAP/LDIF exports** | atc-mgt01 | `*.ldif` already on host — useful for LDAP rebuild |
|
||||
| **MinIO / S3 buckets** | objectscale | Bucket layout + IAM policy docs |
|
||||
| **Network diagram** | docs | VLAN / firewall rules (10.0.10/20/21.x) |
|
||||
|
||||
## Automation
|
||||
|
||||
| Item | Description |
|
||||
|------|-------------|
|
||||
| **CI on Forgejo** | Lint YAML, validate compose, dry-run `docker compose config` |
|
||||
| **Ansible playbook** | `ansible-playbook deploy-lakehouse.yml` from git |
|
||||
| **Health check script** | Cron: curl all `siteMonitor` URLs, alert via Grafana |
|
||||
| **Monthly collect cron** | `collect-fleet-config.sh` + auto-commit branch |
|
||||
|
||||
## Security hardening (lab → prod path)
|
||||
|
||||
- Move all secrets to `.env` / Vault; git only `*.example`
|
||||
- Rotate `atc_cluster` SSH key periodically
|
||||
- Restrict ObjectScale `management_clients` from `0.0.0.0/0`
|
||||
- Enable TLS on Kafka (`SASL_SSL`) if exposed beyond lab VLAN
|
||||
|
||||
## Dashboard enhancements
|
||||
|
||||
- Homepage widget: link to architecture diagram in header logo
|
||||
- Add **DOCS** tab with bookmarks to all `docs/*.md` on Forgejo
|
||||
- Version badge in footer (git commit SHA from build arg)
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pull live Docker/ObjectScale configs into the Lakehouse git repo.
|
||||
# Pull live configs from ATC fleet into the Lakehouse git repo.
|
||||
# Run on atc-docker01 as root (requires SSH mesh).
|
||||
set -euo pipefail
|
||||
|
||||
@@ -7,6 +7,13 @@ REPO="${REPO:-/root/lakehouse}"
|
||||
KEY="${KEY:-/root/.ssh/atc_cluster}"
|
||||
SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10)
|
||||
|
||||
redact_airflow() {
|
||||
sed -E \
|
||||
-e 's/^(fernet_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(internal_api_secret_key = ).*/\1REDACTED/' \
|
||||
-e 's/^(secret_key = ).*/\1REDACTED/'
|
||||
}
|
||||
|
||||
echo "==> db02 docker-compose"
|
||||
"${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml"
|
||||
|
||||
@@ -18,7 +25,23 @@ echo "==> objectscale deploy.yml (redacted)"
|
||||
echo "==> elasticsearch.yml"
|
||||
"${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml"
|
||||
|
||||
echo "==> container inventory"
|
||||
"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json"
|
||||
echo "==> kafka kraft-server.properties"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" \
|
||||
> "$REPO/config/kafka/kraft-server.properties"
|
||||
|
||||
echo "Done. Review and commit: cd $REPO && git diff"
|
||||
echo "==> kafka server.properties (ZK template, reference)"
|
||||
"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" \
|
||||
> "$REPO/config/kafka/server.properties"
|
||||
|
||||
echo "==> airflow.cfg (secrets redacted)"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact_airflow \
|
||||
> "$REPO/config/airflow/airflow.cfg"
|
||||
|
||||
echo "==> airflow generate_data_dag.py"
|
||||
"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" \
|
||||
> "$REPO/config/airflow/generate_data_dag.py"
|
||||
|
||||
echo "==> container inventory (docker01)"
|
||||
"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true
|
||||
|
||||
echo "Done. Review: cd $REPO && git diff"
|
||||
|
||||
Reference in New Issue
Block a user