Lakehouse/scripts/collect/populate-repo.py

#!/usr/bin/env python3
"""Populate Lakehouse git repo with full fleet configs (secrets redacted)."""
import json
import os
import re
import subprocess
import urllib.request
import ssl
from pathlib import Path

REPO = Path(os.environ.get("REPO", "/root/lakehouse"))
KEY = os.environ.get("KEY", "/root/.ssh/atc_cluster")
SSH = ["ssh", "-i", KEY, "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=15"]

PVE = "https://10.0.10.65:8006"
PVE_TOKEN = os.environ.get("PVE_TOKEN", "root@pam!homepage")
PVE_SECRET = os.environ.get("PVE_SECRET", "8890185b-0850-42b7-bab2-a690ea4dc3f1")

HOST_IPS = {
    "atc-docker01": "10.0.21.45", "atc-docker02": "10.0.21.47",
    "atc-mgt01": "10.0.20.104", "atc-kafka": "10.0.21.36", "atc-kafka01": "10.0.21.36",
    "atc-lake01": "10.0.21.50", "atc-elastic": "10.0.21.46", "atc-elastic01": "10.0.21.46",
    "atc-db01": "10.0.20.112", "atc-db02": "10.0.21.51",
    "atc-grafana": "10.0.20.103", "airflow01": "10.0.21.55", "atc-airflow01": "10.0.21.55",
    "atc-portal01": "10.0.21.49", "atc-lama": "10.0.21.39", "atc-lama01": "10.0.21.39",
    "atc-objectscale": "10.0.20.111", "portal01": "10.0.21.49",
}


def ssh(host: str, cmd: str) -> str:
    r = subprocess.run(SSH + [f"root@{host}", cmd], capture_output=True, text=True)
    if r.returncode != 0:
        raise RuntimeError(f"ssh {host}: {r.stderr[:300]}")
    return r.stdout


def redact_secrets(text: str) -> str:
    rules = [
        (r"(connection-password=).+", r"\1REDACTED"),
        (r"(connection-password = ).+", r"\1REDACTED"),
        (r"(cassandra\.password=).+", r"\1REDACTED"),
        (r"(hive\.s3\.aws-access-key=).+", r"\1REDACTED"),
        (r"(hive\.s3\.aws-secret-key=).+", r"\1REDACTED"),
        (r'(S3_ACCESS_KEY = ).+', r'\1"REDACTED"'),
        (r'(S3_SECRET_KEY = ).+', r'\1"REDACTED"'),
        (r"(admin_password = ).+", r"\1REDACTED"),
        (r"(fernet_key = ).+", r"\1REDACTED"),
        (r"(internal_api_secret_key = ).+", r"\1REDACTED"),
        (r"(ssh_password|ansible_become_pass):.*", r"\1: REDACTED"),
    ]
    for pat, repl in rules:
        text = re.sub(pat, repl, text)
    return text


def write(path: Path, content: str):
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content)
    print(f"  wrote {path.relative_to(REPO)}")


def collect_trino():
    d = REPO / "config/trino/catalog"
    files = ssh("atc-lake01", "docker exec trino ls /etc/trino/catalog").split()
    for f in files:
        if not f.endswith(".properties"):
            continue
        raw = ssh("atc-lake01", f"docker exec trino cat /etc/trino/catalog/{f}")
        write(d / f, redact_secrets(raw))


def collect_grafana():
    d = REPO / "config/grafana"
    write(d / "grafana.ini", redact_secrets(ssh("atc-grafana", "cat /etc/grafana/grafana.ini")))
    write(d / "provisioning/dashboards.yml",
          ssh("atc-grafana", "cat /etc/grafana/provisioning/dashboards/dashboards.yml"))


def collect_spark_jobs():
    d = REPO / "config/spark-jobs"
    for path in ssh("atc-lake01", "ls /opt/spark-jobs/*.py 2>/dev/null").split():
        name = os.path.basename(path.strip())
        if name:
            write(d / name, redact_secrets(ssh("atc-lake01", f"cat /opt/spark-jobs/{name}")))


def collect_ldap():
    d = REPO / "config/ldap"
    for f in ["autofs.ldif", "base.ldif", "db.ldif", "openssh.ldif", "setacl.ldif"]:
        try:
            write(d / f, ssh("atc-mgt01", f"cat /root/{f}"))
        except Exception as e:
            print(f"  skip {f}: {e}")


def collect_npm():
    write(REPO / "config/mgt01/npm-compose.yaml",
          ssh("atc-mgt01", "cat /srv/docker/npm/compose.yaml"))


def collect_airflow_scripts():
    dest = REPO / "config/airflow/dags/scripts"
    for path in ssh("atc-airflow01", "ls /root/airflow/dags/scripts/*.py 2>/dev/null").split():
        name = os.path.basename(path.strip())
        if name:
            write(dest / name, ssh("atc-airflow01", f"cat /root/airflow/dags/scripts/{name}"))


def proxmox_inventory():
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    req = urllib.request.Request(
        f"{PVE}/api2/json/cluster/resources?type=vm",
        headers={"Authorization": f"PVEAPIToken={PVE_TOKEN}={PVE_SECRET}"},
    )
    with urllib.request.urlopen(req, context=ctx) as r:
        vms = json.load(r)["data"]
    out = []
    for v in sorted(vms, key=lambda x: x.get("name", "")):
        if v.get("type") != "qemu":
            continue
        name = v.get("name", "")
        ip = HOST_IPS.get(name, "")
        out.append({
            "vmid": v.get("vmid"), "name": name, "status": v.get("status"),
            "node": v.get("node"), "ip": ip or None,
            "maxmem_gb": round(v.get("maxmem", 0) / 1e9, 1),
        })
    write(REPO / "inventory/proxmox-vms.json", json.dumps(out, indent=2))


def main():
    collect_trino()
    collect_grafana()
    collect_spark_jobs()
    collect_ldap()
    collect_npm()
    collect_airflow_scripts()
    proxmox_inventory()


if __name__ == "__main__":
    main()