update (bart)

Add atc-lake01 compose with s3-kafka-consumer; redact DB passwords in connector JSON.

.gitea/workflows/validate.yml

@@ -0,0 +1,46 @@
+name: Validate Lakehouse config
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+    branches: [master, main]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate YAML syntax
+        run: |
+          python3 -c "
+          import yaml, sys
+          from pathlib import Path
+          for p in Path('.').rglob('*.yaml'):
+              if '.git' in str(p): continue
+              try:
+                  yaml.safe_load(p.read_text())
+                  print('OK', p)
+              except Exception as e:
+                  print('FAIL', p, e)
+                  sys.exit(1)
+          for p in Path('.').rglob('*.yml'):
+              if '.git' in str(p) or 'workflows' in str(p): continue
+              try:
+                  yaml.safe_load(p.read_text())
+              except Exception: pass
+          "
+
+      - name: Validate Docker Compose
+        run: |
+          for f in deploy/*.yml compose/*/*.yaml compose/*/*.yml; do
+            [ -f "$f" ] || continue
+            docker compose -f "$f" config >/dev/null && echo "OK $f"
+          done
+
+      - name: Check required docs exist
+        run: |
+          test -f docs/landscape.md
+          test -f docs/network.md
+          test -f README.md

.gitignore

@@ -24,6 +24,6 @@ __pycache__/
 **/.build/
 
 # Backup dumps (bewust buiten git)
-backup/
+/var/backups/
 *.tgz
 *.tar.gz

ansible/README.md

@@ -0,0 +1,9 @@
+# Ansible — ATC Lakehouse
+
+```bash
+# From atc-docker01 (with ansible installed)
+cd /root/lakehouse/ansible
+ansible-playbook -i inventory.ini playbook.yml
+```
+
+Requires SSH mesh (`/root/.ssh/atc_cluster`) on all targets.

ansible/inventory.ini

@@ -0,0 +1,23 @@
+[docker_hosts]
+atc-docker01 ansible_host=10.0.21.45
+atc-docker02 ansible_host=10.0.21.47
+
+[lakehouse]
+atc-kafka01 ansible_host=10.0.21.36
+atc-lake01 ansible_host=10.0.21.50
+atc-airflow01 ansible_host=10.0.21.55
+atc-elastic01 ansible_host=10.0.21.46
+
+[data]
+atc-db01 ansible_host=10.0.20.112
+atc-db02 ansible_host=10.0.21.51
+
+[management]
+atc-mgt01 ansible_host=10.0.20.104
+atc-grafana ansible_host=10.0.20.103
+atc-objectscale ansible_host=10.0.20.111 ansible_user=admin
+
+[all:vars]
+ansible_user=root
+ansible_ssh_private_key_file=/root/.ssh/atc_cluster
+ansible_ssh_common_args='-o StrictHostKeyChecking=no'

ansible/playbook.yml

@@ -0,0 +1,38 @@
+---
+# Deploy ATC Lakehouse from git (run from atc-docker01 or control node with mesh SSH)
+- name: ATC Lakehouse — docker01 stack
+  hosts: atc-docker01
+  become: true
+  tasks:
+    - name: Ensure repo present
+      git:
+        repo: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git
+        dest: /root/lakehouse
+        version: master
+        force: false
+
+    - name: Deploy homepage + RSS + icons
+      command: docker compose -f deploy/docker-compose.homepage.yml up -d --build
+      args:
+        chdir: /root/lakehouse
+
+    - name: Deploy superset
+      command: docker compose -f compose/superset/docker-compose.yaml up -d --build
+      args:
+        chdir: /root/lakehouse
+
+    - name: Revalidate homepage
+      command: docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate
+      changed_when: false
+      failed_when: false
+
+- name: Sync /etc/hosts on fleet
+  hosts: all
+  become: true
+  tasks:
+    - name: Append ATC hosts block
+      blockinfile:
+        path: /etc/hosts
+        marker: "# {mark} ATC Lakehouse lab"
+        block: "{{ lookup('file', '../config/hosts/atc-lab.hosts') }}"
+      when: ansible_user != 'admin'

compose/atc-lake01/docker-compose.yml

@@ -0,0 +1,66 @@
+name: lakehouse
+services:
+  trino:
+    image: trinodb/trino:405
+    container_name: trino
+    ports:
+      - "8089:8080"
+    volumes:
+      - /home/lakehouse/trino/catalog:/etc/trino/catalog
+      - /home/lakehouse/trino/etc/access-control.properties:/etc/trino/access-control.properties
+    deploy:
+      resources:
+        limits:
+          memory: 16G
+    environment:
+      - TRINO_USERNAME=mo
+
+  spark-master:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-master
+    ports:
+      - "8081:8080"
+      - "7077:7077"
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+    environment:
+      - SPARK_MODE=master
+      - SPARK_RPC_AUTHENTICATION_ENABLED=no
+
+  spark-worker:
+    image: bitnamilegacy/spark:latest
+    container_name: spark-worker
+    environment:
+      - SPARK_MODE=worker
+      - SPARK_MASTER_URL=spark://spark-master:7077
+      - SPARK_WORKER_MEMORY=10G
+      - SPARK_RPC_AUTHENTICATION_ENABLED=no
+    deploy:
+      resources:
+        limits:
+          memory: 12G
+
+  kafka-connect:
+    image: debezium/connect:2.5.4.Final
+    container_name: kafka-connect
+    ports:
+      - "8083:8083"
+    environment:
+      BOOTSTRAP_SERVERS: 10.0.21.36:9092
+      GROUP_ID: lakehouse-connect
+      CONFIG_STORAGE_TOPIC: connect-configs
+      OFFSET_STORAGE_TOPIC: connect-offsets
+      STATUS_STORAGE_TOPIC: connect-status
+      CONNECT_REST_ADVERTISED_HOST_NAME: 10.0.21.50
+      CONNECT_REST_PORT: 8083
+    restart: unless-stopped
+
+  s3-kafka-consumer:
+    image: python:3.11-slim
+    container_name: s3-kafka-consumer
+    restart: unless-stopped
+    volumes:
+      - /opt/spark-jobs/s3_consumer_realtime.py:/app/consumer.py:ro
+    command: ["bash", "-c", "pip install -q kafka-python boto3 && python /app/consumer.py"]

compose/forgejo/compose.yaml

@@ -1,19 +0,0 @@
-# Forgejo (lokaal op atc-docker01) — primaire Git staat op atc-mgt01:3001
-services:
-  forgejo:
-    image: codeberg.org/forgejo/forgejo:14
-    container_name: forgejo
-    restart: always
-    environment:
-      USER_UID: "1000"
-      USER_GID: "1000"
-    ports:
-      - "4002:3000"
-      - "222:22"
-    volumes:
-      - forgejo_data:/data
-      - /etc/localtime:/etc/localtime:ro
-
-volumes:
-  forgejo_data:
-    name: forgejo_forgejo

compose/lam/compose.yml

@@ -1,17 +0,0 @@
-# LDAP Account Manager (lokaal op atc-docker01)
-# Productie LDAP/LAM: http://atc-mgt01.dell-atc.lan/lam/
-services:
-  lam:
-    image: ghcr.io/ldapaccountmanager/lam:stable
-    container_name: lam-app-1
-    restart: unless-stopped
-    ports:
-      - "4001:80"
-    environment:
-      DEBIAN_FRONTEND: noninteractive
-    volumes:
-      - lam_data:/var/lib/ldap-account-manager
-
-volumes:
-  lam_data:
-    name: lam_lam

config/README.md

@@ -0,0 +1,19 @@
+# Configuration index
+
+| Directory | Host | Contents |
+|-----------|------|----------|
+| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons |
+| [docker/](docker/) | Multiple | Docker Compose per host |
+| [compose/](../compose/) | atc-docker01 | Source compose stacks |
+| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors |
+| [trino/](trino/) | atc-lake01 | SQL catalog properties |
+| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs |
+| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs |
+| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml |
+| [grafana/](grafana/) | atc-grafana | grafana.ini |
+| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml |
+| [ldap/](ldap/) | atc-mgt01 | LDIF exports |
+| [mgt01/](mgt01/) | atc-mgt01 | NPM compose |
+| [hosts/](hosts/) | All | /etc/hosts snippet |
+
+Refresh all: `./scripts/collect/collect-fleet-config.sh`

config/airflow/README.md

@@ -1,14 +1,35 @@
-# Airflow
+# Airflow — atc-airflow01
 
-| Bestand | Beschrijving |
+| Item | Value |
-|---------|--------------|
+|------|-------|
-| `airflow.cfg` | Referentie-configuratie (deploy naar Airflow home op lake01) |
+| Host | `atc-airflow01` / `10.0.21.55` |
-| `generate_data_dag.py` | DAG voor demo data-generatie |
+| URL | http://atc-airflow01:8080/ or http://10.0.21.55:8080/ |
+| Home | `/root/airflow/` |
+| Config | `/root/airflow/airflow.cfg` |
+| DAGs | `/root/airflow/dags/` |
+| DB | SQLite (`airflow.db` — not in git) |
 
-## URL
+## Files in git
 
-http://atc-lake01.dell-atc.lan:8080/ (of `10.0.21.55:8080` volgens netwerk)
+| File | Notes |
+|------|-------|
+| `airflow.cfg` | Reference copy; `fernet_key` and `internal_api_secret_key` redacted |
+| `generate_data_dag.py` | Synced from live DAGs folder |
+| `../airflow/airflow.cfg` | Older path (legacy); prefer this directory |
 
-## Deploy
+## Restore
 
-Kopieer `generate_data_dag.py` naar de Airflow `dags/` folder op de lake-host en herstart de scheduler/webserver.
+```bash
+# On atc-airflow01
+mkdir -p /root/airflow/dags
+cp airflow.cfg /root/airflow/
+cp generate_data_dag.py /root/airflow/dags/
+# Set real fernet_key in airflow.cfg or via env
+systemctl restart airflow-webserver airflow-scheduler  # if using systemd
+```
+
+## Refresh
+
+```bash
+./scripts/collect/collect-fleet-config.sh
+```

config/airflow/airflow.cfg

@@ -169,7 +169,7 @@ execute_tasks_new_python_interpreter = False
 #
 # Variable: AIRFLOW__CORE__FERNET_KEY
 #
-fernet_key = f4n7uT66HPrl8yxqKHVtme_qINN9YlZ1xUQcZy761MY=
+fernet_key = REDACTED
 
 # Whether to disable pickling dags
 #
@@ -445,7 +445,7 @@ database_access_isolation = False
 #
 # Variable: AIRFLOW__CORE__INTERNAL_API_SECRET_KEY
 #
-internal_api_secret_key = 010RL08807/JBjH4cWzNaw==
+internal_api_secret_key = REDACTED
 
 # The ability to allow testing connections across Airflow UI, API and CLI.
 # Supported options: ``Disabled``, ``Enabled``, ``Hidden``. Default: Disabled

config/airflow/dags/scripts/generate_cassandra_telemetry_data.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake telemetry data for Cassandra
+Generates approximately 1GB of data
+"""
+
+from cassandra.cluster import Cluster
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "9042"
+KEYSPACE = "telemetry"
+TABLE_NAME = "device_metrics"
+
+# Data generation settings
+TARGET_ROWS = 3000000  # Approximately 1GB of data
+BATCH_SIZE = 5000
+
+# Sample data
+METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"]
+DEVICE_PREFIX = "device-"
+
+def generate_fake_device_metric():
+    """Generate a single fake device metric"""
+    device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}"
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                           minutes=random.randint(0, 59))
+    
+    metric_type = random.choice(METRIC_TYPES)
+    metric_value = round(random.uniform(0.0, 100.0), 4)
+    
+    # Generate a long payload field
+    payload = "X" * 200
+    
+    return (device_id, metric_ts, metric_type, metric_value, payload)
+
+def main():
+    print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...")
+    
+    cluster = Cluster([DB_HOST], port=DB_PORT)
+    session = cluster.connect()
+    
+    print(f"Generating {TARGET_ROWS} device metrics...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_device_metric())
+        
+        if len(batch) >= BATCH_SIZE:
+            session.execute(
+                f"""
+                INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+                VALUES (%s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        session.execute(
+            f"""
+            INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+            VALUES (%s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        total_generated += len(batch)
+    
+    session.shutdown()
+    cluster.shutdown()
+    
+    print(f"Completed! Generated {total_generated} device metrics.")
+
+if __name__ == "__main__":
+    main()

config/airflow/dags/scripts/generate_mongodb_events_data.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake event data for MongoDB
+Generates approximately 1GB of data
+"""
+
+import pymongo
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "27017"
+DB_NAME = "supplychain"
+COLLECTION_NAME = "events"
+
+# Data generation settings
+TARGET_DOCUMENTS = 3000000  # Approximately 1GB of data
+BATCH_SIZE = 5000
+
+# Sample data
+EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
+
+def generate_fake_event():
+    """Generate a single fake event"""
+    event_id = uuid.uuid4()
+    event_type = random.choice(EVENT_TYPES)
+    region = random.choice(REGIONS)
+    source = random.choice(SOURCES)
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                     minutes=random.randint(0, 59))
+    
+    amount = random.uniform(100.0, 50000.0)
+    
+    # Generate a long payload field (like the existing data)
+    payload = "X" * 500
+    
+    return {
+        "event_id": event_id,
+        "type": event_type,
+        "region": region,
+        "source": source,
+        "amount": amount,
+        "ts": ts,
+        "payload": payload
+    }
+
+def main():
+    print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
+    
+    client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
+    db = client[DB_NAME]
+    collection = db[COLLECTION_NAME]
+    
+    print(f"Generating {TARGET_DOCUMENTS} events...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_DOCUMENTS):
+        batch.append(generate_fake_event())
+        
+        if len(batch) >= BATCH_SIZE:
+            collection.insert_many(batch)
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} documents...")
+    
+    # Insert remaining documents
+    if batch:
+        collection.insert_many(batch)
+        total_generated += len(batch)
+    
+    client.close()
+    
+    print(f"Completed! Generated {total_generated} events.")
+
+if __name__ == "__main__":
+    main()

config/airflow/dags/scripts/generate_mysql_employee_data.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake employee event data for MySQL
+Generates approximately 1GB of data
+"""
+
+import mysql.connector
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "3306"
+DB_NAME = "hr"
+DB_USER = "mo"
+DB_PASSWORD = "Dell2026!"
+
+# Data generation settings
+TARGET_ROWS = 4000000  # Approximately 1GB of data
+BATCH_SIZE = 10000
+
+# Sample data
+DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"]
+ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"]
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"]
+
+def generate_fake_employee_event():
+    """Generate a single fake employee event"""
+    employee_id = random.randint(1, 100000)
+    department = random.choice(DEPARTMENTS)
+    role_name = random.choice(ROLE_NAMES)
+    region = random.choice(REGIONS)
+    event_type = random.choice(EVENT_TYPES)
+    
+    # Random timestamp within the last 2 years
+    days_ago = random.randint(0, 730)
+    event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                          minutes=random.randint(0, 59))
+    
+    salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None
+    
+    # Generate a long notes field (like the existing data)
+    notes = str(uuid.uuid4()) * 10
+    
+    return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
+
+def main():
+    print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...")
+    
+    conn = mysql.connector.connect(
+        host=DB_HOST,
+        port=DB_PORT,
+        database=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD
+    )
+    cursor = conn.cursor()
+    
+    print(f"Generating {TARGET_ROWS} employee events...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_employee_event())
+        
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                """
+                INSERT INTO employee_events (employee_id, department, role_name, region, 
+                                            event_type, salary_change, event_ts, notes)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            conn.commit()
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        cursor.executemany(
+            """
+            INSERT INTO employee_events (employee_id, department, role_name, region, 
+                                        event_type, salary_change, event_ts, notes)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        conn.commit()
+        total_generated += len(batch)
+    
+    cursor.close()
+    conn.close()
+    
+    print(f"Completed! Generated {total_generated} employee events.")
+
+if __name__ == "__main__":
+    main()

config/airflow/dags/scripts/generate_neo4j_graph_data.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake graph data for Neo4j
+Generates approximately 1GB of data with nodes and relationships
+"""
+
+from neo4j import GraphDatabase
+import random
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "7687"
+DB_USER = "neo4j"
+DB_PASSWORD = "testpwd"
+
+# Data generation settings
+TARGET_NODES = 500000  # Approximately 1GB of data with relationships
+BATCH_SIZE = 1000
+
+# Sample data
+PRODUCT_CATEGORIES = ["Electronics", "Clothing", "Food", "Furniture", "Toys", "Books"]
+SUPPLIER_REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+RELATIONSHIP_TYPES = ["SUPPLIES", "RELATED_TO", "COMPATIBLE_WITH", "PART_OF"]
+
+def generate_fake_product():
+    """Generate a single fake product node"""
+    product_id = str(uuid.uuid4())
+    name = f"Product-{random.randint(1000, 999999)}"
+    category = random.choice(PRODUCT_CATEGORIES)
+    price = round(random.uniform(10.0, 1000.0), 2)
+    stock = random.randint(0, 1000)
+    
+    # Generate a long description field
+    description = "X" * 200
+    
+    return {
+        "product_id": product_id,
+        "name": name,
+        "category": category,
+        "price": price,
+        "stock": stock,
+        "description": description
+    }
+
+def generate_fake_supplier():
+    """Generate a single fake supplier node"""
+    supplier_id = str(uuid.uuid4())
+    name = f"Supplier-{random.randint(1000, 999999)}"
+    region = random.choice(SUPPLIER_REGIONS)
+    rating = round(random.uniform(1.0, 5.0), 1)
+    
+    # Generate a long address field
+    address = "X" * 150
+    
+    return {
+        "supplier_id": supplier_id,
+        "name": name,
+        "region": region,
+        "rating": rating,
+        "address": address
+    }
+
+def main():
+    print(f"Connecting to Neo4j at {DB_HOST}:{DB_PORT}...")
+    
+    driver = GraphDatabase.driver(f"bolt://{DB_HOST}:{DB_PORT}", 
+                                   auth=(DB_USER, DB_PASSWORD))
+    
+    with driver.session() as session:
+        print(f"Generating {TARGET_NODES} product nodes...")
+        print(f"Batch size: {BATCH_SIZE}")
+        
+        total_products = 0
+        total_suppliers = 0
+        product_ids = []
+        
+        # Generate product nodes
+        batch = []
+        for i in range(TARGET_NODES):
+            product = generate_fake_product()
+            batch.append(product)
+            product_ids.append(product["product_id"])
+            
+            if len(batch) >= BATCH_SIZE:
+                session.run(
+                    """
+                    UNWIND $batch as row
+                    CREATE (p:Product {
+                        product_id: row.product_id,
+                        name: row.name,
+                        category: row.category,
+                        price: row.price,
+                        stock: row.stock,
+                        description: row.description
+                    })
+                    """,
+                    batch=batch
+                )
+                total_products += len(batch)
+                batch = []
+                
+                if total_products % 50000 == 0:
+                    print(f"Generated {total_products} product nodes...")
+        
+        # Insert remaining products
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                CREATE (p:Product {
+                    product_id: row.product_id,
+                    name: row.name,
+                    category: row.category,
+                    price: row.price,
+                    stock: row.stock,
+                    description: row.description
+                })
+                """,
+                batch=batch
+            )
+            total_products += len(batch)
+        
+        print(f"Generated {total_products} product nodes.")
+        
+        # Generate supplier nodes (fewer than products)
+        print(f"Generating supplier nodes...")
+        target_suppliers = 10000
+        batch = []
+        supplier_ids = []
+        
+        for i in range(target_suppliers):
+            supplier = generate_fake_supplier()
+            batch.append(supplier)
+            supplier_ids.append(supplier["supplier_id"])
+            
+            if len(batch) >= BATCH_SIZE:
+                session.run(
+                    """
+                    UNWIND $batch as row
+                    CREATE (s:Supplier {
+                        supplier_id: row.supplier_id,
+                        name: row.name,
+                        region: row.region,
+                        rating: row.rating,
+                        address: row.address
+                    })
+                    """,
+                    batch=batch
+                )
+                total_suppliers += len(batch)
+                batch = []
+        
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                CREATE (s:Supplier {
+                    supplier_id: row.supplier_id,
+                    name: row.name,
+                    region: row.region,
+                    rating: row.rating,
+                    address: row.address
+                })
+                """,
+                batch=batch
+            )
+            total_suppliers += len(batch)
+        
+        print(f"Generated {total_suppliers} supplier nodes.")
+        
+        # Create relationships between products and suppliers
+        print(f"Creating relationships...")
+        batch = []
+        total_relationships = 0
+        
+        for product_id in product_ids:
+            # Each product is supplied by 1-3 random suppliers
+            num_suppliers = random.randint(1, 3)
+            for _ in range(num_suppliers):
+                supplier_id = random.choice(supplier_ids)
+                rel_type = random.choice(RELATIONSHIP_TYPES)
+                
+                batch.append({
+                    "product_id": product_id,
+                    "supplier_id": supplier_id,
+                    "rel_type": rel_type
+                })
+                
+                if len(batch) >= BATCH_SIZE:
+                    session.run(
+                        """
+                        UNWIND $batch as row
+                        MATCH (p:Product {product_id: row.product_id})
+                        MATCH (s:Supplier {supplier_id: row.supplier_id})
+                        CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
+                        RETURN rel
+                        """,
+                        batch=batch
+                    )
+                    total_relationships += len(batch)
+                    batch = []
+                    
+                    if total_relationships % 50000 == 0:
+                        print(f"Created {total_relationships} relationships...")
+        
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                MATCH (p:Product {product_id: row.product_id})
+                MATCH (s:Supplier {supplier_id: row.supplier_id})
+                CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
+                RETURN rel
+                """,
+                batch=batch
+            )
+            total_relationships += len(batch)
+        
+        print(f"Created {total_relationships} relationships.")
+    
+    driver.close()
+    print(f"Completed! Generated {total_products} products, {total_suppliers} suppliers, and {total_relationships} relationships.")
+
+if __name__ == "__main__":
+    main()

config/airflow/dags/scripts/generate_postgres_sales_data.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake sales order data for PostgreSQL
+Generates approximately 1GB of data
+"""
+
+import psycopg2
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "5432"
+DB_NAME = "postgres"
+DB_USER = "mo"
+DB_PASSWORD = "Dell2026!"
+
+# Data generation settings
+TARGET_ROWS = 4000000  # Approximately 1GB of data
+BATCH_SIZE = 10000
+
+# Sample data
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
+CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
+ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
+
+def generate_fake_order():
+    """Generate a single fake sales order"""
+    customer_id = random.randint(1, 100000)
+    product_id = random.randint(1, 5000)
+    region = random.choice(REGIONS)
+    sales_channel = random.choice(SALES_CHANNELS)
+    
+    # Random timestamp within the last 2 years
+    days_ago = random.randint(0, 730)
+    order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                          minutes=random.randint(0, 59))
+    
+    amount = round(random.uniform(10.0, 10000.0), 2)
+    currency = random.choice(CURRENCIES)
+    order_status = random.choice(ORDER_STATUSES)
+    
+    # Generate a long notes field (like the existing data)
+    notes = str(uuid.uuid4()) * 10
+    
+    return (customer_id, product_id, region, sales_channel, order_ts, 
+            amount, currency, order_status, notes)
+
+def main():
+    print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
+    
+    conn = psycopg2.connect(
+        host=DB_HOST,
+        port=DB_PORT,
+        database=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD
+    )
+    cursor = conn.cursor()
+    
+    print(f"Generating {TARGET_ROWS} sales orders...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_order())
+        
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                """
+                INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, 
+                                         order_ts, amount, currency, order_status, notes)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            conn.commit()
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        cursor.executemany(
+            """
+            INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, 
+                                     order_ts, amount, currency, order_status, notes)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        conn.commit()
+        total_generated += len(batch)
+    
+    cursor.close()
+    conn.close()
+    
+    print(f"Completed! Generated {total_generated} sales orders.")
+
+if __name__ == "__main__":
+    main()

config/debezium/README.md

@@ -0,0 +1,13 @@
+# Debezium connector definitions
+
+Register on **atc-lake01** via `POST http://10.0.21.50:8083/connectors` with JSON body `{ "name": "...", "config": { ... } }`.
+
+Passwords use `${DEBEZIUM_DB_PASSWORD}` — set on the host, not in git.
+
+| Connector | Source DB | Topic prefix |
+|-----------|-----------|--------------|
+| postgres-sales-connector | postgres @ 10.0.21.51 `public.sales_orders` | postgres_sales |
+| mysql-hr-connector | mysql hr.employee_events | mysql_hr |
+| mongodb-supplychain-connector | mongodb supplychain.events | mongodb_supplychain |
+
+Cassandra CDC skipped (requires table CDC + agent).

config/debezium/connectors/mongodb-supplychain-connector.json

@@ -0,0 +1,12 @@
+{
+  "name": "mongodb-supplychain-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mongodb.MongoDbConnector",
+    "mongodb.hosts": "rs0/10.0.21.51:27017",
+    "mongodb.user": "",
+    "mongodb.password": "${DEBEZIUM_DB_PASSWORD}",
+    "topic.prefix": "mongodb_supplychain",
+    "database.include.list": "supplychain",
+    "collection.include.list": "supplychain.events"
+  }
+}

config/debezium/connectors/mysql-hr-connector.json

@@ -0,0 +1,16 @@
+{
+  "name": "mysql-hr-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "3306",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.server.id": "184054",
+    "topic.prefix": "mysql_hr",
+    "database.include.list": "hr",
+    "table.include.list": "hr.employee_events",
+    "schema.history.internal.kafka.bootstrap.servers": "10.0.21.36:9092",
+    "schema.history.internal.kafka.topic": "schema-changes.hr"
+  }
+}

config/debezium/connectors/postgres-sales-connector.json

@@ -0,0 +1,16 @@
+{
+  "name": "postgres-sales-connector",
+  "config": {
+    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+    "database.hostname": "10.0.21.51",
+    "database.port": "5432",
+    "database.user": "mo",
+    "database.password": "${DEBEZIUM_DB_PASSWORD}",
+    "database.dbname": "postgres",
+    "topic.prefix": "postgres_sales",
+    "schema.include.list": "public",
+    "table.include.list": "public.sales_orders",
+    "plugin.name": "pgoutput",
+    "publication.name": "dbz_publication"
+  }
+}

config/docker/atc-db02/docker-compose.yml

@@ -35,6 +35,19 @@ services:
       - "27017:27017"
     volumes:
       - mongodb_supplychain_data:/data/db
+      - ./init/mongo:/docker-entrypoint-initdb.d:ro
+
+  mongo-express:
+    image: mongo-express
+    container_name: mongo_express
+    depends_on:
+      - mongodb-supplychain
+    ports:
+      - "8081:8081"
+    environment:
+      ME_CONFIG_MONGODB_URL: mongodb://mo:Dell2026%21@mongodb_supplychain:27017/?authSource=admin
+      ME_CONFIG_MONGODB_ENABLE_ADMIN: "true"
+      ME_CONFIG_BASICAUTH: "false"
 
   cassandra-telemetry:
     image: cassandra:4.1

config/docker/atc-db02/init/mongo/01-users.js

@@ -0,0 +1,15 @@
+// Idempotent admin users for lab (mo + bart)
+db = db.getSiblingDB("admin");
+var pw = "Dell2026!";
+["mo", "bart"].forEach(function(name) {
+  if (db.getUser(name) == null) {
+    db.createUser({
+      user: name,
+      pwd: pw,
+      roles: [{ role: "root", db: "admin" }]
+    });
+    print("Created user: " + name);
+  } else {
+    print("User exists: " + name);
+  }
+});

config/grafana/grafana.ini

@@ -0,0 +1,16 @@
+[server]
+http_addr = 0.0.0.0
+http_port = 3000
+
+[security]
+admin_user = admin
+admin_password = REDACTED
+allow_embedding = true
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Viewer
+
+[auth.basic]
+enabled = false

config/grafana/provisioning/dashboards.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'Default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

config/homepage/bookmarks.yaml

@@ -9,6 +9,10 @@
     - Proxmox:
         - abbr: PVE
           href: https://10.0.10.65:8006/
+    - Dockhand:
+        - abbr: DH
+          href: http://atc-docker01.dell-atc.lan:8082/
+
 
 - RSS Feeds:
     - TLDR Data Engineering:

config/homepage/custom.js

@@ -1,32 +1,67 @@
-/* ATC Lakehouse — scanlines + noise only (tabs via settings.yaml) */
+/* ATC Lakehouse — scanlines + noise + architecture link */
 (function () {
+  var ARCH_URL = 'http://atc-docker01.dell-atc.lan:8080/docs/architecture.html';
+
   function injectOverlays() {
     if (!document.querySelector('.palantir-scanline')) {
-      const scan = document.createElement('div');
+      var scan = document.createElement('div');
       scan.className = 'palantir-scanline';
       document.body.appendChild(scan);
     }
     if (!document.querySelector('.palantir-noise')) {
-      const noise = document.createElement('div');
+      var noise = document.createElement('div');
       noise.className = 'palantir-noise';
       document.body.appendChild(noise);
     }
   }
 
+  function linkLakehouseTitle() {
+    document.querySelectorAll('header h1, header p, .greeting, [class*="greeting"]').forEach(function (el) {
+      if (el.dataset.atcLinked) return;
+      if (/lakehouse/i.test(el.textContent || '')) {
+        el.style.cursor = 'pointer';
+        el.title = 'Open architecture diagram';
+        el.addEventListener('click', function () {
+          window.open(ARCH_URL, '_blank');
+        });
+        el.dataset.atcLinked = '1';
+      }
+    });
+  }
+
   function applyServiceColors() {
     document.querySelectorAll('li.service').forEach(function (card) {
-      const desc = card.querySelector('.service-description');
+      var desc = card.querySelector('.service-description');
-      const text = desc ? desc.textContent : '';
+      var text = desc ? desc.textContent : '';
-      const link = card.querySelector('a[href^="http"]');
+      var link = card.querySelector('a[href^="http"]');
       if (text.indexOf('Host:') >= 0 || text.indexOf('Port:') >= 0 || !link) {
         card.classList.add('atc-db-tile');
       }
     });
   }
 
+
+  function showGitSha() {
+    if (document.getElementById('atc-git-sha')) return;
+    fetch('http://atc-mgt01.dell-atc.lan:3001/api/v1/repos/mo/Lakehouse/commits?limit=1')
+      .then(function (r) { return r.json(); })
+      .then(function (d) {
+        var sha = (d && d[0] && d[0].sha) ? d[0].sha.substring(0, 7) : 'git';
+        var el = document.createElement('div');
+        el.id = 'atc-git-sha';
+        el.textContent = 'Lakehouse ' + sha;
+        el.style.cssText = 'position:fixed;bottom:4px;right:8px;font-size:10px;color:#64748b;z-index:9999;font-family:monospace;';
+        document.body.appendChild(el);
+      })
+      .catch(function () {});
+  }
+
   function init() {
     injectOverlays();
     applyServiceColors();
+    linkLakehouseTitle();
+    showGitSha();
+    new MutationObserver(linkLakehouseTitle).observe(document.body, { childList: true, subtree: true });
   }
 
   if (document.readyState === 'loading') {

config/homepage/services.yaml

@@ -81,12 +81,24 @@
         color: "#F46800"
 
 - Infrastructure:
-    - ObjectScale:
+    - ObjectScale UI:
         icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
         href: https://10.0.20.111/
-        description: 10.0.20.111
+        description: ECS admin · https://10.0.20.111 · luna.local
         siteMonitor: https://10.0.20.111/
         color: "#007DB8"
+    - ObjectScale S3:
+        icon: http://atc-docker01.dell-atc.lan:8080/minio.svg
+        href: http://10.0.20.111:9020/
+        description: S3 API :9020 · bucket data · Trino Iceberg + Spark
+        siteMonitor: http://10.0.20.111:9020/
+        color: "#C72C48"
+    - ObjectScale SSH:
+        icon: mdi-console
+        href: https://10.0.20.111/
+        description: admin@atc-objectscale · appliance SSH · see docs/objectscale.md
+        ping: 10.0.20.111
+        color: "#64748b"
     - iDRAC:
         icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
         href: https://10.0.41.102/
@@ -112,6 +124,26 @@
         description: atc-mgt01.dell-atc.lan:3001
         siteMonitor: http://atc-mgt01.dell-atc.lan:3001/
         color: "#F05032"
+    - GPU Lab:
+        icon: mdi-gpu
+        href: http://10.0.20.106:9000/
+        description: atc-gpu-dev VM303 · 4× V100 · model manager + chat
+        siteMonitor: http://10.0.20.106:9000/
+        color: "#76B900"
+    - Dockhand:
+        icon: mdi-docker
+        href: http://atc-docker01.dell-atc.lan:8082/
+        description: Docker control plane · GPU-Dev + Bart-GPU + Mo-GPU
+        siteMonitor: http://atc-docker01.dell-atc.lan:8082/api/health
+        color: "#2496ED"
+
+    - ATC Command Center:
+        icon: mdi-robot-outline
+        href: http://10.0.21.33/
+        description: VM304 MCP · agent hub & ops floor · 10.0.21.33
+        siteMonitor: http://10.0.21.33/
+        color: "#0099cc"
+
     - LDAP Admin:
         icon: http://atc-docker01.dell-atc.lan:8080/ldap.png
         href: http://atc-mgt01.dell-atc.lan/lam/
@@ -130,11 +162,22 @@
         ping: atc-db01.dell-atc.lan
         description: atc-db01:3306 · mysql://USER@atc-db01.dell-atc.lan:3306/DB · mysql -h atc-db01
         color: "#4479A1"
+    - Mongo Express · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
+        href: http://atc-db02.dell-atc.lan:8081/
+        description: Web UI · supplychain · 3M events · users mo & bart
+        siteMonitor: http://atc-db02.dell-atc.lan:8081/
+        ping: atc-db02.dell-atc.lan
+        color: "#47A248"
     - MongoDB · db02:
         icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
         ping: atc-db02.dell-atc.lan
-        description: atc-db02:27017 · mongodb://USER@atc-db02.dell-atc.lan:27017/DB · mongosh
+        description: |
-        color: "#47A248"
+          :27017 · mongodb_supplychain container
+          DB supplychain · collection events (~3M)
+          mongosh: mongodb://mo@atc-db02.dell-atc.lan:27017/supplychain?authSource=admin
+          Trino: mongodb_supplychain.supplychain.events
+        color: "#3d8b40"
     - PostgreSQL · db02:
         icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
         ping: atc-db02.dell-atc.lan
@@ -512,6 +555,39 @@
             href: link
             target: _blank
 
+
+- Documentation:
+    - Architecture Diagram:
+        icon: mdi-sitemap
+        href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
+        description: High-level environment map (Mermaid)
+        color: "#007DB8"
+    - Application Landscape:
+        icon: mdi-book-open-variant
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
+        description: docs/landscape.md in Forgejo
+        color: "#E8752A"
+    - Network Diagram:
+        icon: mdi-lan
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/network.md
+        description: Subnets, ports, SSH mesh
+        color: "#22d3ee"
+    - Disaster Recovery:
+        icon: mdi-backup-restore
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/disaster-recovery.md
+        description: Restore procedures from git
+        color: "#a855f7"
+    - Docker Inventory:
+        icon: mdi-docker
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/docker-inventory.md
+        description: Containers per host
+        color: "#2496ED"
+    - Proxmox VMs:
+        icon: mdi-server
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/inventory/proxmox-vms.json
+        description: VM inventory (JSON)
+        color: "#E57000"
+
 - News & Resources:
     - Dell Technologies:
         icon: http://atc-docker01.dell-atc.lan:8080/dell-technologies.svg

config/homepage/services.yaml.bak-20260603234522

@@ -0,0 +1,635 @@
+---
+# ATC Lakehouse — Dell Technologies FDE Dashboard (Bart & Mo)
+
+
+- Lakehouse · Architecture:
+    - Environment Map:
+        icon: mdi-sitemap
+        href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
+        description: High-level diagram — data flow, hosts, and service map
+        color: "#007DB8"
+    - Git Docs:
+        icon: mdi-book-open-page-variant
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
+        description: Application landscape (Markdown in Forgejo)
+        color: "#E8752A"
+
+- Data Pipeline:
+    - Kafka UI:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
+        href: http://atc-kafka01.dell-atc.lan:9000/
+        description: atc-kafka01.dell-atc.lan:9000
+        siteMonitor: http://atc-kafka01.dell-atc.lan:9000/
+        color: "#E02014"
+    - Debezium:
+        icon: http://atc-docker01.dell-atc.lan:8080/debezium.png
+        href: http://atc-lake01.dell-atc.lan:8083/
+        description: atc-lake01.dell-atc.lan:8083
+        siteMonitor: http://atc-lake01.dell-atc.lan:8083/
+        color: "#4ECDC4"
+    - Spark:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
+        href: http://atc-lake01.dell-atc.lan:8080/
+        description: atc-lake01.dell-atc.lan:8080
+        siteMonitor: http://atc-lake01.dell-atc.lan:8080/
+        color: "#E25A1C"
+    - Trino:
+        icon: http://atc-docker01.dell-atc.lan:8080/trino.svg
+        href: http://atc-lake01.dell-atc.lan:8089/ui/
+        description: atc-lake01.dell-atc.lan:8089/ui/
+        siteMonitor: http://atc-lake01.dell-atc.lan:8089/ui/
+        color: "#DD00A1"
+    - Airflow:
+        icon: http://atc-docker01.dell-atc.lan:8080/apacheairflow.svg
+        href: http://10.0.21.55:8080/
+        description: 10.0.21.55:8080
+        siteMonitor: http://10.0.21.55:8080/health
+        color: "#017CEE"
+
+- Analytics:
+    - Superset:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachesuperset.svg
+        href: http://atc-docker01.dell-atc.lan:8088/
+        description: atc-docker01.dell-atc.lan:8088
+        siteMonitor: http://atc-docker01.dell-atc.lan:8088/health
+        color: "#6c5ce7"
+    - Kibana:
+        icon: http://atc-docker01.dell-atc.lan:8080/kibana.svg
+        href: http://atc-elastic01.dell-atc.lan:5601/
+        description: atc-elastic01.dell-atc.lan:5601
+        siteMonitor: http://atc-elastic01.dell-atc.lan:5601/
+        color: "#F04E98"
+    - Elasticsearch:
+        icon: http://atc-docker01.dell-atc.lan:8080/elasticsearch.svg
+        ping: atc-elastic01.dell-atc.lan
+        description: |
+          Host: atc-elastic01.dell-atc.lan
+          Port: 9200 (no public HTTP)
+          curl: curl http://atc-elastic01.dell-atc.lan:9200
+          Use Kibana for browser UI
+        color: "#005571"
+    - Grafana:
+        icon: http://atc-docker01.dell-atc.lan:8080/grafana.svg
+        href: http://atc-grafana.dell-atc.lan:3000/
+        ping: atc-grafana.dell-atc.lan
+        description: |
+          Host: atc-grafana.dell-atc.lan
+          IP: 10.0.20.103
+          Port: 3000
+          Note: start with systemctl start grafana-server
+        siteMonitor: http://atc-grafana.dell-atc.lan:3000/
+        color: "#F46800"
+
+- Infrastructure:
+    - ObjectScale UI:
+        icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
+        href: https://10.0.20.111/
+        description: ECS admin · https://10.0.20.111 · luna.local
+        siteMonitor: https://10.0.20.111/
+        color: "#007DB8"
+    - ObjectScale S3:
+        icon: http://atc-docker01.dell-atc.lan:8080/minio.svg
+        href: http://10.0.20.111:9020/
+        description: S3 API :9020 · bucket data · Trino Iceberg + Spark
+        siteMonitor: http://10.0.20.111:9020/
+        color: "#C72C48"
+    - ObjectScale SSH:
+        icon: mdi-console
+        href: https://10.0.20.111/
+        description: admin@atc-objectscale · appliance SSH · see docs/objectscale.md
+        ping: 10.0.20.111
+        color: "#64748b"
+    - iDRAC:
+        icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
+        href: https://10.0.41.102/
+        description: Dell iDRAC · 10.0.41.102
+        siteMonitor: https://10.0.41.102/
+        color: "#007DB8"
+    - Proxmox:
+        icon: http://atc-docker01.dell-atc.lan:8080/proxmox.svg
+        href: https://10.0.10.65:8006/
+        description: 10.0.10.65:8006
+        siteMonitor: https://10.0.10.65:8006/
+        color: "#E57000"
+        widget:
+          type: proxmox
+          url: https://10.0.10.65:8006
+          username: "root@pam!homepage"
+          password: "8890185b-0850-42b7-bab2-a690ea4dc3f1"
+          node: pve01
+          fields: ["vms", "lxc", "resources.cpu", "resources.mem"]
+    - Forgejo:
+        icon: http://atc-docker01.dell-atc.lan:8080/git.svg
+        href: http://atc-mgt01.dell-atc.lan:3001/
+        description: atc-mgt01.dell-atc.lan:3001
+        siteMonitor: http://atc-mgt01.dell-atc.lan:3001/
+        color: "#F05032"
+    - LDAP Admin:
+        icon: http://atc-docker01.dell-atc.lan:8080/ldap.png
+        href: http://atc-mgt01.dell-atc.lan/lam/
+        description: atc-mgt01.dell-atc.lan/lam/
+        siteMonitor: http://atc-mgt01.dell-atc.lan/lam/
+        color: "#0984e3"
+
+- Databases:
+    - PostgreSQL · db01:
+        icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
+        ping: atc-db01.dell-atc.lan
+        description: atc-db01:5432 · postgresql://USER@atc-db01.dell-atc.lan:5432/DB · psql -h atc-db01
+        color: "#4169E1"
+    - MySQL · db01:
+        icon: http://atc-docker01.dell-atc.lan:8080/mysql.svg
+        ping: atc-db01.dell-atc.lan
+        description: atc-db01:3306 · mysql://USER@atc-db01.dell-atc.lan:3306/DB · mysql -h atc-db01
+        color: "#4479A1"
+    - Mongo Express · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
+        href: http://atc-db02.dell-atc.lan:8081/
+        description: Web UI · supplychain · 3M events · users mo & bart
+        siteMonitor: http://atc-db02.dell-atc.lan:8081/
+        ping: atc-db02.dell-atc.lan
+        color: "#47A248"
+    - MongoDB · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
+        ping: atc-db02.dell-atc.lan
+        description: |
+          :27017 · mongodb_supplychain container
+          DB supplychain · collection events (~3M)
+          mongosh: mongodb://mo@atc-db02.dell-atc.lan:27017/supplychain?authSource=admin
+          Trino: mongodb_supplychain.supplychain.events
+        color: "#3d8b40"
+    - PostgreSQL · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
+        ping: atc-db02.dell-atc.lan
+        description: atc-db02:5432 · postgresql://USER@atc-db02.dell-atc.lan:5432/DB · psql -h atc-db02
+        color: "#336791"
+    - MySQL · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/mysql.svg
+        ping: atc-db02.dell-atc.lan
+        description: atc-db02:3306 · mysql://USER@atc-db02.dell-atc.lan:3306/DB · mysql -h atc-db02
+        color: "#00758F"
+    - Cassandra · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachecassandra.svg
+        ping: atc-db02.dell-atc.lan
+        description: atc-db02:9042 · cqlsh atc-db02.dell-atc.lan 9042 · cassandra://atc-db02:9042
+        color: "#1287B1"
+    - Neo4j · db02:
+        icon: http://atc-docker01.dell-atc.lan:8080/neo4j.svg
+        href: http://atc-db02.dell-atc.lan:7474/
+        ping: atc-db02.dell-atc.lan
+        description: HTTP :7474 · bolt://atc-db02:7687 · neo4j://atc-db02.dell-atc.lan:7687
+        siteMonitor: http://atc-db02.dell-atc.lan:7474/
+        color: "#008CC1"
+
+- Intel · Hacker News:
+    - HN Front Page:
+        icon: mdi-newspaper
+        href: https://news.ycombinator.com/
+        description: Hacker News — front page
+        color: "#E8752A"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/hn-front
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - HN Data Engineering:
+        icon: mdi-database-search
+        href: https://hn.algolia.com/?q=data%20engineering
+        description: Hacker News — data engineering
+        color: "#3b82f6"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/hn-de
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - HN Kafka:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
+        href: https://hn.algolia.com/?q=kafka
+        description: Hacker News — Kafka & streaming
+        color: "#E02014"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/hn-kafka
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - HN Spark:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
+        href: https://hn.algolia.com/?q=apache+spark
+        description: Hacker News — Apache Spark
+        color: "#E25A1C"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/hn-spark
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Lobsters:
+        icon: mdi-lobster
+        href: https://lobste.rs/
+        description: Computing & infra — curated links
+        color: "#ef4444"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/lobsters
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+
+- Intel · Data Architecture:
+    - Data Eng Weekly:
+        icon: mdi-calendar-week
+        href: https://www.dataengineeringweekly.com/
+        description: Weekly newsletter — pipelines & platforms
+        color: "#22d3ee"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/de-weekly
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Pragmatic Engineer:
+        icon: mdi-account-tie
+        href: https://blog.pragmaticengineer.com/
+        description: Big tech engineering & architecture
+        color: "#a855f7"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/pragmatic
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Martin Fowler:
+        icon: mdi-arch
+        href: https://martinfowler.com/
+        description: Software architecture & design
+        color: "#64748b"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/martinfowler
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - InfoQ:
+        icon: mdi-information-outline
+        href: https://www.infoq.com/
+        description: Architecture, data & dev practices
+        color: "#94a3b8"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/infoq
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - ByteByteGo:
+        icon: mdi-school
+        href: https://blog.bytebytego.com/
+        description: System design — scalable architectures
+        color: "#f59e0b"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/bytebytego
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - TLDR Data Eng:
+        icon: mdi-lightning-bolt
+        href: https://tldr.tech/dataengineering/
+        description: Daily data engineering digest
+        color: "#E8752A"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/tldr-de
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+
+
+    - Seattle Data Guy:
+        icon: mdi-chart-bar
+        href: https://www.seattledataguy.com/
+        description: Practical data engineering tutorials
+        color: "#0ea5e9"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/seattle-de
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - RedMonk:
+        icon: mdi-chart-line
+        href: https://redmonk.com/
+        description: Developer-focused industry analysis
+        color: "#dc2626"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/redmonk
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+
+- Intel · Platforms & Blogs:
+    - Confluent Blog:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
+        href: https://www.confluent.io/blog/
+        description: Kafka & event streaming
+        color: "#007DB8"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/confluent
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Databricks Blog:
+        icon: mdi-layers-triple
+        href: https://www.databricks.com/blog
+        description: Lakehouse & Spark platform
+        color: "#fb923c"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/databricks
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Debezium CDC:
+        icon: http://atc-docker01.dell-atc.lan:8080/debezium.png
+        href: https://debezium.io/blog/
+        description: Change data capture patterns
+        color: "#4ECDC4"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/debezium
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - AWS Big Data:
+        icon: mdi-aws
+        href: https://aws.amazon.com/blogs/big-data/
+        description: AWS analytics & data lakes
+        color: "#FF9900"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/aws-bigdata
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Google Cloud Blog:
+        icon: mdi-google-cloud
+        href: https://cloud.google.com/blog
+        description: GCP data & analytics
+        color: "#4285F4"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/google-cloud
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Cloudflare Eng:
+        icon: mdi-cloud
+        href: https://blog.cloudflare.com/
+        description: Edge, networking & scale
+        color: "#F38020"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/cloudflare
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Meta Engineering:
+        icon: mdi-facebook
+        href: https://engineering.fb.com/
+        description: Large-scale infra & data
+        color: "#1877F2"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/fb-engineering
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+
+    - Airflow Blog:
+        icon: http://atc-docker01.dell-atc.lan:8080/apacheairflow.svg
+        href: https://airflow.apache.org/blog/
+        description: Workflow orchestration & DAGs
+        color: "#017CEE"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/airflow
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - Elastic Blog:
+        icon: http://atc-docker01.dell-atc.lan:8080/elasticsearch.svg
+        href: https://www.elastic.co/blog/
+        description: Search, observability & security
+        color: "#005571"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/elastic
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+    - The New Stack:
+        icon: mdi-newspaper-variant-multiple
+        href: https://thenewstack.io/
+        description: Cloud native & platform engineering
+        color: "#22c55e"
+        widget:
+          type: customapi
+          url: http://atc-docker01.dell-atc.lan:8090/feed/thenewstack
+          refreshInterval: 300000
+          display: dynamic-list
+          mappings:
+            items: items
+            name: title
+            href: link
+            target: _blank
+
+
+- Documentation:
+    - Architecture Diagram:
+        icon: mdi-sitemap
+        href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
+        description: High-level environment map (Mermaid)
+        color: "#007DB8"
+    - Application Landscape:
+        icon: mdi-book-open-variant
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
+        description: docs/landscape.md in Forgejo
+        color: "#E8752A"
+    - Network Diagram:
+        icon: mdi-lan
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/network.md
+        description: Subnets, ports, SSH mesh
+        color: "#22d3ee"
+    - Disaster Recovery:
+        icon: mdi-backup-restore
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/disaster-recovery.md
+        description: Restore procedures from git
+        color: "#a855f7"
+    - Docker Inventory:
+        icon: mdi-docker
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/docker-inventory.md
+        description: Containers per host
+        color: "#2496ED"
+    - Proxmox VMs:
+        icon: mdi-server
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/inventory/proxmox-vms.json
+        description: VM inventory (JSON)
+        color: "#E57000"
+
+- News & Resources:
+    - Dell Technologies:
+        icon: http://atc-docker01.dell-atc.lan:8080/dell-technologies.svg
+        href: https://www.delltechnologies.com/
+        description: Corporate site & solutions
+        color: "#007DB8"
+    - Dell Blog · Data:
+        icon: mdi-post-outline
+        href: https://www.dell.com/en-us/blog/categories/products-solutions-analytics
+        description: Analytics & data solutions
+        color: "#007DB8"
+    - Apache Spark:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
+        href: https://spark.apache.org/docs/latest/
+        description: Spark documentation
+        color: "#E25A1C"
+    - Trino:
+        icon: http://atc-docker01.dell-atc.lan:8080/trino.svg
+        href: https://trino.io/docs/current/
+        description: Distributed SQL engine docs
+        color: "#DD00A1"
+    - Apache Iceberg:
+        icon: mdi-snowflake
+        href: https://iceberg.apache.org/docs/latest/
+        description: Open table format for lakes
+        color: "#38bdf8"
+    - Real Python:
+        icon: mdi-language-python
+        href: https://realpython.com/
+        description: Python tutorials & patterns
+        color: "#3776AB"
+    - MinIO Docs:
+        icon: http://atc-docker01.dell-atc.lan:8080/minio.svg
+        href: https://min.io/docs/minio/linux/index.html
+        description: S3-compatible object storage
+        color: "#C72C48"
+    - TLDR Tech:
+        icon: mdi-lightning-bolt-outline
+        href: https://tldr.tech/
+        description: Daily tech newsletter (web)
+        color: "#E8752A"
+
+- Docker · atc-docker01:
+    - Homepage:
+        icon: mdi-view-dashboard
+        href: http://atc-docker01.dell-atc.lan/
+        description: atc-docker01.dell-atc.lan
+        server: my-docker
+        container: homepage
+        color: "#E8752A"
+    - Superset:
+        icon: http://atc-docker01.dell-atc.lan:8080/apachesuperset.svg
+        href: http://atc-docker01.dell-atc.lan:8088/
+        description: atc-docker01.dell-atc.lan:8088
+        server: my-docker
+        container: superset
+        color: "#6c5ce7"
+    - Forgejo Local:
+        icon: http://atc-docker01.dell-atc.lan:8080/git.svg
+        href: http://atc-docker01.dell-atc.lan:4002/
+        description: atc-docker01.dell-atc.lan:4002
+        siteMonitor: http://atc-docker01.dell-atc.lan:4002/
+        server: my-docker
+        container: forgejo
+        color: "#F05032"

config/homepage/settings.yaml

@@ -27,6 +27,10 @@ layout:
     tab: OPS
     style: row
     columns: 4
+  Object Storage:
+    tab: OPS
+    style: row
+    columns: 3
   Infrastructure:
     tab: OPS
     style: row
@@ -51,6 +55,10 @@ layout:
     tab: FEEDS
     style: row
     columns: 6
+  Documentation:
+    tab: DOCS
+    style: row
+    columns: 4
   News & Resources:
     tab: LINKS
     style: row

config/homepage/widgets.yaml

@@ -25,9 +25,9 @@
     units: metric
     refresh: 3000
 - openmeteo:
-    label: Lab
+    label: Amsterdam
-    latitude: 52.37
+    latitude: 52.374
-    longitude: 4.89
+    longitude: 4.890
     timezone: Europe/Amsterdam
     units: metric
-    cache: 30
+    cache: 15

config/kafka/README.md

@@ -1,25 +1,30 @@
-# Kafka Connect — Debezium connectors
+# Kafka — atc-kafka01
 
-JSON-definities voor CDC van databases naar Kafka.
+| Item | Value |
+|------|-------|
+| Host | `atc-kafka01` / `10.0.21.36` |
+| Mode | **KRaft** (no ZooKeeper) |
+| Broker | `10.0.21.36:9092` |
+| Controller | `localhost:9093` |
+| UI | Kafka UI Docker — port `9000` |
+| Data dir | `/home/kafka/data` |
+| Systemd | `kafka.service` → `kraft/server.properties` |
 
-## Connectors
+## Files
 
-| Bestand | Bron | Topic prefix |
+| File | Purpose |
-|---------|------|--------------|
+|------|---------|
-| `postgres-connector.json` | PostgreSQL op db02 | `postgres-sales` |
+| `kraft-server.properties` | **Live** broker config (systemd) |
-| `mysql-connector.json` | MySQL | — |
+| `server.properties` | Legacy ZK template (not used by current service) |
-| `mongodb-connector.json` | MongoDB | — |
 
-## Deploy
+## Connect
 
-Vervang `PASSWORD_PLACEHOLDER` en pas host/poort aan. Registreer via Kafka Connect REST API op `atc-lake01` (Debezium `:8083`).
+Debezium Connect runs on **atc-lake01:8083** (`BOOTSTRAP_SERVERS=10.0.21.36:9092`).
+
+Connector JSON: `../kafka/*.json` (postgres, mysql, mongodb).
+
+## Refresh
 
 ```bash
-curl -X POST -H "Content-Type: application/json" \
+./scripts/collect/collect-fleet-config.sh
-  --data @postgres-connector.json \
-  http://atc-lake01.dell-atc.lan:8083/connectors/
 ```
-
-## Referentie
-
-- Kafka UI: http://atc-kafka01.dell-atc.lan:9000/

config/kafka/kraft-server.properties

@@ -0,0 +1,35 @@
+# Required KRaft roles
+process.roles=broker,controller
+node.id=1
+controller.quorum.voters=1@localhost:9093
+
+# Listeners
+listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+controller.listener.names=CONTROLLER
+inter.broker.listener.name=PLAINTEXT
+advertised.listeners=PLAINTEXT://10.0.21.36:9092
+listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+
+# Storage - use a clean directory
+log.dirs=/home/kafka/data
+
+# Topic defaults
+num.partitions=3
+default.replication.factor=1
+
+# Retention (7 days)
+log.retention.hours=168
+log.retention.check.interval.ms=300000
+log.segment.bytes=1073741824
+
+# Performance (8 vCPUs)
+num.network.threads=8
+num.io.threads=8
+socket.send.buffer.bytes=102400
+socket.receive.buffer.bytes=102400
+socket.request.max.bytes=104857600
+
+# Transaction state (single node)
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1

config/kafka/mongodb-connector.json

@@ -1 +1,10 @@
-{"connector.class":"io.debezium.connector.mongodb.MongoDbConnector","topic.prefix":"mongodb-supplychain","mongodb.history.kafka.bootstrap.servers":"localhost:9092","mongodb.history.kafka.topic":"schema-changes.supplychain","mongodb.connection.string":"mongodb://10.0.21.51:27017","name":"mongodb-connector","mongodb.name":"supplychain","snapshot.mode":"initial"}
+{
+  "connector.class": "io.debezium.connector.mongodb.MongoDbConnector",
+  "topic.prefix": "mongodb-supplychain",
+  "mongodb.connection.string": "mongodb://mo:Dell2026%21@10.0.21.51:27017/?authSource=admin",
+  "mongodb.history.kafka.bootstrap.servers": "10.0.21.36:9092",
+  "mongodb.history.kafka.topic": "schema-changes.supplychain",
+  "name": "mongodb-connector",
+  "mongodb.name": "supplychain",
+  "snapshot.mode": "initial"
+}

config/kafka/server.properties

@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This configuration file is intended for use in ZK-based mode, where Apache ZooKeeper is required.
+# See kafka.server.KafkaConfig for additional details and defaults
+#
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. If not configured, the host name will be equal to the value of
+# java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+#listeners=PLAINTEXT://:9092
+
+# Listener name, hostname and port the broker will advertise to clients.
+# If not set, it uses the value for "listeners".
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+
+# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
+#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# The number of threads that the server uses for receiving requests from the network and sending responses to the network
+num.network.threads=3
+
+# The number of threads that the server uses for processing requests, which may include disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=102400
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=102400
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Internal Topic Settings  #############################
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+#    1. Durability: Unflushed data may be lost if you are not using replication.
+#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion due to age
+log.retention.hours=168
+
+# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+#log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=300000
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=localhost:2181
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=18000
+
+
+############################# Group Coordinator Settings #############################
+
+# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
+# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
+# The default value for this is 3 seconds.
+# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
+# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
+group.initial.rebalance.delay.ms=0

config/ldap/README.md

@@ -0,0 +1,19 @@
+# Configuration index
+
+| Directory | Host | Contents |
+|-----------|------|----------|
+| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons |
+| [docker/](docker/) | Multiple | Docker Compose per host |
+| [compose/](../compose/) | atc-docker01 | Source compose stacks |
+| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors |
+| [trino/](trino/) | atc-lake01 | SQL catalog properties |
+| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs |
+| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs |
+| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml |
+| [grafana/](grafana/) | atc-grafana | grafana.ini |
+| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml |
+| [ldap/](ldap/) | atc-mgt01 | LDIF exports |
+| [mgt01/](mgt01/) | atc-mgt01 | NPM compose |
+| [hosts/](hosts/) | All | /etc/hosts snippet |
+
+Refresh all: `./scripts/collect/collect-fleet-config.sh`

config/ldap/autofs.ldif

@@ -0,0 +1,7 @@
+dn: cn=autofs,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: autofs
+olcAttributeTypes: ( 1.3.6.1.1.1.1.25 NAME 'automountInformation' DESC 'Information used by the autofs automounter' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE )
+olcObjectClasses: ( 1.3.6.1.1.1.1.13 NAME 'automount' DESC 'An entry in an automounter map' SUP top STRUCTURAL MUST ( cn $ automountInformation $ objectclass ) MAY description )
+olcObjectClasses: ( 1.3.6.1.4.1.2312.4.2.2 NAME 'automountMap' DESC 'A group of related automount objects' SUP top STRUCTURAL MUST ou )
+

config/ldap/base.ldif

@@ -0,0 +1,14 @@
+dn: dc=dell-atc,dc=lan
+objectClass: top
+objectClass: dcObject
+objectClass: organization
+o: Dell ATC Lab
+dc: dell-atc
+
+dn: ou=Users,dc=dell-atc,dc=lan
+objectClass: organizationalUnit
+ou: Users
+
+dn: ou=Groups,dc=dell-atc,dc=lan
+objectClass: organizationalUnit
+ou: Groups

config/ldap/db.ldif

@@ -0,0 +1,14 @@
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcSuffix
+olcSuffix: dc=dell-atc,dc=lan
+
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcRootDN
+olcRootDN: cn=admin,dc=dell-atc,dc=lan
+
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcRootPW
+olcRootPW: {SSHA}MO4kZaN1wIHCgkMw6f0FX0kcQbEe+jmx

config/ldap/openssh.ldif

@@ -0,0 +1,11 @@
+dn: cn=openssh-lpk-openldap,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: openssh-lpk-openldap
+olcAttributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey'
+ DESC 'MANDATORY: OpenSSH Public key'
+ EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 )
+olcObjectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey'
+ DESC 'MANDATORY: OpenSSH LPK objectclass'
+ SUP top AUXILIARY MUST ( sshPublicKey ) )
+
+

config/ldap/setacl.ldif

@@ -0,0 +1,6 @@
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+add: olcAccess
+olcAccess: {0}to attrs=userPassword,shadowLastChange by dn="cn=admin,dc=dell-atc,dc=lan" write by anonymous auth by self write by * none
+olcAccess: {1}to * by dn="cn=admin,dc=dell-atc,dc=lan" write by * read
+

config/mgt01/npm-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  app:
+    image: 'jc21/nginx-proxy-manager:latest'
+    restart: unless-stopped
+    environment:
+      TZ: "Europe/Amsterdam"
+    ports:
+      - '80:80'
+      - '81:81'
+      - '443:443'
+    volumes:
+      - ./data:/data
+      - ./letsencrypt:/etc/letsencrypt
+

config/objectscale/README.md

@@ -27,3 +27,21 @@
 ## Storage
 
 Single-node lab deployment; block device `/dev/sdb` in storage pool `sp1` per deploy.yml.
+
+## management_clients: 0.0.0.0/0 explained
+
+**`0.0.0.0/0` = allow management access from ANY IP address** (no whitelist).
+
+- Fine for isolated lab VLANs behind firewall
+- **Not OK** if management port is reachable from office internet or WAN
+
+**Recommended for ATC lab** (tighten when convenient):
+
+```yaml
+management_clients:
+  - 10.0.10.0/24
+  - 10.0.20.0/24
+  - 10.0.21.0/24
+```
+
+See [docs/objectscale.md](../../docs/objectscale.md) for ports and S3 consumers.

config/objectscale/deploy.yml

@@ -45,10 +45,10 @@ facts:
     # [Required]
     # Password to use with SSH login
     # *** Set to same value as ssh_username to enable SSH public key authentication ***
-    ssh_password: "REDACTED"
+    ssh_password: REDACTED
     # [Required when enabling SSH public key authentication]
     # Password to give to sudo when gaining root access.
-    ansible_become_pass: "REDACTED"
+    ansible_become_pass: REDACTED
     # [Required]
     # Select the type of crypto to use when dealing with ssh public key
     # authentication. Valid values here are:

config/scripts/collect_pipeline_metrics.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Collect Debezium, Kafka CDC, and Spark metrics into postgres monitor schema."""
+import json
+import shlex
+import subprocess
+import urllib.request
+from collections import Counter
+from datetime import datetime, timezone
+
+import psycopg2
+
+PG_DSN = "host=10.0.21.51 dbname=postgres user=mo password=Dell2026!"
+KAFKA = "10.0.21.36:9092"
+DEBEZIUM = "http://localhost:8083"  # Kafka Connect on kafka01; fallback lake01 :8083
+SPARK_MASTER = "http://10.0.21.50:8080"
+
+CDC_TOPICS = [
+    ("PostgreSQL", "postgres-sales.public.sales_orders"),
+    ("MongoDB", "mongodb-supplychain.supplychain.events"),
+]
+
+OP_LABELS = {"c": "INSERT", "u": "UPDATE", "d": "DELETE", "r": "SNAPSHOT", "i": "INSERT"}
+
+
+def fetch_json(url, timeout=10):
+    with urllib.request.urlopen(url, timeout=timeout) as r:
+        return json.loads(r.read().decode())
+
+
+def collect_debezium(cur):
+    connectors = fetch_json(f"{DEBEZIUM}/connectors")
+    cur.execute("DELETE FROM monitor.debezium_connectors")
+    now = datetime.now(timezone.utc)
+    for name in connectors:
+        try:
+            st = fetch_json(f"{DEBEZIUM}/connectors/{name}/status")
+        except Exception as e:
+            cur.execute(
+                """INSERT INTO monitor.debezium_connectors
+                   (connector_name, state, task_state, worker_id, checked_at)
+                   VALUES (%s,%s,%s,%s,%s)""",
+                (name, "ERROR", str(e)[:32], "", now),
+            )
+            continue
+        conn_state = st.get("connector", {}).get("state", "UNKNOWN")
+        tasks = st.get("tasks") or []
+        task_state = tasks[0].get("state", "NONE") if tasks else "NONE"
+        worker = st.get("connector", {}).get("worker_id", "")
+        cur.execute(
+            """INSERT INTO monitor.debezium_connectors
+               (connector_name, state, task_state, worker_id, checked_at)
+               VALUES (%s,%s,%s,%s,%s)""",
+            (name, conn_state, task_state, worker, now),
+        )
+
+
+KAFKA_BIN = "/opt/kafka/bin"
+USE_SSH_KAFKA = False  # set True when running off-host
+
+
+def _kafka_cmd(bin_name, args):
+    parts = [f"{KAFKA_BIN}/{bin_name}"] + list(args)
+    if USE_SSH_KAFKA:
+        remote = " ".join(shlex.quote(p) for p in parts)
+        full = f"ssh -o StrictHostKeyChecking=no root@10.0.21.36 {remote}"
+        return subprocess.check_output(full, shell=True, stderr=subprocess.DEVNULL, timeout=90, text=True)
+    return subprocess.check_output(parts, stderr=subprocess.DEVNULL, timeout=90, text=True)
+
+
+def kafka_end_offsets(topic):
+    try:
+        out = _kafka_cmd(
+            "kafka-run-class.sh",
+            [
+                "kafka.tools.GetOffsetShell",
+                "--broker-list",
+                "localhost:9092",
+                "--topic",
+                topic,
+            ],
+        )
+    except Exception:
+        return []
+    rows = []
+    for line in out.strip().splitlines():
+        parts = line.split(":")
+        if len(parts) >= 3:
+            rows.append((int(parts[1]), int(parts[2])))
+    return rows
+
+
+def sample_topic_messages(topic, max_msgs=3000, tail=5000):
+    """Sample recent messages using kafka-console-consumer from tail."""
+    offsets = kafka_end_offsets(topic)
+    if not offsets:
+        return []
+    # Pick partition 0 for sampling
+    part, end = offsets[0]
+    start = max(0, end - tail)
+    try:
+        out = _kafka_cmd(
+            "kafka-console-consumer.sh",
+            [
+                "--bootstrap-server",
+                "localhost:9092",
+                "--topic",
+                topic,
+                "--partition",
+                str(part),
+                "--offset",
+                str(start),
+                "--max-messages",
+                str(min(max_msgs, tail)),
+                "--timeout-ms",
+                "15000",
+            ],
+        )
+    except Exception:
+        return []
+    return [ln for ln in out.strip().split("\n") if ln.strip()]
+
+
+def parse_debezium_line(line):
+    try:
+        doc = json.loads(line)
+        payload = doc.get("payload") or doc
+        op = payload.get("op") or payload.get("operationType") or "?"
+        src = payload.get("source") or {}
+        table = src.get("table") or src.get("collection") or ""
+        ts_ms = payload.get("ts_ms") or src.get("ts_ms")
+        after = payload.get("after") or {}
+        before = payload.get("before") or {}
+        row = after if after else before
+        key = str(row.get("order_id") or row.get("event_id") or row.get("_id") or "")[:200]
+        detail = str(row.get("region") or row.get("type") or row.get("department") or "")[:200]
+        event_ts = None
+        if ts_ms:
+            event_ts = datetime.fromtimestamp(int(ts_ms) / 1000, tz=timezone.utc)
+        return op, table, key, detail, event_ts
+    except Exception:
+        return None
+
+
+def collect_kafka_cdc(cur):
+    now = datetime.now(timezone.utc)
+    cur.execute("DELETE FROM monitor.kafka_topics")
+    cur.execute("DELETE FROM monitor.cdc_operations")
+    cur.execute("DELETE FROM monitor.cdc_recent_events")
+
+    for source, topic in CDC_TOPICS:
+        for part, end in kafka_end_offsets(topic):
+            cur.execute(
+                """INSERT INTO monitor.kafka_topics (topic, partition_id, end_offset, checked_at)
+                   VALUES (%s,%s,%s,%s)""",
+                (topic, part, end, now),
+            )
+
+        lines = sample_topic_messages(topic, max_msgs=2000, tail=3000)
+        ops = Counter()
+        recent = []
+        for line in lines:
+            parsed = parse_debezium_line(line)
+            if not parsed:
+                continue
+            op, table, key, detail, event_ts = parsed
+            ops[op] += 1
+            if len(recent) < 100:
+                recent.append((op, table, key, detail, event_ts))
+
+        for op, cnt in ops.items():
+            cur.execute(
+                """INSERT INTO monitor.cdc_operations
+                   (source_system, topic, operation, operation_label, event_count, checked_at)
+                   VALUES (%s,%s,%s,%s,%s,%s)""",
+                (source, topic, op, OP_LABELS.get(op, op), cnt, now),
+            )
+
+        for op, table, key, detail, event_ts in recent[:50]:
+            cur.execute(
+                """INSERT INTO monitor.cdc_recent_events
+                   (source_system, topic, operation, operation_label, table_name,
+                    record_key, detail, event_ts, sampled_at)
+                   VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                (
+                    source,
+                    topic,
+                    op,
+                    OP_LABELS.get(op, op),
+                    table,
+                    key,
+                    detail,
+                    event_ts,
+                    now,
+                ),
+            )
+
+
+def collect_spark(cur):
+    now = datetime.now(timezone.utc)
+    cur.execute("DELETE FROM monitor.spark_applications")
+    try:
+        data = fetch_json(f"{SPARK_MASTER}/json/", timeout=5)
+        apps = []
+        if isinstance(data, dict):
+            # Standalone master JSON
+            for a in data.get("activeapps", []) or []:
+                apps.append(a)
+            for a in data.get("completedapps", []) or []:
+                apps.append(a)
+        for a in apps[:20]:
+            cur.execute(
+                """INSERT INTO monitor.spark_applications
+                   (app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
+                   VALUES (%s,%s,%s,%s,%s,%s,%s)
+                   ON CONFLICT (app_id) DO UPDATE SET
+                     app_name=EXCLUDED.app_name, state=EXCLUDED.state,
+                     cores=EXCLUDED.cores, memory_mb=EXCLUDED.memory_mb,
+                     duration_sec=EXCLUDED.duration_sec, checked_at=EXCLUDED.checked_at""",
+                (
+                    a.get("id", "unknown"),
+                    a.get("name", "Spark App"),
+                    "RUNNING" if "attempts" not in a else "COMPLETED",
+                    int(a.get("cores", 0) or 0),
+                    int((a.get("memory", 0) or 0) / 1024 / 1024),
+                    int(a.get("duration", 0) / 1000) if a.get("duration") else 0,
+                    now,
+                ),
+            )
+    except Exception as e:
+        # Placeholder row so dashboard shows Spark host status
+        cur.execute(
+            """INSERT INTO monitor.spark_applications
+               (app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
+               VALUES (%s,%s,%s,%s,%s,%s,%s)
+               ON CONFLICT (app_id) DO UPDATE SET state=EXCLUDED.state, checked_at=EXCLUDED.checked_at""",
+            (
+                "spark-master",
+                f"Spark Master @ {SPARK_MASTER}",
+                "REACHABLE" if "Connection" not in str(e) else "UNREACHABLE",
+                0,
+                0,
+                0,
+                now,
+            ),
+        )
+
+
+def main():
+    conn = psycopg2.connect(PG_DSN)
+    conn.autocommit = True
+    cur = conn.cursor()
+    print("Collecting Debezium...")
+    collect_debezium(cur)
+    print("Collecting Kafka CDC samples...")
+    collect_kafka_cdc(cur)
+    print("Collecting Spark...")
+    collect_spark(cur)
+    cur.close()
+    conn.close()
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

config/spark-jobs/README.md

@@ -0,0 +1 @@
+# Spark jobs

config/spark-jobs/consumer.py

@@ -0,0 +1,38 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+consumer = KafkaConsumer(
+    KAFKA_TOPIC,
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='earliest',
+    value_deserializer=lambda x: json.loads(x.decode('utf-8'))
+)
+
+print("Waiting for messages...")
+for msg in consumer:
+    ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
+    print(f"Written: {key}")

config/spark-jobs/consumer_fixed.py

@@ -0,0 +1,48 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+def deserialize(value):
+    if value is None or len(value) == 0:
+        return None
+    try:
+        return json.loads(value.decode('utf-8'))
+    except:
+        print(f"Skipping non-JSON")
+        return None
+
+consumer = KafkaConsumer(
+    KAFKA_TOPIC,
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='latest',
+    value_deserializer=deserialize
+)
+
+print("Waiting for messages...")
+for msg in consumer:
+    if msg.value is None:
+        continue
+    ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
+    print(f"Written: {key}")

config/spark-jobs/kafka_to_s3.py

@@ -0,0 +1,34 @@
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import col, current_timestamp
+
+spark = SparkSession.builder \
+    .appName("KafkaToS3") \
+    .config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
+    .config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
+    .config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
+    .config("spark.hadoop.fs.s3a.path.style.access", "true") \
+    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
+    .getOrCreate()
+
+spark.sparkContext.setLogLevel("WARN")
+
+df = spark.readStream \
+    .format("kafka") \
+    .option("kafka.bootstrap.servers", "10.0.21.36:9092") \
+    .option("subscribe", "test-lakehouse") \
+    .option("startingOffsets", "latest") \
+    .load()
+
+output = df.select(
+    col("key").cast("string"),
+    col("value").cast("string"),
+    current_timestamp().alias("timestamp")
+)
+
+query = output.writeStream \
+    .outputMode("append") \
+    .format("console") \
+    .start()
+
+print("Streaming started. Press Ctrl+C to stop.")
+query.awaitTermination()

config/spark-jobs/kafka_to_s3_consumer.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+from kafka import KafkaConsumer
+import boto3
+import json
+import os
+import sys
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("=" * 60)
+print(f"Kafka Consumer starting...")
+print(f"Kafka broker: {KAFKA_BROKER}")
+print(f"Kafka topic: {KAFKA_TOPIC}")
+print(f"S3 endpoint: {S3_ENDPOINT}")
+print(f"S3 bucket: {S3_BUCKET}")
+print("=" * 60)
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Kafka consumer
+try:
+    consumer = KafkaConsumer(
+        KAFKA_TOPIC,
+        bootstrap_servers=[KAFKA_BROKER],
+        auto_offset_reset='earliest',
+        enable_auto_commit=True,
+        value_deserializer=lambda x: json.loads(x.decode('utf-8'))
+    )
+    print("Connected to Kafka successfully!")
+except Exception as e:
+    print(f"Failed to connect to Kafka: {e}")
+    sys.exit(1)
+
+print(f"Listening for messages on topic '{KAFKA_TOPIC}'...")
+print("-" * 60)
+
+message_count = 0
+for message in consumer:
+    message_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
+    
+    try:
+        s3.put_object(
+            Bucket=S3_BUCKET,
+            Key=key,
+            Body=json.dumps(message.value, indent=2)
+        )
+        print(f"[{message_count}] Written to s3://{S3_BUCKET}/{key}")
+        print(f"    Data: {json.dumps(message.value)[:100]}...")
+    except Exception as e:
+        print(f"Error writing to S3: {e}")

config/spark-jobs/s3_consumer_all_topics.py

@@ -0,0 +1,69 @@
+from kafka import KafkaConsumer
+from kafka import TopicPartition
+import boto3
+import json
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting S3 consumer for ALL topics...")
+print(f"Kafka: {KAFKA_BROKER}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Consumer zonder specifiek topic
+consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
+
+# Haal alle topics op
+all_topics = consumer.topics()
+print(f"Found topics: {list(all_topics)}")
+
+# Wijs alle partitions van alle topics toe
+for topic in all_topics:
+    partitions = consumer.partitions_for_topic(topic)
+    for partition in partitions:
+        tp = TopicPartition(topic, partition)
+        consumer.assign([tp])
+        consumer.seek_to_beginning()
+        print(f"Assigned: {topic} - partition {partition}")
+
+print("\nReading all messages from all topics...")
+print("-" * 60)
+
+msg_count = 0
+topic_count = {}
+
+for msg in consumer:
+    msg_count += 1
+    topic = msg.topic
+    topic_count[topic] = topic_count.get(topic, 0) + 1
+    
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{topic}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(
+        Bucket=S3_BUCKET,
+        Key=key,
+        Body=json.dumps(data, indent=2)
+    )
+    
+    print(f"[{msg_count}] {topic} -> s3://{S3_BUCKET}/{key}")

config/spark-jobs/s3_consumer_realtime.py

@@ -0,0 +1,45 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Consumer die naar ALLE topics luistert (alleen nieuwe berichten)
+consumer = KafkaConsumer(
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='latest'
+)
+
+# Subscribe op alle topics
+consumer.subscribe(pattern='.*')
+
+print("Listening to ALL topics...")
+print("Waiting for new messages...")
+
+msg_count = 0
+for msg in consumer:
+    msg_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{msg.topic}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(data, indent=2))
+    print(f"[{msg_count}] {msg.topic} -> s3://{S3_BUCKET}/{key}")

config/spark-jobs/s3_consumer_working.py

@@ -0,0 +1,57 @@
+from kafka import KafkaConsumer
+from kafka import TopicPartition
+import boto3
+import json
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting S3 consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Kafka consumer met vaste partition
+consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
+tp = TopicPartition(KAFKA_TOPIC, 0)
+consumer.assign([tp])
+consumer.seek_to_beginning()
+
+print("Reading all messages from beginning...")
+print("-" * 50)
+
+msg_count = 0
+for msg in consumer:
+    msg_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(
+        Bucket=S3_BUCKET,
+        Key=key,
+        Body=json.dumps(data, indent=2)
+    )
+    print(f"[{msg_count}] Written: s3://{S3_BUCKET}/{key}")
+    print(f"    Data: {str(data)[:80]}...")
+
+print(f"Done! Total {msg_count} messages written to S3")

config/spark-jobs/simple_consumer.py

@@ -0,0 +1,12 @@
+from kafka import KafkaConsumer
+import json
+
+consumer = KafkaConsumer(
+    'test-lakehouse',
+    bootstrap_servers=['10.0.21.36:9092'],
+    auto_offset_reset='latest'
+)
+
+print("Waiting for new messages...")
+for msg in consumer:
+    print(f"Received: {msg.value.decode('utf-8')}")

config/superset/Dockerfile

@@ -1,3 +1,7 @@
 FROM apache/superset:latest
-
+USER root
-RUN pip install psycopg2-binary
+RUN pip3 install --no-cache-dir --target=/app/.venv/lib/python3.10/site-packages --no-deps \
+    sqlalchemy-trino==0.5.0 trino==0.337.0 && \
+    pip3 install --no-cache-dir --target=/app/.venv/lib/python3.10/site-packages \
+    requests lz4 orjson python-dateutil pytz tzlocal zstandard charset-normalizer idna urllib3 certifi six greenlet
+USER superset

config/superset/apply_palantir_rich_dashboard.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+"""Apply Palantir styling and enrich ATC Lakehouse Superset dashboard."""
+import json
+import os
+import requests
+
+BASE = "http://127.0.0.1:8088"
+DASH_ID = 1
+CSS_PATH = "/tmp/palantir_dashboard.css"
+
+NEW_CHARTS = [
+    (
+        "Trino · PostgreSQL Sales",
+        "public",
+        "sales_orders",
+        "PostgreSQL · Orders by Channel",
+        "pie",
+        {
+            "metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
+            "groupby": ["sales_channel"],
+            "row_limit": 10,
+        },
+    ),
+    (
+        "Trino · PostgreSQL Sales",
+        "public",
+        "sales_orders",
+        "PostgreSQL · Avg Order by Region",
+        "echarts_timeseries_bar",
+        {
+            "metrics": [
+                {"expressionType": "SQL", "sqlExpression": "AVG(amount)", "label": "Avg Amount"}
+            ],
+            "groupby": ["region"],
+            "row_limit": 15,
+        },
+    ),
+    (
+        "Trino · PostgreSQL Sales",
+        "public",
+        "sales_orders",
+        "PostgreSQL · Status Breakdown",
+        "pie",
+        {
+            "metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
+            "groupby": ["order_status"],
+            "row_limit": 10,
+        },
+    ),
+    (
+        "Trino · MySQL HR",
+        "hr",
+        "employee_events",
+        "MySQL HR · By Event Type",
+        "pie",
+        {
+            "metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
+            "groupby": ["event_type"],
+            "row_limit": 12,
+        },
+    ),
+    (
+        "Trino · MySQL HR",
+        "hr",
+        "employee_events",
+        "MySQL HR · Events per Month",
+        "echarts_timeseries_line",
+        {
+            "metrics": [
+                {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "Events"}
+            ],
+            "groupby": [
+                {
+                    "expressionType": "SQL",
+                    "sqlExpression": "date_trunc('month', event_ts)",
+                    "label": "Month",
+                }
+            ],
+            "row_limit": 24,
+        },
+    ),
+    (
+        "Trino · MongoDB Supply Chain",
+        "supplychain",
+        "events",
+        "MongoDB · Amount by Source",
+        "echarts_timeseries_bar",
+        {
+            "metrics": [
+                {"expressionType": "SQL", "sqlExpression": "SUM(amount)", "label": "Total Amount"}
+            ],
+            "groupby": ["source"],
+            "row_limit": 10,
+        },
+    ),
+    (
+        "Trino · MongoDB Supply Chain",
+        "supplychain",
+        "events",
+        "MongoDB · Events per Month",
+        "echarts_timeseries_line",
+        {
+            "metrics": [
+                {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "Events"}
+            ],
+            "groupby": [
+                {
+                    "expressionType": "SQL",
+                    "sqlExpression": "date_trunc('month', ts)",
+                    "label": "Month",
+                }
+            ],
+            "row_limit": 24,
+        },
+    ),
+    (
+        "Trino · Cassandra Telemetry",
+        "telemetry",
+        "device_metrics",
+        "Cassandra · Avg Metric Over Time",
+        "echarts_timeseries_line",
+        {
+            "metrics": [
+                {
+                    "expressionType": "SQL",
+                    "sqlExpression": "AVG(metric_value)",
+                    "label": "Avg Value",
+                }
+            ],
+            "groupby": [
+                {
+                    "expressionType": "SQL",
+                    "sqlExpression": "date_trunc('day', metric_ts)",
+                    "label": "Day",
+                }
+            ],
+            "row_limit": 30,
+        },
+    ),
+]
+
+SECTIONS = [
+    ("HEADER", "ATC Lakehouse · Federated Data Platform", "Trino · PostgreSQL · MySQL · MongoDB · Cassandra · Dell Technologies FDE"),
+    ("PostgreSQL Sales", "30M orders · postgres_sales.public.sales_orders"),
+    ("MySQL HR", "569K events · mysql_hr.hr.employee_events"),
+    ("MongoDB Supply Chain", "3M events · mongodb_supplychain.supplychain.events"),
+    ("Cassandra Telemetry", "Device metrics · cassandra_telemetry.telemetry.device_metrics"),
+]
+
+
+def session():
+    s = requests.Session()
+    r = s.post(
+        f"{BASE}/api/v1/security/login",
+        json={"username": "admin", "password": "admin", "provider": "db", "refresh": True},
+    )
+    r.raise_for_status()
+    h = {
+        "Authorization": "Bearer " + r.json()["access_token"],
+        "Content-Type": "application/json",
+    }
+    h["X-CSRFToken"] = s.get(f"{BASE}/api/v1/security/csrf_token/", headers=h).json()["result"]
+    h["Referer"] = BASE
+    return s, h
+
+
+def get_db_map(s, h):
+    r = s.get(f"{BASE}/api/v1/database/", headers=h)
+    r.raise_for_status()
+    return {d["database_name"]: d["id"] for d in r.json().get("result", [])}
+
+
+def get_or_create_dataset(s, h, db_id, schema, table):
+    r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
+    for d in r.json().get("result", []):
+        if (
+            d.get("table_name") == table
+            and d.get("schema") == schema
+            and d.get("database", {}).get("id") == db_id
+        ):
+            return d["id"]
+    r = s.post(
+        f"{BASE}/api/v1/dataset/",
+        headers=h,
+        json={"database": db_id, "schema": schema, "table_name": table},
+    )
+    r.raise_for_status()
+    return r.json()["id"]
+
+
+def create_chart(s, h, name, ds_id, viz_type, params):
+    r = s.get(f"{BASE}/api/v1/chart/", headers=h)
+    for c in r.json().get("result", []):
+        if c.get("slice_name") == name:
+            return c["id"]
+    full = {"datasource": f"{ds_id}__table", "viz_type": viz_type, "row_limit": 1000, **params}
+    r = s.post(
+        f"{BASE}/api/v1/chart/",
+        headers=h,
+        json={
+            "slice_name": name,
+            "viz_type": viz_type,
+            "datasource_id": ds_id,
+            "datasource_type": "table",
+            "params": json.dumps(full),
+            "owners": [1, 2, 3],
+        },
+    )
+    if r.status_code not in (200, 201):
+        raise RuntimeError(f"chart {name}: {r.text[:300]}")
+    return r.json()["id"]
+
+
+def build_layout(chart_items):
+    """chart_items: list of (chart_id, name, section) or ('md', title, subtitle)."""
+    layout = {
+        "DASHBOARD_VERSION": "v2",
+        "ROOT_ID": {"type": "ROOT", "id": "ROOT_ID", "children": ["GRID_ID"]},
+        "GRID_ID": {"type": "GRID", "id": "GRID_ID", "children": [], "parents": ["ROOT_ID"]},
+    }
+    row_idx = 0
+
+    def add_row():
+        nonlocal row_idx
+        row_idx += 1
+        rid = f"ROW-{row_idx}"
+        layout["GRID_ID"]["children"].append(rid)
+        layout[rid] = {
+            "type": "ROW",
+            "id": rid,
+            "children": [],
+            "parents": ["ROOT_ID", "GRID_ID"],
+            "meta": {"background": "BACKGROUND_TRANSPARENT"},
+        }
+        return rid
+
+    for item in chart_items:
+        if item[0] == "md":
+            _, title, subtitle = item
+            rid = add_row()
+            mid = f"MARKDOWN-{row_idx}"
+            layout[rid]["children"].append(mid)
+            layout[mid] = {
+                "type": "MARKDOWN",
+                "id": mid,
+                "children": [],
+                "parents": ["ROOT_ID", "GRID_ID", rid],
+                "meta": {
+                    "width": 12,
+                    "height": 12,
+                    "code": f"## {title}\n\n{subtitle}",
+                },
+            }
+        else:
+            cid, name, _section = item
+            rid = add_row()
+            # up to 3 charts per row
+            existing = [
+                k
+                for k in layout[rid]["children"]
+                if k.startswith("CHART-")
+            ]
+            if len(existing) >= 3:
+                rid = add_row()
+            chart_key = f"CHART-explore-{cid}"
+            layout[rid]["children"].append(chart_key)
+            col = len([k for k in layout[rid]["children"] if k.startswith("CHART-")]) - 1
+            layout[chart_key] = {
+                "type": "CHART",
+                "id": chart_key,
+                "children": [],
+                "parents": ["ROOT_ID", "GRID_ID", rid],
+                "meta": {
+                    "width": 4,
+                    "height": 55 if "Total" in name or "Records" in name else 65,
+                    "chartId": cid,
+                    "sliceName": name,
+                },
+            }
+
+    return layout
+
+
+def save_query_contexts():
+    app = __import__("superset.app", fromlist=["create_app"]).create_app()
+    with app.app_context():
+        from flask import g
+        from superset.extensions import db
+        from superset.models.slice import Slice
+        from superset.charts.schemas import ChartDataQueryContextSchema
+        from superset import security_manager
+
+        g.user = security_manager.find_user(username="admin")
+        for sl in db.session.query(Slice).all():
+            try:
+                fd = sl.form_data
+                metric = fd.get("metric")
+                metrics = fd.get("metrics") or ([metric] if metric else [])
+                if not metrics:
+                    metrics = [
+                        {
+                            "expressionType": "SQL",
+                            "sqlExpression": "COUNT(*)",
+                            "label": "COUNT(*)",
+                        }
+                    ]
+                groupby = fd.get("groupby") or []
+                payload = {
+                    "datasource": {"id": sl.datasource_id, "type": sl.datasource_type},
+                    "force": False,
+                    "queries": [
+                        {
+                            "filters": [],
+                            "extras": {"having": "", "where": ""},
+                            "applied_time_extras": {},
+                            "columns": groupby if isinstance(groupby, list) else [],
+                            "metrics": metrics,
+                            "orderby": [],
+                            "annotation_layers": [],
+                            "row_limit": int(fd.get("row_limit") or 1000),
+                            "series_limit": 0,
+                            "order_desc": True,
+                            "url_params": {},
+                            "custom_params": {},
+                            "custom_form_data": {},
+                        }
+                    ],
+                    "form_data": fd,
+                    "result_format": "json",
+                    "result_type": "full",
+                }
+                ChartDataQueryContextSchema().load(payload)
+                sl.query_context = json.dumps(payload)
+                sl.query_context_generation = True
+                db.session.add(sl)
+            except Exception as e:
+                print("qc err", sl.id, e)
+        db.session.commit()
+
+
+def main():
+    s, h = session()
+    db_map = get_db_map(s, h)
+
+    # Create new charts
+    new_ids = []
+    for db_name, schema, table, name, viz, params in NEW_CHARTS:
+        db_id = db_map.get(db_name)
+        if not db_id:
+            print("skip, no db:", db_name)
+            continue
+        ds_id = get_or_create_dataset(s, h, db_id, schema, table)
+        cid = create_chart(s, h, name, ds_id, viz, params)
+        new_ids.append((cid, name, db_name.split("·")[-1].strip()))
+        print("new chart", cid, name)
+
+    # All charts for dashboard
+    r = s.get(f"{BASE}/api/v1/chart/?q=(page:0,page_size:200)", headers=h)
+    all_charts = r.json().get("result", [])
+
+    def sort_key(c):
+        n = c.get("slice_name") or ""
+        if "Lakehouse" in n or "Records per Source" in n:
+            return (0, n)
+        if "PostgreSQL" in n:
+            return (1, n)
+        if "MySQL" in n:
+            return (2, n)
+        if "MongoDB" in n:
+            return (3, n)
+        if "Cassandra" in n:
+            return (4, n)
+        return (5, n)
+
+    all_charts.sort(key=sort_key)
+
+    chart_items = [
+        ("md", "ATC Lakehouse · Federated Data Platform", "Real-time analytics across all Trino catalogs · Dell Technologies"),
+    ]
+    current_section = None
+    for c in all_charts:
+        name = c.get("slice_name") or ""
+        if "PostgreSQL" in name and current_section != "pg":
+            chart_items.append(("md", "PostgreSQL Sales", "30M orders · CDC-enabled · atc-db02"))
+            current_section = "pg"
+        elif "MySQL" in name and current_section != "mysql":
+            chart_items.append(("md", "MySQL HR", "569K employee events · HR domain"))
+            current_section = "mysql"
+        elif "MongoDB" in name and current_section != "mongo":
+            chart_items.append(("md", "MongoDB Supply Chain", "3M supply chain events"))
+            current_section = "mongo"
+        elif "Cassandra" in name and current_section != "cass":
+            chart_items.append(("md", "Cassandra Telemetry", "IoT device metrics"))
+            current_section = "cass"
+        chart_items.append((c["id"], name, current_section))
+
+    chart_ids = [x[0] for x in chart_items if x[0] != "md"]
+    position = build_layout(chart_items)
+
+    css = ""
+    if os.path.exists(CSS_PATH):
+        css = open(CSS_PATH, encoding="utf-8").read()
+
+    chart_configuration = {
+        str(cid): {"id": cid, "crossFilters": {"scope": "global", "chartsInScope": chart_ids}}
+        for cid in chart_ids
+    }
+
+    payload = {
+        "dashboard_title": "ATC Lakehouse · Trino Federated",
+        "published": True,
+        "position_json": json.dumps(position),
+        "css": css,
+        "json_metadata": json.dumps(
+            {
+                "color_scheme": "palantir_ops",
+                "label_colors": {},
+                "refresh_frequency": 120,
+                "timed_refresh_immune_slices": [],
+                "expanded_slices": {},
+                "chart_configuration": chart_configuration,
+                "global_chart_configuration": {
+                    "scope": {"rootPath": ["ROOT_ID"], "excluded": []},
+                    "chartsInScope": chart_ids,
+                },
+                "native_filter_configuration": [],
+                "color_scheme_domain": [],
+                "shared_label_colors": {},
+            }
+        ),
+        "owners": [1, 2, 3],
+    }
+
+    r = s.put(f"{BASE}/api/v1/dashboard/{DASH_ID}", headers=h, json=payload)
+    print("dashboard update", r.status_code)
+    if r.status_code >= 400:
+        print(r.text[:500])
+        return
+
+    for cid in chart_ids:
+        s.put(
+            f"{BASE}/api/v1/chart/{cid}",
+            headers=h,
+            json={"dashboards": [DASH_ID], "owners": [1, 2, 3]},
+        )
+
+    print("Saving query contexts...")
+    save_query_contexts()
+    print(f"Done — {len(chart_ids)} charts, Palantir theme applied to dashboard.")
+
+
+if __name__ == "__main__":
+    main()

config/superset/bootstrap_lakehouse.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import json, sys, requests
+BASE = "http://127.0.0.1:8088"
+HOST = "10.0.21.50:8089"
+USER = "mo"
+SOURCES = [
+  ("Trino · PostgreSQL Sales", f"trino://{USER}@{HOST}/postgres_sales/public", "public", "sales_orders", [
+    ("PostgreSQL · Total Orders", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
+    ("PostgreSQL · Revenue by Region", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"SUM(amount)","label":"Revenue"},"groupby":["region"],"row_limit":20}),
+    ("PostgreSQL · Orders per Month", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Orders"}],"groupby":[{"expressionType":"SQL","sqlExpression":"date_trunc('month', order_ts)","label":"Month"}],"row_limit":24}),
+  ]),
+  ("Trino · MySQL HR", f"trino://{USER}@{HOST}/mysql_hr/hr", "hr", "employee_events", [
+    ("MySQL HR · Total Events", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
+    ("MySQL HR · By Department", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["department"],"row_limit":15}),
+    ("MySQL HR · By Region", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Events"}],"groupby":["region"],"row_limit":20}),
+  ]),
+  ("Trino · MongoDB Supply Chain", f"trino://{USER}@{HOST}/mongodb_supplychain/supplychain", "supplychain", "events", [
+    ("MongoDB · Total Events", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
+    ("MongoDB · By Type", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["type"],"row_limit":10}),
+    ("MongoDB · By Region", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Events"}],"groupby":["region"],"row_limit":10}),
+  ]),
+  ("Trino · Cassandra Telemetry", f"trino://{USER}@{HOST}/cassandra_telemetry/telemetry", "telemetry", "device_metrics", [
+    ("Cassandra · Total Metrics", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
+    ("Cassandra · By Metric Type", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["metric_type"],"row_limit":20}),
+    ("Cassandra · Avg by Device", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"AVG(metric_value)","label":"Avg"}],"groupby":["device_id"],"row_limit":20}),
+  ]),
+]
+OVERVIEW_SQL = """SELECT 'PostgreSQL sales_orders' AS source, COUNT(*) AS records FROM postgres_sales.public.sales_orders
+UNION ALL SELECT 'MySQL employee_events', COUNT(*) FROM mysql_hr.hr.employee_events
+UNION ALL SELECT 'MongoDB events', COUNT(*) FROM mongodb_supplychain.supplychain.events
+UNION ALL SELECT 'Cassandra device_metrics', COUNT(*) FROM cassandra_telemetry.telemetry.device_metrics"""
+
+def headers(s):
+    r=s.post(f"{BASE}/api/v1/security/login",json={"username":"admin","password":"admin","provider":"db","refresh":True}); r.raise_for_status()
+    h={"Authorization":"Bearer "+r.json()["access_token"],"Content-Type":"application/json"}
+    h["X-CSRFToken"]=s.get(f"{BASE}/api/v1/security/csrf_token/",headers=h).json()["result"]; h["Referer"]=BASE; return h
+
+def get_db(s,h,name,uri):
+    r=s.get(f"{BASE}/api/v1/database/",headers=h); r.raise_for_status()
+    for d in r.json().get("result",[]):
+        if d["database_name"]==name: return d["id"]
+    r=s.post(f"{BASE}/api/v1/database/",headers=h,json={"database_name":name,"sqlalchemy_uri":uri,"expose_in_sqllab":True,"allow_run_async":True})
+    if r.status_code not in (200,201): raise SystemExit(r.text)
+    print("DB",name,r.json()["id"]); return r.json()["id"]
+
+def get_ds(s,h,db,schema,table,sql=None):
+    r=s.get(f"{BASE}/api/v1/dataset/",headers=h); r.raise_for_status()
+    for d in r.json().get("result",[]):
+        if d.get("table_name")==table and d.get("schema")==schema and d.get("database",{}).get("id")==db: return d["id"]
+    p={"database":db,"table_name":table,"schema":schema} if not sql else {"database":db,"table_name":table,"sql":sql}
+    r=s.post(f"{BASE}/api/v1/dataset/",headers=h,json=p)
+    if r.status_code not in (200,201): raise SystemExit(r.text)
+    print(" DS",table,r.json()["id"]); return r.json()["id"]
+
+def mk_chart(s,h,name,ds,viz,params):
+    r=s.get(f"{BASE}/api/v1/chart/",headers=h); r.raise_for_status()
+    for c in r.json().get("result",[]):
+        if c.get("slice_name")==name: return c["id"]
+    p={"datasource":f"{ds}__table","viz_type":viz,"row_limit":1000,**params}
+    r=s.post(f"{BASE}/api/v1/chart/",headers=h,json={"slice_name":name,"viz_type":viz,"datasource_id":ds,"datasource_type":"table","params":json.dumps(p)})
+    if r.status_code not in (200,201): raise SystemExit(f"chart {name}: {r.text[:300]}")
+    print("  chart",name,r.json()["id"]); return r.json()["id"]
+
+def mk_dash(s,h,title,cids):
+    layout={"DASHBOARD_VERSION":"v2","ROOT_ID":{"type":"ROOT","id":"ROOT_ID","children":["GRID_ID"]},"GRID_ID":{"type":"GRID","id":"GRID_ID","children":[],"parents":["ROOT_ID"]}}
+    row=col=0
+    for cid in cids:
+        k=f"CHART-{cid}"; x=(col%3)*4; y=row*12
+        layout[k]={"type":"CHART","id":k,"children":[],"meta":{"width":4,"height":10,"chartId":cid},"parents":["ROOT_ID","GRID_ID"]}
+        layout["GRID_ID"]["children"].append(k); col+=1
+        if col%3==0: row+=1
+    payload={"dashboard_title":title,"published":True,"position_json":json.dumps(layout),"json_metadata":"{}"}
+    r=s.get(f"{BASE}/api/v1/dashboard/",headers=h); r.raise_for_status()
+    for d in r.json().get("result",[]):
+        if d.get("dashboard_title")==title:
+            did=d["id"]; s.put(f"{BASE}/api/v1/dashboard/{did}",headers=h,json=payload)
+            for cid in cids: s.put(f"{BASE}/api/v1/chart/{cid}",headers=h,json={"dashboards":[did]})
+            print("Dashboard",did); return did
+    r=s.post(f"{BASE}/api/v1/dashboard/",headers=h,json=payload)
+    if r.status_code not in (200,201): raise SystemExit(r.text)
+    did=r.json()["id"]
+    for cid in cids: s.put(f"{BASE}/api/v1/chart/{cid}",headers=h,json={"dashboards":[did]})
+    print("Dashboard",did); return did
+
+def main():
+    s=requests.Session(); h=headers(s)
+    cids=[]
+    odb=get_db(s,h,"Trino · Lakehouse Overview",f"trino://{USER}@{HOST}/postgres_sales/public")
+    ods=get_ds(s,h,odb,None,"lakehouse_counts",OVERVIEW_SQL)
+    cids.append(mk_chart(s,h,"Lakehouse · Records per Source",ods,"pie",{"metric":{"expressionType":"SQL","sqlExpression":"SUM(records)","label":"Records"},"groupby":["source"],"row_limit":10}))
+    for dbn,uri,sch,tbl,charts in SOURCES:
+        db=get_db(s,h,dbn,uri); ds=get_ds(s,h,db,sch,tbl)
+        for nm,viz,pr in charts: cids.append(mk_chart(s,h,nm,ds,viz,pr))
+    mk_dash(s,h,"ATC Lakehouse · Trino Federated",cids)
+if __name__=="__main__": main()

config/superset/bootstrap_pipeline_dashboard.py

@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+"""Create Superset dashboard for Debezium, Kafka CDC changes, and Spark."""
+import json
+import os
+import requests
+
+BASE = "http://127.0.0.1:8088"
+DASH_TITLE = "ATC Lakehouse · Pipeline & CDC"
+CSS_PATH = "/tmp/palantir_dashboard.css"
+
+MONITOR_URI = "trino://mo@10.0.21.50:8089/postgres_sales/monitor"
+KAFKA_URI = "trino://mo@10.0.21.50:8089/kafka/default"
+
+
+def session():
+    s = requests.Session()
+    r = s.post(
+        f"{BASE}/api/v1/security/login",
+        json={"username": "admin", "password": "admin", "provider": "db", "refresh": True},
+    )
+    r.raise_for_status()
+    h = {"Authorization": "Bearer " + r.json()["access_token"], "Content-Type": "application/json"}
+    h["X-CSRFToken"] = s.get(f"{BASE}/api/v1/security/csrf_token/", headers=h).json()["result"]
+    h["Referer"] = BASE
+    return s, h
+
+
+def get_db(s, h, name, uri):
+    r = s.get(f"{BASE}/api/v1/database/", headers=h)
+    for d in r.json().get("result", []):
+        if d["database_name"] == name:
+            return d["id"]
+    r = s.post(
+        f"{BASE}/api/v1/database/",
+        headers=h,
+        json={"database_name": name, "sqlalchemy_uri": uri, "expose_in_sqllab": True},
+    )
+    r.raise_for_status()
+    return r.json()["id"]
+
+
+def ds_table(s, h, db_id, schema, table):
+    r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
+    for d in r.json().get("result", []):
+        if d.get("table_name") == table and d.get("schema") == schema and d.get("database", {}).get("id") == db_id:
+            return d["id"]
+    r = s.post(
+        f"{BASE}/api/v1/dataset/",
+        headers=h,
+        json={"database": db_id, "schema": schema, "table_name": table},
+    )
+    r.raise_for_status()
+    return r.json()["id"]
+
+
+def ds_sql(s, h, db_id, name, sql):
+    r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
+    for d in r.json().get("result", []):
+        if d.get("table_name") == name:
+            return d["id"]
+    r = s.post(
+        f"{BASE}/api/v1/dataset/",
+        headers=h,
+        json={"database": db_id, "table_name": name, "sql": sql},
+    )
+    r.raise_for_status()
+    return r.json()["id"]
+
+
+def chart(s, h, name, ds_id, viz, params):
+    r = s.get(f"{BASE}/api/v1/chart/", headers=h)
+    for c in r.json().get("result", []):
+        if c.get("slice_name") == name:
+            return c["id"]
+    p = {"datasource": f"{ds_id}__table", "viz_type": viz, "row_limit": 1000, **params}
+    r = s.post(
+        f"{BASE}/api/v1/chart/",
+        headers=h,
+        json={
+            "slice_name": name,
+            "viz_type": viz,
+            "datasource_id": ds_id,
+            "datasource_type": "table",
+            "params": json.dumps(p),
+            "owners": [1, 2, 3],
+        },
+    )
+    r.raise_for_status()
+    return r.json()["id"]
+
+
+def layout(items):
+    L = {
+        "DASHBOARD_VERSION": "v2",
+        "ROOT_ID": {"type": "ROOT", "id": "ROOT_ID", "children": ["GRID_ID"]},
+        "GRID_ID": {"type": "GRID", "id": "GRID_ID", "children": [], "parents": ["ROOT_ID"]},
+    }
+    ri = 0
+
+    def row():
+        nonlocal ri
+        ri += 1
+        rid = f"ROW-{ri}"
+        L["GRID_ID"]["children"].append(rid)
+        L[rid] = {
+            "type": "ROW",
+            "id": rid,
+            "children": [],
+            "parents": ["ROOT_ID", "GRID_ID"],
+            "meta": {"background": "BACKGROUND_TRANSPARENT"},
+        }
+        return rid
+
+    for item in items:
+        if item[0] == "md":
+            rid = row()
+            mid = f"MD-{ri}"
+            L[rid]["children"].append(mid)
+            L[mid] = {
+                "type": "MARKDOWN",
+                "id": mid,
+                "children": [],
+                "parents": ["ROOT_ID", "GRID_ID", rid],
+                "meta": {"width": 12, "height": 10, "code": f"## {item[1]}\n\n{item[2]}"},
+            }
+        else:
+            cid, name = item
+            rid = row()
+            charts_in_row = [k for k in L[rid]["children"] if k.startswith("CHART-")]
+            if len(charts_in_row) >= 3:
+                rid = row()
+            key = f"CHART-{cid}"
+            L[rid]["children"].append(key)
+            hgt = 70 if "Recent" in name or "table" in name.lower() else 55
+            L[key] = {
+                "type": "CHART",
+                "id": key,
+                "children": [],
+                "parents": ["ROOT_ID", "GRID_ID", rid],
+                "meta": {"width": 4 if "Recent" not in name else 12, "height": hgt, "chartId": cid, "sliceName": name},
+            }
+    return L
+
+
+def main():
+    s, h = session()
+    db_mon = get_db(s, h, "Trino · Pipeline Monitor", MONITOR_URI)
+    db_kfk = get_db(s, h, "Trino · Kafka CDC", KAFKA_URI)
+
+    ds_conn = ds_table(s, h, db_mon, "monitor", "debezium_connectors")
+    ds_topics = ds_table(s, h, db_mon, "monitor", "kafka_topics")
+    ds_ops = ds_table(s, h, db_mon, "monitor", "cdc_operations")
+    ds_recent = ds_table(s, h, db_mon, "monitor", "cdc_recent_events")
+    ds_spark = ds_table(s, h, db_mon, "monitor", "spark_applications")
+
+    LIVE_CDC_SQL = """
+SELECT
+  'PostgreSQL' AS source_system,
+  CASE json_extract_scalar(_message, '$.payload.op')
+    WHEN 'c' THEN 'INSERT' WHEN 'u' THEN 'UPDATE' WHEN 'd' THEN 'DELETE' WHEN 'r' THEN 'SNAPSHOT' ELSE 'OTHER'
+  END AS change_type,
+  json_extract_scalar(_message, '$.payload.source.table') AS table_name,
+  json_extract_scalar(_message, '$.payload.after.order_id') AS record_key,
+  _timestamp AS event_time
+FROM kafka.default."postgres-sales.public.sales_orders"
+WHERE _timestamp > current_timestamp - INTERVAL '7' DAY
+LIMIT 500
+"""
+    ds_live = ds_sql(s, h, db_kfk, "live_cdc_postgres_sample", LIVE_CDC_SQL)
+
+    charts = []
+    charts.append(chart(s, h, "Debezium · Connector Status", ds_conn, "table", {}))
+    charts.append(
+        chart(
+            s,
+            h,
+            "Debezium · RUNNING vs State",
+            ds_conn,
+            "pie",
+            {
+                "metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
+                "groupby": ["state"],
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "Kafka · Topic Offsets",
+            ds_topics,
+            "echarts_timeseries_bar",
+            {
+                "metrics": [
+                    {"expressionType": "SQL", "sqlExpression": "SUM(end_offset)", "label": "Messages"}
+                ],
+                "groupby": ["topic"],
+                "row_limit": 20,
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "Kafka · Partitions per Topic",
+            ds_topics,
+            "echarts_timeseries_bar",
+            {
+                "metrics": [
+                    {"expressionType": "SQL", "sqlExpression": "SUM(end_offset)", "label": "Offset"}
+                ],
+                "groupby": ["topic", "partition_id"],
+                "row_limit": 30,
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "CDC · Changes by Type",
+            ds_ops,
+            "pie",
+            {
+                "metric": {"expressionType": "SQL", "sqlExpression": "SUM(event_count)", "label": "Events"},
+                "groupby": ["operation_label"],
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "CDC · Changes per Source",
+            ds_ops,
+            "echarts_timeseries_bar",
+            {
+                "metrics": [
+                    {"expressionType": "SQL", "sqlExpression": "SUM(event_count)", "label": "Events"}
+                ],
+                "groupby": ["source_system", "operation_label"],
+                "row_limit": 20,
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "CDC · Recent Changes (sampled)",
+            ds_recent,
+            "table",
+            {
+                "all_columns": [
+                    "source_system",
+                    "operation_label",
+                    "table_name",
+                    "record_key",
+                    "detail",
+                    "event_ts",
+                ],
+                "row_limit": 50,
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "CDC · Live Stream Sample (PostgreSQL)",
+            ds_live,
+            "table",
+            {
+                "all_columns": ["source_system", "change_type", "table_name", "record_key", "event_time"],
+                "row_limit": 100,
+            },
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "Spark · Applications",
+            ds_spark,
+            "table",
+            {"all_columns": ["app_id", "app_name", "state", "cores", "memory_mb", "duration_sec"]},
+        )
+    )
+    charts.append(
+        chart(
+            s,
+            h,
+            "Pipeline · Total Kafka Messages",
+            ds_topics,
+            "big_number_total",
+            {
+                "metric": {
+                    "expressionType": "SQL",
+                    "sqlExpression": "SUM(end_offset)",
+                    "label": "Total Offset",
+                }
+            },
+        )
+    )
+
+    chart_specs = [
+        ("md", "Pipeline & Change Data Capture", "Debezium → Kafka → Spark · Live CDC visibility"),
+        ("md", "Debezium Connect", "Connector health on kafka01 :8083"),
+        (charts[0], "Debezium · Connector Status"),
+        (charts[1], "Debezium · RUNNING vs State"),
+        ("md", "Apache Kafka", "Topic volume & CDC streams on kafka01"),
+        (charts[2], "Kafka · Topic Offsets"),
+        (charts[3], "Kafka · Partitions per Topic"),
+        (charts[9], "Pipeline · Total Kafka Messages"),
+        ("md", "Data Changes (CDC)", "INSERT / UPDATE / DELETE / SNAPSHOT — gewijzigde data"),
+        (charts[4], "CDC · Changes by Type"),
+        (charts[5], "CDC · Changes per Source"),
+        (charts[6], "CDC · Recent Changes (sampled)"),
+        (charts[7], "CDC · Live Stream Sample (PostgreSQL)"),
+        ("md", "Apache Spark", "Batch & streaming jobs · lake01:8080"),
+        (charts[8], "Spark · Applications"),
+    ]
+    items = chart_specs
+    cids = [c[0] for c in chart_specs if c[0] != "md"]
+    css = open(CSS_PATH).read() if os.path.exists(CSS_PATH) else ""
+
+    payload = {
+        "dashboard_title": DASH_TITLE,
+        "published": True,
+        "position_json": json.dumps(layout(items)),
+        "css": css,
+        "json_metadata": json.dumps(
+            {
+                "color_scheme": "palantir_ops",
+                "refresh_frequency": 120,
+                "chart_configuration": {
+                    str(c): {"id": c, "crossFilters": {"scope": "global", "chartsInScope": cids}}
+                    for c in cids
+                },
+            }
+        ),
+        "owners": [1, 2, 3],
+    }
+
+    r = s.get(f"{BASE}/api/v1/dashboard/", headers=h)
+    dash_id = None
+    for d in r.json().get("result", []):
+        if d.get("dashboard_title") == DASH_TITLE:
+            dash_id = d["id"]
+            break
+    if dash_id:
+        r = s.put(f"{BASE}/api/v1/dashboard/{dash_id}", headers=h, json=payload)
+    else:
+        r = s.post(f"{BASE}/api/v1/dashboard/", headers=h, json=payload)
+        dash_id = r.json()["id"]
+    print("Dashboard", dash_id, r.status_code)
+    for cid in cids:
+        s.put(f"{BASE}/api/v1/chart/{cid}", headers=h, json={"dashboards": [dash_id], "owners": [1, 2, 3]})
+
+    # query contexts
+    os.system("python3 /tmp/fix_charts_qc.py 2>/dev/null || true")
+    print(f"URL: {BASE}/superset/dashboard/{dash_id}/")
+
+
+if __name__ == "__main__":
+    main()

config/superset/collect_pipeline_metrics.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Collect Debezium, Kafka CDC, and Spark metrics into postgres monitor schema."""
+import json
+import shlex
+import subprocess
+import urllib.request
+from collections import Counter
+from datetime import datetime, timezone
+
+import psycopg2
+
+PG_DSN = "host=10.0.21.51 dbname=postgres user=mo password=Dell2026!"
+KAFKA = "10.0.21.36:9092"
+DEBEZIUM = "http://localhost:8083"  # Kafka Connect on kafka01; fallback lake01 :8083
+SPARK_MASTER = "http://10.0.21.50:8080"
+
+CDC_TOPICS = [
+    ("PostgreSQL", "postgres-sales.public.sales_orders"),
+    ("MongoDB", "mongodb-supplychain.supplychain.events"),
+]
+
+OP_LABELS = {"c": "INSERT", "u": "UPDATE", "d": "DELETE", "r": "SNAPSHOT", "i": "INSERT"}
+
+
+def fetch_json(url, timeout=10):
+    with urllib.request.urlopen(url, timeout=timeout) as r:
+        return json.loads(r.read().decode())
+
+
+def collect_debezium(cur):
+    connectors = fetch_json(f"{DEBEZIUM}/connectors")
+    cur.execute("DELETE FROM monitor.debezium_connectors")
+    now = datetime.now(timezone.utc)
+    for name in connectors:
+        try:
+            st = fetch_json(f"{DEBEZIUM}/connectors/{name}/status")
+        except Exception as e:
+            cur.execute(
+                """INSERT INTO monitor.debezium_connectors
+                   (connector_name, state, task_state, worker_id, checked_at)
+                   VALUES (%s,%s,%s,%s,%s)""",
+                (name, "ERROR", str(e)[:32], "", now),
+            )
+            continue
+        conn_state = st.get("connector", {}).get("state", "UNKNOWN")
+        tasks = st.get("tasks") or []
+        task_state = tasks[0].get("state", "NONE") if tasks else "NONE"
+        worker = st.get("connector", {}).get("worker_id", "")
+        cur.execute(
+            """INSERT INTO monitor.debezium_connectors
+               (connector_name, state, task_state, worker_id, checked_at)
+               VALUES (%s,%s,%s,%s,%s)""",
+            (name, conn_state, task_state, worker, now),
+        )
+
+
+KAFKA_BIN = "/opt/kafka/bin"
+USE_SSH_KAFKA = False  # set True when running off-host
+
+
+def _kafka_cmd(bin_name, args):
+    parts = [f"{KAFKA_BIN}/{bin_name}"] + list(args)
+    if USE_SSH_KAFKA:
+        remote = " ".join(shlex.quote(p) for p in parts)
+        full = f"ssh -o StrictHostKeyChecking=no root@10.0.21.36 {remote}"
+        return subprocess.check_output(full, shell=True, stderr=subprocess.DEVNULL, timeout=90, text=True)
+    return subprocess.check_output(parts, stderr=subprocess.DEVNULL, timeout=90, text=True)
+
+
+def kafka_end_offsets(topic):
+    try:
+        out = _kafka_cmd(
+            "kafka-run-class.sh",
+            [
+                "kafka.tools.GetOffsetShell",
+                "--broker-list",
+                "localhost:9092",
+                "--topic",
+                topic,
+            ],
+        )
+    except Exception:
+        return []
+    rows = []
+    for line in out.strip().splitlines():
+        parts = line.split(":")
+        if len(parts) >= 3:
+            rows.append((int(parts[1]), int(parts[2])))
+    return rows
+
+
+def sample_topic_messages(topic, max_msgs=3000, tail=5000):
+    """Sample recent messages using kafka-console-consumer from tail."""
+    offsets = kafka_end_offsets(topic)
+    if not offsets:
+        return []
+    # Pick partition 0 for sampling
+    part, end = offsets[0]
+    start = max(0, end - tail)
+    try:
+        out = _kafka_cmd(
+            "kafka-console-consumer.sh",
+            [
+                "--bootstrap-server",
+                "localhost:9092",
+                "--topic",
+                topic,
+                "--partition",
+                str(part),
+                "--offset",
+                str(start),
+                "--max-messages",
+                str(min(max_msgs, tail)),
+                "--timeout-ms",
+                "15000",
+            ],
+        )
+    except Exception:
+        return []
+    return [ln for ln in out.strip().split("\n") if ln.strip()]
+
+
+def parse_debezium_line(line):
+    try:
+        doc = json.loads(line)
+        payload = doc.get("payload") or doc
+        op = payload.get("op") or payload.get("operationType") or "?"
+        src = payload.get("source") or {}
+        table = src.get("table") or src.get("collection") or ""
+        ts_ms = payload.get("ts_ms") or src.get("ts_ms")
+        after = payload.get("after") or {}
+        before = payload.get("before") or {}
+        row = after if after else before
+        key = str(row.get("order_id") or row.get("event_id") or row.get("_id") or "")[:200]
+        detail = str(row.get("region") or row.get("type") or row.get("department") or "")[:200]
+        event_ts = None
+        if ts_ms:
+            event_ts = datetime.fromtimestamp(int(ts_ms) / 1000, tz=timezone.utc)
+        return op, table, key, detail, event_ts
+    except Exception:
+        return None
+
+
+def collect_kafka_cdc(cur):
+    now = datetime.now(timezone.utc)
+    cur.execute("DELETE FROM monitor.kafka_topics")
+    cur.execute("DELETE FROM monitor.cdc_operations")
+    cur.execute("DELETE FROM monitor.cdc_recent_events")
+
+    for source, topic in CDC_TOPICS:
+        for part, end in kafka_end_offsets(topic):
+            cur.execute(
+                """INSERT INTO monitor.kafka_topics (topic, partition_id, end_offset, checked_at)
+                   VALUES (%s,%s,%s,%s)""",
+                (topic, part, end, now),
+            )
+
+        lines = sample_topic_messages(topic, max_msgs=2000, tail=3000)
+        ops = Counter()
+        recent = []
+        for line in lines:
+            parsed = parse_debezium_line(line)
+            if not parsed:
+                continue
+            op, table, key, detail, event_ts = parsed
+            ops[op] += 1
+            if len(recent) < 100:
+                recent.append((op, table, key, detail, event_ts))
+
+        for op, cnt in ops.items():
+            cur.execute(
+                """INSERT INTO monitor.cdc_operations
+                   (source_system, topic, operation, operation_label, event_count, checked_at)
+                   VALUES (%s,%s,%s,%s,%s,%s)""",
+                (source, topic, op, OP_LABELS.get(op, op), cnt, now),
+            )
+
+        for op, table, key, detail, event_ts in recent[:50]:
+            cur.execute(
+                """INSERT INTO monitor.cdc_recent_events
+                   (source_system, topic, operation, operation_label, table_name,
+                    record_key, detail, event_ts, sampled_at)
+                   VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                (
+                    source,
+                    topic,
+                    op,
+                    OP_LABELS.get(op, op),
+                    table,
+                    key,
+                    detail,
+                    event_ts,
+                    now,
+                ),
+            )
+
+
+def collect_spark(cur):
+    now = datetime.now(timezone.utc)
+    cur.execute("DELETE FROM monitor.spark_applications")
+    try:
+        data = fetch_json(f"{SPARK_MASTER}/json/", timeout=5)
+        apps = []
+        if isinstance(data, dict):
+            # Standalone master JSON
+            for a in data.get("activeapps", []) or []:
+                apps.append(a)
+            for a in data.get("completedapps", []) or []:
+                apps.append(a)
+        for a in apps[:20]:
+            cur.execute(
+                """INSERT INTO monitor.spark_applications
+                   (app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
+                   VALUES (%s,%s,%s,%s,%s,%s,%s)
+                   ON CONFLICT (app_id) DO UPDATE SET
+                     app_name=EXCLUDED.app_name, state=EXCLUDED.state,
+                     cores=EXCLUDED.cores, memory_mb=EXCLUDED.memory_mb,
+                     duration_sec=EXCLUDED.duration_sec, checked_at=EXCLUDED.checked_at""",
+                (
+                    a.get("id", "unknown"),
+                    a.get("name", "Spark App"),
+                    "RUNNING" if "attempts" not in a else "COMPLETED",
+                    int(a.get("cores", 0) or 0),
+                    int((a.get("memory", 0) or 0) / 1024 / 1024),
+                    int(a.get("duration", 0) / 1000) if a.get("duration") else 0,
+                    now,
+                ),
+            )
+    except Exception as e:
+        # Placeholder row so dashboard shows Spark host status
+        cur.execute(
+            """INSERT INTO monitor.spark_applications
+               (app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
+               VALUES (%s,%s,%s,%s,%s,%s,%s)
+               ON CONFLICT (app_id) DO UPDATE SET state=EXCLUDED.state, checked_at=EXCLUDED.checked_at""",
+            (
+                "spark-master",
+                f"Spark Master @ {SPARK_MASTER}",
+                "REACHABLE" if "Connection" not in str(e) else "UNREACHABLE",
+                0,
+                0,
+                0,
+                now,
+            ),
+        )
+
+
+def main():
+    conn = psycopg2.connect(PG_DSN)
+    conn.autocommit = True
+    cur = conn.cursor()
+    print("Collecting Debezium...")
+    collect_debezium(cur)
+    print("Collecting Kafka CDC samples...")
+    collect_kafka_cdc(cur)
+    print("Collecting Spark...")
+    collect_spark(cur)
+    cur.close()
+    conn.close()
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

config/superset/fix_charts_qc.py

@@ -0,0 +1,85 @@
+"""Generate and save query_context for API-created Superset charts."""
+import json
+
+app = __import__("superset.app", fromlist=["create_app"]).create_app()
+
+with app.app_context():
+    from flask import g
+    from superset.extensions import db
+    from superset.models.slice import Slice
+    from superset.models.core import Database
+    from superset.charts.schemas import ChartDataQueryContextSchema
+    from superset import security_manager
+
+    admin = security_manager.find_user(username="admin")
+    g.user = admin
+
+    charts = db.session.query(Slice).order_by(Slice.id).all()
+    for sl in charts:
+        try:
+            fd = sl.form_data
+            metric = fd.get("metric")
+            metrics = fd.get("metrics") or ([metric] if metric else [])
+            if not metrics:
+                metrics = [
+                    {
+                        "expressionType": "SQL",
+                        "sqlExpression": "COUNT(*)",
+                        "label": "COUNT(*)",
+                    }
+                ]
+
+            groupby = fd.get("groupby") or []
+            payload = {
+                "datasource": {
+                    "id": sl.datasource_id,
+                    "type": sl.datasource_type,
+                },
+                "force": False,
+                "queries": [
+                    {
+                        "filters": [],
+                        "extras": {"having": "", "where": ""},
+                        "applied_time_extras": {},
+                        "columns": groupby if isinstance(groupby, list) else [],
+                        "metrics": metrics,
+                        "orderby": [],
+                        "annotation_layers": [],
+                        "row_limit": int(fd.get("row_limit") or 1000),
+                        "series_limit": 0,
+                        "order_desc": True,
+                        "url_params": {},
+                        "custom_params": {},
+                        "custom_form_data": {},
+                    }
+                ],
+                "form_data": fd,
+                "result_format": "json",
+                "result_type": "full",
+            }
+            qc = ChartDataQueryContextSchema().load(payload)
+            ctx = qc.cache_values if hasattr(qc, "cache_values") else None
+            if ctx is None:
+                # fallback: store factory input dict
+                from superset.common.query_context_factory import QueryContextFactory
+
+                factory = QueryContextFactory()
+                ctx = {
+                    "datasource": {
+                        "id": sl.datasource_id,
+                        "type": sl.datasource_type,
+                    },
+                    "force": False,
+                    "queries": payload["queries"],
+                    "form_data": fd,
+                    "result_format": "json",
+                    "result_type": "full",
+                }
+            sl.query_context = json.dumps(ctx) if isinstance(ctx, dict) else json.dumps(payload)
+            sl.query_context_generation = True
+            db.session.add(sl)
+            print("OK", sl.id, sl.slice_name[:50])
+        except Exception as e:
+            print("ERR", sl.id, sl.slice_name[:40], e)
+    db.session.commit()
+    print("committed")

config/superset/palantir_dashboard.css

@@ -0,0 +1,97 @@
+/* ATC Lakehouse dashboard — Palantir OPS overlay */
+.dashboard-wrapper,
+.dashboard,
+.grid-container,
+.dashboard-content,
+.dashboard-component-chart-holder {
+  background: transparent !important;
+}
+
+.dashboard-header-container {
+  background: linear-gradient(135deg, rgba(12, 28, 52, 0.95) 0%, rgba(6, 20, 40, 0.98) 100%) !important;
+  border-bottom: 1px solid rgba(56, 132, 220, 0.25) !important;
+  backdrop-filter: blur(12px);
+}
+
+.dashboard-header .dashboard-title {
+  font-family: 'DM Sans', system-ui, sans-serif !important;
+  font-weight: 700 !important;
+  letter-spacing: -0.02em !important;
+  background: linear-gradient(90deg, #e8eef7 0%, #22d3ee 50%, #fb923c 100%);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+  background-clip: text;
+}
+
+.dashboard-component {
+  background: rgba(12, 28, 52, 0.72) !important;
+  border: 1px solid rgba(56, 132, 220, 0.2) !important;
+  border-radius: 12px !important;
+  box-shadow: 0 4px 24px rgba(0, 0, 0, 0.35), inset 0 1px 0 rgba(255, 255, 255, 0.04) !important;
+  backdrop-filter: blur(8px);
+  transition: border-color 0.2s ease, box-shadow 0.2s ease;
+}
+
+.dashboard-component:hover {
+  border-color: rgba(251, 146, 60, 0.35) !important;
+  box-shadow: 0 8px 32px rgba(37, 99, 235, 0.2) !important;
+}
+
+.chart-header,
+.header-title,
+.header-line {
+  color: #e8eef7 !important;
+  font-family: 'DM Sans', system-ui, sans-serif !important;
+}
+
+.slice_container,
+.chart-container,
+.dashboard-chart-id {
+  background: transparent !important;
+}
+
+/* Big number / KPI tiles */
+.big-number .header-line,
+.big-number-viz .header-line {
+  color: #22d3ee !important;
+  font-size: 2.5rem !important;
+  font-weight: 700 !important;
+  text-shadow: 0 0 24px rgba(34, 211, 238, 0.35);
+}
+
+/* Markdown section headers */
+.dashboard-markdown,
+.markdown-component {
+  background: linear-gradient(90deg, rgba(59, 130, 246, 0.12), transparent) !important;
+  border-left: 3px solid #3b82f6 !important;
+  padding: 12px 16px !important;
+  border-radius: 0 8px 8px 0 !important;
+}
+
+.dashboard-markdown h1,
+.dashboard-markdown h2,
+.markdown-component h1,
+.markdown-component h2 {
+  color: #e8eef7 !important;
+  font-family: 'DM Sans', sans-serif !important;
+  margin: 0 !important;
+}
+
+.dashboard-markdown p,
+.markdown-component p {
+  color: #94a3b8 !important;
+  margin: 4px 0 0 !important;
+}
+
+/* Filter bar */
+.filter-status-pane,
+.dashboard-filters-panel {
+  background: rgba(6, 20, 40, 0.9) !important;
+  border: 1px solid rgba(56, 132, 220, 0.2) !important;
+  border-radius: 10px !important;
+}
+
+/* Grid subtle glow */
+.grid-row {
+  margin-bottom: 8px;
+}

config/superset/save_query_context.py

@@ -0,0 +1,15 @@
+app = __import__("superset.app", fromlist=["create_app"]).create_app()
+
+with app.app_context():
+    from superset.extensions import db
+    from superset.models.slice import Slice
+    charts = db.session.query(Slice).all()
+    for sl in charts:
+        try:
+            sl.query_context = sl.get_query_context()
+            db.session.add(sl)
+            print("saved", sl.id, (sl.slice_name or "")[:50])
+        except Exception as e:
+            print("err", sl.id, e)
+    db.session.commit()
+    print("committed", len(charts), "charts")

config/superset/superset_config.py

@@ -1,39 +1,117 @@
 import os
 
-# Secret key for session signing
+SECRET_KEY = os.environ.get("SUPERSET_SECRET_KEY", "your-secret-key-here")
-SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY', 'your-secret-key-here')
+SQLALCHEMY_DATABASE_URI = "sqlite:////app/superset_home/superset.db"
 
-# Database configuration - use SQLite to avoid psycopg2 issues
-SQLALCHEMY_DATABASE_URI = 'sqlite:////app/superset_home/superset.db'
-
-# Redis cache configuration
 CACHE_CONFIG = {
-    'CACHE_TYPE': 'redis',
+    "CACHE_TYPE": "redis",
-    'CACHE_REDIS_URL': 'redis://redis:6379/0',
+    "CACHE_REDIS_URL": "redis://redis:6379/0",
-    'CACHE_DEFAULT_TIMEOUT': 300
+    "CACHE_DEFAULT_TIMEOUT": 300,
 }
 
-# Enable CSRF protection
 ENABLE_PROXY_FIX = True
-
+TIMEZONE = "Europe/Amsterdam"
-# Feature flags
-FEATURE_FLAGS = {
-    'ENABLE_TEMPLATE_PROCESSING': True,
-    'ALERT_REPORTS': True,
-}
-
-# Row limit
 ROW_LIMIT = 50000
 
-# Viz types
+# Branding — logo must be same-origin (/static/...) for CSP (img-src 'self')
-VIZ_TYPE_DICT = {
+APP_NAME = "Dell"
-    'table': {},
+APP_ICON = "/static/assets/images/dell-logo.svg"
-    'dist_bar': {},
+LOGO_TARGET_PATH = "/superset/welcome/"
-    'line': {},
+LOGO_TOOLTIP = "Dell · ATC Lakehouse"
-    'area': {},
+
-    'pie': {},
+FEATURE_FLAGS = {
-    'number': {},
+    "ENABLE_TEMPLATE_PROCESSING": True,
+    "ALERT_REPORTS": True,
+    "DASHBOARD_NATIVE_FILTERS": True,
+    "DASHBOARD_CROSS_FILTERS": True,
+    "ENABLE_ADVANCED_DATA_TYPES": True,
 }
 
-# Timezone
+# Allow icons server if needed for other assets (optional)
-TIMEZONE = 'Europe/Amsterdam'
+TALISMAN_ENABLED = True
+TALISMAN_CONFIG = {
+    "content_security_policy": {
+        "base-uri": ["'self'"],
+        "default-src": ["'self'"],
+        "img-src": [
+            "'self'",
+            "blob:",
+            "data:",
+            "https://apachesuperset.gateway.scarf.sh",
+            "https://static.scarf.sh/",
+            "http://atc-docker01.dell-atc.lan:8080",
+            "https://atc-docker01.dell-atc.lan:8080",
+        ],
+        "worker-src": ["'self'", "blob:"],
+        "connect-src": ["'self'"],
+        "object-src": "'none'",
+        "style-src": ["'self'", "'unsafe-inline'"],
+        "font-src": ["'self'"],
+        "script-src": ["'self'", "'strict-dynamic'"],
+    },
+    "content_security_policy_nonce_in": ["script-src"],
+    "force_https": False,
+    "frame_options": "SAMEORIGIN",
+}
+
+EXTRA_CATEGORICAL_COLOR_SCHEMES = [
+    {
+        "id": "palantir_ops",
+        "description": "Palantir OPS — blue, cyan, orange, teal",
+        "label_colors": {},
+        "isDefault": True,
+        "colors": [
+            "#3b82f6", "#22d3ee", "#fb923c", "#2dd4bf", "#fbbf24",
+            "#a78bfa", "#f472b6", "#34d399", "#60a5fa", "#94a3b8",
+        ],
+    },
+]
+
+EXTRA_SEQUENTIAL_COLOR_SCHEMES = [
+    {
+        "id": "palantir_blue",
+        "description": "Palantir blue gradient",
+        "isDefault": True,
+        "colors": ["#040c18", "#0c1a30", "#1e40af", "#3b82f6", "#22d3ee", "#7dd3fc"],
+    },
+]
+
+PALANTIR_FONTS = [
+    "https://fonts.googleapis.com/css2?family=DM+Sans:ital,opsz,wght@0,9..40,400;0,9..40,500;0,9..40,600;0,9..40,700&family=JetBrains+Mono:wght@400;500;600&display=swap",
+]
+
+PALANTIR_TOKENS = {
+    "brandAppName": "Dell",
+    "brandLogoAlt": "Dell",
+    "brandLogoUrl": "/static/assets/images/dell-logo.svg",
+    "brandLogoMargin": "8px 12px 8px 0",
+    "brandLogoHref": "/",
+    "brandLogoHeight": "32px",
+    "colorPrimary": "#007DB8",
+    "colorLink": "#22d3ee",
+    "colorSuccess": "#2dd4bf",
+    "colorWarning": "#fbbf24",
+    "colorError": "#f87171",
+    "colorInfo": "#38bdf8",
+    "colorBgBase": "#040c18",
+    "colorBgLayout": "#061428",
+    "colorBgContainer": "#0c1a30",
+    "colorBgElevated": "#0f2444",
+    "colorBorder": "#1e3a5f",
+    "colorBorderSecondary": "rgba(56, 132, 220, 0.22)",
+    "colorText": "#e8eef7",
+    "colorTextSecondary": "#94a3b8",
+    "colorTextTertiary": "#64748b",
+    "fontUrls": PALANTIR_FONTS,
+    "fontFamily": "'DM Sans', Inter, Helvetica, Arial, sans-serif",
+    "fontFamilyCode": "'JetBrains Mono', 'IBM Plex Mono', monospace",
+    "borderRadius": 8,
+    "borderRadiusLG": 12,
+}
+
+THEME_DEFAULT = {
+    "algorithm": "dark",
+    "token": PALANTIR_TOKENS,
+}
+THEME_DARK = None
+ENABLE_UI_THEME_ADMINISTRATION = False

config/trino/README.md

@@ -0,0 +1,19 @@
+# Trino — atc-lake01
+
+| Item | Value |
+|------|-------|
+| UI | http://atc-lake01:8089/ui/ |
+| Container | `trino` (Docker) |
+| Catalogs | `catalog/*.properties` |
+
+## Catalogs
+
+| File | Connector | Source |
+|------|-----------|--------|
+| `postgres_sales.properties` | PostgreSQL | atc-db02:5432 |
+| `mysql_hr.properties` | MySQL | atc-db02:3306 |
+| `mongodb_supplychain.properties` | MongoDB | atc-db02:27017 |
+| `cassandra_telemetry.properties` | Cassandra | atc-db02:9042 |
+| `iceberg.properties` | Iceberg | ObjectScale S3 `10.0.20.111:9020` |
+
+Passwords redacted in git. Refresh: `./scripts/collect/collect-fleet-config.sh`

config/trino/catalog/cassandra_telemetry.properties

@@ -0,0 +1,11 @@
+connector.name=cassandra
+
+cassandra.contact-points=10.0.21.51
+cassandra.native-protocol-port=9042
+
+# Required DC setting for load balancing:
+cassandra.load-policy.dc-aware.local-dc=datacenter1
+
+# If Cassandra auth is enabled, uncomment and set:
+# cassandra.username=mo
+# cassandra.password=REDACTED

config/trino/catalog/iceberg.properties

@@ -0,0 +1,10 @@
+connector.name=iceberg
+iceberg.catalog.type=TESTING_FILE_METASTORE
+hive.metastore.catalog.dir=/tmp/iceberg_metadata
+
+# S3 Connectivity
+hive.s3.endpoint=http://10.0.20.111:9020
+hive.s3.aws-access-key=REDACTED
+hive.s3.aws-secret-key=REDACTED
+hive.s3.path-style-access=true
+hive.s3.ssl.enabled=false

config/trino/catalog/kafka.properties

@@ -0,0 +1,4 @@
+connector.name=kafka
+kafka.nodes=10.0.21.36:9092
+kafka.table-names=postgres-sales.public.sales_orders,mongodb-supplychain.supplychain.events,schema-changes.hr
+kafka.hide-internal-columns=false

config/trino/catalog/mongodb_supplychain.properties

@@ -0,0 +1,3 @@
+connector.name=mongodb
+mongodb.connection-url=mongodb://mo:Dell2026%21@10.0.21.51:27017/?authSource=admin
+mongodb.schema-collection=__trino_schema

config/trino/catalog/mysql_hr.properties

@@ -0,0 +1,4 @@
+connector.name=mysql
+connection-url=jdbc:mysql://10.0.21.51:3306
+connection-user=mo
+connection-password=REDACTED

config/trino/catalog/postgres_sales.properties

@@ -0,0 +1,4 @@
+connector.name=postgresql
+connection-url=jdbc:postgresql://10.0.21.51:5432/postgres
+connection-user=mo
+connection-password=REDACTED

docs/docker-inventory.md

@@ -34,12 +34,23 @@ Compose: `config/docker/atc-db02/docker-compose.yml`
 | Container | Image | Ports |
 |-----------|-------|-------|
 | kafka-connect | debezium/connect:2.5.4 | 8083 |
-| spark-master | bitnami/spark | 7077, 8080 |
+| s3-kafka-consumer | python:3.11-slim | (internal) |
+| spark-master | bitnami/spark | 7077, 8081→8080 |
 | spark-worker | bitnami/spark | (internal) |
 | trino | trinodb/trino:405 | 8089→8080 |
 | spark-temp | apache/spark:3.4.0 | — |
 
-Compose: `config/docker/atc-lake01/docker-compose.yml`
+Debezium connectors: `postgres-sales-connector`, `mysql-hr-connector`, `mongodb-supplychain-connector` (see `config/debezium/connectors/`).
+
+Compose: `compose/atc-lake01/docker-compose.yml`
+
+## atc-airflow01 (10.0.21.55)
+
+| Container | Image | Ports |
+|-----------|-------|-------|
+| airflow | atc-airflow:3.0.6 | 8080 |
+
+DAG `generate_data_all_databases` — data generators for atc-db02. Compose: `/opt/airflow/docker-compose.yml` on host.
 
 ## atc-kafka01 (10.0.21.36)
 
@@ -47,7 +58,7 @@ Compose: `config/docker/atc-lake01/docker-compose.yml`
 |-----------|-------|-------|
 | kafka-ui | provectuslabs/kafka-ui | 9000→8080 |
 
-Kafka broker: native on `:9092` (not containerized in current lab).
+Kafka broker: **KRaft** native on `:9092` — config: `config/kafka/kraft-server.properties`.
 
 ## atc-mgt01 (10.0.20.104)
 

docs/landscape.md

@@ -108,9 +108,9 @@ flowchart TB
 | Spark, Trino, Debezium | Docker (lake01) | `config/docker/atc-lake01/` |
 | Source DBs | Docker (db02) | `config/docker/atc-db02/` |
 | Kafka UI | Docker (kafka01) | `config/docker/atc-kafka01/` |
-| Kafka broker | Native/systemd (kafka01) | `config/kafka/` |
+| Kafka broker | Native KRaft (kafka01) | `config/kafka/kraft-server.properties` |
 | Elasticsearch | Native (elastic01) | `config/elastic/` |
-| Airflow | Native (airflow01) | `config/airflow/` |
+| Airflow | Native (airflow01) | `config/airflow/airflow.cfg`, DAGs |
 | ObjectScale | ECS appliance | `config/objectscale/` |
 | Forgejo | Docker (mgt01) | `compose/forgejo/` |
 

docs/network.md

@@ -0,0 +1,64 @@
+# Network — ATC Lakehouse lab
+
+## Subnets
+
+```mermaid
+flowchart TB
+  subgraph mgmt [10.0.10.0/24 — Management]
+    PVE[pve01 · 10.0.10.65]
+    IDRAC[iDRAC · 10.0.41.102]
+  end
+
+  subgraph data20 [10.0.20.0/24 — Storage & legacy]
+    OS[ObjectScale · 10.0.20.111]
+    DB1[atc-db01 · 10.0.20.112]
+    MGT[atc-mgt01 · 10.0.20.104]
+    GF[atc-grafana · 10.0.20.103]
+  end
+
+  subgraph lake21 [10.0.21.0/24 — Lakehouse compute]
+    D1[atc-docker01 · 10.0.21.45]
+    D2[atc-docker02 · 10.0.21.47]
+    KF[atc-kafka01 · 10.0.21.36]
+    LK[atc-lake01 · 10.0.21.50]
+    DB2[atc-db02 · 10.0.21.51]
+    AF[atc-airflow01 · 10.0.21.55]
+    EL[atc-elastic01 · 10.0.21.46]
+  end
+
+  PVE --> D1 & D2 & KF & LK & DB2 & OS & DB1 & MGT
+```
+
+## DNS
+
+Internal names: `*.dell-atc.lan` (see `config/hosts/atc-lab.hosts`).
+
+| Pattern | Example |
+|---------|---------|
+| Short hostname | `atc-lake01` |
+| FQDN | `atc-lake01.dell-atc.lan` |
+
+## Key ports (east-west)
+
+| Service | Port | Protocol |
+|---------|------|----------|
+| Kafka | 9092 | TCP |
+| Debezium Connect | 8083 | HTTP |
+| Spark UI | 8080 | HTTP |
+| Trino | 8089 | HTTP |
+| PostgreSQL | 5432 | TCP |
+| ObjectScale S3 | 9020 | HTTP |
+| ObjectScale UI | 443 | HTTPS |
+| Homepage | 80 | HTTP |
+
+## SSH mesh
+
+All lab VMs: `/root/.ssh/atc_cluster` — see [ssh-mesh.md](ssh-mesh.md).
+
+ObjectScale uses `admin@` for initial bootstrap; `root@` after key install.
+
+## Firewall notes
+
+- Lab assumes flat L2/L3 trust within `10.0.10/20/21.x`.
+- ObjectScale `management_clients: 0.0.0.0/0` in deploy.yml — tighten for production.
+- Proxmox API `:8006` reachable from docker01 for homepage widget.

docs/objectscale.md

@@ -0,0 +1,57 @@
+# Dell ObjectScale (ECS) — atc-objectscale
+
+| Item | Value |
+|------|-------|
+| Hostname | `luna.local` / `atc-objectscale` |
+| IP | `10.0.20.111` |
+| SSH | `admin@10.0.20.111` (appliance; root via sudo keys) |
+| Management UI | https://10.0.20.111/ (port **443**) |
+| S3 API | http://10.0.20.111:**9020** |
+| Container | `ecs-storageos` (`emccorp/ecs-software`) |
+| Install config | `/opt/emc/ecs-install/deploy.yml` |
+
+## What is `management_clients: 0.0.0.0/0`?
+
+In `deploy.yml`, this setting controls **which client IP addresses may access ECS management ports** (admin API, node management, not the S3 data path).
+
+| Value | Meaning |
+|-------|---------|
+| `0.0.0.0/0` | **Everyone on any network** — entire Internet can reach management ports if routed/firewall allows |
+| `10.0.20.0/24` | Only hosts in `10.0.20.x` subnet |
+| `10.0.21.45` | Only that single host |
+
+**Lab default:** installer often sets `0.0.0.0/0` for convenience (= no IP whitelist).  
+**Production:** restrict to management subnet only, e.g.:
+
+```yaml
+management_clients:
+  - 10.0.10.0/24    # Proxmox / mgmt
+  - 10.0.20.0/24    # storage VLAN
+  - 10.0.21.0/24    # compute VLAN
+```
+
+Changing this requires editing `/opt/emc/ecs-install/deploy.yml` on the appliance and may need an ECS config apply — plan a maintenance window.
+
+## S3 usage in this lab
+
+| Consumer | Bucket / path | Config |
+|----------|---------------|--------|
+| Trino Iceberg | via `iceberg.properties` | `hive.s3.endpoint=http://10.0.20.111:9020` |
+| Spark jobs | bucket `data` | `/opt/spark-jobs/*.py` on lake01 |
+| AWS CLI | `aws --endpoint-url http://10.0.20.111:9020` | keys in Trino catalog (redacted in git) |
+
+## Ports (reference)
+
+| Port | Service |
+|------|---------|
+| 443 | ECS Management UI (HTTPS) |
+| 9020 | S3 API |
+| 4443 | ECS API (nginx) |
+| 9021–9025 | Data head services (internal) |
+
+## Health check
+
+```bash
+curl -sk -o /dev/null -w "%{http_code}\n" https://10.0.20.111/    # expect 200
+curl -s -o /dev/null -w "%{http_code}\n" http://10.0.20.111:9020/  # expect 403 without auth (normal)
+```

docs/recommendations.md

@@ -0,0 +1,33 @@
+# Recommendations — status
+
+## Completed
+
+- [x] Trino catalog properties
+- [x] Grafana ini + dashboard provisioning
+- [x] Spark jobs (`/opt/spark-jobs`)
+- [x] Airflow `airflow.cfg` + DAG scripts
+- [x] Kafka KRaft `server.properties`
+- [x] ObjectScale `deploy.yml`
+- [x] Elasticsearch `elasticsearch.yml`
+- [x] Docker compose per host
+- [x] LDAP LDIF exports (mgt01)
+- [x] NPM compose (mgt01)
+- [x] Proxmox VM inventory JSON
+- [x] SSH mesh (12 hosts + ObjectScale)
+- [x] Architecture diagram page
+- [x] `collect-fleet-config.sh` + `populate-repo.py`
+- [x] Backup script
+- [x] Health check script
+- [x] Cron documentation
+- [x] Ansible playbook + inventory
+- [x] Gitea CI validate workflow
+- [x] Network documentation
+- [x] Homepage DOCS tab
+
+## Optional next steps
+
+- [ ] Kibana `kibana.yml` export
+- [ ] TLS certificates inventory (NPM letsencrypt paths)
+- [ ] Automated weekly git commit via cron (install from `scripts/cron/README.md`)
+- [ ] Vault/external secrets instead of redacted files
+- [ ] Reach remaining hosts: `10.0.21.52`, `.37`, `.38` (no SSH key yet)

inventory/proxmox-vms.json

@@ -0,0 +1,154 @@
+[
+  {
+    "vmid": 103,
+    "name": "DELL-OBSCE-4.2.0.0-OVA",
+    "status": "stopped",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 68.7
+  },
+  {
+    "vmid": 105,
+    "name": "airflow01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.55",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 110,
+    "name": "atc-db01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.20.112",
+    "maxmem_gb": 8.6
+  },
+  {
+    "vmid": 115,
+    "name": "atc-docker01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.45",
+    "maxmem_gb": 8.6
+  },
+  {
+    "vmid": 116,
+    "name": "atc-docker02",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.47",
+    "maxmem_gb": 8.6
+  },
+  {
+    "vmid": 112,
+    "name": "atc-elastic",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.46",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 114,
+    "name": "atc-grafana",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.20.103",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 113,
+    "name": "atc-kafka",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.36",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 108,
+    "name": "atc-lake01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.50",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 111,
+    "name": "atc-lama",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.39",
+    "maxmem_gb": 8.6
+  },
+  {
+    "vmid": 101,
+    "name": "atc-mgt01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.20.104",
+    "maxmem_gb": 8.6
+  },
+  {
+    "vmid": 100,
+    "name": "atc-objectscale",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.20.111",
+    "maxmem_gb": 68.7
+  },
+  {
+    "vmid": 109,
+    "name": "atc-source-systems",
+    "status": "running",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 117,
+    "name": "atc-test",
+    "status": "running",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 4.3
+  },
+  {
+    "vmid": 118,
+    "name": "atc-w11mo",
+    "status": "running",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 17.2
+  },
+  {
+    "vmid": 104,
+    "name": "dataflow01",
+    "status": "running",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 106,
+    "name": "monitor01",
+    "status": "running",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 107,
+    "name": "portal01",
+    "status": "running",
+    "node": "pve01",
+    "ip": "10.0.21.49",
+    "maxmem_gb": 34.4
+  },
+  {
+    "vmid": 102,
+    "name": "rocky9-base",
+    "status": "stopped",
+    "node": "pve01",
+    "ip": null,
+    "maxmem_gb": 34.4
+  }
+]

scripts/backup/backup-lab.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Backup Lakehouse git state + critical Docker volumes to ObjectScale/local path.
+set -euo pipefail
+
+REPO="${REPO:-/root/lakehouse}"
+BACKUP_ROOT="${BACKUP_ROOT:-/var/backups/atc-lakehouse}"
+DATE=$(date +%Y%m%d-%H%M%S)
+DEST="$BACKUP_ROOT/$DATE"
+S3_ENDPOINT="${S3_ENDPOINT:-http://10.0.20.111:9020}"
+S3_BUCKET="${S3_BUCKET:-backups}"
+
+mkdir -p "$DEST"
+
+echo "==> Git archive"
+tar czf "$DEST/lakehouse-git.tgz" -C "$(dirname "$REPO")" "$(basename "$REPO")" \
+  --exclude=".git/objects/pack/*.tmp"
+
+echo "==> Docker volumes (docker01)"
+for vol in forgejo_forgejo superset_superset_home lam_lam; do
+  if docker volume inspect "$vol" &>/dev/null; then
+    docker run --rm -v "${vol}:/data:ro" -v "$DEST:/backup" alpine \
+      tar czf "/backup/${vol}.tgz" -C /data .
+    echo "  backed up $vol"
+  fi
+done
+
+echo "==> Config snapshot via collect"
+"$REPO/scripts/collect/collect-fleet-config.sh" || true
+
+if command -v aws &>/dev/null && [[ -n "${AWS_ACCESS_KEY_ID:-}" ]]; then
+  echo "==> Upload to ObjectScale"
+  aws --endpoint-url "$S3_ENDPOINT" s3 cp "$DEST" "s3://${S3_BUCKET}/lakehouse/${DATE}/" --recursive
+fi
+
+echo "Backup complete: $DEST"
+ls -la "$DEST"

scripts/collect/collect-fleet-config.sh

@@ -1,24 +1,41 @@
 #!/usr/bin/env bash
-# Pull live Docker/ObjectScale configs into the Lakehouse git repo.
+# Pull live configs from entire ATC fleet into git.
-# Run on atc-docker01 as root (requires SSH mesh).
 set -euo pipefail
-
 REPO="${REPO:-/root/lakehouse}"
 KEY="${KEY:-/root/.ssh/atc_cluster}"
-SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10)
+SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=15)
+
+redact() {
+  sed -E \
+    -e 's/^(fernet_key|internal_api_secret_key|admin_password) = .*/\1 = REDACTED/' \
+    -e 's/^(connection-password=).*/\1REDACTED/' \
+    -e 's/(hive\.s3\.aws-access-key=).*/\1REDACTED/' \
+    -e 's/(hive\.s3\.aws-secret-key=).*/\1REDACTED/' \
+    -e 's/(S3_ACCESS_KEY|S3_SECRET_KEY) = .*/\1 = "REDACTED"/' \
+    -e 's/(ssh_password|ansible_become_pass):.*/\1: REDACTED/'
+}
+
+echo "==> Python fleet collector (Trino, Grafana, Spark, LDAP, NPM, Proxmox)"
+python3 "$REPO/scripts/collect/populate-repo.py"
 
 echo "==> db02 docker-compose"
 "${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml"
 
-echo "==> objectscale deploy.yml (redacted)"
+echo "==> objectscale"
-"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | \
+"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | redact > "$REPO/config/objectscale/deploy.yml"
-  sed -E 's/(ssh_password|ansible_become_pass):.*/\1: "REDACTED"/' \
-  > "$REPO/config/objectscale/deploy.yml"
 
-echo "==> elasticsearch.yml"
+echo "==> elasticsearch"
 "${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml"
 
-echo "==> container inventory"
+echo "==> kafka"
-"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json"
+"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" > "$REPO/config/kafka/kraft-server.properties"
+"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" > "$REPO/config/kafka/server.properties"
 
-echo "Done. Review and commit: cd $REPO && git diff"
+echo "==> airflow"
+"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact > "$REPO/config/airflow/airflow.cfg"
+"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" > "$REPO/config/airflow/generate_data_dag.py"
+
+echo "==> docker01 inventory"
+"$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true
+
+echo "Done. git diff && git commit"

scripts/collect/populate-repo.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Populate Lakehouse git repo with full fleet configs (secrets redacted)."""
+import json
+import os
+import re
+import subprocess
+import urllib.request
+import ssl
+from pathlib import Path
+
+REPO = Path(os.environ.get("REPO", "/root/lakehouse"))
+KEY = os.environ.get("KEY", "/root/.ssh/atc_cluster")
+SSH = ["ssh", "-i", KEY, "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=15"]
+
+PVE = "https://10.0.10.65:8006"
+PVE_TOKEN = os.environ.get("PVE_TOKEN", "root@pam!homepage")
+PVE_SECRET = os.environ.get("PVE_SECRET", "8890185b-0850-42b7-bab2-a690ea4dc3f1")
+
+HOST_IPS = {
+    "atc-docker01": "10.0.21.45", "atc-docker02": "10.0.21.47",
+    "atc-mgt01": "10.0.20.104", "atc-kafka": "10.0.21.36", "atc-kafka01": "10.0.21.36",
+    "atc-lake01": "10.0.21.50", "atc-elastic": "10.0.21.46", "atc-elastic01": "10.0.21.46",
+    "atc-db01": "10.0.20.112", "atc-db02": "10.0.21.51",
+    "atc-grafana": "10.0.20.103", "airflow01": "10.0.21.55", "atc-airflow01": "10.0.21.55",
+    "atc-portal01": "10.0.21.49", "atc-lama": "10.0.21.39", "atc-lama01": "10.0.21.39",
+    "atc-objectscale": "10.0.20.111", "portal01": "10.0.21.49",
+}
+
+
+def ssh(host: str, cmd: str) -> str:
+    r = subprocess.run(SSH + [f"root@{host}", cmd], capture_output=True, text=True)
+    if r.returncode != 0:
+        raise RuntimeError(f"ssh {host}: {r.stderr[:300]}")
+    return r.stdout
+
+
+def redact_secrets(text: str) -> str:
+    rules = [
+        (r"(connection-password=).+", r"\1REDACTED"),
+        (r"(connection-password = ).+", r"\1REDACTED"),
+        (r"(cassandra\.password=).+", r"\1REDACTED"),
+        (r"(hive\.s3\.aws-access-key=).+", r"\1REDACTED"),
+        (r"(hive\.s3\.aws-secret-key=).+", r"\1REDACTED"),
+        (r'(S3_ACCESS_KEY = ).+', r'\1"REDACTED"'),
+        (r'(S3_SECRET_KEY = ).+', r'\1"REDACTED"'),
+        (r"(admin_password = ).+", r"\1REDACTED"),
+        (r"(fernet_key = ).+", r"\1REDACTED"),
+        (r"(internal_api_secret_key = ).+", r"\1REDACTED"),
+        (r"(ssh_password|ansible_become_pass):.*", r"\1: REDACTED"),
+    ]
+    for pat, repl in rules:
+        text = re.sub(pat, repl, text)
+    return text
+
+
+def write(path: Path, content: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+    print(f"  wrote {path.relative_to(REPO)}")
+
+
+def collect_trino():
+    d = REPO / "config/trino/catalog"
+    files = ssh("atc-lake01", "docker exec trino ls /etc/trino/catalog").split()
+    for f in files:
+        if not f.endswith(".properties"):
+            continue
+        raw = ssh("atc-lake01", f"docker exec trino cat /etc/trino/catalog/{f}")
+        write(d / f, redact_secrets(raw))
+
+
+def collect_grafana():
+    d = REPO / "config/grafana"
+    write(d / "grafana.ini", redact_secrets(ssh("atc-grafana", "cat /etc/grafana/grafana.ini")))
+    write(d / "provisioning/dashboards.yml",
+          ssh("atc-grafana", "cat /etc/grafana/provisioning/dashboards/dashboards.yml"))
+
+
+def collect_spark_jobs():
+    d = REPO / "config/spark-jobs"
+    for path in ssh("atc-lake01", "ls /opt/spark-jobs/*.py 2>/dev/null").split():
+        name = os.path.basename(path.strip())
+        if name:
+            write(d / name, redact_secrets(ssh("atc-lake01", f"cat /opt/spark-jobs/{name}")))
+
+
+def collect_ldap():
+    d = REPO / "config/ldap"
+    for f in ["autofs.ldif", "base.ldif", "db.ldif", "openssh.ldif", "setacl.ldif"]:
+        try:
+            write(d / f, ssh("atc-mgt01", f"cat /root/{f}"))
+        except Exception as e:
+            print(f"  skip {f}: {e}")
+
+
+def collect_npm():
+    write(REPO / "config/mgt01/npm-compose.yaml",
+          ssh("atc-mgt01", "cat /srv/docker/npm/compose.yaml"))
+
+
+def collect_airflow_scripts():
+    dest = REPO / "config/airflow/dags/scripts"
+    for path in ssh("atc-airflow01", "ls /root/airflow/dags/scripts/*.py 2>/dev/null").split():
+        name = os.path.basename(path.strip())
+        if name:
+            write(dest / name, ssh("atc-airflow01", f"cat /root/airflow/dags/scripts/{name}"))
+
+
+def proxmox_inventory():
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    req = urllib.request.Request(
+        f"{PVE}/api2/json/cluster/resources?type=vm",
+        headers={"Authorization": f"PVEAPIToken={PVE_TOKEN}={PVE_SECRET}"},
+    )
+    with urllib.request.urlopen(req, context=ctx) as r:
+        vms = json.load(r)["data"]
+    out = []
+    for v in sorted(vms, key=lambda x: x.get("name", "")):
+        if v.get("type") != "qemu":
+            continue
+        name = v.get("name", "")
+        ip = HOST_IPS.get(name, "")
+        out.append({
+            "vmid": v.get("vmid"), "name": name, "status": v.get("status"),
+            "node": v.get("node"), "ip": ip or None,
+            "maxmem_gb": round(v.get("maxmem", 0) / 1e9, 1),
+        })
+    write(REPO / "inventory/proxmox-vms.json", json.dumps(out, indent=2))
+
+
+def main():
+    collect_trino()
+    collect_grafana()
+    collect_spark_jobs()
+    collect_ldap()
+    collect_npm()
+    collect_airflow_scripts()
+    proxmox_inventory()
+
+
+if __name__ == "__main__":
+    main()

scripts/cron/README.md

@@ -0,0 +1,14 @@
+# Cron jobs (install on atc-docker01)
+
+```cron
+# Collect fleet configs weekly (Sunday 02:00)
+0 2 * * 0  root  cd /root/lakehouse && ./scripts/collect/collect-fleet-config.sh && git add -A && git diff --staged --quiet || git commit -m "chore: weekly fleet config sync" && git push origin master
+
+# Health check every 15 minutes
+*/15 * * * * root /root/lakehouse/scripts/health/check-services.sh >> /var/log/atc-health.log 2>&1
+
+# Backup daily at 03:00
+0 3 * * * root /root/lakehouse/scripts/backup/backup-lab.sh >> /var/log/atc-backup.log 2>&1
+```
+
+Install: `crontab -e` as root on atc-docker01.

scripts/health/check-services.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Health check all ATC lab HTTP endpoints (from homepage siteMonitors).
+set -u
+
+FAIL=0
+check() {
+  local name="$1" url="$2"
+  local code
+  code=$(curl -sk -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 10 "$url" 2>/dev/null || echo "000")
+  if [[ "$code" =~ ^(200|301|302|303|401|403)$ ]]; then
+    printf "OK   %-28s %s (%s)\n" "$name" "$url" "$code"
+  else
+    printf "FAIL %-28s %s (%s)\n" "$name" "$url" "$code"
+    FAIL=$((FAIL + 1))
+  fi
+}
+
+check "Homepage"      "http://atc-docker01.dell-atc.lan/"
+check "RSS proxy"     "http://atc-docker01.dell-atc.lan:8090/health"
+check "Superset"      "http://atc-docker01.dell-atc.lan:8088/health"
+check "Kafka UI"      "http://atc-kafka01.dell-atc.lan:9000/"
+check "Debezium"      "http://atc-lake01.dell-atc.lan:8083/"
+check "Spark"         "http://atc-lake01.dell-atc.lan:8080/"
+check "Trino"         "http://atc-lake01.dell-atc.lan:8089/ui/"
+check "Airflow"       "http://10.0.21.55:8080/"
+check "Kibana"        "http://atc-elastic01.dell-atc.lan:5601/"
+check "Grafana"       "http://atc-grafana.dell-atc.lan:3000/api/health"
+check "Forgejo"       "http://atc-mgt01.dell-atc.lan:3001/"
+check "ObjectScale"   "https://10.0.20.111/"
+check "Proxmox"       "https://10.0.10.65:8006/"
+
+echo "---"
+[[ $FAIL -eq 0 ]] && echo "All checks passed" && exit 0
+echo "$FAIL check(s) failed" && exit 1