Full lab documentation and infrastructure as code

- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose - Airflow DAG scripts, Proxmox VM inventory, network docs - Ansible playbook, Gitea CI validate workflow - Backup and health-check scripts, cron documentation - Homepage DOCS tab with links to all documentation - Extended collect-fleet-config.sh and populate-repo.py
2026-05-19 23:12:41 +02:00
parent 3c8993a41e
commit df5ec93dc3
46 changed files with 1809 additions and 74 deletions
@@ -0,0 +1,19 @@
+# Configuration index
+
+| Directory | Host | Contents |
+|-----------|------|----------|
+| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons |
+| [docker/](docker/) | Multiple | Docker Compose per host |
+| [compose/](../compose/) | atc-docker01 | Source compose stacks |
+| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors |
+| [trino/](trino/) | atc-lake01 | SQL catalog properties |
+| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs |
+| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs |
+| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml |
+| [grafana/](grafana/) | atc-grafana | grafana.ini |
+| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml |
+| [ldap/](ldap/) | atc-mgt01 | LDIF exports |
+| [mgt01/](mgt01/) | atc-mgt01 | NPM compose |
+| [hosts/](hosts/) | All | /etc/hosts snippet |
+
+Refresh all: `./scripts/collect/collect-fleet-config.sh`
@@ -1193,7 +1193,7 @@ enable_swagger_ui = True
 #
 # Variable: AIRFLOW__API__SECRET_KEY
 #
-secret_key = REDACTED
+secret_key = 010RL08807/JBjH4cWzNaw==

 # Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values
 # except those that have security implications. ``True`` shows all values. ``False`` hides the
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake telemetry data for Cassandra
+Generates approximately 1GB of data
+"""
+
+from cassandra.cluster import Cluster
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "9042"
+KEYSPACE = "telemetry"
+TABLE_NAME = "device_metrics"
+
+# Data generation settings
+TARGET_ROWS = 3000000  # Approximately 1GB of data
+BATCH_SIZE = 5000
+
+# Sample data
+METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"]
+DEVICE_PREFIX = "device-"
+
+def generate_fake_device_metric():
+    """Generate a single fake device metric"""
+    device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}"
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                           minutes=random.randint(0, 59))
+    
+    metric_type = random.choice(METRIC_TYPES)
+    metric_value = round(random.uniform(0.0, 100.0), 4)
+    
+    # Generate a long payload field
+    payload = "X" * 200
+    
+    return (device_id, metric_ts, metric_type, metric_value, payload)
+
+def main():
+    print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...")
+    
+    cluster = Cluster([DB_HOST], port=DB_PORT)
+    session = cluster.connect()
+    
+    print(f"Generating {TARGET_ROWS} device metrics...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_device_metric())
+        
+        if len(batch) >= BATCH_SIZE:
+            session.execute(
+                f"""
+                INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+                VALUES (%s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        session.execute(
+            f"""
+            INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+            VALUES (%s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        total_generated += len(batch)
+    
+    session.shutdown()
+    cluster.shutdown()
+    
+    print(f"Completed! Generated {total_generated} device metrics.")
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake event data for MongoDB
+Generates approximately 1GB of data
+"""
+
+import pymongo
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "27017"
+DB_NAME = "supplychain"
+COLLECTION_NAME = "events"
+
+# Data generation settings
+TARGET_DOCUMENTS = 3000000  # Approximately 1GB of data
+BATCH_SIZE = 5000
+
+# Sample data
+EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
+
+def generate_fake_event():
+    """Generate a single fake event"""
+    event_id = uuid.uuid4()
+    event_type = random.choice(EVENT_TYPES)
+    region = random.choice(REGIONS)
+    source = random.choice(SOURCES)
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                     minutes=random.randint(0, 59))
+    
+    amount = random.uniform(100.0, 50000.0)
+    
+    # Generate a long payload field (like the existing data)
+    payload = "X" * 500
+    
+    return {
+        "event_id": event_id,
+        "type": event_type,
+        "region": region,
+        "source": source,
+        "amount": amount,
+        "ts": ts,
+        "payload": payload
+    }
+
+def main():
+    print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
+    
+    client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
+    db = client[DB_NAME]
+    collection = db[COLLECTION_NAME]
+    
+    print(f"Generating {TARGET_DOCUMENTS} events...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_DOCUMENTS):
+        batch.append(generate_fake_event())
+        
+        if len(batch) >= BATCH_SIZE:
+            collection.insert_many(batch)
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} documents...")
+    
+    # Insert remaining documents
+    if batch:
+        collection.insert_many(batch)
+        total_generated += len(batch)
+    
+    client.close()
+    
+    print(f"Completed! Generated {total_generated} events.")
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake employee event data for MySQL
+Generates approximately 1GB of data
+"""
+
+import mysql.connector
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "3306"
+DB_NAME = "hr"
+DB_USER = "mo"
+DB_PASSWORD = "Dell2026!"
+
+# Data generation settings
+TARGET_ROWS = 4000000  # Approximately 1GB of data
+BATCH_SIZE = 10000
+
+# Sample data
+DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"]
+ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"]
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"]
+
+def generate_fake_employee_event():
+    """Generate a single fake employee event"""
+    employee_id = random.randint(1, 100000)
+    department = random.choice(DEPARTMENTS)
+    role_name = random.choice(ROLE_NAMES)
+    region = random.choice(REGIONS)
+    event_type = random.choice(EVENT_TYPES)
+    
+    # Random timestamp within the last 2 years
+    days_ago = random.randint(0, 730)
+    event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                          minutes=random.randint(0, 59))
+    
+    salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None
+    
+    # Generate a long notes field (like the existing data)
+    notes = str(uuid.uuid4()) * 10
+    
+    return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
+
+def main():
+    print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...")
+    
+    conn = mysql.connector.connect(
+        host=DB_HOST,
+        port=DB_PORT,
+        database=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD
+    )
+    cursor = conn.cursor()
+    
+    print(f"Generating {TARGET_ROWS} employee events...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_employee_event())
+        
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                """
+                INSERT INTO employee_events (employee_id, department, role_name, region, 
+                                            event_type, salary_change, event_ts, notes)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            conn.commit()
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        cursor.executemany(
+            """
+            INSERT INTO employee_events (employee_id, department, role_name, region, 
+                                        event_type, salary_change, event_ts, notes)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        conn.commit()
+        total_generated += len(batch)
+    
+    cursor.close()
+    conn.close()
+    
+    print(f"Completed! Generated {total_generated} employee events.")
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake graph data for Neo4j
+Generates approximately 1GB of data with nodes and relationships
+"""
+
+from neo4j import GraphDatabase
+import random
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "7687"
+DB_USER = "neo4j"
+DB_PASSWORD = "testpwd"
+
+# Data generation settings
+TARGET_NODES = 500000  # Approximately 1GB of data with relationships
+BATCH_SIZE = 1000
+
+# Sample data
+PRODUCT_CATEGORIES = ["Electronics", "Clothing", "Food", "Furniture", "Toys", "Books"]
+SUPPLIER_REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+RELATIONSHIP_TYPES = ["SUPPLIES", "RELATED_TO", "COMPATIBLE_WITH", "PART_OF"]
+
+def generate_fake_product():
+    """Generate a single fake product node"""
+    product_id = str(uuid.uuid4())
+    name = f"Product-{random.randint(1000, 999999)}"
+    category = random.choice(PRODUCT_CATEGORIES)
+    price = round(random.uniform(10.0, 1000.0), 2)
+    stock = random.randint(0, 1000)
+    
+    # Generate a long description field
+    description = "X" * 200
+    
+    return {
+        "product_id": product_id,
+        "name": name,
+        "category": category,
+        "price": price,
+        "stock": stock,
+        "description": description
+    }
+
+def generate_fake_supplier():
+    """Generate a single fake supplier node"""
+    supplier_id = str(uuid.uuid4())
+    name = f"Supplier-{random.randint(1000, 999999)}"
+    region = random.choice(SUPPLIER_REGIONS)
+    rating = round(random.uniform(1.0, 5.0), 1)
+    
+    # Generate a long address field
+    address = "X" * 150
+    
+    return {
+        "supplier_id": supplier_id,
+        "name": name,
+        "region": region,
+        "rating": rating,
+        "address": address
+    }
+
+def main():
+    print(f"Connecting to Neo4j at {DB_HOST}:{DB_PORT}...")
+    
+    driver = GraphDatabase.driver(f"bolt://{DB_HOST}:{DB_PORT}", 
+                                   auth=(DB_USER, DB_PASSWORD))
+    
+    with driver.session() as session:
+        print(f"Generating {TARGET_NODES} product nodes...")
+        print(f"Batch size: {BATCH_SIZE}")
+        
+        total_products = 0
+        total_suppliers = 0
+        product_ids = []
+        
+        # Generate product nodes
+        batch = []
+        for i in range(TARGET_NODES):
+            product = generate_fake_product()
+            batch.append(product)
+            product_ids.append(product["product_id"])
+            
+            if len(batch) >= BATCH_SIZE:
+                session.run(
+                    """
+                    UNWIND $batch as row
+                    CREATE (p:Product {
+                        product_id: row.product_id,
+                        name: row.name,
+                        category: row.category,
+                        price: row.price,
+                        stock: row.stock,
+                        description: row.description
+                    })
+                    """,
+                    batch=batch
+                )
+                total_products += len(batch)
+                batch = []
+                
+                if total_products % 50000 == 0:
+                    print(f"Generated {total_products} product nodes...")
+        
+        # Insert remaining products
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                CREATE (p:Product {
+                    product_id: row.product_id,
+                    name: row.name,
+                    category: row.category,
+                    price: row.price,
+                    stock: row.stock,
+                    description: row.description
+                })
+                """,
+                batch=batch
+            )
+            total_products += len(batch)
+        
+        print(f"Generated {total_products} product nodes.")
+        
+        # Generate supplier nodes (fewer than products)
+        print(f"Generating supplier nodes...")
+        target_suppliers = 10000
+        batch = []
+        supplier_ids = []
+        
+        for i in range(target_suppliers):
+            supplier = generate_fake_supplier()
+            batch.append(supplier)
+            supplier_ids.append(supplier["supplier_id"])
+            
+            if len(batch) >= BATCH_SIZE:
+                session.run(
+                    """
+                    UNWIND $batch as row
+                    CREATE (s:Supplier {
+                        supplier_id: row.supplier_id,
+                        name: row.name,
+                        region: row.region,
+                        rating: row.rating,
+                        address: row.address
+                    })
+                    """,
+                    batch=batch
+                )
+                total_suppliers += len(batch)
+                batch = []
+        
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                CREATE (s:Supplier {
+                    supplier_id: row.supplier_id,
+                    name: row.name,
+                    region: row.region,
+                    rating: row.rating,
+                    address: row.address
+                })
+                """,
+                batch=batch
+            )
+            total_suppliers += len(batch)
+        
+        print(f"Generated {total_suppliers} supplier nodes.")
+        
+        # Create relationships between products and suppliers
+        print(f"Creating relationships...")
+        batch = []
+        total_relationships = 0
+        
+        for product_id in product_ids:
+            # Each product is supplied by 1-3 random suppliers
+            num_suppliers = random.randint(1, 3)
+            for _ in range(num_suppliers):
+                supplier_id = random.choice(supplier_ids)
+                rel_type = random.choice(RELATIONSHIP_TYPES)
+                
+                batch.append({
+                    "product_id": product_id,
+                    "supplier_id": supplier_id,
+                    "rel_type": rel_type
+                })
+                
+                if len(batch) >= BATCH_SIZE:
+                    session.run(
+                        """
+                        UNWIND $batch as row
+                        MATCH (p:Product {product_id: row.product_id})
+                        MATCH (s:Supplier {supplier_id: row.supplier_id})
+                        CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
+                        RETURN rel
+                        """,
+                        batch=batch
+                    )
+                    total_relationships += len(batch)
+                    batch = []
+                    
+                    if total_relationships % 50000 == 0:
+                        print(f"Created {total_relationships} relationships...")
+        
+        if batch:
+            session.run(
+                """
+                UNWIND $batch as row
+                MATCH (p:Product {product_id: row.product_id})
+                MATCH (s:Supplier {supplier_id: row.supplier_id})
+                CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
+                RETURN rel
+                """,
+                batch=batch
+            )
+            total_relationships += len(batch)
+        
+        print(f"Created {total_relationships} relationships.")
+    
+    driver.close()
+    print(f"Completed! Generated {total_products} products, {total_suppliers} suppliers, and {total_relationships} relationships.")
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake sales order data for PostgreSQL
+Generates approximately 1GB of data
+"""
+
+import psycopg2
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "5432"
+DB_NAME = "postgres"
+DB_USER = "mo"
+DB_PASSWORD = "Dell2026!"
+
+# Data generation settings
+TARGET_ROWS = 4000000  # Approximately 1GB of data
+BATCH_SIZE = 10000
+
+# Sample data
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
+CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
+ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
+
+def generate_fake_order():
+    """Generate a single fake sales order"""
+    customer_id = random.randint(1, 100000)
+    product_id = random.randint(1, 5000)
+    region = random.choice(REGIONS)
+    sales_channel = random.choice(SALES_CHANNELS)
+    
+    # Random timestamp within the last 2 years
+    days_ago = random.randint(0, 730)
+    order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                          minutes=random.randint(0, 59))
+    
+    amount = round(random.uniform(10.0, 10000.0), 2)
+    currency = random.choice(CURRENCIES)
+    order_status = random.choice(ORDER_STATUSES)
+    
+    # Generate a long notes field (like the existing data)
+    notes = str(uuid.uuid4()) * 10
+    
+    return (customer_id, product_id, region, sales_channel, order_ts, 
+            amount, currency, order_status, notes)
+
+def main():
+    print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
+    
+    conn = psycopg2.connect(
+        host=DB_HOST,
+        port=DB_PORT,
+        database=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD
+    )
+    cursor = conn.cursor()
+    
+    print(f"Generating {TARGET_ROWS} sales orders...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_order())
+        
+        if len(batch) >= BATCH_SIZE:
+            cursor.executemany(
+                """
+                INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, 
+                                         order_ts, amount, currency, order_status, notes)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            conn.commit()
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        cursor.executemany(
+            """
+            INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, 
+                                     order_ts, amount, currency, order_status, notes)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        conn.commit()
+        total_generated += len(batch)
+    
+    cursor.close()
+    conn.close()
+    
+    print(f"Completed! Generated {total_generated} sales orders.")
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,16 @@
+[server]
+http_addr = 0.0.0.0
+http_port = 3000
+
+[security]
+admin_user = admin
+admin_password = REDACTED
+allow_embedding = true
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Viewer
+
+[auth.basic]
+enabled = false
@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'Default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards
@@ -40,10 +40,27 @@
    });
  }

+
+  function showGitSha() {
+    if (document.getElementById('atc-git-sha')) return;
+    fetch('http://atc-mgt01.dell-atc.lan:3001/api/v1/repos/mo/Lakehouse/commits?limit=1')
+      .then(function (r) { return r.json(); })
+      .then(function (d) {
+        var sha = (d && d[0] && d[0].sha) ? d[0].sha.substring(0, 7) : 'git';
+        var el = document.createElement('div');
+        el.id = 'atc-git-sha';
+        el.textContent = 'Lakehouse ' + sha;
+        el.style.cssText = 'position:fixed;bottom:4px;right:8px;font-size:10px;color:#64748b;z-index:9999;font-family:monospace;';
+        document.body.appendChild(el);
+      })
+      .catch(function () {});
+  }
+
  function init() {
    injectOverlays();
    applyServiceColors();
    linkLakehouseTitle();
+    showGitSha();
    new MutationObserver(linkLakehouseTitle).observe(document.body, { childList: true, subtree: true });
  }

@@ -512,6 +512,39 @@
            href: link
            target: _blank

+
+- Documentation:
+    - Architecture Diagram:
+        icon: mdi-sitemap
+        href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
+        description: High-level environment map (Mermaid)
+        color: "#007DB8"
+    - Application Landscape:
+        icon: mdi-book-open-variant
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
+        description: docs/landscape.md in Forgejo
+        color: "#E8752A"
+    - Network Diagram:
+        icon: mdi-lan
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/network.md
+        description: Subnets, ports, SSH mesh
+        color: "#22d3ee"
+    - Disaster Recovery:
+        icon: mdi-backup-restore
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/disaster-recovery.md
+        description: Restore procedures from git
+        color: "#a855f7"
+    - Docker Inventory:
+        icon: mdi-docker
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/docker-inventory.md
+        description: Containers per host
+        color: "#2496ED"
+    - Proxmox VMs:
+        icon: mdi-server
+        href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/inventory/proxmox-vms.json
+        description: VM inventory (JSON)
+        color: "#E57000"
+
 - News & Resources:
    - Dell Technologies:
        icon: http://atc-docker01.dell-atc.lan:8080/dell-technologies.svg
@@ -51,6 +51,10 @@ layout:
    tab: FEEDS
    style: row
    columns: 6
+  Documentation:
+    tab: DOCS
+    style: row
+    columns: 4
  News & Resources:
    tab: LINKS
    style: row
@@ -0,0 +1,19 @@
+# Configuration index
+
+| Directory | Host | Contents |
+|-----------|------|----------|
+| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons |
+| [docker/](docker/) | Multiple | Docker Compose per host |
+| [compose/](../compose/) | atc-docker01 | Source compose stacks |
+| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors |
+| [trino/](trino/) | atc-lake01 | SQL catalog properties |
+| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs |
+| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs |
+| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml |
+| [grafana/](grafana/) | atc-grafana | grafana.ini |
+| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml |
+| [ldap/](ldap/) | atc-mgt01 | LDIF exports |
+| [mgt01/](mgt01/) | atc-mgt01 | NPM compose |
+| [hosts/](hosts/) | All | /etc/hosts snippet |
+
+Refresh all: `./scripts/collect/collect-fleet-config.sh`
@@ -0,0 +1,7 @@
+dn: cn=autofs,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: autofs
+olcAttributeTypes: ( 1.3.6.1.1.1.1.25 NAME 'automountInformation' DESC 'Information used by the autofs automounter' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE )
+olcObjectClasses: ( 1.3.6.1.1.1.1.13 NAME 'automount' DESC 'An entry in an automounter map' SUP top STRUCTURAL MUST ( cn $ automountInformation $ objectclass ) MAY description )
+olcObjectClasses: ( 1.3.6.1.4.1.2312.4.2.2 NAME 'automountMap' DESC 'A group of related automount objects' SUP top STRUCTURAL MUST ou )
+
@@ -0,0 +1,14 @@
+dn: dc=dell-atc,dc=lan
+objectClass: top
+objectClass: dcObject
+objectClass: organization
+o: Dell ATC Lab
+dc: dell-atc
+
+dn: ou=Users,dc=dell-atc,dc=lan
+objectClass: organizationalUnit
+ou: Users
+
+dn: ou=Groups,dc=dell-atc,dc=lan
+objectClass: organizationalUnit
+ou: Groups
@@ -0,0 +1,14 @@
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcSuffix
+olcSuffix: dc=dell-atc,dc=lan
+
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcRootDN
+olcRootDN: cn=admin,dc=dell-atc,dc=lan
+
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+replace: olcRootPW
+olcRootPW: {SSHA}MO4kZaN1wIHCgkMw6f0FX0kcQbEe+jmx
@@ -0,0 +1,11 @@
+dn: cn=openssh-lpk-openldap,cn=schema,cn=config
+objectClass: olcSchemaConfig
+cn: openssh-lpk-openldap
+olcAttributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey'
+ DESC 'MANDATORY: OpenSSH Public key'
+ EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 )
+olcObjectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey'
+ DESC 'MANDATORY: OpenSSH LPK objectclass'
+ SUP top AUXILIARY MUST ( sshPublicKey ) )
+
+
@@ -0,0 +1,6 @@
+dn: olcDatabase={2}mdb,cn=config
+changetype: modify
+add: olcAccess
+olcAccess: {0}to attrs=userPassword,shadowLastChange by dn="cn=admin,dc=dell-atc,dc=lan" write by anonymous auth by self write by * none
+olcAccess: {1}to * by dn="cn=admin,dc=dell-atc,dc=lan" write by * read
+
@@ -0,0 +1,14 @@
+services:
+  app:
+    image: 'jc21/nginx-proxy-manager:latest'
+    restart: unless-stopped
+    environment:
+      TZ: "Europe/Amsterdam"
+    ports:
+      - '80:80'
+      - '81:81'
+      - '443:443'
+    volumes:
+      - ./data:/data
+      - ./letsencrypt:/etc/letsencrypt
+
@@ -45,10 +45,10 @@ facts:
    # [Required]
    # Password to use with SSH login
    # *** Set to same value as ssh_username to enable SSH public key authentication ***
-    ssh_password: "REDACTED"
+    ssh_password: REDACTED
    # [Required when enabling SSH public key authentication]
    # Password to give to sudo when gaining root access.
-    ansible_become_pass: "REDACTED"
+    ansible_become_pass: REDACTED
    # [Required]
    # Select the type of crypto to use when dealing with ssh public key
    # authentication. Valid values here are:
@@ -0,0 +1 @@
+# Spark jobs
@@ -0,0 +1,38 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+consumer = KafkaConsumer(
+    KAFKA_TOPIC,
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='earliest',
+    value_deserializer=lambda x: json.loads(x.decode('utf-8'))
+)
+
+print("Waiting for messages...")
+for msg in consumer:
+    ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
+    print(f"Written: {key}")
@@ -0,0 +1,48 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+def deserialize(value):
+    if value is None or len(value) == 0:
+        return None
+    try:
+        return json.loads(value.decode('utf-8'))
+    except:
+        print(f"Skipping non-JSON")
+        return None
+
+consumer = KafkaConsumer(
+    KAFKA_TOPIC,
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='latest',
+    value_deserializer=deserialize
+)
+
+print("Waiting for messages...")
+for msg in consumer:
+    if msg.value is None:
+        continue
+    ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
+    print(f"Written: {key}")
@@ -0,0 +1,34 @@
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import col, current_timestamp
+
+spark = SparkSession.builder \
+    .appName("KafkaToS3") \
+    .config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
+    .config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
+    .config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
+    .config("spark.hadoop.fs.s3a.path.style.access", "true") \
+    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
+    .getOrCreate()
+
+spark.sparkContext.setLogLevel("WARN")
+
+df = spark.readStream \
+    .format("kafka") \
+    .option("kafka.bootstrap.servers", "10.0.21.36:9092") \
+    .option("subscribe", "test-lakehouse") \
+    .option("startingOffsets", "latest") \
+    .load()
+
+output = df.select(
+    col("key").cast("string"),
+    col("value").cast("string"),
+    current_timestamp().alias("timestamp")
+)
+
+query = output.writeStream \
+    .outputMode("append") \
+    .format("console") \
+    .start()
+
+print("Streaming started. Press Ctrl+C to stop.")
+query.awaitTermination()
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+from kafka import KafkaConsumer
+import boto3
+import json
+import os
+import sys
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("=" * 60)
+print(f"Kafka Consumer starting...")
+print(f"Kafka broker: {KAFKA_BROKER}")
+print(f"Kafka topic: {KAFKA_TOPIC}")
+print(f"S3 endpoint: {S3_ENDPOINT}")
+print(f"S3 bucket: {S3_BUCKET}")
+print("=" * 60)
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Kafka consumer
+try:
+    consumer = KafkaConsumer(
+        KAFKA_TOPIC,
+        bootstrap_servers=[KAFKA_BROKER],
+        auto_offset_reset='earliest',
+        enable_auto_commit=True,
+        value_deserializer=lambda x: json.loads(x.decode('utf-8'))
+    )
+    print("Connected to Kafka successfully!")
+except Exception as e:
+    print(f"Failed to connect to Kafka: {e}")
+    sys.exit(1)
+
+print(f"Listening for messages on topic '{KAFKA_TOPIC}'...")
+print("-" * 60)
+
+message_count = 0
+for message in consumer:
+    message_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
+    
+    try:
+        s3.put_object(
+            Bucket=S3_BUCKET,
+            Key=key,
+            Body=json.dumps(message.value, indent=2)
+        )
+        print(f"[{message_count}] Written to s3://{S3_BUCKET}/{key}")
+        print(f"    Data: {json.dumps(message.value)[:100]}...")
+    except Exception as e:
+        print(f"Error writing to S3: {e}")
@@ -0,0 +1,69 @@
+from kafka import KafkaConsumer
+from kafka import TopicPartition
+import boto3
+import json
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting S3 consumer for ALL topics...")
+print(f"Kafka: {KAFKA_BROKER}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Consumer zonder specifiek topic
+consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
+
+# Haal alle topics op
+all_topics = consumer.topics()
+print(f"Found topics: {list(all_topics)}")
+
+# Wijs alle partitions van alle topics toe
+for topic in all_topics:
+    partitions = consumer.partitions_for_topic(topic)
+    for partition in partitions:
+        tp = TopicPartition(topic, partition)
+        consumer.assign([tp])
+        consumer.seek_to_beginning()
+        print(f"Assigned: {topic} - partition {partition}")
+
+print("\nReading all messages from all topics...")
+print("-" * 60)
+
+msg_count = 0
+topic_count = {}
+
+for msg in consumer:
+    msg_count += 1
+    topic = msg.topic
+    topic_count[topic] = topic_count.get(topic, 0) + 1
+    
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{topic}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(
+        Bucket=S3_BUCKET,
+        Key=key,
+        Body=json.dumps(data, indent=2)
+    )
+    
+    print(f"[{msg_count}] {topic} -> s3://{S3_BUCKET}/{key}")
@@ -0,0 +1,45 @@
+from kafka import KafkaConsumer
+import boto3
+import json
+from datetime import datetime
+
+KAFKA_BROKER = "10.0.21.36:9092"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Consumer die naar ALLE topics luistert (alleen nieuwe berichten)
+consumer = KafkaConsumer(
+    bootstrap_servers=[KAFKA_BROKER],
+    auto_offset_reset='latest'
+)
+
+# Subscribe op alle topics
+consumer.subscribe(pattern='.*')
+
+print("Listening to ALL topics...")
+print("Waiting for new messages...")
+
+msg_count = 0
+for msg in consumer:
+    msg_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{msg.topic}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(data, indent=2))
+    print(f"[{msg_count}] {msg.topic} -> s3://{S3_BUCKET}/{key}")
@@ -0,0 +1,57 @@
+from kafka import KafkaConsumer
+from kafka import TopicPartition
+import boto3
+import json
+from datetime import datetime
+
+# Configuratie
+KAFKA_BROKER = "10.0.21.36:9092"
+KAFKA_TOPIC = "test-lakehouse"
+S3_ENDPOINT = "http://10.0.20.111:9020"
+S3_BUCKET = "data"
+S3_ACCESS_KEY = "REDACTED"
+S3_SECRET_KEY = "REDACTED"
+
+print("Starting S3 consumer...")
+print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
+print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
+
+# S3 client
+s3 = boto3.client(
+    's3',
+    endpoint_url=S3_ENDPOINT,
+    aws_access_key_id=S3_ACCESS_KEY,
+    aws_secret_access_key=S3_SECRET_KEY,
+    use_ssl=False,
+    verify=False
+)
+
+# Kafka consumer met vaste partition
+consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
+tp = TopicPartition(KAFKA_TOPIC, 0)
+consumer.assign([tp])
+consumer.seek_to_beginning()
+
+print("Reading all messages from beginning...")
+print("-" * 50)
+
+msg_count = 0
+for msg in consumer:
+    msg_count += 1
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
+    key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
+    
+    try:
+        data = json.loads(msg.value.decode('utf-8'))
+    except:
+        data = {"raw": msg.value.decode('utf-8')}
+    
+    s3.put_object(
+        Bucket=S3_BUCKET,
+        Key=key,
+        Body=json.dumps(data, indent=2)
+    )
+    print(f"[{msg_count}] Written: s3://{S3_BUCKET}/{key}")
+    print(f"    Data: {str(data)[:80]}...")
+
+print(f"Done! Total {msg_count} messages written to S3")
@@ -0,0 +1,12 @@
+from kafka import KafkaConsumer
+import json
+
+consumer = KafkaConsumer(
+    'test-lakehouse',
+    bootstrap_servers=['10.0.21.36:9092'],
+    auto_offset_reset='latest'
+)
+
+print("Waiting for new messages...")
+for msg in consumer:
+    print(f"Received: {msg.value.decode('utf-8')}")
@@ -0,0 +1,19 @@
+# Trino — atc-lake01
+
+| Item | Value |
+|------|-------|
+| UI | http://atc-lake01:8089/ui/ |
+| Container | `trino` (Docker) |
+| Catalogs | `catalog/*.properties` |
+
+## Catalogs
+
+| File | Connector | Source |
+|------|-----------|--------|
+| `postgres_sales.properties` | PostgreSQL | atc-db02:5432 |
+| `mysql_hr.properties` | MySQL | atc-db02:3306 |
+| `mongodb_supplychain.properties` | MongoDB | atc-db02:27017 |
+| `cassandra_telemetry.properties` | Cassandra | atc-db02:9042 |
+| `iceberg.properties` | Iceberg | ObjectScale S3 `10.0.20.111:9020` |
+
+Passwords redacted in git. Refresh: `./scripts/collect/collect-fleet-config.sh`
@@ -0,0 +1,11 @@
+connector.name=cassandra
+
+cassandra.contact-points=10.0.21.51
+cassandra.native-protocol-port=9042
+
+# Required DC setting for load balancing:
+cassandra.load-policy.dc-aware.local-dc=datacenter1
+
+# If Cassandra auth is enabled, uncomment and set:
+# cassandra.username=mo
+# cassandra.password=REDACTED
@@ -0,0 +1,10 @@
+connector.name=iceberg
+iceberg.catalog.type=TESTING_FILE_METASTORE
+hive.metastore.catalog.dir=/tmp/iceberg_metadata
+
+# S3 Connectivity
+hive.s3.endpoint=http://10.0.20.111:9020
+hive.s3.aws-access-key=REDACTED
+hive.s3.aws-secret-key=REDACTED
+hive.s3.path-style-access=true
+hive.s3.ssl.enabled=false
@@ -0,0 +1,3 @@
+connector.name=mongodb
+mongodb.connection-url=mongodb://10.0.21.51:27017/
+mongodb.schema-collection=__trino_schema
@@ -0,0 +1,4 @@
+connector.name=mysql
+connection-url=jdbc:mysql://10.0.21.51:3306
+connection-user=mo
+connection-password=REDACTED
@@ -0,0 +1,4 @@
+connector.name=postgresql
+connection-url=jdbc:postgresql://10.0.21.51:5432/postgres
+connection-user=mo
+connection-password=REDACTED