diff --git a/.gitea/workflows/validate.yml b/.gitea/workflows/validate.yml new file mode 100644 index 0000000..87d71bc --- /dev/null +++ b/.gitea/workflows/validate.yml @@ -0,0 +1,46 @@ +name: Validate Lakehouse config + +on: + push: + branches: [master, main] + pull_request: + branches: [master, main] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate YAML syntax + run: | + python3 -c " + import yaml, sys + from pathlib import Path + for p in Path('.').rglob('*.yaml'): + if '.git' in str(p): continue + try: + yaml.safe_load(p.read_text()) + print('OK', p) + except Exception as e: + print('FAIL', p, e) + sys.exit(1) + for p in Path('.').rglob('*.yml'): + if '.git' in str(p) or 'workflows' in str(p): continue + try: + yaml.safe_load(p.read_text()) + except Exception: pass + " + + - name: Validate Docker Compose + run: | + for f in deploy/*.yml compose/*/*.yaml compose/*/*.yml; do + [ -f "$f" ] || continue + docker compose -f "$f" config >/dev/null && echo "OK $f" + done + + - name: Check required docs exist + run: | + test -f docs/landscape.md + test -f docs/network.md + test -f README.md diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..6ca8c3a --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,9 @@ +# Ansible — ATC Lakehouse + +```bash +# From atc-docker01 (with ansible installed) +cd /root/lakehouse/ansible +ansible-playbook -i inventory.ini playbook.yml +``` + +Requires SSH mesh (`/root/.ssh/atc_cluster`) on all targets. diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 0000000..d89f674 --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,23 @@ +[docker_hosts] +atc-docker01 ansible_host=10.0.21.45 +atc-docker02 ansible_host=10.0.21.47 + +[lakehouse] +atc-kafka01 ansible_host=10.0.21.36 +atc-lake01 ansible_host=10.0.21.50 +atc-airflow01 ansible_host=10.0.21.55 +atc-elastic01 ansible_host=10.0.21.46 + +[data] +atc-db01 ansible_host=10.0.20.112 +atc-db02 ansible_host=10.0.21.51 + +[management] +atc-mgt01 ansible_host=10.0.20.104 +atc-grafana ansible_host=10.0.20.103 +atc-objectscale ansible_host=10.0.20.111 ansible_user=admin + +[all:vars] +ansible_user=root +ansible_ssh_private_key_file=/root/.ssh/atc_cluster +ansible_ssh_common_args='-o StrictHostKeyChecking=no' diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 0000000..801ff9e --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,38 @@ +--- +# Deploy ATC Lakehouse from git (run from atc-docker01 or control node with mesh SSH) +- name: ATC Lakehouse — docker01 stack + hosts: atc-docker01 + become: true + tasks: + - name: Ensure repo present + git: + repo: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse.git + dest: /root/lakehouse + version: master + force: false + + - name: Deploy homepage + RSS + icons + command: docker compose -f deploy/docker-compose.homepage.yml up -d --build + args: + chdir: /root/lakehouse + + - name: Deploy superset + command: docker compose -f compose/superset/docker-compose.yaml up -d --build + args: + chdir: /root/lakehouse + + - name: Revalidate homepage + command: docker exec homepage wget -qO- http://127.0.0.1:3000/api/revalidate + changed_when: false + failed_when: false + +- name: Sync /etc/hosts on fleet + hosts: all + become: true + tasks: + - name: Append ATC hosts block + blockinfile: + path: /etc/hosts + marker: "# {mark} ATC Lakehouse lab" + block: "{{ lookup('file', '../config/hosts/atc-lab.hosts') }}" + when: ansible_user != 'admin' diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..1bd2710 --- /dev/null +++ b/config/README.md @@ -0,0 +1,19 @@ +# Configuration index + +| Directory | Host | Contents | +|-----------|------|----------| +| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons | +| [docker/](docker/) | Multiple | Docker Compose per host | +| [compose/](../compose/) | atc-docker01 | Source compose stacks | +| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors | +| [trino/](trino/) | atc-lake01 | SQL catalog properties | +| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs | +| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs | +| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml | +| [grafana/](grafana/) | atc-grafana | grafana.ini | +| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml | +| [ldap/](ldap/) | atc-mgt01 | LDIF exports | +| [mgt01/](mgt01/) | atc-mgt01 | NPM compose | +| [hosts/](hosts/) | All | /etc/hosts snippet | + +Refresh all: `./scripts/collect/collect-fleet-config.sh` diff --git a/config/airflow/airflow.cfg b/config/airflow/airflow.cfg index 2b8d26a..fbb8a50 100644 --- a/config/airflow/airflow.cfg +++ b/config/airflow/airflow.cfg @@ -1193,7 +1193,7 @@ enable_swagger_ui = True # # Variable: AIRFLOW__API__SECRET_KEY # -secret_key = REDACTED +secret_key = 010RL08807/JBjH4cWzNaw== # Expose the configuration file in the web server. Set to ``non-sensitive-only`` to show all values # except those that have security implications. ``True`` shows all values. ``False`` hides the diff --git a/config/airflow/dags/scripts/generate_cassandra_telemetry_data.py b/config/airflow/dags/scripts/generate_cassandra_telemetry_data.py new file mode 100644 index 0000000..58f4129 --- /dev/null +++ b/config/airflow/dags/scripts/generate_cassandra_telemetry_data.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Script to generate fake telemetry data for Cassandra +Generates approximately 1GB of data +""" + +from cassandra.cluster import Cluster +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "9042" +KEYSPACE = "telemetry" +TABLE_NAME = "device_metrics" + +# Data generation settings +TARGET_ROWS = 3000000 # Approximately 1GB of data +BATCH_SIZE = 5000 + +# Sample data +METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"] +DEVICE_PREFIX = "device-" + +def generate_fake_device_metric(): + """Generate a single fake device metric""" + device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}" + + # Random timestamp within the last year + days_ago = random.randint(0, 365) + metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + metric_type = random.choice(METRIC_TYPES) + metric_value = round(random.uniform(0.0, 100.0), 4) + + # Generate a long payload field + payload = "X" * 200 + + return (device_id, metric_ts, metric_type, metric_value, payload) + +def main(): + print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...") + + cluster = Cluster([DB_HOST], port=DB_PORT) + session = cluster.connect() + + print(f"Generating {TARGET_ROWS} device metrics...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_device_metric()) + + if len(batch) >= BATCH_SIZE: + session.execute( + f""" + INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload) + VALUES (%s, %s, %s, %s, %s) + """, + batch + ) + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + session.execute( + f""" + INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload) + VALUES (%s, %s, %s, %s, %s) + """, + batch + ) + total_generated += len(batch) + + session.shutdown() + cluster.shutdown() + + print(f"Completed! Generated {total_generated} device metrics.") + +if __name__ == "__main__": + main() diff --git a/config/airflow/dags/scripts/generate_mongodb_events_data.py b/config/airflow/dags/scripts/generate_mongodb_events_data.py new file mode 100644 index 0000000..b37c6b0 --- /dev/null +++ b/config/airflow/dags/scripts/generate_mongodb_events_data.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Script to generate fake event data for MongoDB +Generates approximately 1GB of data +""" + +import pymongo +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "27017" +DB_NAME = "supplychain" +COLLECTION_NAME = "events" + +# Data generation settings +TARGET_DOCUMENTS = 3000000 # Approximately 1GB of data +BATCH_SIZE = 5000 + +# Sample data +EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"] +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"] + +def generate_fake_event(): + """Generate a single fake event""" + event_id = uuid.uuid4() + event_type = random.choice(EVENT_TYPES) + region = random.choice(REGIONS) + source = random.choice(SOURCES) + + # Random timestamp within the last year + days_ago = random.randint(0, 365) + ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + amount = random.uniform(100.0, 50000.0) + + # Generate a long payload field (like the existing data) + payload = "X" * 500 + + return { + "event_id": event_id, + "type": event_type, + "region": region, + "source": source, + "amount": amount, + "ts": ts, + "payload": payload + } + +def main(): + print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...") + + client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/") + db = client[DB_NAME] + collection = db[COLLECTION_NAME] + + print(f"Generating {TARGET_DOCUMENTS} events...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_DOCUMENTS): + batch.append(generate_fake_event()) + + if len(batch) >= BATCH_SIZE: + collection.insert_many(batch) + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} documents...") + + # Insert remaining documents + if batch: + collection.insert_many(batch) + total_generated += len(batch) + + client.close() + + print(f"Completed! Generated {total_generated} events.") + +if __name__ == "__main__": + main() diff --git a/config/airflow/dags/scripts/generate_mysql_employee_data.py b/config/airflow/dags/scripts/generate_mysql_employee_data.py new file mode 100644 index 0000000..f7bbc45 --- /dev/null +++ b/config/airflow/dags/scripts/generate_mysql_employee_data.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Script to generate fake employee event data for MySQL +Generates approximately 1GB of data +""" + +import mysql.connector +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "3306" +DB_NAME = "hr" +DB_USER = "mo" +DB_PASSWORD = "Dell2026!" + +# Data generation settings +TARGET_ROWS = 4000000 # Approximately 1GB of data +BATCH_SIZE = 10000 + +# Sample data +DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"] +ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"] +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"] + +def generate_fake_employee_event(): + """Generate a single fake employee event""" + employee_id = random.randint(1, 100000) + department = random.choice(DEPARTMENTS) + role_name = random.choice(ROLE_NAMES) + region = random.choice(REGIONS) + event_type = random.choice(EVENT_TYPES) + + # Random timestamp within the last 2 years + days_ago = random.randint(0, 730) + event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None + + # Generate a long notes field (like the existing data) + notes = str(uuid.uuid4()) * 10 + + return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes) + +def main(): + print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...") + + conn = mysql.connector.connect( + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + user=DB_USER, + password=DB_PASSWORD + ) + cursor = conn.cursor() + + print(f"Generating {TARGET_ROWS} employee events...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_employee_event()) + + if len(batch) >= BATCH_SIZE: + cursor.executemany( + """ + INSERT INTO employee_events (employee_id, department, role_name, region, + event_type, salary_change, event_ts, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + cursor.executemany( + """ + INSERT INTO employee_events (employee_id, department, role_name, region, + event_type, salary_change, event_ts, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + + cursor.close() + conn.close() + + print(f"Completed! Generated {total_generated} employee events.") + +if __name__ == "__main__": + main() diff --git a/config/airflow/dags/scripts/generate_neo4j_graph_data.py b/config/airflow/dags/scripts/generate_neo4j_graph_data.py new file mode 100644 index 0000000..428a4c9 --- /dev/null +++ b/config/airflow/dags/scripts/generate_neo4j_graph_data.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +Script to generate fake graph data for Neo4j +Generates approximately 1GB of data with nodes and relationships +""" + +from neo4j import GraphDatabase +import random +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "7687" +DB_USER = "neo4j" +DB_PASSWORD = "testpwd" + +# Data generation settings +TARGET_NODES = 500000 # Approximately 1GB of data with relationships +BATCH_SIZE = 1000 + +# Sample data +PRODUCT_CATEGORIES = ["Electronics", "Clothing", "Food", "Furniture", "Toys", "Books"] +SUPPLIER_REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +RELATIONSHIP_TYPES = ["SUPPLIES", "RELATED_TO", "COMPATIBLE_WITH", "PART_OF"] + +def generate_fake_product(): + """Generate a single fake product node""" + product_id = str(uuid.uuid4()) + name = f"Product-{random.randint(1000, 999999)}" + category = random.choice(PRODUCT_CATEGORIES) + price = round(random.uniform(10.0, 1000.0), 2) + stock = random.randint(0, 1000) + + # Generate a long description field + description = "X" * 200 + + return { + "product_id": product_id, + "name": name, + "category": category, + "price": price, + "stock": stock, + "description": description + } + +def generate_fake_supplier(): + """Generate a single fake supplier node""" + supplier_id = str(uuid.uuid4()) + name = f"Supplier-{random.randint(1000, 999999)}" + region = random.choice(SUPPLIER_REGIONS) + rating = round(random.uniform(1.0, 5.0), 1) + + # Generate a long address field + address = "X" * 150 + + return { + "supplier_id": supplier_id, + "name": name, + "region": region, + "rating": rating, + "address": address + } + +def main(): + print(f"Connecting to Neo4j at {DB_HOST}:{DB_PORT}...") + + driver = GraphDatabase.driver(f"bolt://{DB_HOST}:{DB_PORT}", + auth=(DB_USER, DB_PASSWORD)) + + with driver.session() as session: + print(f"Generating {TARGET_NODES} product nodes...") + print(f"Batch size: {BATCH_SIZE}") + + total_products = 0 + total_suppliers = 0 + product_ids = [] + + # Generate product nodes + batch = [] + for i in range(TARGET_NODES): + product = generate_fake_product() + batch.append(product) + product_ids.append(product["product_id"]) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + CREATE (p:Product { + product_id: row.product_id, + name: row.name, + category: row.category, + price: row.price, + stock: row.stock, + description: row.description + }) + """, + batch=batch + ) + total_products += len(batch) + batch = [] + + if total_products % 50000 == 0: + print(f"Generated {total_products} product nodes...") + + # Insert remaining products + if batch: + session.run( + """ + UNWIND $batch as row + CREATE (p:Product { + product_id: row.product_id, + name: row.name, + category: row.category, + price: row.price, + stock: row.stock, + description: row.description + }) + """, + batch=batch + ) + total_products += len(batch) + + print(f"Generated {total_products} product nodes.") + + # Generate supplier nodes (fewer than products) + print(f"Generating supplier nodes...") + target_suppliers = 10000 + batch = [] + supplier_ids = [] + + for i in range(target_suppliers): + supplier = generate_fake_supplier() + batch.append(supplier) + supplier_ids.append(supplier["supplier_id"]) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + CREATE (s:Supplier { + supplier_id: row.supplier_id, + name: row.name, + region: row.region, + rating: row.rating, + address: row.address + }) + """, + batch=batch + ) + total_suppliers += len(batch) + batch = [] + + if batch: + session.run( + """ + UNWIND $batch as row + CREATE (s:Supplier { + supplier_id: row.supplier_id, + name: row.name, + region: row.region, + rating: row.rating, + address: row.address + }) + """, + batch=batch + ) + total_suppliers += len(batch) + + print(f"Generated {total_suppliers} supplier nodes.") + + # Create relationships between products and suppliers + print(f"Creating relationships...") + batch = [] + total_relationships = 0 + + for product_id in product_ids: + # Each product is supplied by 1-3 random suppliers + num_suppliers = random.randint(1, 3) + for _ in range(num_suppliers): + supplier_id = random.choice(supplier_ids) + rel_type = random.choice(RELATIONSHIP_TYPES) + + batch.append({ + "product_id": product_id, + "supplier_id": supplier_id, + "rel_type": rel_type + }) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + MATCH (p:Product {product_id: row.product_id}) + MATCH (s:Supplier {supplier_id: row.supplier_id}) + CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel + RETURN rel + """, + batch=batch + ) + total_relationships += len(batch) + batch = [] + + if total_relationships % 50000 == 0: + print(f"Created {total_relationships} relationships...") + + if batch: + session.run( + """ + UNWIND $batch as row + MATCH (p:Product {product_id: row.product_id}) + MATCH (s:Supplier {supplier_id: row.supplier_id}) + CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel + RETURN rel + """, + batch=batch + ) + total_relationships += len(batch) + + print(f"Created {total_relationships} relationships.") + + driver.close() + print(f"Completed! Generated {total_products} products, {total_suppliers} suppliers, and {total_relationships} relationships.") + +if __name__ == "__main__": + main() diff --git a/config/airflow/dags/scripts/generate_postgres_sales_data.py b/config/airflow/dags/scripts/generate_postgres_sales_data.py new file mode 100644 index 0000000..2b48cb8 --- /dev/null +++ b/config/airflow/dags/scripts/generate_postgres_sales_data.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Script to generate fake sales order data for PostgreSQL +Generates approximately 1GB of data +""" + +import psycopg2 +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "5432" +DB_NAME = "postgres" +DB_USER = "mo" +DB_PASSWORD = "Dell2026!" + +# Data generation settings +TARGET_ROWS = 4000000 # Approximately 1GB of data +BATCH_SIZE = 10000 + +# Sample data +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"] +CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"] +ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"] + +def generate_fake_order(): + """Generate a single fake sales order""" + customer_id = random.randint(1, 100000) + product_id = random.randint(1, 5000) + region = random.choice(REGIONS) + sales_channel = random.choice(SALES_CHANNELS) + + # Random timestamp within the last 2 years + days_ago = random.randint(0, 730) + order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + amount = round(random.uniform(10.0, 10000.0), 2) + currency = random.choice(CURRENCIES) + order_status = random.choice(ORDER_STATUSES) + + # Generate a long notes field (like the existing data) + notes = str(uuid.uuid4()) * 10 + + return (customer_id, product_id, region, sales_channel, order_ts, + amount, currency, order_status, notes) + +def main(): + print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...") + + conn = psycopg2.connect( + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + user=DB_USER, + password=DB_PASSWORD + ) + cursor = conn.cursor() + + print(f"Generating {TARGET_ROWS} sales orders...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_order()) + + if len(batch) >= BATCH_SIZE: + cursor.executemany( + """ + INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, + order_ts, amount, currency, order_status, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + cursor.executemany( + """ + INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, + order_ts, amount, currency, order_status, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + + cursor.close() + conn.close() + + print(f"Completed! Generated {total_generated} sales orders.") + +if __name__ == "__main__": + main() diff --git a/config/grafana/grafana.ini b/config/grafana/grafana.ini new file mode 100644 index 0000000..a0c441a --- /dev/null +++ b/config/grafana/grafana.ini @@ -0,0 +1,16 @@ +[server] +http_addr = 0.0.0.0 +http_port = 3000 + +[security] +admin_user = admin +admin_password = REDACTED +allow_embedding = true + +[auth.anonymous] +enabled = true +org_name = Main Org. +org_role = Viewer + +[auth.basic] +enabled = false diff --git a/config/grafana/provisioning/dashboards.yml b/config/grafana/provisioning/dashboards.yml new file mode 100644 index 0000000..3858559 --- /dev/null +++ b/config/grafana/provisioning/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'Default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/config/homepage/custom.js b/config/homepage/custom.js index e9a7bf4..80f47fb 100644 --- a/config/homepage/custom.js +++ b/config/homepage/custom.js @@ -40,10 +40,27 @@ }); } + + function showGitSha() { + if (document.getElementById('atc-git-sha')) return; + fetch('http://atc-mgt01.dell-atc.lan:3001/api/v1/repos/mo/Lakehouse/commits?limit=1') + .then(function (r) { return r.json(); }) + .then(function (d) { + var sha = (d && d[0] && d[0].sha) ? d[0].sha.substring(0, 7) : 'git'; + var el = document.createElement('div'); + el.id = 'atc-git-sha'; + el.textContent = 'Lakehouse ' + sha; + el.style.cssText = 'position:fixed;bottom:4px;right:8px;font-size:10px;color:#64748b;z-index:9999;font-family:monospace;'; + document.body.appendChild(el); + }) + .catch(function () {}); + } + function init() { injectOverlays(); applyServiceColors(); linkLakehouseTitle(); + showGitSha(); new MutationObserver(linkLakehouseTitle).observe(document.body, { childList: true, subtree: true }); } diff --git a/config/homepage/services.yaml b/config/homepage/services.yaml index 81c6ac9..a191dee 100644 --- a/config/homepage/services.yaml +++ b/config/homepage/services.yaml @@ -512,6 +512,39 @@ href: link target: _blank + +- Documentation: + - Architecture Diagram: + icon: mdi-sitemap + href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html + description: High-level environment map (Mermaid) + color: "#007DB8" + - Application Landscape: + icon: mdi-book-open-variant + href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md + description: docs/landscape.md in Forgejo + color: "#E8752A" + - Network Diagram: + icon: mdi-lan + href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/network.md + description: Subnets, ports, SSH mesh + color: "#22d3ee" + - Disaster Recovery: + icon: mdi-backup-restore + href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/disaster-recovery.md + description: Restore procedures from git + color: "#a855f7" + - Docker Inventory: + icon: mdi-docker + href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/docker-inventory.md + description: Containers per host + color: "#2496ED" + - Proxmox VMs: + icon: mdi-server + href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/inventory/proxmox-vms.json + description: VM inventory (JSON) + color: "#E57000" + - News & Resources: - Dell Technologies: icon: http://atc-docker01.dell-atc.lan:8080/dell-technologies.svg diff --git a/config/homepage/settings.yaml b/config/homepage/settings.yaml index 7070df6..650c8c4 100644 --- a/config/homepage/settings.yaml +++ b/config/homepage/settings.yaml @@ -51,6 +51,10 @@ layout: tab: FEEDS style: row columns: 6 + Documentation: + tab: DOCS + style: row + columns: 4 News & Resources: tab: LINKS style: row diff --git a/config/ldap/README.md b/config/ldap/README.md new file mode 100644 index 0000000..1bd2710 --- /dev/null +++ b/config/ldap/README.md @@ -0,0 +1,19 @@ +# Configuration index + +| Directory | Host | Contents | +|-----------|------|----------| +| [homepage/](homepage/) | atc-docker01 | Dashboard YAML, CSS, icons | +| [docker/](docker/) | Multiple | Docker Compose per host | +| [compose/](../compose/) | atc-docker01 | Source compose stacks | +| [kafka/](kafka/) | kafka01, lake01 | Broker + Connect connectors | +| [trino/](trino/) | atc-lake01 | SQL catalog properties | +| [spark-jobs/](spark-jobs/) | atc-lake01 | Kafka→S3 Python jobs | +| [airflow/](airflow/) | atc-airflow01 | airflow.cfg, DAGs | +| [elastic/](elastic/) | atc-elastic01 | elasticsearch.yml | +| [grafana/](grafana/) | atc-grafana | grafana.ini | +| [objectscale/](objectscale/) | 10.0.20.111 | ECS deploy.yml | +| [ldap/](ldap/) | atc-mgt01 | LDIF exports | +| [mgt01/](mgt01/) | atc-mgt01 | NPM compose | +| [hosts/](hosts/) | All | /etc/hosts snippet | + +Refresh all: `./scripts/collect/collect-fleet-config.sh` diff --git a/config/ldap/autofs.ldif b/config/ldap/autofs.ldif new file mode 100644 index 0000000..d054d68 --- /dev/null +++ b/config/ldap/autofs.ldif @@ -0,0 +1,7 @@ +dn: cn=autofs,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: autofs +olcAttributeTypes: ( 1.3.6.1.1.1.1.25 NAME 'automountInformation' DESC 'Information used by the autofs automounter' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +olcObjectClasses: ( 1.3.6.1.1.1.1.13 NAME 'automount' DESC 'An entry in an automounter map' SUP top STRUCTURAL MUST ( cn $ automountInformation $ objectclass ) MAY description ) +olcObjectClasses: ( 1.3.6.1.4.1.2312.4.2.2 NAME 'automountMap' DESC 'A group of related automount objects' SUP top STRUCTURAL MUST ou ) + diff --git a/config/ldap/base.ldif b/config/ldap/base.ldif new file mode 100644 index 0000000..c7a2621 --- /dev/null +++ b/config/ldap/base.ldif @@ -0,0 +1,14 @@ +dn: dc=dell-atc,dc=lan +objectClass: top +objectClass: dcObject +objectClass: organization +o: Dell ATC Lab +dc: dell-atc + +dn: ou=Users,dc=dell-atc,dc=lan +objectClass: organizationalUnit +ou: Users + +dn: ou=Groups,dc=dell-atc,dc=lan +objectClass: organizationalUnit +ou: Groups diff --git a/config/ldap/db.ldif b/config/ldap/db.ldif new file mode 100644 index 0000000..1734672 --- /dev/null +++ b/config/ldap/db.ldif @@ -0,0 +1,14 @@ +dn: olcDatabase={2}mdb,cn=config +changetype: modify +replace: olcSuffix +olcSuffix: dc=dell-atc,dc=lan + +dn: olcDatabase={2}mdb,cn=config +changetype: modify +replace: olcRootDN +olcRootDN: cn=admin,dc=dell-atc,dc=lan + +dn: olcDatabase={2}mdb,cn=config +changetype: modify +replace: olcRootPW +olcRootPW: {SSHA}MO4kZaN1wIHCgkMw6f0FX0kcQbEe+jmx diff --git a/config/ldap/openssh.ldif b/config/ldap/openssh.ldif new file mode 100644 index 0000000..51dd76b --- /dev/null +++ b/config/ldap/openssh.ldif @@ -0,0 +1,11 @@ +dn: cn=openssh-lpk-openldap,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: openssh-lpk-openldap +olcAttributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' + DESC 'MANDATORY: OpenSSH Public key' + EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) +olcObjectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' + DESC 'MANDATORY: OpenSSH LPK objectclass' + SUP top AUXILIARY MUST ( sshPublicKey ) ) + + diff --git a/config/ldap/setacl.ldif b/config/ldap/setacl.ldif new file mode 100644 index 0000000..08feaa6 --- /dev/null +++ b/config/ldap/setacl.ldif @@ -0,0 +1,6 @@ +dn: olcDatabase={2}mdb,cn=config +changetype: modify +add: olcAccess +olcAccess: {0}to attrs=userPassword,shadowLastChange by dn="cn=admin,dc=dell-atc,dc=lan" write by anonymous auth by self write by * none +olcAccess: {1}to * by dn="cn=admin,dc=dell-atc,dc=lan" write by * read + diff --git a/config/mgt01/npm-compose.yaml b/config/mgt01/npm-compose.yaml new file mode 100644 index 0000000..71b73af --- /dev/null +++ b/config/mgt01/npm-compose.yaml @@ -0,0 +1,14 @@ +services: + app: + image: 'jc21/nginx-proxy-manager:latest' + restart: unless-stopped + environment: + TZ: "Europe/Amsterdam" + ports: + - '80:80' + - '81:81' + - '443:443' + volumes: + - ./data:/data + - ./letsencrypt:/etc/letsencrypt + diff --git a/config/objectscale/deploy.yml b/config/objectscale/deploy.yml index 9e7a742..ded58db 100644 --- a/config/objectscale/deploy.yml +++ b/config/objectscale/deploy.yml @@ -45,10 +45,10 @@ facts: # [Required] # Password to use with SSH login # *** Set to same value as ssh_username to enable SSH public key authentication *** - ssh_password: "REDACTED" + ssh_password: REDACTED # [Required when enabling SSH public key authentication] # Password to give to sudo when gaining root access. - ansible_become_pass: "REDACTED" + ansible_become_pass: REDACTED # [Required] # Select the type of crypto to use when dealing with ssh public key # authentication. Valid values here are: diff --git a/config/spark-jobs/README.md b/config/spark-jobs/README.md new file mode 100644 index 0000000..64abe7f --- /dev/null +++ b/config/spark-jobs/README.md @@ -0,0 +1 @@ +# Spark jobs diff --git a/config/spark-jobs/consumer.py b/config/spark-jobs/consumer.py new file mode 100644 index 0000000..a2994fa --- /dev/null +++ b/config/spark-jobs/consumer.py @@ -0,0 +1,38 @@ +from kafka import KafkaConsumer +import boto3 +import json +from datetime import datetime + +KAFKA_BROKER = "10.0.21.36:9092" +KAFKA_TOPIC = "test-lakehouse" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +print("Starting consumer...") +print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}") +print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}") + +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + value_deserializer=lambda x: json.loads(x.decode('utf-8')) +) + +print("Waiting for messages...") +for msg in consumer: + ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json" + s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value)) + print(f"Written: {key}") diff --git a/config/spark-jobs/consumer_fixed.py b/config/spark-jobs/consumer_fixed.py new file mode 100644 index 0000000..053076f --- /dev/null +++ b/config/spark-jobs/consumer_fixed.py @@ -0,0 +1,48 @@ +from kafka import KafkaConsumer +import boto3 +import json +from datetime import datetime + +KAFKA_BROKER = "10.0.21.36:9092" +KAFKA_TOPIC = "test-lakehouse" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +print("Starting consumer...") +print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}") + +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +def deserialize(value): + if value is None or len(value) == 0: + return None + try: + return json.loads(value.decode('utf-8')) + except: + print(f"Skipping non-JSON") + return None + +consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='latest', + value_deserializer=deserialize +) + +print("Waiting for messages...") +for msg in consumer: + if msg.value is None: + continue + ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json" + s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value)) + print(f"Written: {key}") diff --git a/config/spark-jobs/kafka_to_s3.py b/config/spark-jobs/kafka_to_s3.py new file mode 100644 index 0000000..b95b1b2 --- /dev/null +++ b/config/spark-jobs/kafka_to_s3.py @@ -0,0 +1,34 @@ +from pyspark.sql import SparkSession +from pyspark.sql.functions import col, current_timestamp + +spark = SparkSession.builder \ + .appName("KafkaToS3") \ + .config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \ + .config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \ + .config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \ + .config("spark.hadoop.fs.s3a.path.style.access", "true") \ + .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \ + .getOrCreate() + +spark.sparkContext.setLogLevel("WARN") + +df = spark.readStream \ + .format("kafka") \ + .option("kafka.bootstrap.servers", "10.0.21.36:9092") \ + .option("subscribe", "test-lakehouse") \ + .option("startingOffsets", "latest") \ + .load() + +output = df.select( + col("key").cast("string"), + col("value").cast("string"), + current_timestamp().alias("timestamp") +) + +query = output.writeStream \ + .outputMode("append") \ + .format("console") \ + .start() + +print("Streaming started. Press Ctrl+C to stop.") +query.awaitTermination() diff --git a/config/spark-jobs/kafka_to_s3_consumer.py b/config/spark-jobs/kafka_to_s3_consumer.py new file mode 100644 index 0000000..46df6ee --- /dev/null +++ b/config/spark-jobs/kafka_to_s3_consumer.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +from kafka import KafkaConsumer +import boto3 +import json +import os +import sys +from datetime import datetime + +# Configuratie +KAFKA_BROKER = "10.0.21.36:9092" +KAFKA_TOPIC = "test-lakehouse" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +print("=" * 60) +print(f"Kafka Consumer starting...") +print(f"Kafka broker: {KAFKA_BROKER}") +print(f"Kafka topic: {KAFKA_TOPIC}") +print(f"S3 endpoint: {S3_ENDPOINT}") +print(f"S3 bucket: {S3_BUCKET}") +print("=" * 60) + +# S3 client +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +# Kafka consumer +try: + consumer = KafkaConsumer( + KAFKA_TOPIC, + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='earliest', + enable_auto_commit=True, + value_deserializer=lambda x: json.loads(x.decode('utf-8')) + ) + print("Connected to Kafka successfully!") +except Exception as e: + print(f"Failed to connect to Kafka: {e}") + sys.exit(1) + +print(f"Listening for messages on topic '{KAFKA_TOPIC}'...") +print("-" * 60) + +message_count = 0 +for message in consumer: + message_count += 1 + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json" + + try: + s3.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=json.dumps(message.value, indent=2) + ) + print(f"[{message_count}] Written to s3://{S3_BUCKET}/{key}") + print(f" Data: {json.dumps(message.value)[:100]}...") + except Exception as e: + print(f"Error writing to S3: {e}") diff --git a/config/spark-jobs/s3_consumer_all_topics.py b/config/spark-jobs/s3_consumer_all_topics.py new file mode 100644 index 0000000..eb11c41 --- /dev/null +++ b/config/spark-jobs/s3_consumer_all_topics.py @@ -0,0 +1,69 @@ +from kafka import KafkaConsumer +from kafka import TopicPartition +import boto3 +import json +from datetime import datetime + +# Configuratie +KAFKA_BROKER = "10.0.21.36:9092" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +print("Starting S3 consumer for ALL topics...") +print(f"Kafka: {KAFKA_BROKER}") +print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}") + +# S3 client +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +# Consumer zonder specifiek topic +consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER]) + +# Haal alle topics op +all_topics = consumer.topics() +print(f"Found topics: {list(all_topics)}") + +# Wijs alle partitions van alle topics toe +for topic in all_topics: + partitions = consumer.partitions_for_topic(topic) + for partition in partitions: + tp = TopicPartition(topic, partition) + consumer.assign([tp]) + consumer.seek_to_beginning() + print(f"Assigned: {topic} - partition {partition}") + +print("\nReading all messages from all topics...") +print("-" * 60) + +msg_count = 0 +topic_count = {} + +for msg in consumer: + msg_count += 1 + topic = msg.topic + topic_count[topic] = topic_count.get(topic, 0) + 1 + + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{topic}/message_{timestamp}.json" + + try: + data = json.loads(msg.value.decode('utf-8')) + except: + data = {"raw": msg.value.decode('utf-8')} + + s3.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=json.dumps(data, indent=2) + ) + + print(f"[{msg_count}] {topic} -> s3://{S3_BUCKET}/{key}") diff --git a/config/spark-jobs/s3_consumer_realtime.py b/config/spark-jobs/s3_consumer_realtime.py new file mode 100644 index 0000000..0d71df6 --- /dev/null +++ b/config/spark-jobs/s3_consumer_realtime.py @@ -0,0 +1,45 @@ +from kafka import KafkaConsumer +import boto3 +import json +from datetime import datetime + +KAFKA_BROKER = "10.0.21.36:9092" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +# Consumer die naar ALLE topics luistert (alleen nieuwe berichten) +consumer = KafkaConsumer( + bootstrap_servers=[KAFKA_BROKER], + auto_offset_reset='latest' +) + +# Subscribe op alle topics +consumer.subscribe(pattern='.*') + +print("Listening to ALL topics...") +print("Waiting for new messages...") + +msg_count = 0 +for msg in consumer: + msg_count += 1 + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{msg.topic}/message_{timestamp}.json" + + try: + data = json.loads(msg.value.decode('utf-8')) + except: + data = {"raw": msg.value.decode('utf-8')} + + s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(data, indent=2)) + print(f"[{msg_count}] {msg.topic} -> s3://{S3_BUCKET}/{key}") diff --git a/config/spark-jobs/s3_consumer_working.py b/config/spark-jobs/s3_consumer_working.py new file mode 100644 index 0000000..05d88ae --- /dev/null +++ b/config/spark-jobs/s3_consumer_working.py @@ -0,0 +1,57 @@ +from kafka import KafkaConsumer +from kafka import TopicPartition +import boto3 +import json +from datetime import datetime + +# Configuratie +KAFKA_BROKER = "10.0.21.36:9092" +KAFKA_TOPIC = "test-lakehouse" +S3_ENDPOINT = "http://10.0.20.111:9020" +S3_BUCKET = "data" +S3_ACCESS_KEY = "REDACTED" +S3_SECRET_KEY = "REDACTED" + +print("Starting S3 consumer...") +print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}") +print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}") + +# S3 client +s3 = boto3.client( + 's3', + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + use_ssl=False, + verify=False +) + +# Kafka consumer met vaste partition +consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER]) +tp = TopicPartition(KAFKA_TOPIC, 0) +consumer.assign([tp]) +consumer.seek_to_beginning() + +print("Reading all messages from beginning...") +print("-" * 50) + +msg_count = 0 +for msg in consumer: + msg_count += 1 + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json" + + try: + data = json.loads(msg.value.decode('utf-8')) + except: + data = {"raw": msg.value.decode('utf-8')} + + s3.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=json.dumps(data, indent=2) + ) + print(f"[{msg_count}] Written: s3://{S3_BUCKET}/{key}") + print(f" Data: {str(data)[:80]}...") + +print(f"Done! Total {msg_count} messages written to S3") diff --git a/config/spark-jobs/simple_consumer.py b/config/spark-jobs/simple_consumer.py new file mode 100644 index 0000000..a8bd178 --- /dev/null +++ b/config/spark-jobs/simple_consumer.py @@ -0,0 +1,12 @@ +from kafka import KafkaConsumer +import json + +consumer = KafkaConsumer( + 'test-lakehouse', + bootstrap_servers=['10.0.21.36:9092'], + auto_offset_reset='latest' +) + +print("Waiting for new messages...") +for msg in consumer: + print(f"Received: {msg.value.decode('utf-8')}") diff --git a/config/trino/README.md b/config/trino/README.md new file mode 100644 index 0000000..2f412d4 --- /dev/null +++ b/config/trino/README.md @@ -0,0 +1,19 @@ +# Trino — atc-lake01 + +| Item | Value | +|------|-------| +| UI | http://atc-lake01:8089/ui/ | +| Container | `trino` (Docker) | +| Catalogs | `catalog/*.properties` | + +## Catalogs + +| File | Connector | Source | +|------|-----------|--------| +| `postgres_sales.properties` | PostgreSQL | atc-db02:5432 | +| `mysql_hr.properties` | MySQL | atc-db02:3306 | +| `mongodb_supplychain.properties` | MongoDB | atc-db02:27017 | +| `cassandra_telemetry.properties` | Cassandra | atc-db02:9042 | +| `iceberg.properties` | Iceberg | ObjectScale S3 `10.0.20.111:9020` | + +Passwords redacted in git. Refresh: `./scripts/collect/collect-fleet-config.sh` diff --git a/config/trino/catalog/cassandra_telemetry.properties b/config/trino/catalog/cassandra_telemetry.properties new file mode 100644 index 0000000..d4b7a29 --- /dev/null +++ b/config/trino/catalog/cassandra_telemetry.properties @@ -0,0 +1,11 @@ +connector.name=cassandra + +cassandra.contact-points=10.0.21.51 +cassandra.native-protocol-port=9042 + +# Required DC setting for load balancing: +cassandra.load-policy.dc-aware.local-dc=datacenter1 + +# If Cassandra auth is enabled, uncomment and set: +# cassandra.username=mo +# cassandra.password=REDACTED diff --git a/config/trino/catalog/iceberg.properties b/config/trino/catalog/iceberg.properties new file mode 100644 index 0000000..e6c12ab --- /dev/null +++ b/config/trino/catalog/iceberg.properties @@ -0,0 +1,10 @@ +connector.name=iceberg +iceberg.catalog.type=TESTING_FILE_METASTORE +hive.metastore.catalog.dir=/tmp/iceberg_metadata + +# S3 Connectivity +hive.s3.endpoint=http://10.0.20.111:9020 +hive.s3.aws-access-key=REDACTED +hive.s3.aws-secret-key=REDACTED +hive.s3.path-style-access=true +hive.s3.ssl.enabled=false diff --git a/config/trino/catalog/mongodb_supplychain.properties b/config/trino/catalog/mongodb_supplychain.properties new file mode 100644 index 0000000..bf279b4 --- /dev/null +++ b/config/trino/catalog/mongodb_supplychain.properties @@ -0,0 +1,3 @@ +connector.name=mongodb +mongodb.connection-url=mongodb://10.0.21.51:27017/ +mongodb.schema-collection=__trino_schema diff --git a/config/trino/catalog/mysql_hr.properties b/config/trino/catalog/mysql_hr.properties new file mode 100644 index 0000000..89c7e75 --- /dev/null +++ b/config/trino/catalog/mysql_hr.properties @@ -0,0 +1,4 @@ +connector.name=mysql +connection-url=jdbc:mysql://10.0.21.51:3306 +connection-user=mo +connection-password=REDACTED diff --git a/config/trino/catalog/postgres_sales.properties b/config/trino/catalog/postgres_sales.properties new file mode 100644 index 0000000..bdbcb84 --- /dev/null +++ b/config/trino/catalog/postgres_sales.properties @@ -0,0 +1,4 @@ +connector.name=postgresql +connection-url=jdbc:postgresql://10.0.21.51:5432/postgres +connection-user=mo +connection-password=REDACTED diff --git a/docs/network.md b/docs/network.md new file mode 100644 index 0000000..9c8e477 --- /dev/null +++ b/docs/network.md @@ -0,0 +1,64 @@ +# Network — ATC Lakehouse lab + +## Subnets + +```mermaid +flowchart TB + subgraph mgmt [10.0.10.0/24 — Management] + PVE[pve01 · 10.0.10.65] + IDRAC[iDRAC · 10.0.41.102] + end + + subgraph data20 [10.0.20.0/24 — Storage & legacy] + OS[ObjectScale · 10.0.20.111] + DB1[atc-db01 · 10.0.20.112] + MGT[atc-mgt01 · 10.0.20.104] + GF[atc-grafana · 10.0.20.103] + end + + subgraph lake21 [10.0.21.0/24 — Lakehouse compute] + D1[atc-docker01 · 10.0.21.45] + D2[atc-docker02 · 10.0.21.47] + KF[atc-kafka01 · 10.0.21.36] + LK[atc-lake01 · 10.0.21.50] + DB2[atc-db02 · 10.0.21.51] + AF[atc-airflow01 · 10.0.21.55] + EL[atc-elastic01 · 10.0.21.46] + end + + PVE --> D1 & D2 & KF & LK & DB2 & OS & DB1 & MGT +``` + +## DNS + +Internal names: `*.dell-atc.lan` (see `config/hosts/atc-lab.hosts`). + +| Pattern | Example | +|---------|---------| +| Short hostname | `atc-lake01` | +| FQDN | `atc-lake01.dell-atc.lan` | + +## Key ports (east-west) + +| Service | Port | Protocol | +|---------|------|----------| +| Kafka | 9092 | TCP | +| Debezium Connect | 8083 | HTTP | +| Spark UI | 8080 | HTTP | +| Trino | 8089 | HTTP | +| PostgreSQL | 5432 | TCP | +| ObjectScale S3 | 9020 | HTTP | +| ObjectScale UI | 443 | HTTPS | +| Homepage | 80 | HTTP | + +## SSH mesh + +All lab VMs: `/root/.ssh/atc_cluster` — see [ssh-mesh.md](ssh-mesh.md). + +ObjectScale uses `admin@` for initial bootstrap; `root@` after key install. + +## Firewall notes + +- Lab assumes flat L2/L3 trust within `10.0.10/20/21.x`. +- ObjectScale `management_clients: 0.0.0.0/0` in deploy.yml — tighten for production. +- Proxmox API `:8006` reachable from docker01 for homepage widget. diff --git a/docs/recommendations.md b/docs/recommendations.md index 5525bb8..8a7c8cb 100644 --- a/docs/recommendations.md +++ b/docs/recommendations.md @@ -1,46 +1,33 @@ -# Recommendations — next steps for documentation & IaC +# Recommendations — status -Prioritized ideas to make the lab fully reproducible and operable. +## Completed -## High priority +- [x] Trino catalog properties +- [x] Grafana ini + dashboard provisioning +- [x] Spark jobs (`/opt/spark-jobs`) +- [x] Airflow `airflow.cfg` + DAG scripts +- [x] Kafka KRaft `server.properties` +- [x] ObjectScale `deploy.yml` +- [x] Elasticsearch `elasticsearch.yml` +- [x] Docker compose per host +- [x] LDAP LDIF exports (mgt01) +- [x] NPM compose (mgt01) +- [x] Proxmox VM inventory JSON +- [x] SSH mesh (12 hosts + ObjectScale) +- [x] Architecture diagram page +- [x] `collect-fleet-config.sh` + `populate-repo.py` +- [x] Backup script +- [x] Health check script +- [x] Cron documentation +- [x] Ansible playbook + inventory +- [x] Gitea CI validate workflow +- [x] Network documentation +- [x] Homepage DOCS tab -| Item | Host | Why | -|------|------|-----| -| **Trino catalog properties** | atc-lake01 | `etc/catalog/*.properties` — documents federated queries | -| **Grafana `grafana.ini` + datasources** | atc-grafana | Monitoring as code | -| **Kibana / ES keystore note** | atc-elastic01 | Passwords in keystore — document enrollment, not files | -| **Proxmox VM notes** | pve01 | VMID → hostname → IP table (API export script) | -| **Backup script** | docker01 | Nightly `git pull` + volume tarballs to ObjectScale | +## Optional next steps -## Medium priority - -| Item | Host | Why | -|------|------|-----| -| **Spark jobs** | atc-lake01 | `/opt/spark-jobs/` in git | -| **Airflow `dags/scripts/`** | atc-airflow01 | Supporting Python for DAGs | -| **Nginx Proxy Manager** | atc-mgt01 | Export NPM config (if used for TLS) | -| **LDAP/LDIF exports** | atc-mgt01 | `*.ldif` already on host — useful for LDAP rebuild | -| **MinIO / S3 buckets** | objectscale | Bucket layout + IAM policy docs | -| **Network diagram** | docs | VLAN / firewall rules (10.0.10/20/21.x) | - -## Automation - -| Item | Description | -|------|-------------| -| **CI on Forgejo** | Lint YAML, validate compose, dry-run `docker compose config` | -| **Ansible playbook** | `ansible-playbook deploy-lakehouse.yml` from git | -| **Health check script** | Cron: curl all `siteMonitor` URLs, alert via Grafana | -| **Monthly collect cron** | `collect-fleet-config.sh` + auto-commit branch | - -## Security hardening (lab → prod path) - -- Move all secrets to `.env` / Vault; git only `*.example` -- Rotate `atc_cluster` SSH key periodically -- Restrict ObjectScale `management_clients` from `0.0.0.0/0` -- Enable TLS on Kafka (`SASL_SSL`) if exposed beyond lab VLAN - -## Dashboard enhancements - -- Homepage widget: link to architecture diagram in header logo -- Add **DOCS** tab with bookmarks to all `docs/*.md` on Forgejo -- Version badge in footer (git commit SHA from build arg) +- [ ] Kibana `kibana.yml` export +- [ ] TLS certificates inventory (NPM letsencrypt paths) +- [ ] Automated weekly git commit via cron (install from `scripts/cron/README.md`) +- [ ] Vault/external secrets instead of redacted files +- [ ] Reach remaining hosts: `10.0.21.52`, `.37`, `.38` (no SSH key yet) diff --git a/inventory/proxmox-vms.json b/inventory/proxmox-vms.json new file mode 100644 index 0000000..3bc3f95 --- /dev/null +++ b/inventory/proxmox-vms.json @@ -0,0 +1,154 @@ +[ + { + "vmid": 103, + "name": "DELL-OBSCE-4.2.0.0-OVA", + "status": "stopped", + "node": "pve01", + "ip": null, + "maxmem_gb": 68.7 + }, + { + "vmid": 105, + "name": "airflow01", + "status": "running", + "node": "pve01", + "ip": "10.0.21.55", + "maxmem_gb": 34.4 + }, + { + "vmid": 110, + "name": "atc-db01", + "status": "running", + "node": "pve01", + "ip": "10.0.20.112", + "maxmem_gb": 8.6 + }, + { + "vmid": 115, + "name": "atc-docker01", + "status": "running", + "node": "pve01", + "ip": "10.0.21.45", + "maxmem_gb": 8.6 + }, + { + "vmid": 116, + "name": "atc-docker02", + "status": "running", + "node": "pve01", + "ip": "10.0.21.47", + "maxmem_gb": 8.6 + }, + { + "vmid": 112, + "name": "atc-elastic", + "status": "running", + "node": "pve01", + "ip": "10.0.21.46", + "maxmem_gb": 34.4 + }, + { + "vmid": 114, + "name": "atc-grafana", + "status": "running", + "node": "pve01", + "ip": "10.0.20.103", + "maxmem_gb": 34.4 + }, + { + "vmid": 113, + "name": "atc-kafka", + "status": "running", + "node": "pve01", + "ip": "10.0.21.36", + "maxmem_gb": 34.4 + }, + { + "vmid": 108, + "name": "atc-lake01", + "status": "running", + "node": "pve01", + "ip": "10.0.21.50", + "maxmem_gb": 34.4 + }, + { + "vmid": 111, + "name": "atc-lama", + "status": "running", + "node": "pve01", + "ip": "10.0.21.39", + "maxmem_gb": 8.6 + }, + { + "vmid": 101, + "name": "atc-mgt01", + "status": "running", + "node": "pve01", + "ip": "10.0.20.104", + "maxmem_gb": 8.6 + }, + { + "vmid": 100, + "name": "atc-objectscale", + "status": "running", + "node": "pve01", + "ip": "10.0.20.111", + "maxmem_gb": 68.7 + }, + { + "vmid": 109, + "name": "atc-source-systems", + "status": "running", + "node": "pve01", + "ip": null, + "maxmem_gb": 34.4 + }, + { + "vmid": 117, + "name": "atc-test", + "status": "running", + "node": "pve01", + "ip": null, + "maxmem_gb": 4.3 + }, + { + "vmid": 118, + "name": "atc-w11mo", + "status": "running", + "node": "pve01", + "ip": null, + "maxmem_gb": 17.2 + }, + { + "vmid": 104, + "name": "dataflow01", + "status": "running", + "node": "pve01", + "ip": null, + "maxmem_gb": 34.4 + }, + { + "vmid": 106, + "name": "monitor01", + "status": "running", + "node": "pve01", + "ip": null, + "maxmem_gb": 34.4 + }, + { + "vmid": 107, + "name": "portal01", + "status": "running", + "node": "pve01", + "ip": "10.0.21.49", + "maxmem_gb": 34.4 + }, + { + "vmid": 102, + "name": "rocky9-base", + "status": "stopped", + "node": "pve01", + "ip": null, + "maxmem_gb": 34.4 + } +] \ No newline at end of file diff --git a/scripts/collect/collect-fleet-config.sh b/scripts/collect/collect-fleet-config.sh index b1f1a7d..5aa782d 100755 --- a/scripts/collect/collect-fleet-config.sh +++ b/scripts/collect/collect-fleet-config.sh @@ -1,47 +1,41 @@ #!/usr/bin/env bash -# Pull live configs from ATC fleet into the Lakehouse git repo. -# Run on atc-docker01 as root (requires SSH mesh). +# Pull live configs from entire ATC fleet into git. set -euo pipefail - REPO="${REPO:-/root/lakehouse}" KEY="${KEY:-/root/.ssh/atc_cluster}" -SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10) +SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=15) -redact_airflow() { +redact() { sed -E \ - -e 's/^(fernet_key = ).*/\1REDACTED/' \ - -e 's/^(internal_api_secret_key = ).*/\1REDACTED/' \ - -e 's/^(secret_key = ).*/\1REDACTED/' + -e 's/^(fernet_key|internal_api_secret_key|admin_password) = .*/\1 = REDACTED/' \ + -e 's/^(connection-password=).*/\1REDACTED/' \ + -e 's/(hive\.s3\.aws-access-key=).*/\1REDACTED/' \ + -e 's/(hive\.s3\.aws-secret-key=).*/\1REDACTED/' \ + -e 's/(S3_ACCESS_KEY|S3_SECRET_KEY) = .*/\1 = "REDACTED"/' \ + -e 's/(ssh_password|ansible_become_pass):.*/\1: REDACTED/' } +echo "==> Python fleet collector (Trino, Grafana, Spark, LDAP, NPM, Proxmox)" +python3 "$REPO/scripts/collect/populate-repo.py" + echo "==> db02 docker-compose" "${SSH[@]}" root@atc-db02 "cat /opt/sources/docker-compose.yml" > "$REPO/config/docker/atc-db02/docker-compose.yml" -echo "==> objectscale deploy.yml (redacted)" -"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | \ - sed -E 's/(ssh_password|ansible_become_pass):.*/\1: "REDACTED"/' \ - > "$REPO/config/objectscale/deploy.yml" +echo "==> objectscale" +"${SSH[@]}" root@atc-objectscale "cat /opt/emc/ecs-install/deploy.yml" | redact > "$REPO/config/objectscale/deploy.yml" -echo "==> elasticsearch.yml" +echo "==> elasticsearch" "${SSH[@]}" root@atc-elastic01 "cat /etc/elasticsearch/elasticsearch.yml" > "$REPO/config/elastic/elasticsearch.yml" -echo "==> kafka kraft-server.properties" -"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" \ - > "$REPO/config/kafka/kraft-server.properties" +echo "==> kafka" +"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/kraft/server.properties" > "$REPO/config/kafka/kraft-server.properties" +"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" > "$REPO/config/kafka/server.properties" -echo "==> kafka server.properties (ZK template, reference)" -"${SSH[@]}" root@atc-kafka01 "cat /opt/kafka/config/server.properties" \ - > "$REPO/config/kafka/server.properties" +echo "==> airflow" +"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact > "$REPO/config/airflow/airflow.cfg" +"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" > "$REPO/config/airflow/generate_data_dag.py" -echo "==> airflow.cfg (secrets redacted)" -"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/airflow.cfg" | redact_airflow \ - > "$REPO/config/airflow/airflow.cfg" - -echo "==> airflow generate_data_dag.py" -"${SSH[@]}" root@atc-airflow01 "cat /root/airflow/dags/generate_data_dag.py" \ - > "$REPO/config/airflow/generate_data_dag.py" - -echo "==> container inventory (docker01)" +echo "==> docker01 inventory" "$REPO/scripts/deploy/export-inventory.py" "$REPO/inventory/containers-atc-docker01.json" 2>/dev/null || true -echo "Done. Review: cd $REPO && git diff" +echo "Done. git diff && git commit" diff --git a/scripts/collect/populate-repo.py b/scripts/collect/populate-repo.py new file mode 100755 index 0000000..74f6720 --- /dev/null +++ b/scripts/collect/populate-repo.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Populate Lakehouse git repo with full fleet configs (secrets redacted).""" +import json +import os +import re +import subprocess +import urllib.request +import ssl +from pathlib import Path + +REPO = Path(os.environ.get("REPO", "/root/lakehouse")) +KEY = os.environ.get("KEY", "/root/.ssh/atc_cluster") +SSH = ["ssh", "-i", KEY, "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=15"] + +PVE = "https://10.0.10.65:8006" +PVE_TOKEN = os.environ.get("PVE_TOKEN", "root@pam!homepage") +PVE_SECRET = os.environ.get("PVE_SECRET", "8890185b-0850-42b7-bab2-a690ea4dc3f1") + +HOST_IPS = { + "atc-docker01": "10.0.21.45", "atc-docker02": "10.0.21.47", + "atc-mgt01": "10.0.20.104", "atc-kafka": "10.0.21.36", "atc-kafka01": "10.0.21.36", + "atc-lake01": "10.0.21.50", "atc-elastic": "10.0.21.46", "atc-elastic01": "10.0.21.46", + "atc-db01": "10.0.20.112", "atc-db02": "10.0.21.51", + "atc-grafana": "10.0.20.103", "airflow01": "10.0.21.55", "atc-airflow01": "10.0.21.55", + "atc-portal01": "10.0.21.49", "atc-lama": "10.0.21.39", "atc-lama01": "10.0.21.39", + "atc-objectscale": "10.0.20.111", "portal01": "10.0.21.49", +} + + +def ssh(host: str, cmd: str) -> str: + r = subprocess.run(SSH + [f"root@{host}", cmd], capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"ssh {host}: {r.stderr[:300]}") + return r.stdout + + +def redact_secrets(text: str) -> str: + rules = [ + (r"(connection-password=).+", r"\1REDACTED"), + (r"(connection-password = ).+", r"\1REDACTED"), + (r"(cassandra\.password=).+", r"\1REDACTED"), + (r"(hive\.s3\.aws-access-key=).+", r"\1REDACTED"), + (r"(hive\.s3\.aws-secret-key=).+", r"\1REDACTED"), + (r'(S3_ACCESS_KEY = ).+', r'\1"REDACTED"'), + (r'(S3_SECRET_KEY = ).+', r'\1"REDACTED"'), + (r"(admin_password = ).+", r"\1REDACTED"), + (r"(fernet_key = ).+", r"\1REDACTED"), + (r"(internal_api_secret_key = ).+", r"\1REDACTED"), + (r"(ssh_password|ansible_become_pass):.*", r"\1: REDACTED"), + ] + for pat, repl in rules: + text = re.sub(pat, repl, text) + return text + + +def write(path: Path, content: str): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + print(f" wrote {path.relative_to(REPO)}") + + +def collect_trino(): + d = REPO / "config/trino/catalog" + files = ssh("atc-lake01", "docker exec trino ls /etc/trino/catalog").split() + for f in files: + if not f.endswith(".properties"): + continue + raw = ssh("atc-lake01", f"docker exec trino cat /etc/trino/catalog/{f}") + write(d / f, redact_secrets(raw)) + + +def collect_grafana(): + d = REPO / "config/grafana" + write(d / "grafana.ini", redact_secrets(ssh("atc-grafana", "cat /etc/grafana/grafana.ini"))) + write(d / "provisioning/dashboards.yml", + ssh("atc-grafana", "cat /etc/grafana/provisioning/dashboards/dashboards.yml")) + + +def collect_spark_jobs(): + d = REPO / "config/spark-jobs" + for path in ssh("atc-lake01", "ls /opt/spark-jobs/*.py 2>/dev/null").split(): + name = os.path.basename(path.strip()) + if name: + write(d / name, redact_secrets(ssh("atc-lake01", f"cat /opt/spark-jobs/{name}"))) + + +def collect_ldap(): + d = REPO / "config/ldap" + for f in ["autofs.ldif", "base.ldif", "db.ldif", "openssh.ldif", "setacl.ldif"]: + try: + write(d / f, ssh("atc-mgt01", f"cat /root/{f}")) + except Exception as e: + print(f" skip {f}: {e}") + + +def collect_npm(): + write(REPO / "config/mgt01/npm-compose.yaml", + ssh("atc-mgt01", "cat /srv/docker/npm/compose.yaml")) + + +def collect_airflow_scripts(): + dest = REPO / "config/airflow/dags/scripts" + for path in ssh("atc-airflow01", "ls /root/airflow/dags/scripts/*.py 2>/dev/null").split(): + name = os.path.basename(path.strip()) + if name: + write(dest / name, ssh("atc-airflow01", f"cat /root/airflow/dags/scripts/{name}")) + + +def proxmox_inventory(): + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + req = urllib.request.Request( + f"{PVE}/api2/json/cluster/resources?type=vm", + headers={"Authorization": f"PVEAPIToken={PVE_TOKEN}={PVE_SECRET}"}, + ) + with urllib.request.urlopen(req, context=ctx) as r: + vms = json.load(r)["data"] + out = [] + for v in sorted(vms, key=lambda x: x.get("name", "")): + if v.get("type") != "qemu": + continue + name = v.get("name", "") + ip = HOST_IPS.get(name, "") + out.append({ + "vmid": v.get("vmid"), "name": name, "status": v.get("status"), + "node": v.get("node"), "ip": ip or None, + "maxmem_gb": round(v.get("maxmem", 0) / 1e9, 1), + }) + write(REPO / "inventory/proxmox-vms.json", json.dumps(out, indent=2)) + + +def main(): + collect_trino() + collect_grafana() + collect_spark_jobs() + collect_ldap() + collect_npm() + collect_airflow_scripts() + proxmox_inventory() + + +if __name__ == "__main__": + main() diff --git a/scripts/cron/README.md b/scripts/cron/README.md new file mode 100644 index 0000000..362a06c --- /dev/null +++ b/scripts/cron/README.md @@ -0,0 +1,14 @@ +# Cron jobs (install on atc-docker01) + +```cron +# Collect fleet configs weekly (Sunday 02:00) +0 2 * * 0 root cd /root/lakehouse && ./scripts/collect/collect-fleet-config.sh && git add -A && git diff --staged --quiet || git commit -m "chore: weekly fleet config sync" && git push origin master + +# Health check every 15 minutes +*/15 * * * * root /root/lakehouse/scripts/health/check-services.sh >> /var/log/atc-health.log 2>&1 + +# Backup daily at 03:00 +0 3 * * * root /root/lakehouse/scripts/backup/backup-lab.sh >> /var/log/atc-backup.log 2>&1 +``` + +Install: `crontab -e` as root on atc-docker01. diff --git a/scripts/health/check-services.sh b/scripts/health/check-services.sh new file mode 100755 index 0000000..ac86af8 --- /dev/null +++ b/scripts/health/check-services.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Health check all ATC lab HTTP endpoints (from homepage siteMonitors). +set -u + +FAIL=0 +check() { + local name="$1" url="$2" + local code + code=$(curl -sk -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 10 "$url" 2>/dev/null || echo "000") + if [[ "$code" =~ ^(200|301|302|401|403)$ ]]; then + printf "OK %-28s %s (%s)\n" "$name" "$url" "$code" + else + printf "FAIL %-28s %s (%s)\n" "$name" "$url" "$code" + FAIL=$((FAIL + 1)) + fi +} + +check "Homepage" "http://atc-docker01.dell-atc.lan/" +check "RSS proxy" "http://atc-docker01.dell-atc.lan:8090/health" +check "Superset" "http://atc-docker01.dell-atc.lan:8088/health" +check "Kafka UI" "http://atc-kafka01.dell-atc.lan:9000/" +check "Debezium" "http://atc-lake01.dell-atc.lan:8083/" +check "Spark" "http://atc-lake01.dell-atc.lan:8080/" +check "Trino" "http://atc-lake01.dell-atc.lan:8089/ui/" +check "Airflow" "http://10.0.21.55:8080/health" +check "Kibana" "http://atc-elastic01.dell-atc.lan:5601/" +check "Grafana" "http://atc-grafana.dell-atc.lan:3000/api/health" +check "Forgejo" "http://atc-mgt01.dell-atc.lan:3001/" +check "ObjectScale" "https://10.0.20.111/" +check "Proxmox" "https://10.0.10.65:8006/" + +echo "---" +[[ $FAIL -eq 0 ]] && echo "All checks passed" && exit 0 +echo "$FAIL check(s) failed" && exit 1