From 26606fb05c16c7c58db7608b75acf69e0e01e0e7 Mon Sep 17 00:00:00 2001 From: Lakehouse Admin Date: Tue, 19 May 2026 13:03:34 +0200 Subject: [PATCH] Initial commit: Lakehouse configuration files --- config/homepage/custom.css | 52 ++++ config/homepage/proxmox.yaml | 5 + config/homepage/services.yaml | 97 ++++++++ config/kafka/mongodb-connector.json | 1 + config/kafka/mysql-connector.json | 1 + config/kafka/postgres-connector.json | 1 + .../generate_cassandra_telemetry_data.py | 90 +++++++ .../generate_mongodb_events_data.py | 89 +++++++ .../generate_mysql_employee_data.py | 106 ++++++++ .../generate_neo4j_graph_data.py | 227 ++++++++++++++++++ .../generate_postgres_sales_data.py | 108 +++++++++ config/superset/Dockerfile | 3 + config/superset/docker-compose.yaml | 25 ++ config/superset/superset_config.py | 39 +++ 14 files changed, 844 insertions(+) create mode 100644 config/homepage/custom.css create mode 100644 config/homepage/proxmox.yaml create mode 100644 config/homepage/services.yaml create mode 100644 config/kafka/mongodb-connector.json create mode 100644 config/kafka/mysql-connector.json create mode 100644 config/kafka/postgres-connector.json create mode 100755 config/scripts/data-generation/generate_cassandra_telemetry_data.py create mode 100755 config/scripts/data-generation/generate_mongodb_events_data.py create mode 100755 config/scripts/data-generation/generate_mysql_employee_data.py create mode 100755 config/scripts/data-generation/generate_neo4j_graph_data.py create mode 100755 config/scripts/data-generation/generate_postgres_sales_data.py create mode 100644 config/superset/Dockerfile create mode 100644 config/superset/docker-compose.yaml create mode 100644 config/superset/superset_config.py diff --git a/config/homepage/custom.css b/config/homepage/custom.css new file mode 100644 index 0000000..29e8350 --- /dev/null +++ b/config/homepage/custom.css @@ -0,0 +1,52 @@ +/* Palantir-style Dark Theme with Colorful Accents */ +:root { + --background-color: #0a0e1a; + --card-background: #111827; + --card-hover: #1f2937; + --text-primary: #f9fafb; + --text-secondary: #9ca3af; + --accent-blue: #3b82f6; + --accent-purple: #8b5cf6; + --accent-pink: #ec4899; + --accent-green: #10b981; + --accent-orange: #f59e0b; + --accent-cyan: #06b6d4; +} + +body { + background-color: var(--background-color); + color: var(--text-primary); +} + +.service-card { + background: linear-gradient(135deg, var(--card-background) 0%, #1a2332 100%); + border: 1px solid rgba(59, 130, 246, 0.2); + transition: all 0.3s ease; +} + +.service-card:hover { + background: linear-gradient(135deg, var(--card-hover) 0%, #2d3748 100%); + border-color: var(--accent-blue); + transform: translateY(-2px); + box-shadow: 0 10px 40px rgba(59, 130, 246, 0.3); +} + +.group-header { + background: linear-gradient(90deg, var(--accent-blue), var(--accent-purple)); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + font-weight: bold; + text-transform: uppercase; + letter-spacing: 1px; +} + +/* Colorful icons based on service type */ +.icon-kafka { filter: drop-shadow(0 0 8px var(--accent-blue)); } +.icon-spark { filter: drop-shadow(0 0 8px var(--accent-orange)); } +.icon-trino { filter: drop-shadow(0 0 8px var(--accent-cyan)); } +.icon-airflow { filter: drop-shadow(0 0 8px var(--accent-green)); } +.icon-elasticsearch { filter: drop-shadow(0 0 8px var(--accent-purple)); } +.icon-kibana { filter: drop-shadow(0 0 8px var(--accent-pink)); } +.icon-grafana { filter: drop-shadow(0 0 8px var(--accent-orange)); } +.icon-proxmox { filter: drop-shadow(0 0 8px var(--accent-blue)); } +.icon-git { filter: drop-shadow(0 0 8px var(--accent-green)); } diff --git a/config/homepage/proxmox.yaml b/config/homepage/proxmox.yaml new file mode 100644 index 0000000..653a4f7 --- /dev/null +++ b/config/homepage/proxmox.yaml @@ -0,0 +1,5 @@ +--- +pve: + url: https://10.0.10.65:8006 + username: root@pam + password: Dell2026! diff --git a/config/homepage/services.yaml b/config/homepage/services.yaml new file mode 100644 index 0000000..78f0812 --- /dev/null +++ b/config/homepage/services.yaml @@ -0,0 +1,97 @@ +--- +# Palantir-style Data Pipeline Dashboard +# Organized by system type with colorful layout + +- Data Pipeline: + - Kafka UI: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kafka.svg + href: http://10.0.21.36:9000/ + description: Kafka Topics & Management + - Debezium Connect: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/debezium.svg + href: http://10.0.21.50:8083/ + description: CDC Connectors + - Spark Master: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-spark.svg + href: http://10.0.21.50:8080/ + description: Spark Jobs & Monitoring + - Trino: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/trino.svg + href: http://10.0.21.50:8089/ + description: Distributed SQL Query Engine + +- Source Systems: + - MongoDB: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mongodb.svg + href: mongodb://10.0.21.51:27017 + description: MongoDB Supply Chain Data + - PostgreSQL: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/postgresql.svg + href: postgresql://10.0.21.51:5432 + description: PostgreSQL Sales Data + - Cassandra: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-cassandra.svg + href: cassandra://10.0.21.51:9042 + description: Cassandra Telemetry Data + - MySQL: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mysql.svg + href: mysql://10.0.21.51:3306 + description: MySQL HR Data + - Neo4j: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/neo4j.svg + href: http://10.0.21.51:7474/ + description: Neo4j Graph Database + +- Storage: + - ObjectScale S3: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/minio.svg + href: http://10.0.20.111:9020/ + description: S3 Object Storage + - ObjectScale Web: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/object-storage.svg + href: https://10.0.20.111/ + description: ObjectScale Management UI + +- Orchestration: + - Airflow: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-airflow.svg + href: http://10.0.21.55:8080/ + description: Workflow Orchestration + +- Analytics & Visualization: + - Superset: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-superset.svg + href: http://10.0.21.45:8088/ + description: Business Intelligence & Dashboards + - Elasticsearch: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/elasticsearch.svg + href: http://10.0.21.46:9200/ + description: Search & Analytics Engine + - Kibana: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kibana.svg + href: http://10.0.21.46:5601/ + description: Data Visualization + - Grafana: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/grafana.svg + href: http://10.0.20.103/ + description: Metrics & Monitoring + +- Management: + - Proxmox: + icon: proxmox.png + href: http://10.0.10.65:8006/ + description: Virtualization Platform + - Git/Forgejo: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/git.svg + href: http://atc-mgt01.dell-atc.lan:3001/ + description: Git Repository + - ATC LDAP: + icon: /icons/lam.png + href: http://atc-mgt01.dell-atc.lan/lam/ + description: LDAP Management + +- Development: + - Homepage Config: + icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/homepage.svg + href: http://atc-docker01.dell-atc.lan/ + description: Dashboard Configuration diff --git a/config/kafka/mongodb-connector.json b/config/kafka/mongodb-connector.json new file mode 100644 index 0000000..5bba347 --- /dev/null +++ b/config/kafka/mongodb-connector.json @@ -0,0 +1 @@ +{"connector.class":"io.debezium.connector.mongodb.MongoDbConnector","topic.prefix":"mongodb-supplychain","mongodb.history.kafka.bootstrap.servers":"localhost:9092","mongodb.history.kafka.topic":"schema-changes.supplychain","mongodb.connection.string":"mongodb://10.0.21.51:27017","name":"mongodb-connector","mongodb.name":"supplychain","snapshot.mode":"initial"} \ No newline at end of file diff --git a/config/kafka/mysql-connector.json b/config/kafka/mysql-connector.json new file mode 100644 index 0000000..32f3e5a --- /dev/null +++ b/config/kafka/mysql-connector.json @@ -0,0 +1 @@ +{"connector.class":"io.debezium.connector.mysql.MySqlConnector","database.user":"mo","topic.prefix":"mysql-hr","schema.history.internal.kafka.topic":"schema-changes.hr","database.server.id":"184054","database.hostname":"10.0.21.51","database.password":"PASSWORD_PLACEHOLDER","name":"mysql-connector","schema.history.internal.kafka.bootstrap.servers":"localhost:9092","database.port":"3306","database.include.list":"hr","snapshot.mode":"schema_only"} \ No newline at end of file diff --git a/config/kafka/postgres-connector.json b/config/kafka/postgres-connector.json new file mode 100644 index 0000000..b65986d --- /dev/null +++ b/config/kafka/postgres-connector.json @@ -0,0 +1 @@ +{"connector.class":"io.debezium.connector.postgresql.PostgresConnector","database.user":"mo","database.dbname":"postgres","topic.prefix":"postgres-sales","database.hostname":"10.0.21.51","database.password":"PASSWORD_PLACEHOLDER","database.history.kafka.bootstrap.servers":"localhost:9092","database.history.kafka.topic":"schema-changes.sales","name":"postgres-connector","table.include.list":"public.sales_orders","database.port":"5432","plugin.name":"pgoutput"} \ No newline at end of file diff --git a/config/scripts/data-generation/generate_cassandra_telemetry_data.py b/config/scripts/data-generation/generate_cassandra_telemetry_data.py new file mode 100755 index 0000000..58f4129 --- /dev/null +++ b/config/scripts/data-generation/generate_cassandra_telemetry_data.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Script to generate fake telemetry data for Cassandra +Generates approximately 1GB of data +""" + +from cassandra.cluster import Cluster +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "9042" +KEYSPACE = "telemetry" +TABLE_NAME = "device_metrics" + +# Data generation settings +TARGET_ROWS = 3000000 # Approximately 1GB of data +BATCH_SIZE = 5000 + +# Sample data +METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"] +DEVICE_PREFIX = "device-" + +def generate_fake_device_metric(): + """Generate a single fake device metric""" + device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}" + + # Random timestamp within the last year + days_ago = random.randint(0, 365) + metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + metric_type = random.choice(METRIC_TYPES) + metric_value = round(random.uniform(0.0, 100.0), 4) + + # Generate a long payload field + payload = "X" * 200 + + return (device_id, metric_ts, metric_type, metric_value, payload) + +def main(): + print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...") + + cluster = Cluster([DB_HOST], port=DB_PORT) + session = cluster.connect() + + print(f"Generating {TARGET_ROWS} device metrics...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_device_metric()) + + if len(batch) >= BATCH_SIZE: + session.execute( + f""" + INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload) + VALUES (%s, %s, %s, %s, %s) + """, + batch + ) + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + session.execute( + f""" + INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload) + VALUES (%s, %s, %s, %s, %s) + """, + batch + ) + total_generated += len(batch) + + session.shutdown() + cluster.shutdown() + + print(f"Completed! Generated {total_generated} device metrics.") + +if __name__ == "__main__": + main() diff --git a/config/scripts/data-generation/generate_mongodb_events_data.py b/config/scripts/data-generation/generate_mongodb_events_data.py new file mode 100755 index 0000000..b37c6b0 --- /dev/null +++ b/config/scripts/data-generation/generate_mongodb_events_data.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Script to generate fake event data for MongoDB +Generates approximately 1GB of data +""" + +import pymongo +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "27017" +DB_NAME = "supplychain" +COLLECTION_NAME = "events" + +# Data generation settings +TARGET_DOCUMENTS = 3000000 # Approximately 1GB of data +BATCH_SIZE = 5000 + +# Sample data +EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"] +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"] + +def generate_fake_event(): + """Generate a single fake event""" + event_id = uuid.uuid4() + event_type = random.choice(EVENT_TYPES) + region = random.choice(REGIONS) + source = random.choice(SOURCES) + + # Random timestamp within the last year + days_ago = random.randint(0, 365) + ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + amount = random.uniform(100.0, 50000.0) + + # Generate a long payload field (like the existing data) + payload = "X" * 500 + + return { + "event_id": event_id, + "type": event_type, + "region": region, + "source": source, + "amount": amount, + "ts": ts, + "payload": payload + } + +def main(): + print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...") + + client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/") + db = client[DB_NAME] + collection = db[COLLECTION_NAME] + + print(f"Generating {TARGET_DOCUMENTS} events...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_DOCUMENTS): + batch.append(generate_fake_event()) + + if len(batch) >= BATCH_SIZE: + collection.insert_many(batch) + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} documents...") + + # Insert remaining documents + if batch: + collection.insert_many(batch) + total_generated += len(batch) + + client.close() + + print(f"Completed! Generated {total_generated} events.") + +if __name__ == "__main__": + main() diff --git a/config/scripts/data-generation/generate_mysql_employee_data.py b/config/scripts/data-generation/generate_mysql_employee_data.py new file mode 100755 index 0000000..f7bbc45 --- /dev/null +++ b/config/scripts/data-generation/generate_mysql_employee_data.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Script to generate fake employee event data for MySQL +Generates approximately 1GB of data +""" + +import mysql.connector +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "3306" +DB_NAME = "hr" +DB_USER = "mo" +DB_PASSWORD = "Dell2026!" + +# Data generation settings +TARGET_ROWS = 4000000 # Approximately 1GB of data +BATCH_SIZE = 10000 + +# Sample data +DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"] +ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"] +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"] + +def generate_fake_employee_event(): + """Generate a single fake employee event""" + employee_id = random.randint(1, 100000) + department = random.choice(DEPARTMENTS) + role_name = random.choice(ROLE_NAMES) + region = random.choice(REGIONS) + event_type = random.choice(EVENT_TYPES) + + # Random timestamp within the last 2 years + days_ago = random.randint(0, 730) + event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None + + # Generate a long notes field (like the existing data) + notes = str(uuid.uuid4()) * 10 + + return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes) + +def main(): + print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...") + + conn = mysql.connector.connect( + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + user=DB_USER, + password=DB_PASSWORD + ) + cursor = conn.cursor() + + print(f"Generating {TARGET_ROWS} employee events...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_employee_event()) + + if len(batch) >= BATCH_SIZE: + cursor.executemany( + """ + INSERT INTO employee_events (employee_id, department, role_name, region, + event_type, salary_change, event_ts, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + cursor.executemany( + """ + INSERT INTO employee_events (employee_id, department, role_name, region, + event_type, salary_change, event_ts, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + + cursor.close() + conn.close() + + print(f"Completed! Generated {total_generated} employee events.") + +if __name__ == "__main__": + main() diff --git a/config/scripts/data-generation/generate_neo4j_graph_data.py b/config/scripts/data-generation/generate_neo4j_graph_data.py new file mode 100755 index 0000000..428a4c9 --- /dev/null +++ b/config/scripts/data-generation/generate_neo4j_graph_data.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +""" +Script to generate fake graph data for Neo4j +Generates approximately 1GB of data with nodes and relationships +""" + +from neo4j import GraphDatabase +import random +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "7687" +DB_USER = "neo4j" +DB_PASSWORD = "testpwd" + +# Data generation settings +TARGET_NODES = 500000 # Approximately 1GB of data with relationships +BATCH_SIZE = 1000 + +# Sample data +PRODUCT_CATEGORIES = ["Electronics", "Clothing", "Food", "Furniture", "Toys", "Books"] +SUPPLIER_REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +RELATIONSHIP_TYPES = ["SUPPLIES", "RELATED_TO", "COMPATIBLE_WITH", "PART_OF"] + +def generate_fake_product(): + """Generate a single fake product node""" + product_id = str(uuid.uuid4()) + name = f"Product-{random.randint(1000, 999999)}" + category = random.choice(PRODUCT_CATEGORIES) + price = round(random.uniform(10.0, 1000.0), 2) + stock = random.randint(0, 1000) + + # Generate a long description field + description = "X" * 200 + + return { + "product_id": product_id, + "name": name, + "category": category, + "price": price, + "stock": stock, + "description": description + } + +def generate_fake_supplier(): + """Generate a single fake supplier node""" + supplier_id = str(uuid.uuid4()) + name = f"Supplier-{random.randint(1000, 999999)}" + region = random.choice(SUPPLIER_REGIONS) + rating = round(random.uniform(1.0, 5.0), 1) + + # Generate a long address field + address = "X" * 150 + + return { + "supplier_id": supplier_id, + "name": name, + "region": region, + "rating": rating, + "address": address + } + +def main(): + print(f"Connecting to Neo4j at {DB_HOST}:{DB_PORT}...") + + driver = GraphDatabase.driver(f"bolt://{DB_HOST}:{DB_PORT}", + auth=(DB_USER, DB_PASSWORD)) + + with driver.session() as session: + print(f"Generating {TARGET_NODES} product nodes...") + print(f"Batch size: {BATCH_SIZE}") + + total_products = 0 + total_suppliers = 0 + product_ids = [] + + # Generate product nodes + batch = [] + for i in range(TARGET_NODES): + product = generate_fake_product() + batch.append(product) + product_ids.append(product["product_id"]) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + CREATE (p:Product { + product_id: row.product_id, + name: row.name, + category: row.category, + price: row.price, + stock: row.stock, + description: row.description + }) + """, + batch=batch + ) + total_products += len(batch) + batch = [] + + if total_products % 50000 == 0: + print(f"Generated {total_products} product nodes...") + + # Insert remaining products + if batch: + session.run( + """ + UNWIND $batch as row + CREATE (p:Product { + product_id: row.product_id, + name: row.name, + category: row.category, + price: row.price, + stock: row.stock, + description: row.description + }) + """, + batch=batch + ) + total_products += len(batch) + + print(f"Generated {total_products} product nodes.") + + # Generate supplier nodes (fewer than products) + print(f"Generating supplier nodes...") + target_suppliers = 10000 + batch = [] + supplier_ids = [] + + for i in range(target_suppliers): + supplier = generate_fake_supplier() + batch.append(supplier) + supplier_ids.append(supplier["supplier_id"]) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + CREATE (s:Supplier { + supplier_id: row.supplier_id, + name: row.name, + region: row.region, + rating: row.rating, + address: row.address + }) + """, + batch=batch + ) + total_suppliers += len(batch) + batch = [] + + if batch: + session.run( + """ + UNWIND $batch as row + CREATE (s:Supplier { + supplier_id: row.supplier_id, + name: row.name, + region: row.region, + rating: row.rating, + address: row.address + }) + """, + batch=batch + ) + total_suppliers += len(batch) + + print(f"Generated {total_suppliers} supplier nodes.") + + # Create relationships between products and suppliers + print(f"Creating relationships...") + batch = [] + total_relationships = 0 + + for product_id in product_ids: + # Each product is supplied by 1-3 random suppliers + num_suppliers = random.randint(1, 3) + for _ in range(num_suppliers): + supplier_id = random.choice(supplier_ids) + rel_type = random.choice(RELATIONSHIP_TYPES) + + batch.append({ + "product_id": product_id, + "supplier_id": supplier_id, + "rel_type": rel_type + }) + + if len(batch) >= BATCH_SIZE: + session.run( + """ + UNWIND $batch as row + MATCH (p:Product {product_id: row.product_id}) + MATCH (s:Supplier {supplier_id: row.supplier_id}) + CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel + RETURN rel + """, + batch=batch + ) + total_relationships += len(batch) + batch = [] + + if total_relationships % 50000 == 0: + print(f"Created {total_relationships} relationships...") + + if batch: + session.run( + """ + UNWIND $batch as row + MATCH (p:Product {product_id: row.product_id}) + MATCH (s:Supplier {supplier_id: row.supplier_id}) + CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel + RETURN rel + """, + batch=batch + ) + total_relationships += len(batch) + + print(f"Created {total_relationships} relationships.") + + driver.close() + print(f"Completed! Generated {total_products} products, {total_suppliers} suppliers, and {total_relationships} relationships.") + +if __name__ == "__main__": + main() diff --git a/config/scripts/data-generation/generate_postgres_sales_data.py b/config/scripts/data-generation/generate_postgres_sales_data.py new file mode 100755 index 0000000..2b48cb8 --- /dev/null +++ b/config/scripts/data-generation/generate_postgres_sales_data.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Script to generate fake sales order data for PostgreSQL +Generates approximately 1GB of data +""" + +import psycopg2 +import random +from datetime import datetime, timedelta +import uuid +import sys + +# Database connection details +DB_HOST = "10.0.21.51" +DB_PORT = "5432" +DB_NAME = "postgres" +DB_USER = "mo" +DB_PASSWORD = "Dell2026!" + +# Data generation settings +TARGET_ROWS = 4000000 # Approximately 1GB of data +BATCH_SIZE = 10000 + +# Sample data +REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"] +SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"] +CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"] +ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"] + +def generate_fake_order(): + """Generate a single fake sales order""" + customer_id = random.randint(1, 100000) + product_id = random.randint(1, 5000) + region = random.choice(REGIONS) + sales_channel = random.choice(SALES_CHANNELS) + + # Random timestamp within the last 2 years + days_ago = random.randint(0, 730) + order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), + minutes=random.randint(0, 59)) + + amount = round(random.uniform(10.0, 10000.0), 2) + currency = random.choice(CURRENCIES) + order_status = random.choice(ORDER_STATUSES) + + # Generate a long notes field (like the existing data) + notes = str(uuid.uuid4()) * 10 + + return (customer_id, product_id, region, sales_channel, order_ts, + amount, currency, order_status, notes) + +def main(): + print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...") + + conn = psycopg2.connect( + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + user=DB_USER, + password=DB_PASSWORD + ) + cursor = conn.cursor() + + print(f"Generating {TARGET_ROWS} sales orders...") + print(f"Batch size: {BATCH_SIZE}") + + total_generated = 0 + batch = [] + + for i in range(TARGET_ROWS): + batch.append(generate_fake_order()) + + if len(batch) >= BATCH_SIZE: + cursor.executemany( + """ + INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, + order_ts, amount, currency, order_status, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + batch = [] + + if total_generated % 100000 == 0: + print(f"Generated {total_generated} rows...") + + # Insert remaining rows + if batch: + cursor.executemany( + """ + INSERT INTO sales_orders (customer_id, product_id, region, sales_channel, + order_ts, amount, currency, order_status, notes) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + batch + ) + conn.commit() + total_generated += len(batch) + + cursor.close() + conn.close() + + print(f"Completed! Generated {total_generated} sales orders.") + +if __name__ == "__main__": + main() diff --git a/config/superset/Dockerfile b/config/superset/Dockerfile new file mode 100644 index 0000000..521c1a4 --- /dev/null +++ b/config/superset/Dockerfile @@ -0,0 +1,3 @@ +FROM apache/superset:latest + +RUN pip install psycopg2-binary diff --git a/config/superset/docker-compose.yaml b/config/superset/docker-compose.yaml new file mode 100644 index 0000000..db3878c --- /dev/null +++ b/config/superset/docker-compose.yaml @@ -0,0 +1,25 @@ +services: + superset: + build: . + container_name: superset + restart: unless-stopped + ports: + - "8088:8088" + environment: + - SUPERSET_SECRET_KEY=your-secret-key-here + - SUPERSET_LOAD_EXAMPLES=no + volumes: + - ./superset_config.py:/app/pythonpath/superset_config.py + - superset_home:/app/superset_home + depends_on: + - redis + + redis: + image: redis:7 + container_name: superset_redis + restart: unless-stopped + ports: + - "6379:6379" + +volumes: + superset_home: diff --git a/config/superset/superset_config.py b/config/superset/superset_config.py new file mode 100644 index 0000000..0ebd1d2 --- /dev/null +++ b/config/superset/superset_config.py @@ -0,0 +1,39 @@ +import os + +# Secret key for session signing +SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY', 'your-secret-key-here') + +# Database configuration - use SQLite to avoid psycopg2 issues +SQLALCHEMY_DATABASE_URI = 'sqlite:////app/superset_home/superset.db' + +# Redis cache configuration +CACHE_CONFIG = { + 'CACHE_TYPE': 'redis', + 'CACHE_REDIS_URL': 'redis://redis:6379/0', + 'CACHE_DEFAULT_TIMEOUT': 300 +} + +# Enable CSRF protection +ENABLE_PROXY_FIX = True + +# Feature flags +FEATURE_FLAGS = { + 'ENABLE_TEMPLATE_PROCESSING': True, + 'ALERT_REPORTS': True, +} + +# Row limit +ROW_LIMIT = 50000 + +# Viz types +VIZ_TYPE_DICT = { + 'table': {}, + 'dist_bar': {}, + 'line': {}, + 'area': {}, + 'pie': {}, + 'number': {}, +} + +# Timezone +TIMEZONE = 'Europe/Amsterdam'