ATC/Lakehouse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: Lakehouse configuration files

Browse Source
This commit is contained in:
Lakehouse Admin
2026-05-19 13:03:34 +02:00
commit 26606fb05c
14 changed files with 844 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/homepage/custom.css
+52
View File
@@ -0,0 +1,52 @@
/* Palantir-style Dark Theme with Colorful Accents */
:root {
--background-color: #0a0e1a;
--card-background: #111827;
--card-hover: #1f2937;
--text-primary: #f9fafb;
--text-secondary: #9ca3af;
--accent-blue: #3b82f6;
--accent-purple: #8b5cf6;
--accent-pink: #ec4899;
--accent-green: #10b981;
--accent-orange: #f59e0b;
--accent-cyan: #06b6d4;
}
body {
background-color: var(--background-color);
color: var(--text-primary);
}
.service-card {
background: linear-gradient(135deg, var(--card-background) 0%, #1a2332 100%);
border: 1px solid rgba(59, 130, 246, 0.2);
transition: all 0.3s ease;
}
.service-card:hover {
background: linear-gradient(135deg, var(--card-hover) 0%, #2d3748 100%);
border-color: var(--accent-blue);
transform: translateY(-2px);
box-shadow: 0 10px 40px rgba(59, 130, 246, 0.3);
}
.group-header {
background: linear-gradient(90deg, var(--accent-blue), var(--accent-purple));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
font-weight: bold;
text-transform: uppercase;
letter-spacing: 1px;
}
/* Colorful icons based on service type */
.icon-kafka { filter: drop-shadow(0 0 8px var(--accent-blue)); }
.icon-spark { filter: drop-shadow(0 0 8px var(--accent-orange)); }
.icon-trino { filter: drop-shadow(0 0 8px var(--accent-cyan)); }
.icon-airflow { filter: drop-shadow(0 0 8px var(--accent-green)); }
.icon-elasticsearch { filter: drop-shadow(0 0 8px var(--accent-purple)); }
.icon-kibana { filter: drop-shadow(0 0 8px var(--accent-pink)); }
.icon-grafana { filter: drop-shadow(0 0 8px var(--accent-orange)); }
.icon-proxmox { filter: drop-shadow(0 0 8px var(--accent-blue)); }
.icon-git { filter: drop-shadow(0 0 8px var(--accent-green)); }
config/homepage/proxmox.yaml
+5
View File
@@ -0,0 +1,5 @@
---
pve:
url: https://10.0.10.65:8006
username: root@pam
password: Dell2026!
config/homepage/services.yaml
+97
View File
@@ -0,0 +1,97 @@
---
# Palantir-style Data Pipeline Dashboard
# Organized by system type with colorful layout
- Data Pipeline:
- Kafka UI:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kafka.svg
href: http://10.0.21.36:9000/
description: Kafka Topics & Management
- Debezium Connect:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/debezium.svg
href: http://10.0.21.50:8083/
description: CDC Connectors
- Spark Master:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-spark.svg
href: http://10.0.21.50:8080/
description: Spark Jobs & Monitoring
- Trino:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/trino.svg
href: http://10.0.21.50:8089/
description: Distributed SQL Query Engine
- Source Systems:
- MongoDB:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mongodb.svg
href: mongodb://10.0.21.51:27017
description: MongoDB Supply Chain Data
- PostgreSQL:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/postgresql.svg
href: postgresql://10.0.21.51:5432
description: PostgreSQL Sales Data
- Cassandra:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-cassandra.svg
href: cassandra://10.0.21.51:9042
description: Cassandra Telemetry Data
- MySQL:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/mysql.svg
href: mysql://10.0.21.51:3306
description: MySQL HR Data
- Neo4j:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/neo4j.svg
href: http://10.0.21.51:7474/
description: Neo4j Graph Database
- Storage:
- ObjectScale S3:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/minio.svg
href: http://10.0.20.111:9020/
description: S3 Object Storage
- ObjectScale Web:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/object-storage.svg
href: https://10.0.20.111/
description: ObjectScale Management UI
- Orchestration:
- Airflow:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-airflow.svg
href: http://10.0.21.55:8080/
description: Workflow Orchestration
- Analytics & Visualization:
- Superset:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/apache-superset.svg
href: http://10.0.21.45:8088/
description: Business Intelligence & Dashboards
- Elasticsearch:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/elasticsearch.svg
href: http://10.0.21.46:9200/
description: Search & Analytics Engine
- Kibana:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/kibana.svg
href: http://10.0.21.46:5601/
description: Data Visualization
- Grafana:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/grafana.svg
href: http://10.0.20.103/
description: Metrics & Monitoring
- Management:
- Proxmox:
icon: proxmox.png
href: http://10.0.10.65:8006/
description: Virtualization Platform
- Git/Forgejo:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/git.svg
href: http://atc-mgt01.dell-atc.lan:3001/
description: Git Repository
- ATC LDAP:
icon: /icons/lam.png
href: http://atc-mgt01.dell-atc.lan/lam/
description: LDAP Management
- Development:
- Homepage Config:
icon: https://raw.githubusercontent.com/walkxcode/dashboard-icons/main/svg/homepage.svg
href: http://atc-docker01.dell-atc.lan/
description: Dashboard Configuration
config/kafka/mongodb-connector.json
+1
View File
@@ -0,0 +1 @@
{"connector.class":"io.debezium.connector.mongodb.MongoDbConnector","topic.prefix":"mongodb-supplychain","mongodb.history.kafka.bootstrap.servers":"localhost:9092","mongodb.history.kafka.topic":"schema-changes.supplychain","mongodb.connection.string":"mongodb://10.0.21.51:27017","name":"mongodb-connector","mongodb.name":"supplychain","snapshot.mode":"initial"}
config/kafka/mysql-connector.json
+1
View File
@@ -0,0 +1 @@
{"connector.class":"io.debezium.connector.mysql.MySqlConnector","database.user":"mo","topic.prefix":"mysql-hr","schema.history.internal.kafka.topic":"schema-changes.hr","database.server.id":"184054","database.hostname":"10.0.21.51","database.password":"PASSWORD_PLACEHOLDER","name":"mysql-connector","schema.history.internal.kafka.bootstrap.servers":"localhost:9092","database.port":"3306","database.include.list":"hr","snapshot.mode":"schema_only"}
config/kafka/postgres-connector.json
+1
View File
@@ -0,0 +1 @@
{"connector.class":"io.debezium.connector.postgresql.PostgresConnector","database.user":"mo","database.dbname":"postgres","topic.prefix":"postgres-sales","database.hostname":"10.0.21.51","database.password":"PASSWORD_PLACEHOLDER","database.history.kafka.bootstrap.servers":"localhost:9092","database.history.kafka.topic":"schema-changes.sales","name":"postgres-connector","table.include.list":"public.sales_orders","database.port":"5432","plugin.name":"pgoutput"}
config/scripts/data-generation/generate_cassandra_telemetry_data.py
Executable
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""
Script to generate fake telemetry data for Cassandra
Generates approximately 1GB of data
"""
from cassandra.cluster import Cluster
import random
from datetime import datetime, timedelta
import uuid
import sys
# Database connection details
DB_HOST = "10.0.21.51"
DB_PORT = "9042"
KEYSPACE = "telemetry"
TABLE_NAME = "device_metrics"
# Data generation settings
TARGET_ROWS = 3000000 # Approximately 1GB of data
BATCH_SIZE = 5000
# Sample data
METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"]
DEVICE_PREFIX = "device-"
def generate_fake_device_metric():
"""Generate a single fake device metric"""
device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}"
# Random timestamp within the last year
days_ago = random.randint(0, 365)
metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
metric_type = random.choice(METRIC_TYPES)
metric_value = round(random.uniform(0.0, 100.0), 4)
# Generate a long payload field
payload = "X" * 200
return (device_id, metric_ts, metric_type, metric_value, payload)
def main():
print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...")
cluster = Cluster([DB_HOST], port=DB_PORT)
session = cluster.connect()
print(f"Generating {TARGET_ROWS} device metrics...")
print(f"Batch size: {BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_ROWS):
batch.append(generate_fake_device_metric())
if len(batch) >= BATCH_SIZE:
session.execute(
f"""
INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
VALUES (%s, %s, %s, %s, %s)
""",
batch
)
total_generated += len(batch)
batch = []
if total_generated % 100000 == 0:
print(f"Generated {total_generated} rows...")
# Insert remaining rows
if batch:
session.execute(
f"""
INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
VALUES (%s, %s, %s, %s, %s)
""",
batch
)
total_generated += len(batch)
session.shutdown()
cluster.shutdown()
print(f"Completed! Generated {total_generated} device metrics.")
if __name__ == "__main__":
main()
config/scripts/data-generation/generate_mongodb_events_data.py
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""
Script to generate fake event data for MongoDB
Generates approximately 1GB of data
"""
import pymongo
import random
from datetime import datetime, timedelta
import uuid
import sys
# Database connection details
DB_HOST = "10.0.21.51"
DB_PORT = "27017"
DB_NAME = "supplychain"
COLLECTION_NAME = "events"
# Data generation settings
TARGET_DOCUMENTS = 3000000 # Approximately 1GB of data
BATCH_SIZE = 5000
# Sample data
EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
def generate_fake_event():
"""Generate a single fake event"""
event_id = uuid.uuid4()
event_type = random.choice(EVENT_TYPES)
region = random.choice(REGIONS)
source = random.choice(SOURCES)
# Random timestamp within the last year
days_ago = random.randint(0, 365)
ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
amount = random.uniform(100.0, 50000.0)
# Generate a long payload field (like the existing data)
payload = "X" * 500
return {
"event_id": event_id,
"type": event_type,
"region": region,
"source": source,
"amount": amount,
"ts": ts,
"payload": payload
}
def main():
print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
db = client[DB_NAME]
collection = db[COLLECTION_NAME]
print(f"Generating {TARGET_DOCUMENTS} events...")
print(f"Batch size: {BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_DOCUMENTS):
batch.append(generate_fake_event())
if len(batch) >= BATCH_SIZE:
collection.insert_many(batch)
total_generated += len(batch)
batch = []
if total_generated % 100000 == 0:
print(f"Generated {total_generated} documents...")
# Insert remaining documents
if batch:
collection.insert_many(batch)
total_generated += len(batch)
client.close()
print(f"Completed! Generated {total_generated} events.")
if __name__ == "__main__":
main()
config/scripts/data-generation/generate_mysql_employee_data.py
Executable
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""
Script to generate fake employee event data for MySQL
Generates approximately 1GB of data
"""
import mysql.connector
import random
from datetime import datetime, timedelta
import uuid
import sys
# Database connection details
DB_HOST = "10.0.21.51"
DB_PORT = "3306"
DB_NAME = "hr"
DB_USER = "mo"
DB_PASSWORD = "Dell2026!"
# Data generation settings
TARGET_ROWS = 4000000 # Approximately 1GB of data
BATCH_SIZE = 10000
# Sample data
DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"]
ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"]
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"]
def generate_fake_employee_event():
"""Generate a single fake employee event"""
employee_id = random.randint(1, 100000)
department = random.choice(DEPARTMENTS)
role_name = random.choice(ROLE_NAMES)
region = random.choice(REGIONS)
event_type = random.choice(EVENT_TYPES)
# Random timestamp within the last 2 years
days_ago = random.randint(0, 730)
event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None
# Generate a long notes field (like the existing data)
notes = str(uuid.uuid4()) * 10
return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
def main():
print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...")
conn = mysql.connector.connect(
host=DB_HOST,
port=DB_PORT,
database=DB_NAME,
user=DB_USER,
password=DB_PASSWORD
)
cursor = conn.cursor()
print(f"Generating {TARGET_ROWS} employee events...")
print(f"Batch size: {BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_ROWS):
batch.append(generate_fake_employee_event())
if len(batch) >= BATCH_SIZE:
cursor.executemany(
"""
INSERT INTO employee_events (employee_id, department, role_name, region,
event_type, salary_change, event_ts, notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""",
batch
)
conn.commit()
total_generated += len(batch)
batch = []
if total_generated % 100000 == 0:
print(f"Generated {total_generated} rows...")
# Insert remaining rows
if batch:
cursor.executemany(
"""
INSERT INTO employee_events (employee_id, department, role_name, region,
event_type, salary_change, event_ts, notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""",
batch
)
conn.commit()
total_generated += len(batch)
cursor.close()
conn.close()
print(f"Completed! Generated {total_generated} employee events.")
if __name__ == "__main__":
main()
config/scripts/data-generation/generate_neo4j_graph_data.py
Executable
+227
View File
@@ -0,0 +1,227 @@
#!/usr/bin/env python3
"""
Script to generate fake graph data for Neo4j
Generates approximately 1GB of data with nodes and relationships
"""
from neo4j import GraphDatabase
import random
import uuid
import sys
# Database connection details
DB_HOST = "10.0.21.51"
DB_PORT = "7687"
DB_USER = "neo4j"
DB_PASSWORD = "testpwd"
# Data generation settings
TARGET_NODES = 500000 # Approximately 1GB of data with relationships
BATCH_SIZE = 1000
# Sample data
PRODUCT_CATEGORIES = ["Electronics", "Clothing", "Food", "Furniture", "Toys", "Books"]
SUPPLIER_REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
RELATIONSHIP_TYPES = ["SUPPLIES", "RELATED_TO", "COMPATIBLE_WITH", "PART_OF"]
def generate_fake_product():
"""Generate a single fake product node"""
product_id = str(uuid.uuid4())
name = f"Product-{random.randint(1000, 999999)}"
category = random.choice(PRODUCT_CATEGORIES)
price = round(random.uniform(10.0, 1000.0), 2)
stock = random.randint(0, 1000)
# Generate a long description field
description = "X" * 200
return {
"product_id": product_id,
"name": name,
"category": category,
"price": price,
"stock": stock,
"description": description
}
def generate_fake_supplier():
"""Generate a single fake supplier node"""
supplier_id = str(uuid.uuid4())
name = f"Supplier-{random.randint(1000, 999999)}"
region = random.choice(SUPPLIER_REGIONS)
rating = round(random.uniform(1.0, 5.0), 1)
# Generate a long address field
address = "X" * 150
return {
"supplier_id": supplier_id,
"name": name,
"region": region,
"rating": rating,
"address": address
}
def main():
print(f"Connecting to Neo4j at {DB_HOST}:{DB_PORT}...")
driver = GraphDatabase.driver(f"bolt://{DB_HOST}:{DB_PORT}",
auth=(DB_USER, DB_PASSWORD))
with driver.session() as session:
print(f"Generating {TARGET_NODES} product nodes...")
print(f"Batch size: {BATCH_SIZE}")
total_products = 0
total_suppliers = 0
product_ids = []
# Generate product nodes
batch = []
for i in range(TARGET_NODES):
product = generate_fake_product()
batch.append(product)
product_ids.append(product["product_id"])
if len(batch) >= BATCH_SIZE:
session.run(
"""
UNWIND $batch as row
CREATE (p:Product {
product_id: row.product_id,
name: row.name,
category: row.category,
price: row.price,
stock: row.stock,
description: row.description
})
""",
batch=batch
)
total_products += len(batch)
batch = []
if total_products % 50000 == 0:
print(f"Generated {total_products} product nodes...")
# Insert remaining products
if batch:
session.run(
"""
UNWIND $batch as row
CREATE (p:Product {
product_id: row.product_id,
name: row.name,
category: row.category,
price: row.price,
stock: row.stock,
description: row.description
})
""",
batch=batch
)
total_products += len(batch)
print(f"Generated {total_products} product nodes.")
# Generate supplier nodes (fewer than products)
print(f"Generating supplier nodes...")
target_suppliers = 10000
batch = []
supplier_ids = []
for i in range(target_suppliers):
supplier = generate_fake_supplier()
batch.append(supplier)
supplier_ids.append(supplier["supplier_id"])
if len(batch) >= BATCH_SIZE:
session.run(
"""
UNWIND $batch as row
CREATE (s:Supplier {
supplier_id: row.supplier_id,
name: row.name,
region: row.region,
rating: row.rating,
address: row.address
})
""",
batch=batch
)
total_suppliers += len(batch)
batch = []
if batch:
session.run(
"""
UNWIND $batch as row
CREATE (s:Supplier {
supplier_id: row.supplier_id,
name: row.name,
region: row.region,
rating: row.rating,
address: row.address
})
""",
batch=batch
)
total_suppliers += len(batch)
print(f"Generated {total_suppliers} supplier nodes.")
# Create relationships between products and suppliers
print(f"Creating relationships...")
batch = []
total_relationships = 0
for product_id in product_ids:
# Each product is supplied by 1-3 random suppliers
num_suppliers = random.randint(1, 3)
for _ in range(num_suppliers):
supplier_id = random.choice(supplier_ids)
rel_type = random.choice(RELATIONSHIP_TYPES)
batch.append({
"product_id": product_id,
"supplier_id": supplier_id,
"rel_type": rel_type
})
if len(batch) >= BATCH_SIZE:
session.run(
"""
UNWIND $batch as row
MATCH (p:Product {product_id: row.product_id})
MATCH (s:Supplier {supplier_id: row.supplier_id})
CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
RETURN rel
""",
batch=batch
)
total_relationships += len(batch)
batch = []
if total_relationships % 50000 == 0:
print(f"Created {total_relationships} relationships...")
if batch:
session.run(
"""
UNWIND $batch as row
MATCH (p:Product {product_id: row.product_id})
MATCH (s:Supplier {supplier_id: row.supplier_id})
CALL apoc.create.relationship(p, row.rel_type, {}, s) YIELD rel
RETURN rel
""",
batch=batch
)
total_relationships += len(batch)
print(f"Created {total_relationships} relationships.")
driver.close()
print(f"Completed! Generated {total_products} products, {total_suppliers} suppliers, and {total_relationships} relationships.")
if __name__ == "__main__":
main()
config/scripts/data-generation/generate_postgres_sales_data.py
Executable
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
"""
Script to generate fake sales order data for PostgreSQL
Generates approximately 1GB of data
"""
import psycopg2
import random
from datetime import datetime, timedelta
import uuid
import sys
# Database connection details
DB_HOST = "10.0.21.51"
DB_PORT = "5432"
DB_NAME = "postgres"
DB_USER = "mo"
DB_PASSWORD = "Dell2026!"
# Data generation settings
TARGET_ROWS = 4000000 # Approximately 1GB of data
BATCH_SIZE = 10000
# Sample data
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
def generate_fake_order():
"""Generate a single fake sales order"""
customer_id = random.randint(1, 100000)
product_id = random.randint(1, 5000)
region = random.choice(REGIONS)
sales_channel = random.choice(SALES_CHANNELS)
# Random timestamp within the last 2 years
days_ago = random.randint(0, 730)
order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
amount = round(random.uniform(10.0, 10000.0), 2)
currency = random.choice(CURRENCIES)
order_status = random.choice(ORDER_STATUSES)
# Generate a long notes field (like the existing data)
notes = str(uuid.uuid4()) * 10
return (customer_id, product_id, region, sales_channel, order_ts,
amount, currency, order_status, notes)
def main():
print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
conn = psycopg2.connect(
host=DB_HOST,
port=DB_PORT,
database=DB_NAME,
user=DB_USER,
password=DB_PASSWORD
)
cursor = conn.cursor()
print(f"Generating {TARGET_ROWS} sales orders...")
print(f"Batch size: {BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_ROWS):
batch.append(generate_fake_order())
if len(batch) >= BATCH_SIZE:
cursor.executemany(
"""
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
order_ts, amount, currency, order_status, notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
batch
)
conn.commit()
total_generated += len(batch)
batch = []
if total_generated % 100000 == 0:
print(f"Generated {total_generated} rows...")
# Insert remaining rows
if batch:
cursor.executemany(
"""
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
order_ts, amount, currency, order_status, notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
batch
)
conn.commit()
total_generated += len(batch)
cursor.close()
conn.close()
print(f"Completed! Generated {total_generated} sales orders.")
if __name__ == "__main__":
main()
config/superset/Dockerfile
+3
View File
@@ -0,0 +1,3 @@
FROM apache/superset:latest
RUN pip install psycopg2-binary
config/superset/docker-compose.yaml
+25
View File
@@ -0,0 +1,25 @@
services:
superset:
build: .
container_name: superset
restart: unless-stopped
ports:
- "8088:8088"
environment:
- SUPERSET_SECRET_KEY=your-secret-key-here
- SUPERSET_LOAD_EXAMPLES=no
volumes:
- ./superset_config.py:/app/pythonpath/superset_config.py
- superset_home:/app/superset_home
depends_on:
- redis
redis:
image: redis:7
container_name: superset_redis
restart: unless-stopped
ports:
- "6379:6379"
volumes:
superset_home:
config/superset/superset_config.py
+39
View File
@@ -0,0 +1,39 @@
import os
# Secret key for session signing
SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY', 'your-secret-key-here')
# Database configuration - use SQLite to avoid psycopg2 issues
SQLALCHEMY_DATABASE_URI = 'sqlite:////app/superset_home/superset.db'
# Redis cache configuration
CACHE_CONFIG = {
'CACHE_TYPE': 'redis',
'CACHE_REDIS_URL': 'redis://redis:6379/0',
'CACHE_DEFAULT_TIMEOUT': 300
}
# Enable CSRF protection
ENABLE_PROXY_FIX = True
# Feature flags
FEATURE_FLAGS = {
'ENABLE_TEMPLATE_PROCESSING': True,
'ALERT_REPORTS': True,
}
# Row limit
ROW_LIMIT = 50000
# Viz types
VIZ_TYPE_DICT = {
'table': {},
'dist_bar': {},
'line': {},
'area': {},
'pie': {},
'number': {},
}
# Timezone
TIMEZONE = 'Europe/Amsterdam'