Lakehouse Admin
df5ec93dc3
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose - Airflow DAG scripts, Proxmox VM inventory, network docs - Ansible playbook, Gitea CI validate workflow - Backup and health-check scripts, cron documentation - Homepage DOCS tab with links to all documentation - Extended collect-fleet-config.sh and populate-repo.py
109 lines
3.2 KiB
Python
109 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to generate fake sales order data for PostgreSQL
|
|
Generates approximately 1GB of data
|
|
"""
|
|
|
|
import psycopg2
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import sys
|
|
|
|
# Database connection details
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "5432"
|
|
DB_NAME = "postgres"
|
|
DB_USER = "mo"
|
|
DB_PASSWORD = "Dell2026!"
|
|
|
|
# Data generation settings
|
|
TARGET_ROWS = 4000000 # Approximately 1GB of data
|
|
BATCH_SIZE = 10000
|
|
|
|
# Sample data
|
|
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
|
|
SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
|
|
CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
|
|
ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
|
|
|
|
def generate_fake_order():
|
|
"""Generate a single fake sales order"""
|
|
customer_id = random.randint(1, 100000)
|
|
product_id = random.randint(1, 5000)
|
|
region = random.choice(REGIONS)
|
|
sales_channel = random.choice(SALES_CHANNELS)
|
|
|
|
# Random timestamp within the last 2 years
|
|
days_ago = random.randint(0, 730)
|
|
order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
|
|
amount = round(random.uniform(10.0, 10000.0), 2)
|
|
currency = random.choice(CURRENCIES)
|
|
order_status = random.choice(ORDER_STATUSES)
|
|
|
|
# Generate a long notes field (like the existing data)
|
|
notes = str(uuid.uuid4()) * 10
|
|
|
|
return (customer_id, product_id, region, sales_channel, order_ts,
|
|
amount, currency, order_status, notes)
|
|
|
|
def main():
|
|
print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
|
|
|
|
conn = psycopg2.connect(
|
|
host=DB_HOST,
|
|
port=DB_PORT,
|
|
database=DB_NAME,
|
|
user=DB_USER,
|
|
password=DB_PASSWORD
|
|
)
|
|
cursor = conn.cursor()
|
|
|
|
print(f"Generating {TARGET_ROWS} sales orders...")
|
|
print(f"Batch size: {BATCH_SIZE}")
|
|
|
|
total_generated = 0
|
|
batch = []
|
|
|
|
for i in range(TARGET_ROWS):
|
|
batch.append(generate_fake_order())
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
cursor.executemany(
|
|
"""
|
|
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
|
|
order_ts, amount, currency, order_status, notes)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
batch = []
|
|
|
|
if total_generated % 100000 == 0:
|
|
print(f"Generated {total_generated} rows...")
|
|
|
|
# Insert remaining rows
|
|
if batch:
|
|
cursor.executemany(
|
|
"""
|
|
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
|
|
order_ts, amount, currency, order_status, notes)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
print(f"Completed! Generated {total_generated} sales orders.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|