Lakehouse Admin
df5ec93dc3
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose - Airflow DAG scripts, Proxmox VM inventory, network docs - Ansible playbook, Gitea CI validate workflow - Backup and health-check scripts, cron documentation - Homepage DOCS tab with links to all documentation - Extended collect-fleet-config.sh and populate-repo.py
107 lines
3.3 KiB
Python
107 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to generate fake employee event data for MySQL
|
|
Generates approximately 1GB of data
|
|
"""
|
|
|
|
import mysql.connector
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import sys
|
|
|
|
# Database connection details
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "3306"
|
|
DB_NAME = "hr"
|
|
DB_USER = "mo"
|
|
DB_PASSWORD = "Dell2026!"
|
|
|
|
# Data generation settings
|
|
TARGET_ROWS = 4000000 # Approximately 1GB of data
|
|
BATCH_SIZE = 10000
|
|
|
|
# Sample data
|
|
DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"]
|
|
ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"]
|
|
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
|
|
EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"]
|
|
|
|
def generate_fake_employee_event():
|
|
"""Generate a single fake employee event"""
|
|
employee_id = random.randint(1, 100000)
|
|
department = random.choice(DEPARTMENTS)
|
|
role_name = random.choice(ROLE_NAMES)
|
|
region = random.choice(REGIONS)
|
|
event_type = random.choice(EVENT_TYPES)
|
|
|
|
# Random timestamp within the last 2 years
|
|
days_ago = random.randint(0, 730)
|
|
event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
|
|
salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None
|
|
|
|
# Generate a long notes field (like the existing data)
|
|
notes = str(uuid.uuid4()) * 10
|
|
|
|
return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes)
|
|
|
|
def main():
|
|
print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...")
|
|
|
|
conn = mysql.connector.connect(
|
|
host=DB_HOST,
|
|
port=DB_PORT,
|
|
database=DB_NAME,
|
|
user=DB_USER,
|
|
password=DB_PASSWORD
|
|
)
|
|
cursor = conn.cursor()
|
|
|
|
print(f"Generating {TARGET_ROWS} employee events...")
|
|
print(f"Batch size: {BATCH_SIZE}")
|
|
|
|
total_generated = 0
|
|
batch = []
|
|
|
|
for i in range(TARGET_ROWS):
|
|
batch.append(generate_fake_employee_event())
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
cursor.executemany(
|
|
"""
|
|
INSERT INTO employee_events (employee_id, department, role_name, region,
|
|
event_type, salary_change, event_ts, notes)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
batch = []
|
|
|
|
if total_generated % 100000 == 0:
|
|
print(f"Generated {total_generated} rows...")
|
|
|
|
# Insert remaining rows
|
|
if batch:
|
|
cursor.executemany(
|
|
"""
|
|
INSERT INTO employee_events (employee_id, department, role_name, region,
|
|
event_type, salary_change, event_ts, notes)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
print(f"Completed! Generated {total_generated} employee events.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|