mo
91 lines
2.4 KiB
Python
91 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to generate fake event data for MongoDB
|
|
Generates approximately 1GB of data
|
|
"""
|
|
|
|
import pymongo
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import sys
|
|
|
|
# Database connection details
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "27017"
|
|
DB_NAME = "supplychain"
|
|
COLLECTION_NAME = "events"
|
|
|
|
# Data generation settings
|
|
import os
|
|
TARGET_DOCUMENTS = int(os.getenv("GEN_ROWS", "5000")) # light, configurable via GEN_ROWS
|
|
BATCH_SIZE = min(5000, max(500, TARGET_DOCUMENTS))
|
|
|
|
# Sample data
|
|
EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
|
|
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
|
|
SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
|
|
|
|
def generate_fake_event():
|
|
"""Generate a single fake event"""
|
|
event_id = str(uuid.uuid4())
|
|
event_type = random.choice(EVENT_TYPES)
|
|
region = random.choice(REGIONS)
|
|
source = random.choice(SOURCES)
|
|
|
|
# Random timestamp within the last year
|
|
days_ago = random.randint(0, 365)
|
|
ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
|
|
amount = random.uniform(100.0, 50000.0)
|
|
|
|
# Generate a long payload field (like the existing data)
|
|
payload = "X" * 500
|
|
|
|
return {
|
|
"event_id": event_id,
|
|
"type": event_type,
|
|
"region": region,
|
|
"source": source,
|
|
"amount": amount,
|
|
"ts": ts,
|
|
"payload": payload
|
|
}
|
|
|
|
def main():
|
|
print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
|
|
|
|
client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
|
|
db = client[DB_NAME]
|
|
collection = db[COLLECTION_NAME]
|
|
|
|
print(f"Generating {TARGET_DOCUMENTS} events...")
|
|
print(f"Batch size: {BATCH_SIZE}")
|
|
|
|
total_generated = 0
|
|
batch = []
|
|
|
|
for i in range(TARGET_DOCUMENTS):
|
|
batch.append(generate_fake_event())
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
collection.insert_many(batch)
|
|
total_generated += len(batch)
|
|
batch = []
|
|
|
|
if total_generated % 100000 == 0:
|
|
print(f"Generated {total_generated} documents...")
|
|
|
|
# Insert remaining documents
|
|
if batch:
|
|
collection.insert_many(batch)
|
|
total_generated += len(batch)
|
|
|
|
client.close()
|
|
|
|
print(f"Completed! Generated {total_generated} events.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|