infra: fix mongodb uuid encoding + combined DAG honors rows conf

2026-06-26 08:41:44 +00:00
parent 897ccaa9f3
commit 9cbdccb403
3 changed files with 166 additions and 1 deletions
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake event data for MongoDB
+Generates approximately 1GB of data
+"""
+
+import pymongo
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "27017"
+DB_NAME = "supplychain"
+COLLECTION_NAME = "events"
+
+# Data generation settings
+import os
+TARGET_DOCUMENTS = int(os.getenv("GEN_ROWS", "5000"))  # light, configurable via GEN_ROWS
+BATCH_SIZE = min(5000, max(500, TARGET_DOCUMENTS))
+
+# Sample data
+EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
+REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
+SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
+
+def generate_fake_event():
+    """Generate a single fake event"""
+    event_id = str(uuid.uuid4())
+    event_type = random.choice(EVENT_TYPES)
+    region = random.choice(REGIONS)
+    source = random.choice(SOURCES)
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                     minutes=random.randint(0, 59))
+    
+    amount = random.uniform(100.0, 50000.0)
+    
+    # Generate a long payload field (like the existing data)
+    payload = "X" * 500
+    
+    return {
+        "event_id": event_id,
+        "type": event_type,
+        "region": region,
+        "source": source,
+        "amount": amount,
+        "ts": ts,
+        "payload": payload
+    }
+
+def main():
+    print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
+    
+    client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
+    db = client[DB_NAME]
+    collection = db[COLLECTION_NAME]
+    
+    print(f"Generating {TARGET_DOCUMENTS} events...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_DOCUMENTS):
+        batch.append(generate_fake_event())
+        
+        if len(batch) >= BATCH_SIZE:
+            collection.insert_many(batch)
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} documents...")
+    
+    # Insert remaining documents
+    if batch:
+        collection.insert_many(batch)
+        total_generated += len(batch)
+    
+    client.close()
+    
+    print(f"Completed! Generated {total_generated} events.")
+
+if __name__ == "__main__":
+    main()