infra: mirror light per-source Airflow DAGs + generators (GEN_ROWS), fix script path

2026-06-26 01:07:23 +00:00
parent 9400f9410c
commit 62b416d210
6 changed files with 690 additions and 0 deletions
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Script to generate fake telemetry data for Cassandra
+Generates approximately 1GB of data
+"""
+
+from cassandra.cluster import Cluster
+import random
+from datetime import datetime, timedelta
+import uuid
+import sys
+
+# Database connection details
+DB_HOST = "10.0.21.51"
+DB_PORT = "9042"
+KEYSPACE = "telemetry"
+TABLE_NAME = "device_metrics"
+
+# Data generation settings
+import os
+TARGET_ROWS = int(os.getenv("GEN_ROWS", "5000"))  # light, configurable via GEN_ROWS
+BATCH_SIZE = min(5000, max(500, TARGET_ROWS))
+
+# Sample data
+METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"]
+DEVICE_PREFIX = "device-"
+
+def generate_fake_device_metric():
+    """Generate a single fake device metric"""
+    device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}"
+    
+    # Random timestamp within the last year
+    days_ago = random.randint(0, 365)
+    metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23), 
+                                           minutes=random.randint(0, 59))
+    
+    metric_type = random.choice(METRIC_TYPES)
+    metric_value = round(random.uniform(0.0, 100.0), 4)
+    
+    # Generate a long payload field
+    payload = "X" * 200
+    
+    return (device_id, metric_ts, metric_type, metric_value, payload)
+
+def main():
+    print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...")
+    
+    cluster = Cluster([DB_HOST], port=DB_PORT)
+    session = cluster.connect()
+    
+    print(f"Generating {TARGET_ROWS} device metrics...")
+    print(f"Batch size: {BATCH_SIZE}")
+    
+    total_generated = 0
+    batch = []
+    
+    for i in range(TARGET_ROWS):
+        batch.append(generate_fake_device_metric())
+        
+        if len(batch) >= BATCH_SIZE:
+            session.execute(
+                f"""
+                INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+                VALUES (%s, %s, %s, %s, %s)
+                """,
+                batch
+            )
+            total_generated += len(batch)
+            batch = []
+            
+            if total_generated % 100000 == 0:
+                print(f"Generated {total_generated} rows...")
+    
+    # Insert remaining rows
+    if batch:
+        session.execute(
+            f"""
+            INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
+            VALUES (%s, %s, %s, %s, %s)
+            """,
+            batch
+        )
+        total_generated += len(batch)
+    
+    session.shutdown()
+    cluster.shutdown()
+    
+    print(f"Completed! Generated {total_generated} device metrics.")
+
+if __name__ == "__main__":
+    main()