Lakehouse Admin
df5ec93dc3
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose - Airflow DAG scripts, Proxmox VM inventory, network docs - Ansible playbook, Gitea CI validate workflow - Backup and health-check scripts, cron documentation - Homepage DOCS tab with links to all documentation - Extended collect-fleet-config.sh and populate-repo.py
91 lines
2.5 KiB
Python
91 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to generate fake telemetry data for Cassandra
|
|
Generates approximately 1GB of data
|
|
"""
|
|
|
|
from cassandra.cluster import Cluster
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import sys
|
|
|
|
# Database connection details
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "9042"
|
|
KEYSPACE = "telemetry"
|
|
TABLE_NAME = "device_metrics"
|
|
|
|
# Data generation settings
|
|
TARGET_ROWS = 3000000 # Approximately 1GB of data
|
|
BATCH_SIZE = 5000
|
|
|
|
# Sample data
|
|
METRIC_TYPES = ["temperature", "humidity", "pressure", "voltage", "current"]
|
|
DEVICE_PREFIX = "device-"
|
|
|
|
def generate_fake_device_metric():
|
|
"""Generate a single fake device metric"""
|
|
device_id = f"{DEVICE_PREFIX}{random.randint(1, 50000)}"
|
|
|
|
# Random timestamp within the last year
|
|
days_ago = random.randint(0, 365)
|
|
metric_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
|
|
metric_type = random.choice(METRIC_TYPES)
|
|
metric_value = round(random.uniform(0.0, 100.0), 4)
|
|
|
|
# Generate a long payload field
|
|
payload = "X" * 200
|
|
|
|
return (device_id, metric_ts, metric_type, metric_value, payload)
|
|
|
|
def main():
|
|
print(f"Connecting to Cassandra at {DB_HOST}:{DB_PORT}...")
|
|
|
|
cluster = Cluster([DB_HOST], port=DB_PORT)
|
|
session = cluster.connect()
|
|
|
|
print(f"Generating {TARGET_ROWS} device metrics...")
|
|
print(f"Batch size: {BATCH_SIZE}")
|
|
|
|
total_generated = 0
|
|
batch = []
|
|
|
|
for i in range(TARGET_ROWS):
|
|
batch.append(generate_fake_device_metric())
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
session.execute(
|
|
f"""
|
|
INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
total_generated += len(batch)
|
|
batch = []
|
|
|
|
if total_generated % 100000 == 0:
|
|
print(f"Generated {total_generated} rows...")
|
|
|
|
# Insert remaining rows
|
|
if batch:
|
|
session.execute(
|
|
f"""
|
|
INSERT INTO {KEYSPACE}.{TABLE_NAME} (device_id, metric_ts, metric_type, metric_value, payload)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
""",
|
|
batch
|
|
)
|
|
total_generated += len(batch)
|
|
|
|
session.shutdown()
|
|
cluster.shutdown()
|
|
|
|
print(f"Completed! Generated {total_generated} device metrics.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|