mo
944635ffe1
Source generators (postgres/mysql/mongodb) now emit realistic PII (name/email/phone/ip/iban/address/national_id/dob). deploy/mask_pii_setup.py ALTERs the source tables and backfills a bounded sample so Debezium CDC and OpenMetadata PII auto-classification see real sensitive values.
79 lines
2.6 KiB
Python
79 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate fake event data for MongoDB.
|
|
Light + configurable via GEN_ROWS. Now also embeds contact PII fields
|
|
(contact_name/email/phone) for masking + PII classification demos.
|
|
"""
|
|
|
|
import pymongo
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import os
|
|
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "27017"
|
|
DB_NAME = "supplychain"
|
|
COLLECTION_NAME = "events"
|
|
|
|
TARGET_DOCUMENTS = int(os.getenv("GEN_ROWS", "5000"))
|
|
BATCH_SIZE = min(5000, max(500, TARGET_DOCUMENTS))
|
|
|
|
EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
|
|
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
|
|
SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
|
|
|
|
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
|
|
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
|
|
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
|
|
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
|
|
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
|
|
|
|
|
|
def _name():
|
|
return f"{random.choice(FIRST)} {random.choice(LAST)}"
|
|
|
|
|
|
def generate_fake_event():
|
|
event_id = str(uuid.uuid4())
|
|
days_ago = random.randint(0, 365)
|
|
ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
nm = _name()
|
|
return {
|
|
"event_id": event_id,
|
|
"type": random.choice(EVENT_TYPES),
|
|
"region": random.choice(REGIONS),
|
|
"source": random.choice(SOURCES),
|
|
"amount": random.uniform(100.0, 50000.0),
|
|
"ts": ts,
|
|
"contact_name": nm,
|
|
"contact_email": f"{nm.lower().replace(' ', '.')}{random.randint(1, 999)}@{random.choice(DOMAINS)}",
|
|
"contact_phone": f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}",
|
|
"payload": "X" * 500,
|
|
}
|
|
|
|
|
|
def main():
|
|
print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
|
|
client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
|
|
collection = client[DB_NAME][COLLECTION_NAME]
|
|
print(f"Generating {TARGET_DOCUMENTS} events (with PII)... batch={BATCH_SIZE}")
|
|
total_generated = 0
|
|
batch = []
|
|
for i in range(TARGET_DOCUMENTS):
|
|
batch.append(generate_fake_event())
|
|
if len(batch) >= BATCH_SIZE:
|
|
collection.insert_many(batch)
|
|
total_generated += len(batch)
|
|
batch = []
|
|
if batch:
|
|
collection.insert_many(batch)
|
|
total_generated += len(batch)
|
|
client.close()
|
|
print(f"Completed! Generated {total_generated} events.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|