mo
944635ffe1
Source generators (postgres/mysql/mongodb) now emit realistic PII (name/email/phone/ip/iban/address/national_id/dob). deploy/mask_pii_setup.py ALTERs the source tables and backfills a bounded sample so Debezium CDC and OpenMetadata PII auto-classification see real sensitive values.
108 lines
4.2 KiB
Python
108 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generate fake sales order data for PostgreSQL.
|
|
Light + configurable via GEN_ROWS. Now also populates realistic PII columns
|
|
(customer_name/email/phone/ip, billing_iban, shipping_address) so the masking
|
|
ETL + OpenMetadata PII auto-classification have real sensitive data to work on.
|
|
"""
|
|
|
|
import psycopg2
|
|
import random
|
|
from datetime import datetime, timedelta
|
|
import uuid
|
|
import os
|
|
|
|
DB_HOST = "10.0.21.51"
|
|
DB_PORT = "5432"
|
|
DB_NAME = "postgres"
|
|
DB_USER = "mo"
|
|
DB_PASSWORD = "Dell2026!"
|
|
|
|
TARGET_ROWS = int(os.getenv("GEN_ROWS", "5000"))
|
|
BATCH_SIZE = min(10000, max(500, TARGET_ROWS))
|
|
|
|
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
|
|
SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
|
|
CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
|
|
ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
|
|
|
|
# ── lightweight PII generator (no external Faker dependency) ─────────────────
|
|
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
|
|
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
|
|
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
|
|
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
|
|
STREETS = ["Kerkstraat", "Dorpsweg", "Molenpad", "Stationsplein", "Industrieweg", "Lindelaan",
|
|
"Beukenhof", "Havenstraat", "Parkweg", "Schoolstraat"]
|
|
CITIES = ["Amsterdam", "Rotterdam", "Utrecht", "Eindhoven", "Den Haag", "Groningen", "Tilburg", "Almere"]
|
|
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
|
|
|
|
|
|
def _name():
|
|
return f"{random.choice(FIRST)} {random.choice(LAST)}"
|
|
|
|
|
|
def _pii():
|
|
nm = _name()
|
|
return (
|
|
nm,
|
|
f"{nm.lower().replace(' ', '.')}{random.randint(1, 999)}@{random.choice(DOMAINS)}",
|
|
f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}",
|
|
".".join(str(random.randint(1, 254)) for _ in range(4)),
|
|
f"NL{random.randint(10, 99)}DELL{random.randint(10**9, 10**10 - 1)}",
|
|
f"{random.choice(STREETS)} {random.randint(1, 320)}, {random.randint(1000, 9999)} {random.choice(CITIES)}",
|
|
)
|
|
|
|
|
|
def generate_fake_order():
|
|
customer_id = random.randint(1, 100000)
|
|
product_id = random.randint(1, 5000)
|
|
region = random.choice(REGIONS)
|
|
sales_channel = random.choice(SALES_CHANNELS)
|
|
days_ago = random.randint(0, 730)
|
|
order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
|
|
minutes=random.randint(0, 59))
|
|
amount = round(random.uniform(10.0, 10000.0), 2)
|
|
currency = random.choice(CURRENCIES)
|
|
order_status = random.choice(ORDER_STATUSES)
|
|
notes = str(uuid.uuid4()) * 10
|
|
name, email, phone, ip, iban, address = _pii()
|
|
return (customer_id, product_id, region, sales_channel, order_ts,
|
|
amount, currency, order_status, notes,
|
|
name, email, phone, ip, iban, address)
|
|
|
|
|
|
INSERT_SQL = """
|
|
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
|
|
order_ts, amount, currency, order_status, notes,
|
|
customer_name, customer_email, customer_phone,
|
|
customer_ip, billing_iban, shipping_address)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
|
|
|
|
def main():
|
|
print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
|
|
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
|
|
cursor = conn.cursor()
|
|
print(f"Generating {TARGET_ROWS} sales orders (with PII)... batch={BATCH_SIZE}")
|
|
total_generated = 0
|
|
batch = []
|
|
for i in range(TARGET_ROWS):
|
|
batch.append(generate_fake_order())
|
|
if len(batch) >= BATCH_SIZE:
|
|
cursor.executemany(INSERT_SQL, batch)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
batch = []
|
|
if batch:
|
|
cursor.executemany(INSERT_SQL, batch)
|
|
conn.commit()
|
|
total_generated += len(batch)
|
|
cursor.close()
|
|
conn.close()
|
|
print(f"Completed! Generated {total_generated} sales orders.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|