ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(masking): add PII columns to generators + backfill (Faker-style, no deps)

Browse Source 
Source generators (postgres/mysql/mongodb) now emit realistic PII
(name/email/phone/ip/iban/address/national_id/dob). deploy/mask_pii_setup.py
ALTERs the source tables and backfills a bounded sample so Debezium CDC and
OpenMetadata PII auto-classification see real sensitive values.
This commit is contained in:
mo
2026-06-27 02:50:28 +02:00
parent c70639c46d
commit 944635ffe1
5 changed files with 564 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/airflow/mask_pii_setup.py
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Add realistic PII columns to the source tables and backfill a bounded sample.
Runs inside the atc-agents api container (has psycopg2/pymysql/pymongo + network
to 10.0.21.51). Idempotent: ADD COLUMN IF NOT EXISTS / duplicate-column tolerant.
Backfills the most recent N rows so OpenMetadata auto-classification samples real
PII values. New generator rows also get PII (see updated generator scripts).
"""
import os
import random
import psycopg2
import pymysql
import pymongo
DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51")
DB_USER = os.getenv("SRC_DB_USER", "mo")
DB_PASS = os.getenv("SRC_DB_PASSWORD", "Dell2026!")
N = int(os.getenv("PII_BACKFILL_ROWS", "4000"))
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
STREETS = ["Kerkstraat", "Dorpsweg", "Molenpad", "Stationsplein", "Industrieweg", "Lindelaan",
"Beukenhof", "Havenstraat", "Parkweg", "Schoolstraat"]
CITIES = ["Amsterdam", "Rotterdam", "Utrecht", "Eindhoven", "Den Haag", "Groningen", "Tilburg", "Almere"]
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
def full_name():
return f"{random.choice(FIRST)} {random.choice(LAST)}"
def email(name):
base = name.lower().replace(" ", ".")
return f"{base}{random.randint(1, 999)}@{random.choice(DOMAINS)}"
def phone():
return f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}"
def ip():
return ".".join(str(random.randint(1, 254)) for _ in range(4))
def iban():
return f"NL{random.randint(10, 99)}DELL{random.randint(10**9, 10**10 - 1)}"
def national_id():
return str(random.randint(100000000, 999999999))
def address():
return f"{random.choice(STREETS)} {random.randint(1, 320)}, {random.randint(1000, 9999)} {random.choice(CITIES)}"
def dob():
return f"{random.randint(1960, 2002)}-{random.randint(1, 12):02d}-{random.randint(1, 28):02d}"
def do_postgres():
print("== PostgreSQL sales_orders ==")
conn = psycopg2.connect(host=DB_HOST, dbname="postgres", user=DB_USER, password=DB_PASS, port=5432, connect_timeout=10)
conn.autocommit = True
cur = conn.cursor()
for col, typ in [("customer_name", "text"), ("customer_email", "text"), ("customer_phone", "text"),
("customer_ip", "text"), ("billing_iban", "text"), ("shipping_address", "text")]:
cur.execute(f"ALTER TABLE public.sales_orders ADD COLUMN IF NOT EXISTS {col} {typ}")
print(" columns ensured")
cur.execute("SELECT order_id FROM public.sales_orders ORDER BY order_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for oid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid))
cur.executemany(
"UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, "
"customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
cur.close()
conn.close()
def do_mysql():
print("== MySQL employee_events ==")
conn = pymysql.connect(host=DB_HOST, user=DB_USER, password=DB_PASS, database="hr", port=3306,
connect_timeout=10, autocommit=True)
cur = conn.cursor()
for col, typ in [("employee_name", "varchar(120)"), ("employee_email", "varchar(160)"),
("employee_phone", "varchar(40)"), ("national_id", "varchar(20)"),
("home_address", "varchar(255)"), ("date_of_birth", "date")]:
try:
cur.execute(f"ALTER TABLE employee_events ADD COLUMN {col} {typ}")
except Exception as e:
if "Duplicate column" not in str(e):
raise
print(" columns ensured")
cur.execute("SELECT event_id FROM employee_events ORDER BY event_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for eid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid))
cur.executemany(
"UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, "
"national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
cur.close()
conn.close()
def do_mongo():
print("== MongoDB events ==")
uri = os.getenv("SRC_MONGO_URI", f"mongodb://{DB_HOST}:27017/?replicaSet=rs0")
client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=10000)
coll = client["supplychain"]["events"]
docs = list(coll.find({}, {"_id": 1}).sort("_id", -1).limit(N))
n = 0
for d in docs:
nm = full_name()
coll.update_one({"_id": d["_id"]}, {"$set": {
"contact_name": nm, "contact_email": email(nm), "contact_phone": phone(),
}})
n += 1
print(f" updated {n} docs with PII fields")
client.close()
if __name__ == "__main__":
for fn in (do_postgres, do_mysql, do_mongo):
try:
fn()
except Exception as e:
print(f" ERROR in {fn.__name__}: {e}")
print("done")
deploy/airflow/scripts/generate_mongodb_events_data.py
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""
Generate fake event data for MongoDB.
Light + configurable via GEN_ROWS. Now also embeds contact PII fields
(contact_name/email/phone) for masking + PII classification demos.
"""
import pymongo
import random
from datetime import datetime, timedelta
import uuid
import os
DB_HOST = "10.0.21.51"
DB_PORT = "27017"
DB_NAME = "supplychain"
COLLECTION_NAME = "events"
TARGET_DOCUMENTS = int(os.getenv("GEN_ROWS", "5000"))
BATCH_SIZE = min(5000, max(500, TARGET_DOCUMENTS))
EVENT_TYPES = ["INSERT", "UPDATE", "DELETE", "CREATE", "MODIFY"]
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
SOURCES = ["ERP", "WMS", "CRM", "SCM", "TMS"]
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
def _name():
return f"{random.choice(FIRST)} {random.choice(LAST)}"
def generate_fake_event():
event_id = str(uuid.uuid4())
days_ago = random.randint(0, 365)
ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
nm = _name()
return {
"event_id": event_id,
"type": random.choice(EVENT_TYPES),
"region": random.choice(REGIONS),
"source": random.choice(SOURCES),
"amount": random.uniform(100.0, 50000.0),
"ts": ts,
"contact_name": nm,
"contact_email": f"{nm.lower().replace(' ', '.')}{random.randint(1, 999)}@{random.choice(DOMAINS)}",
"contact_phone": f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}",
"payload": "X" * 500,
}
def main():
print(f"Connecting to MongoDB at {DB_HOST}:{DB_PORT}...")
client = pymongo.MongoClient(f"mongodb://{DB_HOST}:{DB_PORT}/")
collection = client[DB_NAME][COLLECTION_NAME]
print(f"Generating {TARGET_DOCUMENTS} events (with PII)... batch={BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_DOCUMENTS):
batch.append(generate_fake_event())
if len(batch) >= BATCH_SIZE:
collection.insert_many(batch)
total_generated += len(batch)
batch = []
if batch:
collection.insert_many(batch)
total_generated += len(batch)
client.close()
print(f"Completed! Generated {total_generated} events.")
if __name__ == "__main__":
main()
deploy/airflow/scripts/generate_mysql_employee_data.py
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""
Generate fake employee event data for MySQL.
Light + configurable via GEN_ROWS. Now also populates realistic PII columns
(employee_name/email/phone, national_id, home_address, date_of_birth).
"""
import mysql.connector
import random
from datetime import datetime, timedelta
import uuid
import os
DB_HOST = "10.0.21.51"
DB_PORT = "3306"
DB_NAME = "hr"
DB_USER = "mo"
DB_PASSWORD = "Dell2026!"
TARGET_ROWS = int(os.getenv("GEN_ROWS", "5000"))
BATCH_SIZE = min(10000, max(500, TARGET_ROWS))
DEPARTMENTS = ["HR", "Operations", "Sales", "Marketing", "Finance", "IT", "Engineering", "Legal"]
ROLE_NAMES = ["Analyst", "Lead", "Manager", "Consultant", "Director", "Engineer", "Specialist", "Coordinator"]
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
EVENT_TYPES = ["TRANSFER", "PROMOTION", "TERMINATION", "HIRED", "SALARY_CHANGE", "DEPARTMENT_CHANGE"]
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
STREETS = ["Kerkstraat", "Dorpsweg", "Molenpad", "Stationsplein", "Industrieweg", "Lindelaan",
"Beukenhof", "Havenstraat", "Parkweg", "Schoolstraat"]
CITIES = ["Amsterdam", "Rotterdam", "Utrecht", "Eindhoven", "Den Haag", "Groningen", "Tilburg", "Almere"]
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "company.eu", "contact.io"]
def _name():
return f"{random.choice(FIRST)} {random.choice(LAST)}"
def _pii():
nm = _name()
return (
nm,
f"{nm.lower().replace(' ', '.')}{random.randint(1, 999)}@{random.choice(DOMAINS)}",
f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}",
str(random.randint(100000000, 999999999)),
f"{random.choice(STREETS)} {random.randint(1, 320)}, {random.randint(1000, 9999)} {random.choice(CITIES)}",
f"{random.randint(1960, 2002)}-{random.randint(1, 12):02d}-{random.randint(1, 28):02d}",
)
def generate_fake_employee_event():
employee_id = random.randint(1, 100000)
department = random.choice(DEPARTMENTS)
role_name = random.choice(ROLE_NAMES)
region = random.choice(REGIONS)
event_type = random.choice(EVENT_TYPES)
days_ago = random.randint(0, 730)
event_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
salary_change = round(random.uniform(1000.0, 20000.0), 2) if random.random() > 0.3 else None
notes = str(uuid.uuid4()) * 10
name, email, phone, nid, address, dob = _pii()
return (employee_id, department, role_name, region, event_type, salary_change, event_ts, notes,
name, email, phone, nid, address, dob)
INSERT_SQL = """
INSERT INTO employee_events (employee_id, department, role_name, region,
event_type, salary_change, event_ts, notes,
employee_name, employee_email, employee_phone,
national_id, home_address, date_of_birth)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
def main():
print(f"Connecting to MySQL at {DB_HOST}:{DB_PORT}...")
conn = mysql.connector.connect(host=DB_HOST, port=DB_PORT, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
cursor = conn.cursor()
print(f"Generating {TARGET_ROWS} employee events (with PII)... batch={BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_ROWS):
batch.append(generate_fake_employee_event())
if len(batch) >= BATCH_SIZE:
cursor.executemany(INSERT_SQL, batch)
conn.commit()
total_generated += len(batch)
batch = []
if batch:
cursor.executemany(INSERT_SQL, batch)
conn.commit()
total_generated += len(batch)
cursor.close()
conn.close()
print(f"Completed! Generated {total_generated} employee events.")
if __name__ == "__main__":
main()
deploy/airflow/scripts/generate_postgres_sales_data.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""
Generate fake sales order data for PostgreSQL.
Light + configurable via GEN_ROWS. Now also populates realistic PII columns
(customer_name/email/phone/ip, billing_iban, shipping_address) so the masking
ETL + OpenMetadata PII auto-classification have real sensitive data to work on.
"""
import psycopg2
import random
from datetime import datetime, timedelta
import uuid
import os
DB_HOST = "10.0.21.51"
DB_PORT = "5432"
DB_NAME = "postgres"
DB_USER = "mo"
DB_PASSWORD = "Dell2026!"
TARGET_ROWS = int(os.getenv("GEN_ROWS", "5000"))
BATCH_SIZE = min(10000, max(500, TARGET_ROWS))
REGIONS = ["EU", "APAC", "LATAM", "NA", "EMEA"]
SALES_CHANNELS = ["STORE", "ONLINE", "MOBILE", "B2B"]
CURRENCIES = ["EUR", "USD", "GBP", "JPY", "CNY"]
ORDER_STATUSES = ["SHIPPED", "PENDING", "CANCELLED", "RETURNED", "DELIVERED"]
# ── lightweight PII generator (no external Faker dependency) ─────────────────
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
STREETS = ["Kerkstraat", "Dorpsweg", "Molenpad", "Stationsplein", "Industrieweg", "Lindelaan",
"Beukenhof", "Havenstraat", "Parkweg", "Schoolstraat"]
CITIES = ["Amsterdam", "Rotterdam", "Utrecht", "Eindhoven", "Den Haag", "Groningen", "Tilburg", "Almere"]
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
def _name():
return f"{random.choice(FIRST)} {random.choice(LAST)}"
def _pii():
nm = _name()
return (
nm,
f"{nm.lower().replace(' ', '.')}{random.randint(1, 999)}@{random.choice(DOMAINS)}",
f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}",
".".join(str(random.randint(1, 254)) for _ in range(4)),
f"NL{random.randint(10, 99)}DELL{random.randint(10**9, 10**10 - 1)}",
f"{random.choice(STREETS)} {random.randint(1, 320)}, {random.randint(1000, 9999)} {random.choice(CITIES)}",
)
def generate_fake_order():
customer_id = random.randint(1, 100000)
product_id = random.randint(1, 5000)
region = random.choice(REGIONS)
sales_channel = random.choice(SALES_CHANNELS)
days_ago = random.randint(0, 730)
order_ts = datetime.now() - timedelta(days=days_ago, hours=random.randint(0, 23),
minutes=random.randint(0, 59))
amount = round(random.uniform(10.0, 10000.0), 2)
currency = random.choice(CURRENCIES)
order_status = random.choice(ORDER_STATUSES)
notes = str(uuid.uuid4()) * 10
name, email, phone, ip, iban, address = _pii()
return (customer_id, product_id, region, sales_channel, order_ts,
amount, currency, order_status, notes,
name, email, phone, ip, iban, address)
INSERT_SQL = """
INSERT INTO sales_orders (customer_id, product_id, region, sales_channel,
order_ts, amount, currency, order_status, notes,
customer_name, customer_email, customer_phone,
customer_ip, billing_iban, shipping_address)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
def main():
print(f"Connecting to PostgreSQL at {DB_HOST}:{DB_PORT}...")
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
cursor = conn.cursor()
print(f"Generating {TARGET_ROWS} sales orders (with PII)... batch={BATCH_SIZE}")
total_generated = 0
batch = []
for i in range(TARGET_ROWS):
batch.append(generate_fake_order())
if len(batch) >= BATCH_SIZE:
cursor.executemany(INSERT_SQL, batch)
conn.commit()
total_generated += len(batch)
batch = []
if batch:
cursor.executemany(INSERT_SQL, batch)
conn.commit()
total_generated += len(batch)
cursor.close()
conn.close()
print(f"Completed! Generated {total_generated} sales orders.")
if __name__ == "__main__":
main()
deploy/mask_pii_setup.py
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Add realistic PII columns to the source tables and backfill a bounded sample.
Runs inside the atc-agents api container (has psycopg2/pymysql/pymongo + network
to 10.0.21.51). Idempotent: ADD COLUMN IF NOT EXISTS / duplicate-column tolerant.
Backfills the most recent N rows so OpenMetadata auto-classification samples real
PII values. New generator rows also get PII (see updated generator scripts).
"""
import os
import random
import psycopg2
import pymysql
import pymongo
DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51")
DB_USER = os.getenv("SRC_DB_USER", "mo")
DB_PASS = os.getenv("SRC_DB_PASSWORD", "Dell2026!")
N = int(os.getenv("PII_BACKFILL_ROWS", "4000"))
FIRST = ["Sophie", "Liam", "Emma", "Noah", "Julia", "Lucas", "Mila", "Daan", "Anna", "Sem",
"Eva", "Finn", "Tess", "Bram", "Lotte", "Max", "Sara", "Thijs", "Nina", "Ruben"]
LAST = ["de Vries", "Jansen", "Bakker", "Visser", "Smit", "Meijer", "Mulder", "Bos",
"Vos", "Peters", "Hendriks", "Dijkstra", "Kok", "Willems", "Maas", "Koning"]
STREETS = ["Kerkstraat", "Dorpsweg", "Molenpad", "Stationsplein", "Industrieweg", "Lindelaan",
"Beukenhof", "Havenstraat", "Parkweg", "Schoolstraat"]
CITIES = ["Amsterdam", "Rotterdam", "Utrecht", "Eindhoven", "Den Haag", "Groningen", "Tilburg", "Almere"]
DOMAINS = ["example.com", "mail.nl", "acme-corp.eu", "shopmail.com", "contact.io"]
def full_name():
return f"{random.choice(FIRST)} {random.choice(LAST)}"
def email(name):
base = name.lower().replace(" ", ".")
return f"{base}{random.randint(1, 999)}@{random.choice(DOMAINS)}"
def phone():
return f"+31 6 {random.randint(10, 99)} {random.randint(100000, 999999)}"
def ip():
return ".".join(str(random.randint(1, 254)) for _ in range(4))
def iban():
return f"NL{random.randint(10, 99)}DELL{random.randint(10**9, 10**10 - 1)}"
def national_id():
return str(random.randint(100000000, 999999999))
def address():
return f"{random.choice(STREETS)} {random.randint(1, 320)}, {random.randint(1000, 9999)} {random.choice(CITIES)}"
def dob():
return f"{random.randint(1960, 2002)}-{random.randint(1, 12):02d}-{random.randint(1, 28):02d}"
def do_postgres():
print("== PostgreSQL sales_orders ==")
conn = psycopg2.connect(host=DB_HOST, dbname="postgres", user=DB_USER, password=DB_PASS, port=5432, connect_timeout=10)
conn.autocommit = True
cur = conn.cursor()
for col, typ in [("customer_name", "text"), ("customer_email", "text"), ("customer_phone", "text"),
("customer_ip", "text"), ("billing_iban", "text"), ("shipping_address", "text")]:
cur.execute(f"ALTER TABLE public.sales_orders ADD COLUMN IF NOT EXISTS {col} {typ}")
print(" columns ensured")
cur.execute("SELECT order_id FROM public.sales_orders ORDER BY order_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for oid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), ip(), iban(), address(), oid))
cur.executemany(
"UPDATE public.sales_orders SET customer_name=%s, customer_email=%s, customer_phone=%s, "
"customer_ip=%s, billing_iban=%s, shipping_address=%s WHERE order_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
cur.close()
conn.close()
def do_mysql():
print("== MySQL employee_events ==")
conn = pymysql.connect(host=DB_HOST, user=DB_USER, password=DB_PASS, database="hr", port=3306,
connect_timeout=10, autocommit=True)
cur = conn.cursor()
for col, typ in [("employee_name", "varchar(120)"), ("employee_email", "varchar(160)"),
("employee_phone", "varchar(40)"), ("national_id", "varchar(20)"),
("home_address", "varchar(255)"), ("date_of_birth", "date")]:
try:
cur.execute(f"ALTER TABLE employee_events ADD COLUMN {col} {typ}")
except Exception as e:
if "Duplicate column" not in str(e):
raise
print(" columns ensured")
cur.execute("SELECT event_id FROM employee_events ORDER BY event_id DESC LIMIT %s", (N,))
ids = [r[0] for r in cur.fetchall()]
rows = []
for eid in ids:
nm = full_name()
rows.append((nm, email(nm), phone(), national_id(), address(), dob(), eid))
cur.executemany(
"UPDATE employee_events SET employee_name=%s, employee_email=%s, employee_phone=%s, "
"national_id=%s, home_address=%s, date_of_birth=%s WHERE event_id=%s", rows)
print(f" backfilled {len(rows)} rows with PII")
cur.close()
conn.close()
def do_mongo():
print("== MongoDB events ==")
uri = os.getenv("SRC_MONGO_URI", f"mongodb://{DB_HOST}:27017/?replicaSet=rs0")
client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=10000)
coll = client["supplychain"]["events"]
docs = list(coll.find({}, {"_id": 1}).sort("_id", -1).limit(N))
n = 0
for d in docs:
nm = full_name()
coll.update_one({"_id": d["_id"]}, {"$set": {
"contact_name": nm, "contact_email": email(nm), "contact_phone": phone(),
}})
n += 1
print(f" updated {n} docs with PII fields")
client.close()
if __name__ == "__main__":
for fn in (do_postgres, do_mysql, do_mongo):
try:
fn()
except Exception as e:
print(f" ERROR in {fn.__name__}: {e}")
print("done")