mo
67 lines
2.0 KiB
Python
67 lines
2.0 KiB
Python
"""Lightweight change-event emitter to Kafka for sources without a native
|
|
Debezium connector (Cassandra 4.1 with cdc disabled, Neo4j 4.4 Community).
|
|
|
|
This performs an application-level dual-write: as the generators insert data,
|
|
each new record is also published to Kafka as a Debezium-style change event.
|
|
The platform's all-topics S3 consumer then archives these to object storage,
|
|
so Cassandra/Neo4j changes flow DB -> Kafka -> S3 just like the CDC sources.
|
|
|
|
Best-effort: never raises, so data generation cannot be broken by Kafka issues.
|
|
"""
|
|
import json
|
|
import os
|
|
import time
|
|
|
|
KAFKA_BOOTSTRAP = os.getenv("CDC_KAFKA_BOOTSTRAP", "10.0.21.36:9092")
|
|
|
|
_producer = None
|
|
|
|
|
|
def _default(o):
|
|
# make datetimes / uuids / Decimals JSON-serializable
|
|
try:
|
|
import datetime
|
|
if isinstance(o, (datetime.datetime, datetime.date)):
|
|
return o.isoformat()
|
|
except Exception:
|
|
pass
|
|
return str(o)
|
|
|
|
|
|
def _get_producer():
|
|
global _producer
|
|
if _producer is None:
|
|
from kafka import KafkaProducer
|
|
_producer = KafkaProducer(
|
|
bootstrap_servers=KAFKA_BOOTSTRAP,
|
|
value_serializer=lambda v: json.dumps(v, default=_default).encode("utf-8"),
|
|
linger_ms=50,
|
|
acks=1,
|
|
retries=3,
|
|
request_timeout_ms=20000,
|
|
)
|
|
return _producer
|
|
|
|
|
|
def emit_changes(topic, db, table, records):
|
|
"""Publish a list of dict records as change events. Returns count sent."""
|
|
if not records:
|
|
return 0
|
|
try:
|
|
p = _get_producer()
|
|
now = int(time.time() * 1000)
|
|
for r in records:
|
|
evt = {
|
|
"op": "c",
|
|
"ts_ms": now,
|
|
"source": {"connector": "app-cdc", "db": db, "table": table},
|
|
"after": r,
|
|
}
|
|
p.send(topic, evt)
|
|
p.flush(timeout=30)
|
|
print(f"[cdc] emitted {len(records)} change events -> {topic}")
|
|
return len(records)
|
|
except Exception as e:
|
|
print(f"[cdc] emit warning ({topic}): {str(e)[:200]}")
|
|
return 0
|