feat(cdc): app-level Cassandra+Neo4j change events to Kafka (->S3) on generate

2026-06-26 09:21:17 +00:00
parent f4cd33b784
commit be2bbcbd92
3 changed files with 103 additions and 1 deletions
@@ -0,0 +1,66 @@
+"""Lightweight change-event emitter to Kafka for sources without a native
+Debezium connector (Cassandra 4.1 with cdc disabled, Neo4j 4.4 Community).
+
+This performs an application-level dual-write: as the generators insert data,
+each new record is also published to Kafka as a Debezium-style change event.
+The platform's all-topics S3 consumer then archives these to object storage,
+so Cassandra/Neo4j changes flow DB -> Kafka -> S3 just like the CDC sources.
+
+Best-effort: never raises, so data generation cannot be broken by Kafka issues.
+"""
+import json
+import os
+import time
+
+KAFKA_BOOTSTRAP = os.getenv("CDC_KAFKA_BOOTSTRAP", "10.0.21.36:9092")
+
+_producer = None
+
+
+def _default(o):
+    # make datetimes / uuids / Decimals JSON-serializable
+    try:
+        import datetime
+        if isinstance(o, (datetime.datetime, datetime.date)):
+            return o.isoformat()
+    except Exception:
+        pass
+    return str(o)
+
+
+def _get_producer():
+    global _producer
+    if _producer is None:
+        from kafka import KafkaProducer
+        _producer = KafkaProducer(
+            bootstrap_servers=KAFKA_BOOTSTRAP,
+            value_serializer=lambda v: json.dumps(v, default=_default).encode("utf-8"),
+            linger_ms=50,
+            acks=1,
+            retries=3,
+            request_timeout_ms=20000,
+        )
+    return _producer
+
+
+def emit_changes(topic, db, table, records):
+    """Publish a list of dict records as change events. Returns count sent."""
+    if not records:
+        return 0
+    try:
+        p = _get_producer()
+        now = int(time.time() * 1000)
+        for r in records:
+            evt = {
+                "op": "c",
+                "ts_ms": now,
+                "source": {"connector": "app-cdc", "db": db, "table": table},
+                "after": r,
+            }
+            p.send(topic, evt)
+        p.flush(timeout=30)
+        print(f"[cdc] emitted {len(records)} change events -> {topic}")
+        return len(records)
+    except Exception as e:
+        print(f"[cdc] emit warning ({topic}): {str(e)[:200]}")
+        return 0