feat(cdc): app-level Cassandra+Neo4j change events to Kafka (->S3) on generate

infra/airflow/scripts/generate_neo4j_graph_data.py

@@ -8,6 +8,16 @@ from neo4j import GraphDatabase
 import random
 import uuid
 import sys
+import os
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+try:
+    from cdc_emit import emit_changes
+except Exception:
+    def emit_changes(*a, **k):
+        return 0
+
+NODES_TOPIC = "neo4j_graph.cdc.nodes"
+RELS_TOPIC = "neo4j_graph.cdc.relationships"
 
 # Database connection details
 DB_HOST = "10.0.21.51"
@@ -121,6 +131,7 @@ def main():
                     batch=batch
                 )
                 total_products += len(batch)
+                emit_changes(NODES_TOPIC, "graph", "Product", batch)
                 batch = []
                 
                 if total_products % 50000 == 0:
@@ -143,6 +154,7 @@ def main():
                 batch=batch
             )
             total_products += len(batch)
+            emit_changes(NODES_TOPIC, "graph", "Product", batch)
         
         print(f"Generated {total_products} product nodes.")
         
@@ -172,6 +184,7 @@ def main():
                     batch=batch
                 )
                 total_suppliers += len(batch)
+                emit_changes(NODES_TOPIC, "graph", "Supplier", batch)
                 batch = []
         
         if batch:
@@ -189,6 +202,7 @@ def main():
                 batch=batch
             )
             total_suppliers += len(batch)
+            emit_changes(NODES_TOPIC, "graph", "Supplier", batch)
         
         print(f"Generated {total_suppliers} supplier nodes.")
         
@@ -213,6 +227,7 @@ def main():
                 if len(batch) >= BATCH_SIZE:
                     _flush_rels(session, batch)
                     total_relationships += len(batch)
+                    emit_changes(RELS_TOPIC, "graph", "RELATIONSHIP", batch)
                     batch = []
                     
                     if total_relationships % 50000 == 0:
@@ -221,6 +236,7 @@ def main():
         if batch:
             _flush_rels(session, batch)
             total_relationships += len(batch)
+            emit_changes(RELS_TOPIC, "graph", "RELATIONSHIP", batch)
         
         print(f"Created {total_relationships} relationships.")