feat(pii): masking for Cassandra & Neo4j (graph nodes + live property read) + MongoDB payload/free-text
This commit is contained in:
+12
-4
@@ -34,9 +34,11 @@ NODES: list[dict[str, Any]] = [
|
||||
{"id": "generator", "label": "Data Generator", "sub": "Airflow DAGs", "kind": "generator", "x": 9, "y": 34},
|
||||
{"id": "hdfs", "label": "Hadoop HDFS", "sub": "historical_sales", "kind": "hadoop", "x": 9, "y": 80},
|
||||
# col 1 — source databases
|
||||
{"id": "postgres", "label": "PostgreSQL", "sub": "sales_orders", "kind": "source", "x": 28, "y": 16},
|
||||
{"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 38},
|
||||
{"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 60},
|
||||
{"id": "postgres", "label": "PostgreSQL", "sub": "sales_orders", "kind": "source", "x": 28, "y": 12},
|
||||
{"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 28},
|
||||
{"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 44},
|
||||
{"id": "cassandra", "label": "Cassandra", "sub": "device_metrics", "kind": "source", "x": 28, "y": 60},
|
||||
{"id": "neo4j", "label": "Neo4j", "sub": "Product · Supplier graph", "kind": "source", "x": 28, "y": 76},
|
||||
# col 2 — change data capture
|
||||
{"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 24},
|
||||
{"id": "spark", "label": "Apache Spark", "sub": "Streaming · batch", "kind": "compute", "x": 62, "y": 38,
|
||||
@@ -57,9 +59,13 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "generator", "to": "postgres", "kind": "generate", "movement_id": "gen_postgres"},
|
||||
{"from": "generator", "to": "mysql", "kind": "generate", "movement_id": "gen_mysql"},
|
||||
{"from": "generator", "to": "mongodb", "kind": "generate", "movement_id": "gen_mongodb"},
|
||||
{"from": "generator", "to": "cassandra", "kind": "generate"},
|
||||
{"from": "generator", "to": "neo4j", "kind": "generate"},
|
||||
{"from": "postgres", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "mysql", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "mongodb", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "cassandra", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "neo4j", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "kafka", "to": "spark", "kind": "stream"},
|
||||
{"from": "spark", "to": "iceberg_curated", "kind": "movement", "movement_id": "spark_to_curated"},
|
||||
{"from": "spark", "to": "s3_cdc", "kind": "movement", "movement_id": "spark_to_s3"},
|
||||
@@ -77,6 +83,8 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "postgres", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "mysql", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "mongodb", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "cassandra", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "neo4j", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "trino", "to": "openmetadata", "kind": "catalog"},
|
||||
]
|
||||
|
||||
@@ -165,7 +173,7 @@ async def _build() -> dict[str, Any]:
|
||||
used = spark.get("cores_used") or 0
|
||||
metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
|
||||
node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
|
||||
elif n["id"] in ("postgres", "mysql", "mongodb"):
|
||||
elif n["id"] in ("postgres", "mysql", "mongodb", "cassandra", "neo4j"):
|
||||
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
|
||||
elif n["id"] == "iceberg_hadoop":
|
||||
c = _iceberg_hadoop_count()
|
||||
|
||||
Reference in New Issue
Block a user