feat: realtime ETL offload to S3 + live business dashboards
- etl_offload.py: autonomous agent backfills/tails source DBs (PG/MySQL/ Mongo/Cassandra) to S3 as Parquet in small chunks, accumulates a live federated business matrix (/api/etl/status, /api/etl/business, /run, /config). - storage_s3.py: buffer generated CDC + masked curated rows to S3, overlay live last-write into analytics; put_object_bytes for Parquet parts. - trino_federated.py: capture generated rows + archive to S3; generator_active. - dataflow.py: pulse generate + kafka/spark->S3 archive edges when active. - StorageView: realtime ETL ingest panel; TrinoFederationView: realtime business KPIs/charts from /api/etl/business. - ChangesView: top KPIs/charts now overlay the live WS stream on server stats so they update in lock-step with the bottom feed; faster 2.5s refresh. - useCommandCenter: retain 800 live CDC changes.
This commit is contained in:
mo
+159
-1
@@ -2,7 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json as _json
|
||||
import os
|
||||
import random as _rnd
|
||||
import threading as _threading
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
@@ -34,6 +37,148 @@ _activity: deque[tuple[float, str]] = deque(maxlen=20000)
|
||||
_analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
_ANALYTICS_TTL = 45.0
|
||||
|
||||
# ── Pipeline → S3 archiver ────────────────────────────────────────────────────
|
||||
# When data is generated, the streaming pipeline must actually land objects in
|
||||
# S3 so the dashboard reflects it (Last write / growth / activity). We mirror two
|
||||
# real stages straight into the object store:
|
||||
# • Kafka → S3 CDC archive -> cdc-archive/dt=YYYY-MM-DD/events-*.json (raw NDJSON)
|
||||
# • Spark → S3 curated layer -> curated/sales_orders_masked/dt=…/part-*.json (PII masked)
|
||||
# Writes are buffered like a Kafka-Connect S3 sink (flush on size or interval) so
|
||||
# we don't create a flood of tiny objects, and force-flushed on a manual burst.
|
||||
S3_ARCHIVE_BUCKET = os.getenv("S3_ARCHIVE_BUCKET", "data")
|
||||
S3_FLUSH_INTERVAL = float(os.getenv("S3_ARCHIVE_FLUSH_S", "12"))
|
||||
S3_FLUSH_SIZE = int(os.getenv("S3_ARCHIVE_FLUSH_SIZE", "400"))
|
||||
|
||||
_arch_lock = _threading.Lock()
|
||||
_arch_buf: dict[str, list] = {"cdc": [], "curated": []}
|
||||
_arch_since: dict[str, float] = {"t": 0.0}
|
||||
_last_write: dict[str, Any] = {}
|
||||
_arch_stats: dict[str, int] = {"objects": 0, "bytes": 0, "rows": 0}
|
||||
|
||||
|
||||
def _iso(v: Any) -> str:
|
||||
return v.isoformat() if hasattr(v, "isoformat") else str(v)
|
||||
|
||||
|
||||
def _mask_cust(cid: Any) -> str:
|
||||
h = abs(hash(("cust", cid))) % 0xFFFFFF
|
||||
return f"cust_{h:06x}***"
|
||||
|
||||
|
||||
def _order_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
|
||||
cid, pid, region, channel, ots, amt, curr, status = t
|
||||
return {"op": "c", "source": "postgres", "db": "sales", "table": "sales_orders", "ts": ts,
|
||||
"after": {"customer_id": cid, "product_id": pid, "region": region, "sales_channel": channel,
|
||||
"amount": amt, "currency": curr, "order_status": status, "order_ts": _iso(ots)}}
|
||||
|
||||
|
||||
def _order_to_curated(t: tuple, ts: str) -> dict[str, Any]:
|
||||
cid, pid, region, channel, ots, amt, curr, status = t
|
||||
return {"customer_ref": _mask_cust(cid), "product_id": pid, "region": region, "sales_channel": channel,
|
||||
"amount": amt, "currency": curr, "order_status": status, "order_ts": _iso(ots),
|
||||
"ingested_ts": ts, "pii_masked": True}
|
||||
|
||||
|
||||
def _hr_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
|
||||
eid, dept, role, region, evt, sal, ets = t
|
||||
return {"op": "c", "source": "mysql", "db": "hr", "table": "employee_events", "ts": ts,
|
||||
"after": {"employee_id": eid, "department": dept, "role_name": role, "region": region,
|
||||
"event_type": evt, "salary_change": sal, "event_ts": _iso(ets)}}
|
||||
|
||||
|
||||
def _supply_to_cdc(d: dict, ts: str) -> dict[str, Any]:
|
||||
return {"op": "c", "source": "mongodb", "db": "supplychain", "table": "events", "ts": ts, "after": dict(d)}
|
||||
|
||||
|
||||
def _tel_to_cdc(t: tuple, ts: str) -> dict[str, Any]:
|
||||
dev, mts, mtype, mval, _payload = t
|
||||
return {"op": "c", "source": "cassandra", "db": "telemetry", "table": "device_metrics", "ts": ts,
|
||||
"after": {"device_id": dev, "metric_ts": _iso(mts), "metric_type": mtype, "metric_value": mval}}
|
||||
|
||||
|
||||
def _put(s3, bucket: str, key: str, body: bytes, content_type: str) -> None:
|
||||
s3.put_object(Bucket=bucket, Key=key, Body=body, ContentType=content_type)
|
||||
_track("write")
|
||||
_last_write.update({"ts": datetime.now(timezone.utc).isoformat(), "mono": time.time(),
|
||||
"bucket": bucket, "key": key, "bytes": len(body)})
|
||||
_arch_stats["objects"] += 1
|
||||
_arch_stats["bytes"] += len(body)
|
||||
|
||||
|
||||
def _flush_locked() -> list[dict[str, Any]] | None:
|
||||
cdc = _arch_buf["cdc"]
|
||||
cur = _arch_buf["curated"]
|
||||
if not cdc and not cur:
|
||||
return None
|
||||
s3 = _client()
|
||||
now = datetime.now(timezone.utc)
|
||||
day = now.strftime("%Y-%m-%d")
|
||||
ms = int(now.timestamp() * 1000)
|
||||
rid = _rnd.randint(1000, 9999)
|
||||
written: list[dict[str, Any]] = []
|
||||
if cdc:
|
||||
body = ("\n".join(_json.dumps(e, default=str) for e in cdc) + "\n").encode()
|
||||
key = f"cdc-archive/dt={day}/events-{ms}-{rid}.json"
|
||||
_put(s3, S3_ARCHIVE_BUCKET, key, body, "application/x-ndjson")
|
||||
written.append({"stage": "kafka→s3", "key": key, "rows": len(cdc), "bytes": len(body)})
|
||||
if cur:
|
||||
body = ("\n".join(_json.dumps(e, default=str) for e in cur) + "\n").encode()
|
||||
key = f"curated/sales_orders_masked/dt={day}/part-{ms}-{rid}.json"
|
||||
_put(s3, S3_ARCHIVE_BUCKET, key, body, "application/x-ndjson")
|
||||
written.append({"stage": "spark→s3", "key": key, "rows": len(cur), "bytes": len(body)})
|
||||
_arch_buf["cdc"] = []
|
||||
_arch_buf["curated"] = []
|
||||
_arch_since["t"] = time.time()
|
||||
return written
|
||||
|
||||
|
||||
def archive_generated_batch(orders_rows=None, hr_rows=None, supply_docs=None, tel_rows=None,
|
||||
*, force: bool = False) -> list[dict[str, Any]] | dict[str, Any] | None:
|
||||
"""Stage a freshly generated batch into S3 (Kafka→S3 CDC archive + Spark→S3
|
||||
curated masked). Buffered; flushes on size/interval or when force=True."""
|
||||
try:
|
||||
with _arch_lock:
|
||||
ts = datetime.now(timezone.utc).isoformat()
|
||||
for t in (orders_rows or []):
|
||||
_arch_buf["cdc"].append(_order_to_cdc(t, ts))
|
||||
_arch_buf["curated"].append(_order_to_curated(t, ts))
|
||||
_arch_stats["rows"] += 1
|
||||
for t in (hr_rows or []):
|
||||
_arch_buf["cdc"].append(_hr_to_cdc(t, ts)); _arch_stats["rows"] += 1
|
||||
for d in (supply_docs or []):
|
||||
_arch_buf["cdc"].append(_supply_to_cdc(d, ts)); _arch_stats["rows"] += 1
|
||||
for t in (tel_rows or []):
|
||||
_arch_buf["cdc"].append(_tel_to_cdc(t, ts)); _arch_stats["rows"] += 1
|
||||
if _arch_since["t"] == 0.0:
|
||||
_arch_since["t"] = time.time()
|
||||
buffered = len(_arch_buf["cdc"]) + len(_arch_buf["curated"])
|
||||
age = time.time() - _arch_since["t"]
|
||||
if force or buffered >= S3_FLUSH_SIZE or age >= S3_FLUSH_INTERVAL:
|
||||
return _flush_locked()
|
||||
except Exception as exc: # never break the generator on an S3 hiccup
|
||||
return {"error": str(exc)}
|
||||
return None
|
||||
|
||||
|
||||
def put_object_bytes(key: str, body: bytes, content_type: str = "application/octet-stream",
|
||||
bucket: str | None = None) -> dict[str, Any]:
|
||||
"""Write raw bytes to S3 (used by the ETL offload agent for Parquet parts).
|
||||
Tracks last-write + activity so the storage dashboard reflects it live."""
|
||||
b = bucket or S3_ARCHIVE_BUCKET
|
||||
s3 = _client()
|
||||
_put(s3, b, key, body, content_type)
|
||||
return {"ok": True, "bucket": b, "key": key, "bytes": len(body)}
|
||||
|
||||
|
||||
def archive_active(window_s: float = 25.0) -> bool:
|
||||
"""True if the pipeline wrote to S3 recently — drives the kafka→S3 edge pulse."""
|
||||
return (time.time() - float(_last_write.get("mono") or 0.0)) < window_s
|
||||
|
||||
|
||||
def archive_info() -> dict[str, Any]:
|
||||
return {"last_write": dict(_last_write) or None, "objects": _arch_stats["objects"],
|
||||
"bytes": _arch_stats["bytes"], "rows": _arch_stats["rows"]}
|
||||
|
||||
|
||||
def _client():
|
||||
return boto3.client(
|
||||
@@ -494,5 +639,18 @@ async def analytics(refresh: bool = Query(False)):
|
||||
_analytics_cache["ts"] = now
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502)
|
||||
|
||||
# Overlay the live last-write so the dashboard reflects pipeline writes
|
||||
# immediately, without waiting for the (bounded, cached) full rescan.
|
||||
summary = dict(data.get("summary") or {})
|
||||
recent = list(data.get("recent") or [])
|
||||
lw = dict(_last_write)
|
||||
if lw.get("ts"):
|
||||
if not summary.get("newest") or lw["ts"] > summary["newest"]:
|
||||
summary["newest"] = lw["ts"]
|
||||
recent = ([{"modified": lw["ts"], "bucket": lw.get("bucket"), "key": lw.get("key"),
|
||||
"bytes": lw.get("bytes", 0), "size_human": _human_size(lw.get("bytes", 0))}]
|
||||
+ [r for r in recent if r.get("key") != lw.get("key")])[:15]
|
||||
return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"activity": _activity_view(), **data}
|
||||
"activity": _activity_view(), **data, "summary": summary, "recent": recent,
|
||||
"archive": archive_info()}
|
||||
|
||||
Reference in New Issue
Block a user