feat: realtime ETL offload to S3 + live business dashboards
- etl_offload.py: autonomous agent backfills/tails source DBs (PG/MySQL/ Mongo/Cassandra) to S3 as Parquet in small chunks, accumulates a live federated business matrix (/api/etl/status, /api/etl/business, /run, /config). - storage_s3.py: buffer generated CDC + masked curated rows to S3, overlay live last-write into analytics; put_object_bytes for Parquet parts. - trino_federated.py: capture generated rows + archive to S3; generator_active. - dataflow.py: pulse generate + kafka/spark->S3 archive edges when active. - StorageView: realtime ETL ingest panel; TrinoFederationView: realtime business KPIs/charts from /api/etl/business. - ChangesView: top KPIs/charts now overlay the live WS stream on server stats so they update in lock-step with the bottom feed; faster 2.5s refresh. - useCommandCenter: retain 800 live CDC changes.
This commit is contained in:
mo
+44
-11
@@ -453,25 +453,30 @@ _TEL_INSERT_TPL = ("INSERT INTO {ks}.device_metrics "
|
||||
def _gen_orders(n: int):
|
||||
rows, by_r, by_s, val = _order_rows(n)
|
||||
_gen_pg().cursor().executemany(_PG_INSERT, rows)
|
||||
return by_r, by_s, val
|
||||
return rows, by_r, by_s, val
|
||||
|
||||
|
||||
def _gen_hr(n: int):
|
||||
_gen_mysql().cursor().executemany(_MYSQL_INSERT, _hr_rows(n))
|
||||
rows = _hr_rows(n)
|
||||
_gen_mysql().cursor().executemany(_MYSQL_INSERT, rows)
|
||||
return rows
|
||||
|
||||
|
||||
def _gen_supply(n: int):
|
||||
docs = _supply_docs(n)
|
||||
if docs:
|
||||
_gen_mongo()["events"].insert_many(docs)
|
||||
_gen_mongo()["events"].insert_many([dict(d) for d in docs])
|
||||
return docs
|
||||
|
||||
|
||||
def _gen_tel(n: int):
|
||||
import sql_console as s
|
||||
sess = _gen_cass()
|
||||
cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
|
||||
for row in _tel_rows(n):
|
||||
rows = _tel_rows(n)
|
||||
for row in rows:
|
||||
sess.execute(cql, row)
|
||||
return rows
|
||||
|
||||
|
||||
def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any]:
|
||||
@@ -482,10 +487,15 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
|
||||
by_r: dict[str, int] = {}
|
||||
by_s: dict[str, int] = {}
|
||||
val = 0.0
|
||||
order_built: list = []
|
||||
hr_built: list = []
|
||||
supply_built: list = []
|
||||
tel_built: list = []
|
||||
if orders > 0:
|
||||
try:
|
||||
import psycopg2
|
||||
rows, by_r, by_s, val = _order_rows(orders)
|
||||
order_built = rows
|
||||
c = psycopg2.connect(host=s.DB_HOST, port=s.PG_PORT, user=s.PG_USER, password=s.PG_PASS, dbname=s.PG_DB, connect_timeout=8)
|
||||
try:
|
||||
c.autocommit = True
|
||||
@@ -498,9 +508,10 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
|
||||
if hr > 0:
|
||||
try:
|
||||
import pymysql
|
||||
hr_built = _hr_rows(hr)
|
||||
c = pymysql.connect(host=s.DB_HOST, port=s.MYSQL_PORT, user=s.MYSQL_USER, password=s.MYSQL_PASS, database=s.MYSQL_DB, connect_timeout=8, autocommit=True)
|
||||
try:
|
||||
c.cursor().executemany(_MYSQL_INSERT, _hr_rows(hr))
|
||||
c.cursor().executemany(_MYSQL_INSERT, hr_built)
|
||||
out["hr_events"] = hr
|
||||
finally:
|
||||
c.close()
|
||||
@@ -508,9 +519,10 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
|
||||
pass
|
||||
if supply > 0:
|
||||
try:
|
||||
supply_built = _supply_docs(supply)
|
||||
cli = s._mongo_client()
|
||||
try:
|
||||
cli[s.MONGO_DB]["events"].insert_many(_supply_docs(supply))
|
||||
cli[s.MONGO_DB]["events"].insert_many([dict(d) for d in supply_built])
|
||||
out["supply_events"] = supply
|
||||
finally:
|
||||
cli.close()
|
||||
@@ -518,17 +530,28 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
|
||||
pass
|
||||
if tel > 0:
|
||||
try:
|
||||
tel_built = _tel_rows(tel)
|
||||
cluster = s._cass_cluster()
|
||||
sess = cluster.connect()
|
||||
try:
|
||||
cql = _TEL_INSERT_TPL.format(ks=s.CASS_KS)
|
||||
for row in _tel_rows(tel):
|
||||
for row in tel_built:
|
||||
sess.execute(cql, row)
|
||||
out["telemetry"] = tel
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
# Land the generated batch in S3 through the pipeline stages (Kafka→S3 CDC
|
||||
# archive + Spark→S3 curated masked) so Object Storage reflects it at once.
|
||||
try:
|
||||
from storage_s3 import archive_generated_batch
|
||||
archive_generated_batch(order_built if out["orders"] else [],
|
||||
hr_built if out["hr_events"] else [],
|
||||
supply_built if out["supply_events"] else [],
|
||||
tel_built if out["telemetry"] else [], force=True)
|
||||
except Exception:
|
||||
pass
|
||||
# fold into the live counters + feed so the dashboard reflects it instantly
|
||||
with _gen_lock:
|
||||
c = _GEN["counts"]
|
||||
@@ -561,22 +584,32 @@ def _gen_tick():
|
||||
by_r: dict[str, int] = {}
|
||||
by_s: dict[str, int] = {}
|
||||
val = 0.0
|
||||
order_built: list = []
|
||||
hr_built: list = []
|
||||
supply_built: list = []
|
||||
tel_built: list = []
|
||||
try:
|
||||
by_r, by_s, val = _gen_orders(no)
|
||||
order_built, by_r, by_s, val = _gen_orders(no)
|
||||
except Exception:
|
||||
_gen_reset("pg"); no = 0
|
||||
try:
|
||||
_gen_hr(nh)
|
||||
hr_built = _gen_hr(nh)
|
||||
except Exception:
|
||||
_gen_reset("mysql"); nh = 0
|
||||
try:
|
||||
_gen_supply(ns)
|
||||
supply_built = _gen_supply(ns)
|
||||
except Exception:
|
||||
_gen_reset("mongo"); ns = 0
|
||||
try:
|
||||
_gen_tel(nt)
|
||||
tel_built = _gen_tel(nt)
|
||||
except Exception:
|
||||
_gen_reset("cass"); nt = 0
|
||||
# stream the batch into S3 (buffered like a Kafka-Connect S3 sink)
|
||||
try:
|
||||
from storage_s3 import archive_generated_batch
|
||||
archive_generated_batch(order_built, hr_built, supply_built, tel_built, force=False)
|
||||
except Exception:
|
||||
pass
|
||||
with _gen_lock:
|
||||
c = _GEN["counts"]
|
||||
c["orders"] += no
|
||||
|
||||
Reference in New Issue
Block a user