feat: Data Explorer tab + manual data entry with live CDC
Add a dedicated business-data view (separate from infra/search): - New "Data Explorer" tab: KPIs (customers, employees, products, source row totals, sample revenue) + charts driven by Elasticsearch aggregations (revenue over time, by region/channel/status, top customers & products, HR by department/role/event, supply chain by type, telemetry averages), with region + free-text filters. - Backend /api/search/business endpoint: battery of ES aggregations with numeric/date index template (so amount sums and date histograms work), source totals via cheap planner estimates; biased the orders sample to rows carrying customer names so people are searchable/visible. Manual data entry on source systems: - "Insert row" action in the Data Hub browser opens a column-aware form; POST /api/sql/row/insert writes to Postgres/MySQL/Mongo/Cassandra/Neo4j. - Inserts into CDC sources (PG/MySQL/Mongo) are captured by Debezium and streamed to Kafka in real time; UI flags this and pulses the data flow.
This commit is contained in:
mo
+221
-3
@@ -39,6 +39,11 @@ BUSINESS_TABLES = {
|
||||
("mongodb_supplychain", "supplychain", "events"),
|
||||
("cassandra_telemetry", "telemetry", "device_metrics"),
|
||||
}
|
||||
# prefer rows that actually carry the business identifiers (names are sparse in raw tables)
|
||||
BUSINESS_WHERE = {
|
||||
("postgres_sales", "public", "sales_orders"): "customer_name IS NOT NULL",
|
||||
("mysql_hr", "hr", "employee_events"): "employee_name IS NOT NULL",
|
||||
}
|
||||
# de-duplicated business entities so users can search people/customers by name + id
|
||||
ENTITY_QUERIES = [
|
||||
(
|
||||
@@ -61,6 +66,41 @@ ENTITY_QUERIES = [
|
||||
),
|
||||
]
|
||||
|
||||
# business index names (federated source data + entity rollups)
|
||||
IDX_ORDERS = "atc-postgres-sales-public-sales-orders"
|
||||
IDX_HR = "atc-mysql-hr-hr-employee-events"
|
||||
IDX_SUPPLY = "atc-mongodb-supplychain-supplychain-events"
|
||||
IDX_TELEMETRY = "atc-cassandra-telemetry-telemetry-device-metrics"
|
||||
IDX_CUSTOMERS = "atc-customers"
|
||||
IDX_EMPLOYEES = "atc-employees"
|
||||
IDX_PRODUCTS = "atc-products"
|
||||
|
||||
# lenient multi-format date parsing for the timestamp columns in the data
|
||||
_DATE_FMT = (
|
||||
"yyyy-MM-dd HH:mm:ss.SSS zzz||yyyy-MM-dd HH:mm:ss zzz||"
|
||||
"yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||"
|
||||
"strict_date_optional_time||epoch_millis"
|
||||
)
|
||||
_NUM = {"type": "double", "ignore_malformed": True}
|
||||
_DATE = {"type": "date", "format": _DATE_FMT, "ignore_malformed": True}
|
||||
# index template so measures are numeric (sum/avg) and timestamps are real dates
|
||||
ATC_TEMPLATE = {
|
||||
"index_patterns": [f"{INDEX_PREFIX}*"],
|
||||
"priority": 200,
|
||||
"template": {
|
||||
"settings": {"number_of_replicas": 0, "refresh_interval": "5s"},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"amount": _NUM, "total_amount": _NUM, "salary_change": _NUM,
|
||||
"unit_price": _NUM, "metric_value": _NUM, "quantity": _NUM,
|
||||
"revenue": _NUM, "order_count": _NUM,
|
||||
"order_ts": _DATE, "event_ts": _DATE, "metric_ts": _DATE,
|
||||
"order_date": _DATE, "ts": _DATE,
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
|
||||
@@ -300,6 +340,154 @@ async def es_search(q: str = Query(..., min_length=1), size: int = Query(10, le=
|
||||
return await search_query({"q": q, "size": size})
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Business data overview (data abstraction — charts & filters over the real data)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
_biz_cache: dict[str, Any] = {}
|
||||
_BIZ_TTL = 30.0
|
||||
_totals_cache: dict[str, Any] = {"at": 0.0, "data": {}}
|
||||
_TOTALS_TTL = 600.0
|
||||
|
||||
|
||||
def _buckets(agg: dict, key: str, metric: str | None = None) -> list[dict]:
|
||||
out = []
|
||||
for b in (agg.get(key, {}) or {}).get("buckets", []):
|
||||
item = {"key": b.get("key_as_string", b.get("key")), "count": b.get("doc_count", 0)}
|
||||
if metric and metric in b:
|
||||
item["value"] = round(b[metric].get("value") or 0, 2)
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
async def _es_aggs(client: httpx.AsyncClient, index: str, filt: dict, aggs: dict) -> dict:
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{ELASTICSEARCH_URL}/{index}/_search",
|
||||
params={"ignore_unavailable": "true", "allow_no_indices": "true"},
|
||||
auth=_auth(),
|
||||
json={"size": 0, "query": filt, "aggs": aggs, "track_total_hits": True},
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
return {}
|
||||
return r.json()
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def _source_totals() -> dict:
|
||||
"""True row counts in the source systems via cheap planner estimates (cached)."""
|
||||
if time.time() - _totals_cache["at"] < _TOTALS_TTL and _totals_cache["data"]:
|
||||
return _totals_cache["data"]
|
||||
out: dict[str, Any] = {}
|
||||
try:
|
||||
import sql_console as s
|
||||
out["orders"] = s._table_row_count("postgres", "public.sales_orders")
|
||||
out["hr_events"] = s._table_row_count("mysql", "hr.employee_events")
|
||||
out["supply_events"] = s._table_row_count("mongodb", "supplychain.events")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
_totals_cache.update({"at": time.time(), "data": out})
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/business")
|
||||
async def business_overview(q: str = Query(""), region: str = Query("")):
|
||||
cache_key = f"{q}|{region}"
|
||||
cached = _biz_cache.get(cache_key)
|
||||
if cached and time.time() - cached["at"] < _BIZ_TTL:
|
||||
return cached["data"]
|
||||
|
||||
must: list[dict] = []
|
||||
if q.strip():
|
||||
must.append({"query_string": {"query": q, "lenient": True, "default_operator": "AND"}})
|
||||
region_must = must + ([{"term": {"region.keyword": region}}] if region.strip() else [])
|
||||
filt = {"bool": {"must": must or [{"match_all": {}}]}}
|
||||
rfilt = {"bool": {"must": region_must or [{"match_all": {}}]}}
|
||||
|
||||
async with httpx.AsyncClient(timeout=20.0, verify=False) as client:
|
||||
orders = await _es_aggs(client, IDX_ORDERS, rfilt, {
|
||||
"by_region": {"terms": {"field": "region.keyword", "size": 12}, "aggs": {"rev": {"sum": {"field": "amount"}}}},
|
||||
"by_channel": {"terms": {"field": "sales_channel.keyword", "size": 12}, "aggs": {"rev": {"sum": {"field": "amount"}}}},
|
||||
"by_status": {"terms": {"field": "order_status.keyword", "size": 12}},
|
||||
"top_customers": {"terms": {"field": "customer_name.keyword", "size": 10}, "aggs": {"spend": {"sum": {"field": "amount"}}}},
|
||||
"top_products": {"terms": {"field": "product_id", "size": 10}, "aggs": {"rev": {"sum": {"field": "amount"}}}},
|
||||
"over_time": {"date_histogram": {"field": "order_ts", "calendar_interval": "month", "min_doc_count": 0}, "aggs": {"rev": {"sum": {"field": "amount"}}}},
|
||||
"revenue": {"sum": {"field": "amount"}},
|
||||
"avg_order": {"avg": {"field": "amount"}},
|
||||
})
|
||||
hr = await _es_aggs(client, IDX_HR, rfilt, {
|
||||
"by_department": {"terms": {"field": "department.keyword", "size": 15}},
|
||||
"by_role": {"terms": {"field": "role_name.keyword", "size": 12}},
|
||||
"by_event": {"terms": {"field": "event_type.keyword", "size": 12}},
|
||||
})
|
||||
supply = await _es_aggs(client, IDX_SUPPLY, rfilt, {
|
||||
"by_type": {"terms": {"field": "type.keyword", "size": 15}},
|
||||
"by_region": {"terms": {"field": "region.keyword", "size": 12}},
|
||||
})
|
||||
telemetry = await _es_aggs(client, IDX_TELEMETRY, filt, {
|
||||
"by_metric": {"terms": {"field": "metric_type.keyword", "size": 12}, "aggs": {"avg": {"avg": {"field": "metric_value"}}}},
|
||||
})
|
||||
|
||||
async def _count(index: str, body_filt: dict) -> int:
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{ELASTICSEARCH_URL}/{index}/_count",
|
||||
params={"ignore_unavailable": "true", "allow_no_indices": "true"},
|
||||
auth=_auth(), json={"query": body_filt},
|
||||
)
|
||||
return int(r.json().get("count", 0)) if r.status_code < 400 else 0
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
|
||||
customers = await _count(IDX_CUSTOMERS, rfilt)
|
||||
employees = await _count(IDX_EMPLOYEES, rfilt)
|
||||
products = await _count(IDX_PRODUCTS, rfilt)
|
||||
|
||||
o_agg = orders.get("aggregations", {})
|
||||
hr_agg = hr.get("aggregations", {})
|
||||
sup_agg = supply.get("aggregations", {})
|
||||
tel_agg = telemetry.get("aggregations", {})
|
||||
orders_indexed = orders.get("hits", {}).get("total", {}).get("value", 0)
|
||||
|
||||
data = {
|
||||
"ok": True,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"filters": {"q": q, "region": region},
|
||||
"kpis": {
|
||||
"customers": customers,
|
||||
"employees": employees,
|
||||
"products": products,
|
||||
"orders_indexed": orders_indexed,
|
||||
"revenue_indexed": round((o_agg.get("revenue", {}) or {}).get("value") or 0, 2),
|
||||
"avg_order": round((o_agg.get("avg_order", {}) or {}).get("value") or 0, 2),
|
||||
"telemetry_indexed": telemetry.get("hits", {}).get("total", {}).get("value", 0),
|
||||
"supply_indexed": supply.get("hits", {}).get("total", {}).get("value", 0),
|
||||
"source_totals": _source_totals(),
|
||||
},
|
||||
"orders": {
|
||||
"by_region": _buckets(o_agg, "by_region", "rev"),
|
||||
"by_channel": _buckets(o_agg, "by_channel", "rev"),
|
||||
"by_status": _buckets(o_agg, "by_status"),
|
||||
"top_customers": _buckets(o_agg, "top_customers", "spend"),
|
||||
"top_products": _buckets(o_agg, "top_products", "rev"),
|
||||
"over_time": _buckets(o_agg, "over_time", "rev"),
|
||||
},
|
||||
"hr": {
|
||||
"by_department": _buckets(hr_agg, "by_department"),
|
||||
"by_role": _buckets(hr_agg, "by_role"),
|
||||
"by_event": _buckets(hr_agg, "by_event"),
|
||||
},
|
||||
"supply": {
|
||||
"by_type": _buckets(sup_agg, "by_type"),
|
||||
"by_region": _buckets(sup_agg, "by_region"),
|
||||
},
|
||||
"telemetry": {"by_metric": _buckets(tel_agg, "by_metric", "avg")},
|
||||
"regions": [b["key"] for b in _buckets(o_agg, "by_region")],
|
||||
}
|
||||
_biz_cache[cache_key] = {"at": time.time(), "data": data}
|
||||
return data
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Indexer (Trino-federated → ES atc-* indices)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
@@ -345,8 +533,9 @@ def _meta(catalog: str, schema: str, table: str, source: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _index_trino_table(sqlmod, client: httpx.Client, catalog: str, schema: str, table: str, limit: int = ROWS_PER_TABLE) -> int:
|
||||
res = sqlmod._run_trino(f'SELECT * FROM "{catalog}"."{schema}"."{table}" LIMIT {limit}', limit)
|
||||
def _index_trino_table(sqlmod, client: httpx.Client, catalog: str, schema: str, table: str, limit: int = ROWS_PER_TABLE, where: str | None = None) -> int:
|
||||
where_sql = f" WHERE {where}" if where else ""
|
||||
res = sqlmod._run_trino(f'SELECT * FROM "{catalog}"."{schema}"."{table}"{where_sql} LIMIT {limit}', limit)
|
||||
if not res.get("ok"):
|
||||
raise RuntimeError(res.get("error", "query failed")[:200])
|
||||
cols = res.get("columns", [])
|
||||
@@ -447,11 +636,39 @@ def _index_catalog(client: httpx.Client) -> int:
|
||||
return written
|
||||
|
||||
|
||||
def _ensure_template(client: httpx.Client) -> None:
|
||||
try:
|
||||
client.put(f"{ELASTICSEARCH_URL}/_index_template/atc-data", json=ATC_TEMPLATE)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_reindex_state["errors"].append(f"template: {str(exc)[:120]}")
|
||||
|
||||
|
||||
def _drop_atc_indices(client: httpx.Client) -> None:
|
||||
"""Delete existing atc-* indices so they are re-created with the new
|
||||
numeric/date mappings from the template (mappings can't be changed in place)."""
|
||||
try:
|
||||
r = client.get(f"{ELASTICSEARCH_URL}/_cat/indices/{INDEX_PREFIX}*?format=json&h=index")
|
||||
for it in (r.json() if r.status_code < 400 else []):
|
||||
nm = it.get("index")
|
||||
if nm:
|
||||
try:
|
||||
client.delete(f"{ELASTICSEARCH_URL}/{nm}")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_reindex_state["errors"].append(f"cleanup: {str(exc)[:120]}")
|
||||
|
||||
|
||||
def _reindex_worker():
|
||||
import sql_console as sqlmod
|
||||
try:
|
||||
client = _es_client(timeout=90.0)
|
||||
with client:
|
||||
# numeric/date mappings + fresh indices
|
||||
_reindex_state["current"] = "preparing mappings"
|
||||
_ensure_template(client)
|
||||
_drop_atc_indices(client)
|
||||
|
||||
# catalog/metadata
|
||||
_reindex_state["current"] = "catalog inventory"
|
||||
_reindex_state["total_docs"] += _index_catalog(client)
|
||||
@@ -484,7 +701,8 @@ def _reindex_worker():
|
||||
_reindex_state["current"] = f"{catalog}.{schema}.{table}"
|
||||
try:
|
||||
cap = ROWS_BUSINESS if (catalog, schema, table) in BUSINESS_TABLES or catalog == "iceberg" else ROWS_PER_TABLE
|
||||
n = _index_trino_table(sqlmod, client, catalog, schema, table, cap)
|
||||
where = BUSINESS_WHERE.get((catalog, schema, table))
|
||||
n = _index_trino_table(sqlmod, client, catalog, schema, table, cap, where)
|
||||
idx = _sanitize_index(catalog, schema, table)
|
||||
_reindex_state["indices"][idx] = n
|
||||
_reindex_state["total_docs"] += n
|
||||
|
||||
Reference in New Issue
Block a user