feat: federated query spans all 5 databases (not just 3)
The marquee panel only joined 3 region-keyed sources. Add a "one SQL across every database" reach matrix that fans a single Trino query out to PostgreSQL, MySQL, MongoDB, Cassandra and the Hadoop/Iceberg lake in one UNION ALL (telemetry has no region, so a per-source summary is used instead of a misleading join). - New MATRIX_SQL + concurrent execution alongside the region scorecard so total latency stays ~ the slower query - Federated tab shows the 5-source matrix (records + headline metric per engine) above the relabelled 3-source region scorecard
This commit is contained in:
+41
-2
@@ -68,6 +68,25 @@ MARQUEE_SQL = (
|
|||||||
"ORDER BY o.revenue DESC NULLS LAST"
|
"ORDER BY o.revenue DESC NULLS LAST"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Federated reach - ONE SQL touching every database/engine in the stack.
|
||||||
|
# Telemetry has no region dimension, so instead of a misleading join we
|
||||||
|
# summarise each source side-by-side in a single UNION ALL query.
|
||||||
|
MATRIX_SQL = (
|
||||||
|
"SELECT 1 ord, 'PostgreSQL' source, 'postgres_sales' catalog, 'public.sales_orders' dataset,\n"
|
||||||
|
" count(*) records, CAST(sum(amount) AS double) metric, 'total revenue' metric_label\n"
|
||||||
|
"FROM postgres_sales.public.sales_orders\n"
|
||||||
|
"UNION ALL SELECT 2,'MySQL','mysql_hr','hr.employee_events',count(*),\n"
|
||||||
|
" CAST(count(DISTINCT employee_id) AS double),'distinct employees' FROM mysql_hr.hr.employee_events\n"
|
||||||
|
"UNION ALL SELECT 3,'MongoDB','mongodb_supplychain','supplychain.events',count(*),\n"
|
||||||
|
" CAST(sum(amount) AS double),'event value' FROM mongodb_supplychain.supplychain.events\n"
|
||||||
|
"UNION ALL SELECT 4,'Cassandra','cassandra_telemetry','telemetry.device_metrics',count(*),\n"
|
||||||
|
" CAST(avg(metric_value) AS double),'avg metric value' FROM cassandra_telemetry.telemetry.device_metrics\n"
|
||||||
|
"UNION ALL SELECT 5,'Hadoop / Iceberg','iceberg','hadoop.orders_ext',count(*),\n"
|
||||||
|
" CAST(sum(amount) AS double),'lake revenue' FROM iceberg.hadoop.orders_ext\n"
|
||||||
|
"ORDER BY ord"
|
||||||
|
)
|
||||||
|
MATRIX_CATALOGS = ["postgres_sales", "mysql_hr", "mongodb_supplychain", "cassandra_telemetry", "iceberg"]
|
||||||
|
|
||||||
|
|
||||||
def _trino(sql: str, limit: int = 500) -> dict[str, Any]:
|
def _trino(sql: str, limit: int = 500) -> dict[str, Any]:
|
||||||
import sql_console as s
|
import sql_console as s
|
||||||
@@ -98,15 +117,35 @@ def _load_marquee() -> None:
|
|||||||
|
|
||||||
def _marquee_worker() -> None:
|
def _marquee_worker() -> None:
|
||||||
try:
|
try:
|
||||||
|
# Run the two federated queries concurrently so the total wait stays
|
||||||
|
# close to the slower of the two (region scorecard ~ matrix).
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
def timed(sql: str) -> tuple[dict, int]:
|
||||||
a = time.time()
|
a = time.time()
|
||||||
res = _trino(MARQUEE_SQL, 50)
|
r = _trino(sql, 50)
|
||||||
elapsed = int((time.time() - a) * 1000)
|
return r, int((time.time() - a) * 1000)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=2) as ex:
|
||||||
|
f_region = ex.submit(timed, MARQUEE_SQL)
|
||||||
|
f_matrix = ex.submit(timed, MATRIX_SQL)
|
||||||
|
res, elapsed = f_region.result()
|
||||||
|
mres, melapsed = f_matrix.result()
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
"ok": res.get("ok", False),
|
"ok": res.get("ok", False),
|
||||||
"sql": MARQUEE_SQL,
|
"sql": MARQUEE_SQL,
|
||||||
"catalogs": ["postgres_sales", "mysql_hr", "mongodb_supplychain"],
|
"catalogs": ["postgres_sales", "mysql_hr", "mongodb_supplychain"],
|
||||||
"elapsed_ms": elapsed,
|
"elapsed_ms": elapsed,
|
||||||
"rows": _rows_as_dicts(res) if res.get("ok") else [],
|
"rows": _rows_as_dicts(res) if res.get("ok") else [],
|
||||||
|
"matrix": {
|
||||||
|
"ok": mres.get("ok", False),
|
||||||
|
"sql": MATRIX_SQL,
|
||||||
|
"catalogs": MATRIX_CATALOGS,
|
||||||
|
"elapsed_ms": melapsed,
|
||||||
|
"rows": _rows_as_dicts(mres) if mres.get("ok") else [],
|
||||||
|
"error": None if mres.get("ok") else str(mres.get("error", ""))[:300],
|
||||||
|
},
|
||||||
"error": None if res.get("ok") else str(res.get("error", ""))[:300],
|
"error": None if res.get("ok") else str(res.get("error", ""))[:300],
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -189,6 +189,9 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
|
|||||||
|
|
||||||
const m = marquee?.marquee
|
const m = marquee?.marquee
|
||||||
const totals = catalogs?.source_totals || {}
|
const totals = catalogs?.source_totals || {}
|
||||||
|
const mxRows: any[] = m?.matrix?.rows || []
|
||||||
|
const mxMax = Math.max(1, ...mxRows.map((r) => Number(r.records) || 0))
|
||||||
|
const mxCatalogs: string[] = m?.matrix?.catalogs || ['postgres_sales', 'mysql_hr', 'mongodb_supplychain', 'cassandra_telemetry', 'iceberg']
|
||||||
|
|
||||||
const body = (
|
const body = (
|
||||||
<>
|
<>
|
||||||
@@ -218,7 +221,64 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
|
|||||||
</Panel>
|
</Panel>
|
||||||
|
|
||||||
<Panel
|
<Panel
|
||||||
title="Cross-source federated query — one SQL, three databases"
|
title="One SQL across every database"
|
||||||
|
subtitle={m?.matrix?.elapsed_ms != null ? `${mxCatalogs.length} databases · ${(m.matrix.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : `${mxCatalogs.length} databases`}
|
||||||
|
icon={Network}
|
||||||
|
>
|
||||||
|
<p className="mb-2 text-[10px] text-foreground-muted">
|
||||||
|
A single Trino query fanned out to <span className="text-docker">five engines at once</span> — relational, document, wide-column and the Hadoop lakehouse — no copies, no ETL.
|
||||||
|
</p>
|
||||||
|
<div className="mb-2 flex flex-wrap items-center gap-1.5">
|
||||||
|
{mxCatalogs.map((c) => (
|
||||||
|
<span key={c} className="rounded-full bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">{c}</span>
|
||||||
|
))}
|
||||||
|
<button type="button" onClick={refreshMarquee} className="ml-auto inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
|
||||||
|
{marquee?.running ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} re-run
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<pre className="mb-2 overflow-x-auto rounded-md border border-border bg-surface p-2 font-mono text-[9px] leading-relaxed text-foreground-muted">{m?.matrix?.sql}</pre>
|
||||||
|
{mxRows.length ? (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-[10px]">
|
||||||
|
<thead>
|
||||||
|
<tr className="border-b border-border text-left text-foreground-muted">
|
||||||
|
<th className="py-1 pr-3">Source</th>
|
||||||
|
<th className="py-1 pr-3">Catalog · dataset</th>
|
||||||
|
<th className="py-1 pr-3">Records</th>
|
||||||
|
<th className="py-1 pr-3 text-right">Headline metric</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{mxRows.map((r, i) => {
|
||||||
|
const isMoney = /revenue|value/i.test(r.metric_label || '')
|
||||||
|
const pct = Math.max(3, (Number(r.records) / mxMax) * 100)
|
||||||
|
const color = COLORS[i % COLORS.length]
|
||||||
|
return (
|
||||||
|
<tr key={i} className="border-b border-border/40">
|
||||||
|
<td className="py-1 pr-3 font-medium text-foreground">
|
||||||
|
<span className="inline-flex items-center gap-1.5"><span className="h-2 w-2 rounded-full" style={{ backgroundColor: color }} />{r.source}</span>
|
||||||
|
</td>
|
||||||
|
<td className="py-1 pr-3 font-mono text-foreground-muted">{r.catalog}<span className="text-foreground-faint"> · {r.dataset}</span></td>
|
||||||
|
<td className="py-1 pr-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<div className="relative h-2.5 w-24 overflow-hidden rounded bg-surface-overlay"><div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: color }} /></div>
|
||||||
|
<span className="font-mono text-foreground">{fmtNum(r.records)}</span>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td className="py-1 pr-3 text-right font-mono text-emerald-400">{isMoney ? fmtMoney(r.metric) : fmtNum(r.metric)} <span className="text-foreground-faint">{r.metric_label}</span></td>
|
||||||
|
</tr>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<p className="py-4 text-center text-[10px] text-foreground-faint">{marquee?.running ? <><Loader2 className="mr-1 inline h-3 w-3 animate-spin" /> Federating across all databases…</> : 'No result yet — click re-run.'}</p>
|
||||||
|
)}
|
||||||
|
</Panel>
|
||||||
|
|
||||||
|
<Panel
|
||||||
|
title="Region scorecard — live join across 3 OLTP sources"
|
||||||
subtitle={m?.elapsed_ms != null ? `${(m.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : ''}
|
subtitle={m?.elapsed_ms != null ? `${(m.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : ''}
|
||||||
icon={Network}
|
icon={Network}
|
||||||
>
|
>
|
||||||
|
|||||||
Reference in New Issue
Block a user