feat(hadoop): real Hive engine + filterable, detailed analytics with lineage
Deploys Apache Hive 3.1.3 (Derby metastore + external table over the HDFS historical CSV, MapReduce exec) on the Hadoop master, so the engine comparison shows a REAL measured Hive latency (~5.2s) next to live Trino (~0.3s); Impala stays clearly-labelled representative. The API re-measures Hive over SSH on a 30-min TTL (cached + persisted, with a committed seed). Adds filters (year/region/category/channel), a region×category heatmap, Trino exec stats, and a "where is this data read from" lineage panel (Trino->S3/Iceberg/Parquet with snapshot+files, Hive->HDFS/CSV with namenode+files). Mounts host SSH key read-only into the api container for the live Hive benchmark.
This commit is contained in:
mo
+1
-1
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py .
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py hive_bench_seed.json .
|
||||
RUN mkdir -p /data
|
||||
ENV DATABASE_URL=sqlite:////data/atc-agents.db
|
||||
EXPOSE 3201
|
||||
|
||||
+282
-110
@@ -1,67 +1,113 @@
|
||||
"""Hadoop lakehouse analytics for the Command Center 'Hadoop' tab.
|
||||
|
||||
Runs live analytical queries on the curated lakehouse table
|
||||
(iceberg.hadoop.historical_sales, stored as Parquet on Dell ECS S3) via Trino,
|
||||
and exposes a query-engine comparison: Trino is measured live, while Impala and
|
||||
Hive are shown as clearly-labelled *representative* figures (those engines are
|
||||
not deployed on this platform). Results are cached briefly so the UI stays snappy.
|
||||
- /api/hadoop/filters -> available filter values (years/regions/categories/channels)
|
||||
- /api/hadoop/analytics -> live, filterable KPIs + breakdowns via Trino (with exec stats)
|
||||
- /api/hadoop/source -> data lineage: exactly where each engine reads from
|
||||
- /api/hadoop/engines -> engine comparison. Trino is measured live; Hive is a REAL
|
||||
measurement (Apache Hive 3.1.3, MapReduce) refreshed over SSH
|
||||
from the cluster; Impala is a clearly-labelled representative.
|
||||
|
||||
Trino reads the curated table iceberg.hadoop.historical_sales (Parquet on Dell ECS S3);
|
||||
Hive reads lake.historical_sales (CSV on HDFS). Both ~25k rows -> fair latency contrast.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
import os
|
||||
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
TABLE = "iceberg.hadoop.historical_sales"
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")
|
||||
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870")
|
||||
|
||||
HIVE_SSH_HOST = os.getenv("HIVE_SSH_HOST", "10.0.21.61")
|
||||
HIVE_SSH_USER = os.getenv("HIVE_SSH_USER", "root")
|
||||
HIVE_BENCH_CMD = os.getenv("HIVE_BENCH_CMD", "/opt/hive/bench.sh")
|
||||
HIVE_BENCH_TTL = float(os.getenv("HIVE_BENCH_TTL", "1800")) # 30 min
|
||||
_SEED = Path(__file__).with_name("hive_bench_seed.json")
|
||||
_PERSIST = Path("/data/hive_bench.json")
|
||||
|
||||
router = APIRouter(prefix="/api/hadoop", tags=["hadoop"])
|
||||
|
||||
_cache: dict[str, Any] = {}
|
||||
_TTL = 60.0 # seconds
|
||||
_TTL = 45.0
|
||||
|
||||
# dimension -> Iceberg column
|
||||
DIMS = {"years": "order_year", "regions": "region", "categories": "product_category", "channels": "channel"}
|
||||
|
||||
|
||||
async def _trino(sql: str, deadline_s: float = 30.0) -> list[list[Any]]:
|
||||
"""Execute a Trino statement and return all rows (list of lists)."""
|
||||
headers = {
|
||||
"X-Trino-User": TRINO_USER,
|
||||
"X-Trino-Catalog": "iceberg",
|
||||
"X-Trino-Schema": "hadoop",
|
||||
}
|
||||
# --------------------------------------------------------------------------- Trino
|
||||
async def _trino(sql: str, deadline_s: float = 30.0) -> dict[str, Any]:
|
||||
headers = {"X-Trino-User": TRINO_USER, "X-Trino-Catalog": "iceberg", "X-Trino-Schema": "hadoop"}
|
||||
rows: list[list[Any]] = []
|
||||
cols: list[str] | None = None
|
||||
stats: dict[str, Any] = {}
|
||||
start = time.monotonic()
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=headers)
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
while True:
|
||||
err = payload.get("error")
|
||||
if err:
|
||||
raise RuntimeError(err.get("message", str(err)))
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(payload["error"].get("message", str(payload["error"])))
|
||||
if payload.get("columns") and cols is None:
|
||||
cols = [c["name"] for c in payload["columns"]]
|
||||
rows.extend(payload.get("data", []) or [])
|
||||
if payload.get("stats"):
|
||||
stats = payload["stats"]
|
||||
nxt = payload.get("nextUri")
|
||||
if not nxt:
|
||||
break
|
||||
if time.monotonic() - start > deadline_s:
|
||||
raise TimeoutError("Trino query exceeded deadline")
|
||||
await asyncio.sleep(0.05)
|
||||
await asyncio.sleep(0.04)
|
||||
rr = await client.get(nxt, headers={"X-Trino-User": TRINO_USER})
|
||||
rr.raise_for_status()
|
||||
payload = rr.json()
|
||||
return rows
|
||||
return {"rows": rows, "columns": cols or [], "stats": stats}
|
||||
|
||||
|
||||
def _cached(key: str):
|
||||
def _trino_stats(stats: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"processed_rows": stats.get("processedRows"),
|
||||
"processed_bytes": stats.get("processedBytes"),
|
||||
"elapsed_ms": stats.get("elapsedTimeMillis"),
|
||||
"cpu_ms": stats.get("cpuTimeMillis"),
|
||||
"peak_memory_bytes": stats.get("peakMemoryBytes"),
|
||||
"splits": stats.get("totalSplits"),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- filters
|
||||
def _parse(v: str | None) -> list[str]:
|
||||
return [x.strip() for x in (v or "").split(",") if x.strip()]
|
||||
|
||||
|
||||
def _where(years, regions, categories, channels) -> str:
|
||||
clauses = []
|
||||
yrs = [y for y in years if y.lstrip("-").isdigit()]
|
||||
if yrs:
|
||||
clauses.append(f"order_year IN ({','.join(yrs)})")
|
||||
for vals, col in ((regions, "region"), (categories, "product_category"), (channels, "channel")):
|
||||
if vals:
|
||||
esc = ",".join("'" + v.replace("'", "''") + "'" for v in vals)
|
||||
clauses.append(f"{col} IN ({esc})")
|
||||
return ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
|
||||
|
||||
def _cached(key: str, ttl: float = _TTL):
|
||||
item = _cache.get(key)
|
||||
if item and (time.time() - item["ts"] < _TTL):
|
||||
if item and (time.time() - item["ts"] < ttl):
|
||||
return item["data"]
|
||||
return None
|
||||
|
||||
@@ -71,42 +117,62 @@ def _store(key: str, data: Any):
|
||||
return data
|
||||
|
||||
|
||||
@router.get("/filters")
|
||||
async def filters():
|
||||
cached = _cached("filters", ttl=300)
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
try:
|
||||
out = {}
|
||||
q = await asyncio.gather(
|
||||
_trino(f"SELECT DISTINCT order_year FROM {TABLE} ORDER BY 1"),
|
||||
_trino(f"SELECT DISTINCT region FROM {TABLE} ORDER BY 1"),
|
||||
_trino(f"SELECT DISTINCT product_category FROM {TABLE} ORDER BY 1"),
|
||||
_trino(f"SELECT DISTINCT channel FROM {TABLE} ORDER BY 1"),
|
||||
)
|
||||
out = {
|
||||
"ok": True,
|
||||
"years": [r[0] for r in q[0]["rows"]],
|
||||
"regions": [r[0] for r in q[1]["rows"]],
|
||||
"categories": [r[0] for r in q[2]["rows"]],
|
||||
"channels": [r[0] for r in q[3]["rows"]],
|
||||
}
|
||||
return JSONResponse(_store("filters", out))
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@router.get("/analytics")
|
||||
async def analytics():
|
||||
cached = _cached("analytics")
|
||||
async def analytics(
|
||||
years: str | None = Query(None),
|
||||
regions: str | None = Query(None),
|
||||
categories: str | None = Query(None),
|
||||
channels: str | None = Query(None),
|
||||
):
|
||||
where = _where(_parse(years), _parse(regions), _parse(categories), _parse(channels))
|
||||
key = "an:" + where
|
||||
cached = _cached(key)
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
try:
|
||||
kpis_q = (
|
||||
f"SELECT count(*) AS orders, sum(amount) AS revenue, avg(amount) AS aov, "
|
||||
f"sum(quantity) AS units, count(DISTINCT region) AS regions, "
|
||||
f"min(order_year) AS min_y, max(order_year) AS max_y FROM {TABLE}"
|
||||
f"SELECT count(*), sum(amount), avg(amount), sum(quantity), count(DISTINCT region), "
|
||||
f"min(order_year), max(order_year), avg(unit_price) FROM {TABLE} {where}"
|
||||
)
|
||||
by_year_q = (
|
||||
f"SELECT order_year, sum(amount) AS revenue, count(*) AS orders "
|
||||
f"FROM {TABLE} GROUP BY order_year ORDER BY order_year"
|
||||
kpi_res = await _trino(kpis_q)
|
||||
k = kpi_res["rows"][0] if kpi_res["rows"] else [0] * 8
|
||||
by_year, by_region, by_category, by_channel, by_status = await asyncio.gather(
|
||||
_trino(f"SELECT order_year, sum(amount), count(*) FROM {TABLE} {where} GROUP BY order_year ORDER BY order_year"),
|
||||
_trino(f"SELECT region, sum(amount), count(*) FROM {TABLE} {where} GROUP BY region ORDER BY 2 DESC"),
|
||||
_trino(f"SELECT product_category, sum(amount), count(*) FROM {TABLE} {where} GROUP BY product_category ORDER BY 2 DESC"),
|
||||
_trino(f"SELECT channel, sum(amount), count(*) FROM {TABLE} {where} GROUP BY channel ORDER BY 2 DESC"),
|
||||
_trino(f"SELECT region, product_category, sum(amount) FROM {TABLE} {where} GROUP BY region, product_category ORDER BY 1,2"),
|
||||
)
|
||||
by_region_q = (
|
||||
f"SELECT region, sum(amount) AS revenue FROM {TABLE} "
|
||||
f"GROUP BY region ORDER BY revenue DESC"
|
||||
)
|
||||
by_category_q = (
|
||||
f"SELECT product_category, sum(amount) AS revenue FROM {TABLE} "
|
||||
f"GROUP BY product_category ORDER BY revenue DESC"
|
||||
)
|
||||
by_channel_q = (
|
||||
f"SELECT channel, sum(amount) AS revenue, count(*) AS orders FROM {TABLE} "
|
||||
f"GROUP BY channel ORDER BY revenue DESC"
|
||||
)
|
||||
kpis, by_year, by_region, by_category, by_channel = await asyncio.gather(
|
||||
_trino(kpis_q), _trino(by_year_q), _trino(by_region_q),
|
||||
_trino(by_category_q), _trino(by_channel_q),
|
||||
)
|
||||
k = kpis[0] if kpis else [0, 0, 0, 0, 0, None, None]
|
||||
data = {
|
||||
"ok": True,
|
||||
"table": TABLE,
|
||||
"location": "s3://data/hadoop/historical_sales (Iceberg/Parquet on Dell ECS)",
|
||||
"filtered": bool(where),
|
||||
"where": where,
|
||||
"kpis": {
|
||||
"orders": int(k[0] or 0),
|
||||
"revenue": float(k[1] or 0),
|
||||
@@ -115,76 +181,182 @@ async def analytics():
|
||||
"regions": int(k[4] or 0),
|
||||
"year_min": k[5],
|
||||
"year_max": k[6],
|
||||
"avg_unit_price": float(k[7] or 0),
|
||||
},
|
||||
"by_year": [{"year": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_year],
|
||||
"by_region": [{"region": r[0], "revenue": float(r[1] or 0)} for r in by_region],
|
||||
"by_category": [{"category": r[0], "revenue": float(r[1] or 0)} for r in by_category],
|
||||
"by_channel": [{"channel": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_channel],
|
||||
"by_year": [{"year": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_year["rows"]],
|
||||
"by_region": [{"region": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_region["rows"]],
|
||||
"by_category": [{"category": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_category["rows"]],
|
||||
"by_channel": [{"channel": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_channel["rows"]],
|
||||
"matrix": [{"region": r[0], "category": r[1], "revenue": float(r[2] or 0)} for r in by_status["rows"]],
|
||||
"trino_stats": _trino_stats(kpi_res["stats"]),
|
||||
}
|
||||
return JSONResponse(_store("analytics", data))
|
||||
return JSONResponse(_store(key, data))
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=200)
|
||||
return JSONResponse({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@router.get("/source")
|
||||
async def source():
|
||||
cached = _cached("source", ttl=120)
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
trino_src: dict[str, Any] = {
|
||||
"engine": "Trino",
|
||||
"catalog": "iceberg",
|
||||
"schema": "hadoop",
|
||||
"table": "historical_sales",
|
||||
"format": "Apache Iceberg · Parquet",
|
||||
"storage": "Dell ECS S3 (object store)",
|
||||
"location": "s3://data/hadoop/historical_sales",
|
||||
"s3_endpoint": S3_ENDPOINT,
|
||||
"metastore": "Iceberg (file metadata on S3)",
|
||||
}
|
||||
try:
|
||||
f = await _trino('SELECT count(*), coalesce(sum(file_size_in_bytes),0) FROM iceberg.hadoop."historical_sales$files"')
|
||||
trino_src["data_files"] = int(f["rows"][0][0]) if f["rows"] else None
|
||||
trino_src["data_bytes"] = int(f["rows"][0][1]) if f["rows"] else None
|
||||
s = await _trino('SELECT snapshot_id, committed_at FROM iceberg.hadoop."historical_sales$snapshots" ORDER BY committed_at DESC LIMIT 1')
|
||||
if s["rows"]:
|
||||
trino_src["snapshot_id"] = str(s["rows"][0][0])
|
||||
trino_src["snapshot_at"] = str(s["rows"][0][1])
|
||||
c = await _trino(f"SELECT count(*) FROM {TABLE}")
|
||||
trino_src["rows"] = int(c["rows"][0][0]) if c["rows"] else None
|
||||
except Exception as e:
|
||||
trino_src["error"] = str(e)
|
||||
|
||||
hv = _hive_cache["data"] or {}
|
||||
hive_src = {
|
||||
"engine": "Hive",
|
||||
"database": "lake",
|
||||
"table": "historical_sales",
|
||||
"format": hv.get("storage", "HDFS · CSV (TEXTFILE)"),
|
||||
"storage": "HDFS (Hadoop cluster)",
|
||||
"location": hv.get("hdfs_location", "hdfs://atc-hadoop-m01:8020/data/historical/sales_orders"),
|
||||
"namenode": HDFS_NN_URL,
|
||||
"metastore": "Hive Metastore (Derby)",
|
||||
"exec": hv.get("exec", "MapReduce"),
|
||||
"data_files": hv.get("hdfs_files"),
|
||||
"data_bytes": hv.get("hdfs_size_bytes"),
|
||||
"rows": hv.get("rows"),
|
||||
"measured_at": hv.get("measured_at"),
|
||||
}
|
||||
data = {"ok": True, "sources": [trino_src, hive_src], "hdfs_namenode": HDFS_NN_URL}
|
||||
return JSONResponse(_store("source", data))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- Hive (real, via SSH)
|
||||
def _load_seed() -> dict[str, Any]:
|
||||
for p in (_PERSIST, _SEED):
|
||||
try:
|
||||
if p.exists():
|
||||
return json.loads(p.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
_hive_cache: dict[str, Any] = {"data": _load_seed(), "ts": 0.0, "refreshing": False}
|
||||
|
||||
|
||||
def _ssh_hive_bench() -> dict[str, Any]:
|
||||
import paramiko
|
||||
|
||||
cli = paramiko.SSHClient()
|
||||
cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
cli.connect(HIVE_SSH_HOST, username=HIVE_SSH_USER, timeout=10, look_for_keys=True, allow_agent=False)
|
||||
try:
|
||||
_stdin, stdout, _stderr = cli.exec_command(HIVE_BENCH_CMD, timeout=120)
|
||||
out = stdout.read().decode("utf-8", "replace").strip()
|
||||
finally:
|
||||
cli.close()
|
||||
line = [ln for ln in out.splitlines() if ln.strip().startswith("{")]
|
||||
return json.loads(line[-1]) if line else {}
|
||||
|
||||
|
||||
async def _refresh_hive() -> None:
|
||||
if _hive_cache["refreshing"]:
|
||||
return
|
||||
_hive_cache["refreshing"] = True
|
||||
try:
|
||||
res = await asyncio.to_thread(_ssh_hive_bench)
|
||||
if res and res.get("ok"):
|
||||
_hive_cache["data"] = res
|
||||
_hive_cache["ts"] = time.time()
|
||||
try:
|
||||
_PERSIST.write_text(json.dumps(res))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
_hive_cache["refreshing"] = False
|
||||
|
||||
|
||||
@router.get("/engines")
|
||||
async def engines():
|
||||
"""Live-measured Trino latency vs representative Impala/Hive figures."""
|
||||
cached = _cached("engines")
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
bench_sql = (
|
||||
f"SELECT region, product_category, sum(amount) AS revenue, "
|
||||
f"avg(unit_price) AS avg_price, count(*) AS n "
|
||||
f"FROM {TABLE} GROUP BY region, product_category ORDER BY revenue DESC"
|
||||
)
|
||||
# Trino measured live (same workload as Hive: revenue by region)
|
||||
bench_sql = f"SELECT region, sum(amount) AS revenue, count(*) AS n FROM {TABLE} GROUP BY region ORDER BY revenue DESC"
|
||||
trino_entry: dict[str, Any] = {"name": "Trino", "measured": True, "ok": False}
|
||||
try:
|
||||
# warm + measure (best of 2 to dampen JIT/scheduling noise)
|
||||
await _trino(bench_sql)
|
||||
await _trino(bench_sql) # warm
|
||||
t0 = time.monotonic()
|
||||
await _trino(bench_sql)
|
||||
res = await _trino(bench_sql)
|
||||
trino_ms = round((time.monotonic() - t0) * 1000)
|
||||
|
||||
cnt = await _trino(f"SELECT count(*) FROM {TABLE}")
|
||||
rows_scanned = int(cnt[0][0]) if cnt else 0
|
||||
|
||||
# Representative multipliers for an analytical aggregate over columnar
|
||||
# Parquet of this size. Impala (MPP C++ daemons) is in the same league
|
||||
# as Trino; Hive (Tez/MR batch) pays heavy job-startup cost.
|
||||
data = {
|
||||
st = _trino_stats(res["stats"])
|
||||
trino_entry.update({
|
||||
"ok": True,
|
||||
"benchmark_sql": bench_sql,
|
||||
"rows_scanned": rows_scanned,
|
||||
"measured_engine": "Trino",
|
||||
"engines": [
|
||||
{
|
||||
"name": "Trino",
|
||||
"measured": True,
|
||||
"latency_ms": trino_ms,
|
||||
"model": "MPP · in-memory pipelined",
|
||||
"storage": "Iceberg / Parquet on S3",
|
||||
"best_for": "Interactive federated SQL & lakehouse BI",
|
||||
"note": "Live query on iceberg.hadoop.historical_sales",
|
||||
},
|
||||
{
|
||||
"name": "Impala",
|
||||
"measured": False,
|
||||
"latency_ms": max(1, round(trino_ms * 0.9)),
|
||||
"model": "MPP · C++ daemons (LLVM codegen)",
|
||||
"storage": "Parquet on HDFS / S3 (HMS)",
|
||||
"best_for": "Low-latency interactive BI on Hadoop",
|
||||
"note": "Representative — no Impala daemon deployed",
|
||||
},
|
||||
{
|
||||
"name": "Hive",
|
||||
"measured": False,
|
||||
"latency_ms": max(1, round(trino_ms * 8)),
|
||||
"model": "Batch · Tez / MapReduce",
|
||||
"storage": "ORC / Parquet on HDFS (HMS)",
|
||||
"best_for": "Large ETL / batch transforms",
|
||||
"note": "Representative — no HiveServer2 deployed",
|
||||
},
|
||||
],
|
||||
}
|
||||
return JSONResponse(_store("engines", data))
|
||||
"latency_ms": trino_ms,
|
||||
"model": "MPP · in-memory pipelined",
|
||||
"storage": "Iceberg / Parquet on S3",
|
||||
"best_for": "Interactive federated SQL & lakehouse BI",
|
||||
"note": "Live query on iceberg.hadoop.historical_sales",
|
||||
"processed_rows": st["processed_rows"],
|
||||
"processed_bytes": st["processed_bytes"],
|
||||
"by_region": [{"region": r[0], "revenue": float(r[1] or 0), "n": int(r[2] or 0)} for r in res["rows"]],
|
||||
})
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=200)
|
||||
trino_entry["error"] = str(e)
|
||||
trino_ms = 0
|
||||
|
||||
# Hive: REAL measurement, cached + refreshed over SSH
|
||||
if time.time() - _hive_cache["ts"] > HIVE_BENCH_TTL:
|
||||
asyncio.create_task(_refresh_hive())
|
||||
hv = _hive_cache["data"] or {}
|
||||
hive_ms = int(hv.get("query_time_ms") or 0)
|
||||
hive_entry = {
|
||||
"name": "Hive",
|
||||
"measured": bool(hv.get("ok")),
|
||||
"ok": bool(hv.get("ok")),
|
||||
"latency_ms": hive_ms,
|
||||
"wall_ms": hv.get("wall_ms"),
|
||||
"model": f"Batch · {hv.get('exec', 'MapReduce')}",
|
||||
"storage": hv.get("storage", "HDFS · CSV (TEXTFILE)"),
|
||||
"best_for": "Large ETL / batch transforms",
|
||||
"note": f"Live measurement · Apache Hive {hv.get('version', '3.1.3')} on HDFS",
|
||||
"version": hv.get("version"),
|
||||
"measured_at": hv.get("measured_at"),
|
||||
"rows": hv.get("rows"),
|
||||
"by_region": hv.get("by_region", []),
|
||||
}
|
||||
|
||||
# Impala: representative (not deployed on this cluster)
|
||||
base = trino_ms or 300
|
||||
impala_entry = {
|
||||
"name": "Impala",
|
||||
"measured": False,
|
||||
"ok": True,
|
||||
"latency_ms": max(1, round(base * 0.9)),
|
||||
"model": "MPP · C++ daemons (LLVM codegen)",
|
||||
"storage": "Parquet on HDFS / S3 (HMS)",
|
||||
"best_for": "Low-latency interactive BI on Hadoop",
|
||||
"note": "Representative — Impala daemons not deployed on this cluster",
|
||||
}
|
||||
|
||||
data = {
|
||||
"ok": True,
|
||||
"benchmark": "Revenue by region over ~25k historical rows",
|
||||
"trino_sql": bench_sql,
|
||||
"hive_sql": hv.get("query"),
|
||||
"engines": [trino_entry, hive_entry, impala_entry],
|
||||
}
|
||||
return JSONResponse(data)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"engine":"Hive","ok":true,"attempts":1,"version":"3.1.3","exec":"MapReduce (local)","storage":"HDFS · CSV (TEXTFILE)","table":"lake.historical_sales","hdfs_location":"hdfs://atc-hadoop-m01:8020/data/historical/sales_orders","hdfs_size_bytes":2496372,"hdfs_files":5,"query":"SELECT region, round(sum(amount),2) AS revenue, count(*) AS n FROM lake.historical_sales GROUP BY region ORDER BY revenue DESC","query_time_ms":5340,"wall_ms":14440,"rows":25000,"measured_at":"2026-06-26T17:00:37Z","by_region":[{"region":"LATAM","revenue":2.549512032E7,"n":5050},{"region":"EMEA","revenue":2.525153502E7,"n":5047},{"region":"NA","revenue":2.516295049E7,"n":5021},{"region":"EU","revenue":2.489941114E7,"n":4976},{"region":"APAC","revenue":2.474656452E7,"n":4906}]}
|
||||
Reference in New Issue
Block a user