feat(hadoop): embedded lakehouse analytics + Impala/Hive vs Trino comparison
Adds an Analytics sub-tab to the Hadoop view with live KPIs and revenue breakdowns (by year/region/category/channel) queried from Trino over iceberg.hadoop.historical_sales, plus a query-engine comparison panel. Trino latency is measured live; Impala and Hive are shown as clearly-labelled representative figures (those engines are not deployed). New cached endpoints /api/hadoop/analytics and /api/hadoop/engines.
This commit is contained in:
mo
+1
-1
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py .
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py .
|
||||
RUN mkdir -p /data
|
||||
ENV DATABASE_URL=sqlite:////data/atc-agents.db
|
||||
EXPOSE 3201
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Hadoop lakehouse analytics for the Command Center 'Hadoop' tab.
|
||||
|
||||
Runs live analytical queries on the curated lakehouse table
|
||||
(iceberg.hadoop.historical_sales, stored as Parquet on Dell ECS S3) via Trino,
|
||||
and exposes a query-engine comparison: Trino is measured live, while Impala and
|
||||
Hive are shown as clearly-labelled *representative* figures (those engines are
|
||||
not deployed on this platform). Results are cached briefly so the UI stays snappy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
import os
|
||||
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
TABLE = "iceberg.hadoop.historical_sales"
|
||||
|
||||
router = APIRouter(prefix="/api/hadoop", tags=["hadoop"])
|
||||
|
||||
_cache: dict[str, Any] = {}
|
||||
_TTL = 60.0 # seconds
|
||||
|
||||
|
||||
async def _trino(sql: str, deadline_s: float = 30.0) -> list[list[Any]]:
|
||||
"""Execute a Trino statement and return all rows (list of lists)."""
|
||||
headers = {
|
||||
"X-Trino-User": TRINO_USER,
|
||||
"X-Trino-Catalog": "iceberg",
|
||||
"X-Trino-Schema": "hadoop",
|
||||
}
|
||||
rows: list[list[Any]] = []
|
||||
start = time.monotonic()
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=headers)
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
while True:
|
||||
err = payload.get("error")
|
||||
if err:
|
||||
raise RuntimeError(err.get("message", str(err)))
|
||||
rows.extend(payload.get("data", []) or [])
|
||||
nxt = payload.get("nextUri")
|
||||
if not nxt:
|
||||
break
|
||||
if time.monotonic() - start > deadline_s:
|
||||
raise TimeoutError("Trino query exceeded deadline")
|
||||
await asyncio.sleep(0.05)
|
||||
rr = await client.get(nxt, headers={"X-Trino-User": TRINO_USER})
|
||||
rr.raise_for_status()
|
||||
payload = rr.json()
|
||||
return rows
|
||||
|
||||
|
||||
def _cached(key: str):
|
||||
item = _cache.get(key)
|
||||
if item and (time.time() - item["ts"] < _TTL):
|
||||
return item["data"]
|
||||
return None
|
||||
|
||||
|
||||
def _store(key: str, data: Any):
|
||||
_cache[key] = {"data": data, "ts": time.time()}
|
||||
return data
|
||||
|
||||
|
||||
@router.get("/analytics")
|
||||
async def analytics():
|
||||
cached = _cached("analytics")
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
try:
|
||||
kpis_q = (
|
||||
f"SELECT count(*) AS orders, sum(amount) AS revenue, avg(amount) AS aov, "
|
||||
f"sum(quantity) AS units, count(DISTINCT region) AS regions, "
|
||||
f"min(order_year) AS min_y, max(order_year) AS max_y FROM {TABLE}"
|
||||
)
|
||||
by_year_q = (
|
||||
f"SELECT order_year, sum(amount) AS revenue, count(*) AS orders "
|
||||
f"FROM {TABLE} GROUP BY order_year ORDER BY order_year"
|
||||
)
|
||||
by_region_q = (
|
||||
f"SELECT region, sum(amount) AS revenue FROM {TABLE} "
|
||||
f"GROUP BY region ORDER BY revenue DESC"
|
||||
)
|
||||
by_category_q = (
|
||||
f"SELECT product_category, sum(amount) AS revenue FROM {TABLE} "
|
||||
f"GROUP BY product_category ORDER BY revenue DESC"
|
||||
)
|
||||
by_channel_q = (
|
||||
f"SELECT channel, sum(amount) AS revenue, count(*) AS orders FROM {TABLE} "
|
||||
f"GROUP BY channel ORDER BY revenue DESC"
|
||||
)
|
||||
kpis, by_year, by_region, by_category, by_channel = await asyncio.gather(
|
||||
_trino(kpis_q), _trino(by_year_q), _trino(by_region_q),
|
||||
_trino(by_category_q), _trino(by_channel_q),
|
||||
)
|
||||
k = kpis[0] if kpis else [0, 0, 0, 0, 0, None, None]
|
||||
data = {
|
||||
"ok": True,
|
||||
"table": TABLE,
|
||||
"location": "s3://data/hadoop/historical_sales (Iceberg/Parquet on Dell ECS)",
|
||||
"kpis": {
|
||||
"orders": int(k[0] or 0),
|
||||
"revenue": float(k[1] or 0),
|
||||
"aov": float(k[2] or 0),
|
||||
"units": int(k[3] or 0),
|
||||
"regions": int(k[4] or 0),
|
||||
"year_min": k[5],
|
||||
"year_max": k[6],
|
||||
},
|
||||
"by_year": [{"year": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_year],
|
||||
"by_region": [{"region": r[0], "revenue": float(r[1] or 0)} for r in by_region],
|
||||
"by_category": [{"category": r[0], "revenue": float(r[1] or 0)} for r in by_category],
|
||||
"by_channel": [{"channel": r[0], "revenue": float(r[1] or 0), "orders": int(r[2] or 0)} for r in by_channel],
|
||||
}
|
||||
return JSONResponse(_store("analytics", data))
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=200)
|
||||
|
||||
|
||||
@router.get("/engines")
|
||||
async def engines():
|
||||
"""Live-measured Trino latency vs representative Impala/Hive figures."""
|
||||
cached = _cached("engines")
|
||||
if cached is not None:
|
||||
return JSONResponse(cached)
|
||||
bench_sql = (
|
||||
f"SELECT region, product_category, sum(amount) AS revenue, "
|
||||
f"avg(unit_price) AS avg_price, count(*) AS n "
|
||||
f"FROM {TABLE} GROUP BY region, product_category ORDER BY revenue DESC"
|
||||
)
|
||||
try:
|
||||
# warm + measure (best of 2 to dampen JIT/scheduling noise)
|
||||
await _trino(bench_sql)
|
||||
t0 = time.monotonic()
|
||||
await _trino(bench_sql)
|
||||
trino_ms = round((time.monotonic() - t0) * 1000)
|
||||
|
||||
cnt = await _trino(f"SELECT count(*) FROM {TABLE}")
|
||||
rows_scanned = int(cnt[0][0]) if cnt else 0
|
||||
|
||||
# Representative multipliers for an analytical aggregate over columnar
|
||||
# Parquet of this size. Impala (MPP C++ daemons) is in the same league
|
||||
# as Trino; Hive (Tez/MR batch) pays heavy job-startup cost.
|
||||
data = {
|
||||
"ok": True,
|
||||
"benchmark_sql": bench_sql,
|
||||
"rows_scanned": rows_scanned,
|
||||
"measured_engine": "Trino",
|
||||
"engines": [
|
||||
{
|
||||
"name": "Trino",
|
||||
"measured": True,
|
||||
"latency_ms": trino_ms,
|
||||
"model": "MPP · in-memory pipelined",
|
||||
"storage": "Iceberg / Parquet on S3",
|
||||
"best_for": "Interactive federated SQL & lakehouse BI",
|
||||
"note": "Live query on iceberg.hadoop.historical_sales",
|
||||
},
|
||||
{
|
||||
"name": "Impala",
|
||||
"measured": False,
|
||||
"latency_ms": max(1, round(trino_ms * 0.9)),
|
||||
"model": "MPP · C++ daemons (LLVM codegen)",
|
||||
"storage": "Parquet on HDFS / S3 (HMS)",
|
||||
"best_for": "Low-latency interactive BI on Hadoop",
|
||||
"note": "Representative — no Impala daemon deployed",
|
||||
},
|
||||
{
|
||||
"name": "Hive",
|
||||
"measured": False,
|
||||
"latency_ms": max(1, round(trino_ms * 8)),
|
||||
"model": "Batch · Tez / MapReduce",
|
||||
"storage": "ORC / Parquet on HDFS (HMS)",
|
||||
"best_for": "Large ETL / batch transforms",
|
||||
"note": "Representative — no HiveServer2 deployed",
|
||||
},
|
||||
],
|
||||
}
|
||||
return JSONResponse(_store("engines", data))
|
||||
except Exception as e:
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=200)
|
||||
@@ -39,6 +39,7 @@ from presentation_static import get_static_deck, list_static_decks
|
||||
from storage_s3 import router as storage_s3_router
|
||||
from hdfs_api import router as hdfs_router
|
||||
from pipeline_ops import router as pipeline_router
|
||||
from hadoop_analytics import router as hadoop_router
|
||||
from elasticsearch_api import router as elasticsearch_router
|
||||
from sql_console import router as sql_router
|
||||
from ssh_terminal import ssh_session
|
||||
@@ -721,6 +722,7 @@ app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
|
||||
app.include_router(storage_s3_router)
|
||||
app.include_router(hdfs_router)
|
||||
app.include_router(pipeline_router)
|
||||
app.include_router(hadoop_router)
|
||||
app.include_router(elasticsearch_router)
|
||||
app.include_router(sql_router)
|
||||
app.add_middleware(
|
||||
|
||||
Reference in New Issue
Block a user