feat(hadoop): real Hive engine + filterable, detailed analytics with lineage
Deploys Apache Hive 3.1.3 (Derby metastore + external table over the HDFS historical CSV, MapReduce exec) on the Hadoop master, so the engine comparison shows a REAL measured Hive latency (~5.2s) next to live Trino (~0.3s); Impala stays clearly-labelled representative. The API re-measures Hive over SSH on a 30-min TTL (cached + persisted, with a committed seed). Adds filters (year/region/category/channel), a region×category heatmap, Trino exec stats, and a "where is this data read from" lineage panel (Trino->S3/Iceberg/Parquet with snapshot+files, Hive->HDFS/CSV with namenode+files). Mounts host SSH key read-only into the api container for the live Hive benchmark.
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Real Hive benchmark on the HDFS historical dataset. Emits one JSON line.
|
||||
# Retries because Hive's LocalJobRunner can flake on a cold first job.
|
||||
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
|
||||
export HIVE_HOME=/opt/hive
|
||||
export HADOOP_HOME=/usr/lib/hadoop
|
||||
export HADOOP_CONF_DIR=/opt/hive/hadoopconf
|
||||
export HADOOP_USER_NAME=hdfs
|
||||
export HADOOP_CLIENT_OPTS="-Xmx3g"
|
||||
export PATH=$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
|
||||
|
||||
Q="SELECT region, round(sum(amount),2) AS revenue, count(*) AS n FROM lake.historical_sales GROUP BY region ORDER BY revenue DESC"
|
||||
ISO=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
HSIZE=$(hdfs dfs -du -s /data/historical/sales_orders 2>/dev/null | awk '{print $1}'); [ -z "$HSIZE" ] && HSIZE=0
|
||||
HFILES=$(hdfs dfs -ls -R /data/historical/sales_orders 2>/dev/null | grep -c "part-"); [ -z "$HFILES" ] && HFILES=0
|
||||
|
||||
ERR=$(mktemp); OUT=$(mktemp); RC=1; WALL=0; TT=0
|
||||
for attempt in 1 2 3; do
|
||||
T0=$(date +%s%3N)
|
||||
$HIVE_HOME/bin/hive -e "$Q" >"$OUT" 2>"$ERR"
|
||||
RC=$?
|
||||
T1=$(date +%s%3N)
|
||||
WALL=$((T1-T0))
|
||||
if [ $RC -eq 0 ] && [ -s "$OUT" ]; then
|
||||
TT=$(grep -oE "Time taken: [0-9.]+ seconds" "$ERR" | grep -oE "[0-9.]+" | sort -nr | head -1)
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
[ -z "$TT" ] && TT=0
|
||||
QMS=$(awk "BEGIN{printf \"%d\", $TT*1000}")
|
||||
ROWS_JSON=$(awk -F'\t' 'NF>=3{printf "%s{\"region\":\"%s\",\"revenue\":%s,\"n\":%s}", (c++? ",":""), $1,$2,$3}' "$OUT")
|
||||
cat <<JSON
|
||||
{"engine":"Hive","ok":$([ $RC -eq 0 ] && echo true || echo false),"attempts":$attempt,"version":"3.1.3","exec":"MapReduce (local)","storage":"HDFS · CSV (TEXTFILE)","table":"lake.historical_sales","hdfs_location":"hdfs://atc-hadoop-m01:8020/data/historical/sales_orders","hdfs_size_bytes":$HSIZE,"hdfs_files":$HFILES,"query":"$Q","query_time_ms":$QMS,"wall_ms":$WALL,"rows":25000,"measured_at":"$ISO","by_region":[${ROWS_JSON}]}
|
||||
JSON
|
||||
rm -f "$ERR" "$OUT"
|
||||
Reference in New Issue
Block a user