mo
96d490807a
Deploys Apache Hive 3.1.3 (Derby metastore + external table over the HDFS historical CSV, MapReduce exec) on the Hadoop master, so the engine comparison shows a REAL measured Hive latency (~5.2s) next to live Trino (~0.3s); Impala stays clearly-labelled representative. The API re-measures Hive over SSH on a 30-min TTL (cached + persisted, with a committed seed). Adds filters (year/region/category/channel), a region×category heatmap, Trino exec stats, and a "where is this data read from" lineage panel (Trino->S3/Iceberg/Parquet with snapshot+files, Hive->HDFS/CSV with namenode+files). Mounts host SSH key read-only into the api container for the live Hive benchmark.
60 lines
2.7 KiB
Bash
60 lines
2.7 KiB
Bash
#!/bin/bash
|
|
set -e
|
|
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
|
|
export HIVE_HOME=/opt/hive
|
|
# Bigtop hadoop: pick a HADOOP_HOME that contains bin/hadoop
|
|
if [ -x /usr/lib/hadoop/bin/hadoop ]; then export HADOOP_HOME=/usr/lib/hadoop; else export HADOOP_HOME=/usr; fi
|
|
export HADOOP_CONF_DIR=/etc/hadoop/conf
|
|
export PATH=$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
|
|
echo "JAVA_HOME=$JAVA_HOME HADOOP_HOME=$HADOOP_HOME"
|
|
|
|
echo "=== guava fix ==="
|
|
rm -f /opt/hive/lib/guava-19.0.jar
|
|
cp -f /usr/lib/hadoop/lib/guava-27.0-jre.jar /opt/hive/lib/ 2>/dev/null || cp -f /usr/lib/hadoop/client/guava-27.0-jre.jar /opt/hive/lib/
|
|
ls /opt/hive/lib/guava-*.jar
|
|
|
|
echo "=== hive-site.xml ==="
|
|
cat > /opt/hive/conf/hive-site.xml <<'XML'
|
|
<?xml version="1.0"?>
|
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
<configuration>
|
|
<property><name>javax.jdo.option.ConnectionURL</name><value>jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true</value></property>
|
|
<property><name>javax.jdo.option.ConnectionDriverName</name><value>org.apache.derby.jdbc.EmbeddedDriver</value></property>
|
|
<property><name>hive.metastore.warehouse.dir</name><value>/user/hive/warehouse</value></property>
|
|
<property><name>hive.execution.engine</name><value>mr</value></property>
|
|
<property><name>mapreduce.framework.name</name><value>local</value></property>
|
|
<property><name>hive.exec.mode.local.auto</name><value>true</value></property>
|
|
<property><name>hive.exec.submitviachild</name><value>false</value></property>
|
|
<property><name>hive.metastore.schema.verification</name><value>false</value></property>
|
|
<property><name>datanucleus.schema.autoCreateAll</name><value>true</value></property>
|
|
<property><name>hive.server2.enable.doAs</name><value>false</value></property>
|
|
<property><name>hive.stats.autogather</name><value>false</value></property>
|
|
<property><name>hive.metastore.event.db.notification.api.auth</name><value>false</value></property>
|
|
</configuration>
|
|
XML
|
|
|
|
cat > /opt/hive/conf/hive-env.sh <<ENV
|
|
export JAVA_HOME=$JAVA_HOME
|
|
export HADOOP_HOME=$HADOOP_HOME
|
|
export HADOOP_CONF_DIR=$HADOOP_CONF_DIR
|
|
export HIVE_HOME=$HIVE_HOME
|
|
ENV
|
|
|
|
# reusable wrapper for running hive non-interactively
|
|
cat > /opt/hive/runhive.sh <<WRAP
|
|
#!/bin/bash
|
|
export JAVA_HOME=$JAVA_HOME
|
|
export HIVE_HOME=$HIVE_HOME
|
|
export HADOOP_HOME=$HADOOP_HOME
|
|
export HADOOP_CONF_DIR=$HADOOP_CONF_DIR
|
|
export HADOOP_CLIENT_OPTS="-Xmx2g \$HADOOP_CLIENT_OPTS"
|
|
export PATH=\$JAVA_HOME/bin:\$HIVE_HOME/bin:\$PATH
|
|
exec hive "\$@"
|
|
WRAP
|
|
chmod +x /opt/hive/runhive.sh
|
|
|
|
echo "=== init derby metastore schema ==="
|
|
rm -rf /opt/hive/metastore_db
|
|
cd /opt/hive
|
|
$HIVE_HOME/bin/schematool -dbType derby -initSchema >/tmp/schematool.log 2>&1 && echo "schema init OK" || { echo "schema init FAILED"; tail -25 /tmp/schematool.log; exit 1; }
|