ClickHouse/benchmark/hardware.sh

128 lines
3.3 KiB
Bash
Raw Normal View History

#!/bin/bash -e
2020-09-22 03:13:28 +00:00
if [[ -n $1 ]]; then
SCALE=$1
else
SCALE=100
fi
2020-09-22 03:13:28 +00:00
TABLE="hits_${SCALE}m_obfuscated"
DATASET="${TABLE}_v1.tar.xz"
QUERIES_FILE="queries.sql"
TRIES=3
2021-09-22 00:22:57 +00:00
AMD64_BIN_URL="https://builds.clickhouse.com/master/amd64/clickhouse"
AARCH64_BIN_URL="https://builds.clickhouse.com/master/aarch64/clickhouse"
2020-09-22 03:17:09 +00:00
2020-09-25 00:55:57 +00:00
# Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'.
FASTER_DOWNLOAD=wget
if command -v axel >/dev/null; then
FASTER_DOWNLOAD=axel
2020-09-22 03:06:08 +00:00
else
echo "It's recommended to install 'axel' for faster downloads."
fi
if command -v pixz >/dev/null; then
TAR_PARAMS='-Ipixz'
else
echo "It's recommended to install 'pixz' for faster decompression of the dataset."
fi
2020-09-22 03:13:28 +00:00
mkdir -p clickhouse-benchmark-$SCALE
pushd clickhouse-benchmark-$SCALE
if [[ ! -f clickhouse ]]; then
CPU=$(uname -m)
if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then
2020-09-22 03:17:09 +00:00
$FASTER_DOWNLOAD "$AMD64_BIN_URL"
elif [[ $CPU == aarch64 ]]; then
2020-09-22 03:17:09 +00:00
$FASTER_DOWNLOAD "$AARCH64_BIN_URL"
else
echo "Unsupported CPU type: $CPU"
exit 1
fi
fi
chmod a+x clickhouse
if [[ ! -f $QUERIES_FILE ]]; then
wget "https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/$QUERIES_FILE"
fi
if [[ ! -d data ]]; then
if [[ ! -f $DATASET ]]; then
$FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET"
fi
2020-09-25 00:29:42 +00:00
2020-09-22 03:06:08 +00:00
tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET
fi
2020-09-25 01:03:17 +00:00
uptime
echo "Starting clickhouse-server"
./clickhouse server > server.log 2>&1 &
PID=$!
function finish {
kill $PID
wait
}
trap finish EXIT
echo "Waiting for clickhouse-server to start"
for i in {1..30}; do
sleep 1
2020-09-22 02:22:53 +00:00
./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" 2>/dev/null && break || echo '.'
if [[ $i == 30 ]]; then exit 1; fi
done
echo
echo "Will perform benchmark. Results:"
echo
cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do
sync
echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
echo -n "["
for i in $(seq 1 $TRIES); do
2021-07-27 22:09:29 +00:00
RES=$(./clickhouse client --max_memory_usage 100G --time --format=Null --query="$query" 2>&1 ||:)
[[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
[[ "$i" != $TRIES ]] && echo -n ", "
done
echo "],"
done
echo
echo "Benchmark complete. System info:"
2020-09-22 02:22:53 +00:00
echo
2020-09-25 00:29:42 +00:00
echo '----Version, build id-----------'
./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())"
2020-09-25 00:36:06 +00:00
./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw
./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))"
echo '----CPU-------------------------'
2020-09-25 01:06:21 +00:00
cat /proc/cpuinfo | grep -i -F 'model name' | uniq
lscpu
echo '----Block Devices---------------'
lsblk
echo '----Disk Free and Total--------'
df -h .
echo '----Memory Free and Total-------'
free -h
echo '----Physical Memory Amount------'
cat /proc/meminfo | grep MemTotal
echo '----RAID Info-------------------'
cat /proc/mdstat
#echo '----PCI-------------------------'
#lspci
#echo '----All Hardware Info-----------'
#lshw
echo '--------------------------------'
echo