thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Correclty process Druid errors. Add restart after each query as some queries leave degraded state of Druid

Browse Source
This commit is contained in:
Ilya Yatsishin 2022-07-08 20:14:58 +00:00
parent cef3f1d4ae
commit c2157ff536
4 changed files with 56 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/druid/benchmark.sh
View File

@ -18,7 +18,9 @@ sed -i 's MaxDirectMemorySize=1g MaxDirectMemorySize=5g g' apache-druid-$VERSION
# Disable cache to test query performance
sed -i 's druid.historical.cache.useCache=true druid.historical.cache.useCache=false g' apache-druid-$VERSION/conf/druid/single-server/medium/historical/runtime.properties
sed -i 's druid.historical.cache.populateCache=true druid.historical.cache.populateCache=false g' apache-druid-$VERSION/conf/druid/single-server/medium/historical/runtime.properties
sed -i 's druid.processing.buffer.sizeBytes=500MiB druid.processing.buffer.sizeBytes=1000MiB g' apache-druid-$VERSION/conf/druid/single-server/medium/historical/runtime.properties
echo "druid.query.groupBy.maxMergingDictionarySize=5000000000" >> apache-druid-$VERSION/conf/druid/single-server/medium/historical/runtime.properties
# Druid launcher does not start Druid as a daemon. Run it in background
./apache-druid-${VERSION}/bin/start-single-server-medium &

1
benchmark/druid/check.json Normal file
View File

@ -0,0 +1 @@
{"query": "SELECT COUNT(*) from hits"}

82
benchmark/druid/results/c6a.4xlarge.json
View File

@ -11,48 +11,48 @@
"data_size": 45188608472,
"result": [
[0.179351, 0.040782, 0.026180],
[0.305672, 0.123548, 0.112512],
[0.187141, 0.072973, 0.071330],
[1.451711, 0.060012, 0.056498],
[1.819509, 1.405169, 1.377259],
[2.993129, 2.383569, 2.725656],
[0.061959, 0.016261, 0.010215],
[0.285995, 0.143307, 0.131404],
[3.561017, 2.008080, 2.019541],
[2.549107, 2.432374, 2.419320],
[0.302380, 0.235096, 0.178124],
[0.404714, 0.227526, 0.209203],
[0.866275, 0.745312, 0.715667],
[2.929871, 2.943361, 2.793336],
[2.031640, 1.839312, 1.598095],
[5.304266, 5.124386, 5.266312],
[4.370606, 3.720471, 3.679889],
[3.603636, 3.407057, 3.503119],
[0.030800, 0.004050, 0.003638],
[0.963703, 0.898970, 0.902731],
[18.642888, 18.434240, 18.451905],
[0.053461, 0.014235, 0.015694],
[0.040045, 0.020073, 0.021030],
[7.561157, 6.471910, 6.453816],
[0.111474, 0.019374, 0.027832],
[0.016728, 0.015407, 0.013660],
[0.013881, 0.012076, 0.012084],
[0.022610, 0.015202, 0.015416],
[0.408549, 0.275624, 0.275657],
[0.365362, 0.159893, 0.151254],
[1.380429, 0.136963, 0.131130],
[3.126491, 2.462627, 2.431463],
[9.350598, 7.418928, 7.213518],
[null, null, null],
[0.006903, 0.004498, 0.003752],
[1.758529, 1.328889, 1.393226],
[2.106015, 1.906621, 1.653751],
[2.708141, 2.064777, 1.868830],
[7.690441, 7.650803, 7.350766],
[17.321933, 18.088249, 18.109673],
[0.532669, 0.278820, 0.270483],
[7.541045, 6.631755, 6.555070],
[8.634310, 7.615905, 7.573183],
[1.650839, 0.832760, 0.881859],
[1.680002, 0.504994, 0.512728],
[2.406642, 1.939866, 1.667841],
[8.632186, 7.264837, 7.358885],
[61.095584, 59.243483, 59.111986],
[0.006551, 0.005535, 0.004936],
[0.010023, 0.010525, 0.009739],
[0.009157, 0.008836, 0.008432],
[null, null, null],
[4.378734, 3.495874, 3.754953],
[0.623633, 0.603495, 0.580749],
[0.551690, 0.483025, 0.494859],
[0.516076, 0.189721, 0.184861],
[0.005613, 0.005072, 0.004628],
[0.375339, 0.276121, 0.264940],
[0.207965, 0.187909, 0.183420],
[0.176383, 0.107790, 0.106120]
[0.005462, 0.004723, 0.004469],
[0.006079, 0.005037, 0.005726],
[null, null, null],
[null, null, null],
[0.009679, 0.009139, 0.011461],
[0.006833, 0.005752, 0.005962],
[null, null, null],
[null, null, null],
[0.011282, 0.009306, 0.009612],
[null, null, null],
[0.042850, 0.047030, 0.040549],
[0.013658, 0.013335, 0.013686],
[0.018262, 0.012727, 0.012357],
[0.013259, 0.011685, 0.011177],
[0.006448, 0.005744, 0.005489],
[0.006924, 0.008225, 0.006115],
[60.505400, 59.499392, 59.221125],
[0.010123, 0.008507, 0.008990],
[0.010177, 0.008739, 0.009381],
[0.011097, 0.009424, 0.013203],
[null, null, null],
[0.014453, 0.013647, 0.013853],
[0.017409, 0.013077, 0.012445],
[0.010489, 0.009483, 0.009735]
]
}

16
benchmark/druid/run.sh
View File

@ -3,14 +3,22 @@
TRIES=3
cat queries.sql | while read query; do
sync
for i in $(seq 1 100); do
CHECK=$(curl -o /dev/null -w '%{http_code}' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @check.json })
if [[ "$CHECK" == "200" ]]; then
break
fi
sleep 1
done
echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
echo -n "["
for i in $(seq 1 $TRIES); do
echo "{\"query\":\"$query\"}"| sed -e 's EventTime __time g' | tr -d ';' > query.json
RES=$(curl -o /dev/null -s -w '%{time_total}\n' -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @query.json)
[[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
echo "{\"query\":\"$query\", \"context\": {\"timeout\": 1000000} }"| sed -e 's EventTime __time g' | tr -d ';' > query.json
curl -o /dev/null -w '%{http_code} %{time_total}\n' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @query.json | awk '{ if($1=="200") printf $2; else printf "null"; }'
[[ "$i" != $TRIES ]] && echo -n ", "
done
echo "],"
# Ugly hack to measure independently queries. Otherwise some queries make Druid degraded and results are incorrect. For example after Q13 even SELECT 1 works for 7 seconds
pkill -f historical
sleep 3
done