Minor changes

benchmark/druid/benchmark.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 sudo apt-get update
-sudo apt install openjdk-8-jdk -y
+sudo apt install -y openjdk-8-jdk python2
 sudo update-alternatives --config java
 
 # Install
@@ -31,7 +31,8 @@ gzip -d hits.tsv.gz
 
 ./apache-druid-${VERSION}/bin/post-index-task --file ingest.json --url http://localhost:8081
 
-# The loading time should be checked from the logs
+# The command above will fail due to timeout but still continue to run in background.
+# The loading time should be checked from the logs.
 
 # Run the queries
 ./run.sh

benchmark/druid/results/c6a.4xlarge.json

@@ -3,7 +3,7 @@
     "date": "2022-07-01",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
-    "comment": "",
+    "comment": "Druid is killed and restarted after every query. Otherwise some queries make Druid degraded and results are incorrect. For example after Q13 even SELECT 1 works for 7 seconds",
 
     "tags": ["Java", "column-oriented"],
 

benchmark/druid/run.sh

@@ -4,20 +4,19 @@ TRIES=3
 cat queries.sql | while read query; do
     sync
     for i in $(seq 1 100); do
-        CHECK=$(curl -o - -w '%{http_code}' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @check.json })
-	if [[ "$CHECK" == "[{\"EXPR\$0\":99997497}]"$'\n'"200" ]]; then
-	    break
-	fi
-	sleep 1
+        CHECK=$(curl -o /dev/null -w '%{http_code}' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @check.json })
+        [[ "$CHECK" == "200" ]] && break
+        sleep 1
     done
     echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
-    echo "{\"query\":\"$query\", \"context\": {\"timeout\": 1000000} }"| sed -e 's EventTime __time g' | tr -d ';' > query.json
     echo -n "["
     for i in $(seq 1 $TRIES); do
-	curl -o /dev/null -w '%{http_code} %{time_total}\n' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @query.json | awk '{ if($1=="200") printf $2; else printf "null"; }'
+        echo "{\"query\":\"$query\", \"context\": {\"timeout\": 1000000} }"| sed -e 's EventTime __time g' | tr -d ';' > query.json
+        curl -w '%{http_code} %{time_total}\n' -s -XPOST -H'Content-Type: application/json' http://localhost:8888/druid/v2/sql/ -d @query.json | awk '{ if ($1!="200") { printf "null" } }'
         [[ "$i" != $TRIES ]] && echo -n ", "
     done
     echo "],"
+
     # Ugly hack to measure independently queries. Otherwise some queries make Druid degraded and results are incorrect. For example after Q13 even SELECT 1 works for 7 seconds
     pkill -f historical
     sleep 3

benchmark/elasticsearch/benchmark.sh

@@ -16,6 +16,9 @@ export PASSWORD='...'
 
 curl -k -XGET 'https://localhost:9200' -u "elastic:${PASSWORD}"
 
+# This will create an index.
+curl -k -XPUT -u "elastic:${PASSWORD}" 'https://localhost:9200/hits'
+
 wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.json.gz'
 gzip -d hits.json.gz