From a25996ed8d44743608932725f5a87421202ed9cf Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Nov 2020 02:10:19 +0300 Subject: [PATCH 1/5] Upload perf test results to the CI database --- docker/test/performance-comparison/compare.sh | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index d0cf12baa9e..20720e0b976 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1074,6 +1074,56 @@ wait unset IFS } +function upload_results +{ + if ! [ -v CHPC_DATABASE_URL ] + then + echo Database for test results is not specified, will not upload them. + return 0 + fi + + # Surprisingly, clickhouse-client doesn't understand --host 127.0.0.1:9000 + # so I have to do this instead. I tried to use Poco URI parser for this, + # but it's also broken and can't parse host:port. + IFS=':' read host port <<<"${CHPC_DATABASE_URL}" + + upload_client=(clickhouse-client + --host "${host}" + --port "${port}" + --secure + --user "${CHPC_DATABASE_USER}" + --password "${CHPC_DATABASE_PASSWORD}" + --config "ch/tests/config/client_config.xml" + --database perftest + -m + --date_time_input_format=best_effort) + + set +x # Don't show password in the log + cat "report/all-query-metrics.tsv" | "${upload_client[@]}" --query " + insert into query_metrics_tmp + select + toDate(event_time) event_date, + toDateTime('$(cd ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time, + $PR_TO_TEST pr_number, + '$REF_SHA' old_sha, + '$SHA_TO_TEST' new_sha, + test, + query_index, + query_display_name, + metric_name, + old_value, + new_value, + diff, + stat_threshold + from input('metric_name text, old_value float, new_value float, diff float, + ratio_display_text text, stat_threshold float, + test text, query_index int, query_display_name text') + settings date_time_input_format='best_effort' + format TSV + settings date_time_input_format='best_effort'" + set -x +} + # Check that local and client are in PATH clickhouse-local --version > /dev/null clickhouse-client --version > /dev/null @@ -1145,6 +1195,9 @@ case "$stage" in time "$script_dir/report.py" --report=all-queries > all-queries.html 2> >(tee -a report/errors.log 1>&2) ||: time "$script_dir/report.py" > report.html ;& +"upload_results") + time upload_results ||: + ;& esac # Print some final debug info to help debug Weirdness, of which there is plenty. From 4245c470ac579d1d5f7b13083b32fd4a78c7a4fd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 3 Nov 2020 17:37:54 +0300 Subject: [PATCH 2/5] fixes --- docker/test/performance-comparison/compare.sh | 11 ++++++----- docker/test/performance-comparison/entrypoint.sh | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 20720e0b976..ca52fbd5891 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1085,7 +1085,7 @@ function upload_results # Surprisingly, clickhouse-client doesn't understand --host 127.0.0.1:9000 # so I have to do this instead. I tried to use Poco URI parser for this, # but it's also broken and can't parse host:port. - IFS=':' read host port <<<"${CHPC_DATABASE_URL}" + IFS=':' read -r host port <<<"${CHPC_DATABASE_URL}" upload_client=(clickhouse-client --host "${host}" @@ -1096,14 +1096,14 @@ function upload_results --config "ch/tests/config/client_config.xml" --database perftest -m - --date_time_input_format=best_effort) + "--date_time_input_format=best_effort") set +x # Don't show password in the log - cat "report/all-query-metrics.tsv" | "${upload_client[@]}" --query " + "${upload_client[@]}" --query " insert into query_metrics_tmp select toDate(event_time) event_date, - toDateTime('$(cd ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time, + toDateTime('$(cd right/ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time, $PR_TO_TEST pr_number, '$REF_SHA' old_sha, '$SHA_TO_TEST' new_sha, @@ -1120,7 +1120,8 @@ function upload_results test text, query_index int, query_display_name text') settings date_time_input_format='best_effort' format TSV - settings date_time_input_format='best_effort'" + settings date_time_input_format='best_effort' + " < "report/all-query-metrics.tsv" set -x } diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index ed2e542eadd..570a1c21514 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -121,6 +121,9 @@ set +e PATH="$(readlink -f right/)":"$PATH" export PATH +export REF_PR +export REF_SHA + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ From 5cb708a50ab8025bf7110d4c64b38a8494c62e44 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 4 Nov 2020 04:19:53 +0300 Subject: [PATCH 3/5] fixup --- docker/test/performance-comparison/compare.sh | 74 +++++++++---------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index ca52fbd5891..08462298093 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1083,45 +1083,41 @@ function upload_results fi # Surprisingly, clickhouse-client doesn't understand --host 127.0.0.1:9000 - # so I have to do this instead. I tried to use Poco URI parser for this, - # but it's also broken and can't parse host:port. - IFS=':' read -r host port <<<"${CHPC_DATABASE_URL}" - - upload_client=(clickhouse-client - --host "${host}" - --port "${port}" - --secure - --user "${CHPC_DATABASE_USER}" - --password "${CHPC_DATABASE_PASSWORD}" - --config "ch/tests/config/client_config.xml" - --database perftest - -m - "--date_time_input_format=best_effort") - - set +x # Don't show password in the log - "${upload_client[@]}" --query " - insert into query_metrics_tmp - select - toDate(event_time) event_date, - toDateTime('$(cd right/ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time, - $PR_TO_TEST pr_number, - '$REF_SHA' old_sha, - '$SHA_TO_TEST' new_sha, - test, - query_index, - query_display_name, - metric_name, - old_value, - new_value, - diff, - stat_threshold - from input('metric_name text, old_value float, new_value float, diff float, - ratio_display_text text, stat_threshold float, - test text, query_index int, query_display_name text') - settings date_time_input_format='best_effort' - format TSV - settings date_time_input_format='best_effort' - " < "report/all-query-metrics.tsv" + # so I have to extract host and port with clickhouse-local. I tried to use + # Poco URI parser to support this in the client, but it's broken and can't + # parse host:port. + set +x # Don't show password in the log + clickhouse-client \ + $(clickhouse-local --query "with '${CHPC_DATABASE_URL}' as url select '--host ' || domain(url) || ' --port ' || toString(port(url)) format TSV") \ + --secure \ + --user "${CHPC_DATABASE_USER}" \ + --password "${CHPC_DATABASE_PASSWORD}" \ + --config "ch/tests/config/client_config.xml" \ + --database perftest \ + --date_time_input_format=best_effort \ + --query " + insert into query_metrics_tmp + select + toDate(event_time) event_date, + toDateTime('$(cd right/ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time, + $PR_TO_TEST pr_number, + '$REF_SHA' old_sha, + '$SHA_TO_TEST' new_sha, + test, + query_index, + query_display_name, + metric_name, + old_value, + new_value, + diff, + stat_threshold + from input('metric_name text, old_value float, new_value float, diff float, + ratio_display_text text, stat_threshold float, + test text, query_index int, query_display_name text') + settings date_time_input_format='best_effort' + format TSV + settings date_time_input_format='best_effort' +" < report/all-query-metrics.tsv # Don't leave whitespace after INSERT: https://github.com/ClickHouse/ClickHouse/issues/16652 set -x } From 8d5fe1148c721a3065cbebcea5f02ae9689876f4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 4 Nov 2020 18:07:34 +0300 Subject: [PATCH 4/5] add client config --- docker/test/performance-comparison/compare.sh | 2 +- .../config/client_config.xml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 docker/test/performance-comparison/config/client_config.xml diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 08462298093..60d226f025d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1092,7 +1092,7 @@ function upload_results --secure \ --user "${CHPC_DATABASE_USER}" \ --password "${CHPC_DATABASE_PASSWORD}" \ - --config "ch/tests/config/client_config.xml" \ + --config "right/config/client_config.xml" \ --database perftest \ --date_time_input_format=best_effort \ --query " diff --git a/docker/test/performance-comparison/config/client_config.xml b/docker/test/performance-comparison/config/client_config.xml new file mode 100644 index 00000000000..9f590389dc3 --- /dev/null +++ b/docker/test/performance-comparison/config/client_config.xml @@ -0,0 +1,17 @@ + + + + + true + true + sslv2,sslv3 + true + + AcceptCertificateHandler + + + + From ac487ab26a15b530ae3b8c543b6249da545b8ce1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 5 Nov 2020 23:45:58 +0300 Subject: [PATCH 5/5] Update compare.sh --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 60d226f025d..7b436833225 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1096,7 +1096,7 @@ function upload_results --database perftest \ --date_time_input_format=best_effort \ --query " - insert into query_metrics_tmp + insert into query_metrics_v1 select toDate(event_time) event_date, toDateTime('$(cd right/ch && git show -s --format=%ci "$SHA_TO_TEST" | cut -d' ' -f-2)') event_time,