ClickHouse/docker/test/performance-comparison/entrypoint.sh

164 lines
5.9 KiB
Bash
Raw Normal View History

2019-12-26 21:33:10 +00:00
#!/bin/bash
2020-01-10 14:06:07 +00:00
set -ex
2019-12-26 21:33:10 +00:00
2021-06-09 11:40:38 +00:00
CHPC_CHECK_START_TIMESTAMP="$(date +%s)"
export CHPC_CHECK_START_TIMESTAMP
2021-06-08 14:12:47 +00:00
# Use the packaged repository to find the revision we will compare to.
2020-03-12 12:51:09 +00:00
function find_reference_sha
{
2020-07-28 13:15:41 +00:00
git -C right/ch log -1 origin/master
git -C right/ch log -1 pr
2020-03-12 12:51:09 +00:00
# Go back from the revision to be tested, trying to find the closest published
# testing release. The PR branch may be either pull/*/head which is the
# author's branch, or pull/*/merge, which is head merged with some master
# automatically by Github. We will use a merge base with master as a reference
# for tesing (or some older commit). A caveat is that if we're testing the
# master, the merge base is the tested commit itself, so we have to step back
# once.
2020-07-27 11:20:45 +00:00
start_ref=$(git -C right/ch merge-base origin/master pr)
2020-08-05 23:33:32 +00:00
if [ "$PR_TO_TEST" == "0" ]
2020-03-12 12:51:09 +00:00
then
start_ref=$start_ref~
2020-03-12 12:51:09 +00:00
fi
# Loop back to find a commit that actually has a published perf test package.
2020-03-12 12:51:09 +00:00
while :
do
2020-03-16 14:54:17 +00:00
# FIXME the original idea was to compare to a closest testing tag, which
# is a version that is verified to work correctly. However, we're having
# some test stability issues now, and the testing release can't roll out
# for more that a weak already because of that. Temporarily switch to
# using just closest master, so that we can go on.
#ref_tag=$(git -C ch describe --match='v*-testing' --abbrev=0 --first-parent "$start_ref")
ref_tag="$start_ref"
2020-03-12 12:51:09 +00:00
echo Reference tag is "$ref_tag"
# We use annotated tags which have their own shas, so we have to further
# dereference the tag to get the commit it points to, hence the '~0' thing.
2020-07-27 11:20:45 +00:00
REF_SHA=$(git -C right/ch rev-parse "$ref_tag~0")
2020-03-12 12:51:09 +00:00
# FIXME sometimes we have testing tags on commits without published builds.
# Normally these are documentation commits. Loop to skip them.
# Historically there were various path for the performance test package,
# test all of them.
unset found
for path in "https://clickhouse-builds.s3.yandex.net/0/$REF_SHA/"{,clickhouse_build_check/}"performance/performance.tgz"
do
if curl --fail --head "$path"
then
found="$path"
break
fi
done
if [ -n "$found" ] ; then break; fi
2020-03-12 12:51:09 +00:00
start_ref="$REF_SHA~"
done
REF_PR=0
}
chown nobody workspace output
chgrp nogroup workspace output
chmod 777 workspace output
cd workspace
# Download the package for the version we are going to test
for path in "https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/"{,clickhouse_build_check/}"performance/performance.tgz"
do
if curl --fail --head "$path"
then
right_path="$path"
fi
done
mkdir right
wget -nv -nd -c "$right_path" -O- | tar -C right --strip-components=1 -zxv
2020-03-12 12:51:09 +00:00
# Find reference revision if not specified explicitly
if [ "$REF_SHA" == "" ]; then find_reference_sha; fi
if [ "$REF_SHA" == "" ]; then echo Reference SHA is not specified ; exit 1 ; fi
if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi
2020-01-27 12:35:56 +00:00
# Show what we're testing
2020-02-03 17:06:17 +00:00
(
2020-07-27 11:20:45 +00:00
git -C right/ch log -1 --decorate "$REF_SHA" ||:
2020-02-03 17:06:17 +00:00
) | tee left-commit.txt
2020-03-12 12:51:09 +00:00
2020-02-03 17:06:17 +00:00
(
2020-07-27 11:20:45 +00:00
git -C right/ch log -1 --decorate "$SHA_TO_TEST" ||:
echo
echo Real tested commit is:
git -C right/ch log -1 --decorate "pr"
2020-02-03 17:06:17 +00:00
) | tee right-commit.txt
2020-01-10 14:06:07 +00:00
2020-05-22 08:54:35 +00:00
if [ "$PR_TO_TEST" != "0" ]
then
2020-06-05 14:30:26 +00:00
# If the PR only changes the tests and nothing else, prepare a list of these
# tests for use by compare.sh. Compare to merge base, because master might be
# far in the future and have unrelated test changes.
2020-07-27 11:20:45 +00:00
base=$(git -C right/ch merge-base pr origin/master)
git -C right/ch diff --name-only "$base" pr -- . | tee all-changed-files.txt
git -C right/ch diff --name-only "$base" pr -- tests/performance | tee changed-test-definitions.txt
git -C right/ch diff --name-only "$base" pr -- docker/test/performance-comparison | tee changed-test-scripts.txt
git -C right/ch diff --name-only "$base" pr -- :!tests/performance :!docker/test/performance-comparison | tee other-changed-files.txt
2020-05-22 08:54:35 +00:00
fi
2020-02-25 19:51:09 +00:00
2020-01-14 19:05:58 +00:00
# Set python output encoding so that we can print queries with Russian letters.
export PYTHONIOENCODING=utf-8
2020-04-02 18:44:58 +00:00
# By default, use the main comparison script from the tested package, so that we
# can change it in PRs.
script_path="right/scripts"
if [ -v CHPC_LOCAL_SCRIPT ]
then
script_path=".."
fi
2020-01-16 19:39:07 +00:00
# Even if we have some errors, try our best to save the logs.
set +e
2020-03-18 01:28:57 +00:00
2020-04-17 15:47:01 +00:00
# Use clickhouse-client and clickhouse-local from the right server.
PATH="$(readlink -f right/)":"$PATH"
export PATH
2020-11-03 14:37:54 +00:00
export REF_PR
export REF_SHA
2021-08-18 22:21:23 +00:00
# Try to collect some core dumps. I've seen two patterns in Sandbox:
# 1) |/home/zomb-sandbox/venv/bin/python /home/zomb-sandbox/client/sandbox/bin/coredumper.py %e %p %g %u %s %P %c
# Not sure what this script does (puts them to sandbox resources, logs some messages?),
# and it's not accessible from inside docker anyway.
# 2) something like %e.%p.core.dmp. The dump should end up in the workspace directory.
# At least we remove the ulimit and then try to pack some common file names into output.
2021-08-18 04:00:51 +00:00
ulimit -c unlimited
2021-08-18 22:21:23 +00:00
cat /proc/sys/kernel/core_pattern
2021-08-18 07:58:21 +00:00
2020-04-17 15:47:01 +00:00
# Start the main comparison script.
2020-03-18 01:28:57 +00:00
{ \
time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \
2020-04-02 18:44:58 +00:00
time stage=configure "$script_path"/compare.sh ; \
2020-03-18 01:28:57 +00:00
} 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee compare.log
2019-12-26 21:33:10 +00:00
2020-02-27 17:57:08 +00:00
# Stop the servers to free memory. Normally they are restarted before getting
# the profile info, so they shouldn't use much, but if the comparison script
# fails in the middle, this might not be the case.
2020-02-27 20:02:50 +00:00
for _ in {1..30}
2020-02-27 17:57:08 +00:00
do
2020-02-27 20:02:50 +00:00
killall clickhouse || break
sleep 1
2020-02-27 17:57:08 +00:00
done
2020-02-25 19:51:09 +00:00
dmesg -T > dmesg.log
2020-02-18 17:30:10 +00:00
2021-08-17 10:30:51 +00:00
ls -lath
2020-06-18 03:14:18 +00:00
7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \
{right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \
report analyze benchmark metrics \
2021-08-18 22:21:23 +00:00
./*.core.dmp ./*.core
2020-06-18 03:14:18 +00:00
2019-12-26 21:33:10 +00:00
cp compare.log /output