From 7bbbb19b481a72a9078fa8fe9c120251bea89e28 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:05:50 +0300 Subject: [PATCH 1/6] try to collect some core dumps in perf tests --- docker/test/performance-comparison/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 614debce1c1..fd25a673c85 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -144,8 +144,11 @@ done dmesg -T > dmesg.log +cat /proc/sys/kernel/core_pattern + 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ - report analyze benchmark metrics + report analyze benchmark metrics \ + ./*.core cp compare.log /output From c9404c1fb89f13ef984173e371e45bf313cf4340 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:30:51 +0300 Subject: [PATCH 2/6] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index fd25a673c85..1d03d953ccd 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -146,9 +146,11 @@ dmesg -T > dmesg.log cat /proc/sys/kernel/core_pattern +ls -lath + 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ report analyze benchmark metrics \ - ./*.core + ./*.core.dmp cp compare.log /output From 8199399159b34d3ce6421f7775113b316a4a64fb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 18 Aug 2021 07:00:51 +0300 Subject: [PATCH 3/6] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 1d03d953ccd..406cb6d8f9d 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -127,6 +127,8 @@ export PATH export REF_PR export REF_SHA +ulimit -c unlimited + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ From 59e3cb18f4e53c453951267b5599afeb664290d8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 18 Aug 2021 10:58:21 +0300 Subject: [PATCH 4/6] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 406cb6d8f9d..19af56e3299 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -129,6 +129,7 @@ export REF_SHA ulimit -c unlimited + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ From 7f15c5c55e48b5d050eeb1177733a48e22e4fc20 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 19 Aug 2021 01:21:23 +0300 Subject: [PATCH 5/6] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 19af56e3299..1295e5567fb 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -127,8 +127,14 @@ export PATH export REF_PR export REF_SHA +# Try to collect some core dumps. I've seen two patterns in Sandbox: +# 1) |/home/zomb-sandbox/venv/bin/python /home/zomb-sandbox/client/sandbox/bin/coredumper.py %e %p %g %u %s %P %c +# Not sure what this script does (puts them to sandbox resources, logs some messages?), +# and it's not accessible from inside docker anyway. +# 2) something like %e.%p.core.dmp. The dump should end up in the workspace directory. +# At least we remove the ulimit and then try to pack some common file names into output. ulimit -c unlimited - +cat /proc/sys/kernel/core_pattern # Start the main comparison script. { \ @@ -147,13 +153,11 @@ done dmesg -T > dmesg.log -cat /proc/sys/kernel/core_pattern - ls -lath 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ report analyze benchmark metrics \ - ./*.core.dmp + ./*.core.dmp ./*.core cp compare.log /output From db53638a95532f31d99d9664cfab619e8510a2ce Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 19 Aug 2021 01:21:51 +0300 Subject: [PATCH 6/6] Update download.sh --- docker/test/performance-comparison/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index bd72547ec1c..49323c28700 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -13,7 +13,7 @@ left_sha=$2 # right_pr=$3 not used for now right_sha=$4 -datasets=${CHPC_DATASETS:-"hits1 hits10 hits100 values"} +datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"} declare -A dataset_paths dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar"