mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Improvements in coverage images
This commit is contained in:
parent
bd5d2d89ad
commit
78860bf2bd
@ -7,8 +7,10 @@ ENV SOURCE_DIR=/build
|
|||||||
ENV OUTPUT_DIR=/output
|
ENV OUTPUT_DIR=/output
|
||||||
ENV IGNORE='.*contrib.*'
|
ENV IGNORE='.*contrib.*'
|
||||||
|
|
||||||
CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-10 CXX=clang++-10 cmake .. && cd /; \
|
RUN apt-get update && apt-get install cmake --yes --no-install-recommends
|
||||||
|
|
||||||
|
CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /; \
|
||||||
dpkg -i /package_folder/clickhouse-common-static_*.deb; \
|
dpkg -i /package_folder/clickhouse-common-static_*.deb; \
|
||||||
llvm-profdata-10 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
|
llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
|
||||||
llvm-cov-10 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
|
llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
|
||||||
genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}
|
genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
# docker build -t yandex/clickhouse-stateful-test-with-coverage .
|
# docker build -t yandex/clickhouse-stateful-test-with-coverage .
|
||||||
FROM yandex/clickhouse-stateless-test
|
FROM yandex/clickhouse-stateless-test-with-coverage
|
||||||
|
|
||||||
RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list
|
RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list
|
||||||
|
|
||||||
RUN apt-get update -y \
|
RUN apt-get update -y \
|
||||||
&& env DEBIAN_FRONTEND=noninteractive \
|
&& env DEBIAN_FRONTEND=noninteractive \
|
||||||
apt-get install --yes --no-install-recommends \
|
apt-get install --yes --no-install-recommends \
|
||||||
python3-requests
|
python3-requests procps psmisc
|
||||||
|
|
||||||
COPY s3downloader /s3downloader
|
COPY s3downloader /s3downloader
|
||||||
COPY run.sh /run.sh
|
COPY run.sh /run.sh
|
||||||
|
@ -1,40 +1,44 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
kill_clickhouse () {
|
kill_clickhouse () {
|
||||||
kill "$(pgrep -u clickhouse)" 2>/dev/null
|
echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
|
pkill -f "clickhouse-server" 2>/dev/null
|
||||||
|
|
||||||
for _ in {1..10}
|
|
||||||
|
for _ in {1..120}
|
||||||
do
|
do
|
||||||
if ! kill -0 "$(pgrep -u clickhouse)"; then
|
if ! pkill -0 -f "clickhouse-server" ; then break ; fi
|
||||||
echo "No clickhouse process"
|
echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
break
|
sleep 1
|
||||||
else
|
|
||||||
echo "Process $(pgrep -u clickhouse) still alive"
|
|
||||||
sleep 10
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if pkill -0 -f "clickhouse-server"
|
||||||
|
then
|
||||||
|
pstree -apgT
|
||||||
|
jobs
|
||||||
|
echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
start_clickhouse () {
|
start_clickhouse () {
|
||||||
LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
|
LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
|
||||||
}
|
counter=0
|
||||||
|
until clickhouse-client --query "SELECT 1"
|
||||||
wait_llvm_profdata () {
|
|
||||||
while kill -0 "$(pgrep llvm-profdata-10)"
|
|
||||||
do
|
do
|
||||||
echo "Waiting for profdata $(pgrep llvm-profdata-10) still alive"
|
if [ "$counter" -gt 120 ]
|
||||||
sleep 3
|
then
|
||||||
|
echo "Cannot start clickhouse-server"
|
||||||
|
cat /var/log/clickhouse-server/stdout.log
|
||||||
|
tail -n1000 /var/log/clickhouse-server/stderr.log
|
||||||
|
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
counter=$((counter + 1))
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
merge_client_files_in_background () {
|
|
||||||
client_files=$(ls /client_*profraw 2>/dev/null)
|
|
||||||
if [ -n "$client_files" ]
|
|
||||||
then
|
|
||||||
llvm-profdata-10 merge -sparse "$client_files" -o "merged_client_$(date +%s).profraw"
|
|
||||||
rm "$client_files"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
chmod 777 /
|
chmod 777 /
|
||||||
|
|
||||||
@ -51,26 +55,7 @@ chmod 777 -R /var/log/clickhouse-server/
|
|||||||
# install test configs
|
# install test configs
|
||||||
/usr/share/clickhouse-test/config/install.sh
|
/usr/share/clickhouse-test/config/install.sh
|
||||||
|
|
||||||
function start()
|
start_clickhouse
|
||||||
{
|
|
||||||
counter=0
|
|
||||||
until clickhouse-client --query "SELECT 1"
|
|
||||||
do
|
|
||||||
if [ "$counter" -gt 120 ]
|
|
||||||
then
|
|
||||||
echo "Cannot start clickhouse-server"
|
|
||||||
cat /var/log/clickhouse-server/stdout.log
|
|
||||||
tail -n1000 /var/log/clickhouse-server/stderr.log
|
|
||||||
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
timeout 120 service clickhouse-server start
|
|
||||||
sleep 0.5
|
|
||||||
counter=$((counter + 1))
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
start
|
|
||||||
|
|
||||||
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
|
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
|
||||||
if ! /s3downloader --dataset-names $DATASETS; then
|
if ! /s3downloader --dataset-names $DATASETS; then
|
||||||
@ -81,10 +66,6 @@ fi
|
|||||||
|
|
||||||
chmod 777 -R /var/lib/clickhouse
|
chmod 777 -R /var/lib/clickhouse
|
||||||
|
|
||||||
while /bin/true; do
|
|
||||||
merge_client_files_in_background
|
|
||||||
sleep 2
|
|
||||||
done &
|
|
||||||
|
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW DATABASES"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW DATABASES"
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
||||||
@ -93,14 +74,13 @@ LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "CREATE DA
|
|||||||
kill_clickhouse
|
kill_clickhouse
|
||||||
start_clickhouse
|
start_clickhouse
|
||||||
|
|
||||||
sleep 10
|
|
||||||
|
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||||
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
||||||
|
|
||||||
|
|
||||||
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
||||||
SKIP_LIST_OPT="--use-skip-list"
|
SKIP_LIST_OPT="--use-skip-list"
|
||||||
fi
|
fi
|
||||||
@ -113,11 +93,6 @@ LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard -
|
|||||||
|
|
||||||
kill_clickhouse
|
kill_clickhouse
|
||||||
|
|
||||||
wait_llvm_profdata
|
|
||||||
|
|
||||||
sleep 3
|
sleep 3
|
||||||
|
|
||||||
wait_llvm_profdata # 100% merged all parts
|
|
||||||
|
|
||||||
|
|
||||||
cp /*.profraw /profraw ||:
|
cp /*.profraw /profraw ||:
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
# docker build -t yandex/clickhouse-stateless-with-coverage-test .
|
# docker build -t yandex/clickhouse-stateless-test-with-coverage .
|
||||||
# TODO: that can be based on yandex/clickhouse-stateless-test (llvm version and CMD differs)
|
# TODO: that can be based on yandex/clickhouse-stateless-test (llvm version and CMD differs)
|
||||||
FROM yandex/clickhouse-test-base
|
FROM yandex/clickhouse-test-base
|
||||||
|
|
||||||
@ -28,7 +28,9 @@ RUN apt-get update -y \
|
|||||||
lsof \
|
lsof \
|
||||||
unixodbc \
|
unixodbc \
|
||||||
wget \
|
wget \
|
||||||
qemu-user-static
|
qemu-user-static \
|
||||||
|
procps \
|
||||||
|
psmisc
|
||||||
|
|
||||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||||
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
|
&& wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
|
||||||
|
@ -2,27 +2,41 @@
|
|||||||
|
|
||||||
kill_clickhouse () {
|
kill_clickhouse () {
|
||||||
echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
|
echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
kill "$(pgrep -u clickhouse)" 2>/dev/null
|
pkill -f "clickhouse-server" 2>/dev/null
|
||||||
|
|
||||||
for _ in {1..10}
|
|
||||||
|
for _ in {1..120}
|
||||||
do
|
do
|
||||||
if ! kill -0 "$(pgrep -u clickhouse)"; then
|
if ! pkill -0 -f "clickhouse-server" ; then break ; fi
|
||||||
echo "No clickhouse process" | ts '%Y-%m-%d %H:%M:%S'
|
echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
break
|
sleep 1
|
||||||
else
|
|
||||||
echo "Process $(pgrep -u clickhouse) still alive" | ts '%Y-%m-%d %H:%M:%S'
|
|
||||||
sleep 10
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "Will try to send second kill signal for sure"
|
if pkill -0 -f "clickhouse-server"
|
||||||
kill "$(pgrep -u clickhouse)" 2>/dev/null
|
then
|
||||||
sleep 5
|
pstree -apgT
|
||||||
echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
|
jobs
|
||||||
|
echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S'
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
start_clickhouse () {
|
start_clickhouse () {
|
||||||
LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
|
LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
|
||||||
|
counter=0
|
||||||
|
until clickhouse-client --query "SELECT 1"
|
||||||
|
do
|
||||||
|
if [ "$counter" -gt 120 ]
|
||||||
|
then
|
||||||
|
echo "Cannot start clickhouse-server"
|
||||||
|
cat /var/log/clickhouse-server/stdout.log
|
||||||
|
tail -n1000 /var/log/clickhouse-server/stderr.log
|
||||||
|
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 0.5
|
||||||
|
counter=$((counter + 1))
|
||||||
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
chmod 777 /
|
chmod 777 /
|
||||||
@ -44,9 +58,6 @@ chmod 777 -R /var/log/clickhouse-server/
|
|||||||
|
|
||||||
start_clickhouse
|
start_clickhouse
|
||||||
|
|
||||||
sleep 10
|
|
||||||
|
|
||||||
|
|
||||||
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
||||||
SKIP_LIST_OPT="--use-skip-list"
|
SKIP_LIST_OPT="--use-skip-list"
|
||||||
fi
|
fi
|
||||||
|
Loading…
Reference in New Issue
Block a user