diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9fccee89c1d..338d2e9f20c 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,6 +14,13 @@ unset (_current_dir_name) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") +if (WITH_COVERAGE) + set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) + separate_arguments(WITHOUT_COVERAGE_LIST) + # disable coverage for contib files and build with optimisations + add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) +endif() + if (SANITIZE STREQUAL "undefined") # 3rd-party libraries usually not intended to work with UBSan. add_compile_options(-fno-sanitize=undefined) diff --git a/docker/images.json b/docker/images.json index 16747cc3725..e7300972b6d 100644 --- a/docker/images.json +++ b/docker/images.json @@ -17,10 +17,6 @@ "name": "yandex/clickhouse-unbundled-builder", "dependent": [] }, - "docker/test/coverage": { - "name": "yandex/clickhouse-coverage", - "dependent": [] - }, "docker/test/compatibility/centos": { "name": "yandex/clickhouse-test-old-centos", "dependent": [] @@ -48,27 +44,22 @@ "docker/test/stateless": { "name": "yandex/clickhouse-stateless-test", "dependent": [ - "docker/test/stateful" + "docker/test/stateful", + "docker/test/coverage" ] }, "docker/test/stateless_pytest": { "name": "yandex/clickhouse-stateless-pytest", "dependent": [] }, - "docker/test/stateless_with_coverage": { - "name": "yandex/clickhouse-stateless-test-with-coverage", - "dependent": [ - "docker/test/stateful_with_coverage" - ] - }, "docker/test/stateful": { "name": "yandex/clickhouse-stateful-test", "dependent": [ "docker/test/stress" ] }, - "docker/test/stateful_with_coverage": { - "name": "yandex/clickhouse-stateful-test-with-coverage", + "docker/test/coverage": { + "name": "yandex/clickhouse-test-coverage", "dependent": [] }, "docker/test/unit": { @@ -143,9 +134,7 @@ "name": "yandex/clickhouse-test-base", "dependent": [ "docker/test/stateless", - "docker/test/stateless_with_coverage", - "docker/test/stateless_pytest", - "docker/test/coverage" + "docker/test/stateless_pytest" ] }, "docker/packager/unbundled": { diff --git a/docker/test/coverage/Dockerfile b/docker/test/coverage/Dockerfile index cea1a63cf6f..681f65e0f6f 100644 --- a/docker/test/coverage/Dockerfile +++ b/docker/test/coverage/Dockerfile @@ -1,16 +1,18 @@ -# docker build -t yandex/clickhouse-coverage . -ARG PARENT_TAG=latest -FROM yandex/clickhouse-test-base:${PARENT_TAG} +# docker build -t yandex/clickhouse-test-coverage . +FROM yandex/clickhouse-stateless-test +RUN apt-get update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + cmake + +COPY s3downloader /s3downloader +COPY run.sh /run.sh + +ENV DATASETS="hits visits" ENV COVERAGE_DIR=/coverage_reports ENV SOURCE_DIR=/build ENV OUTPUT_DIR=/output ENV IGNORE='.*contrib.*' -RUN apt-get update && apt-get install cmake --yes --no-install-recommends - -CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /; \ - dpkg -i /package_folder/clickhouse-common-static_*.deb; \ - llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \ - llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \ - genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR} +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/coverage/run.sh b/docker/test/coverage/run.sh new file mode 100755 index 00000000000..e2369a28a9a --- /dev/null +++ b/docker/test/coverage/run.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +kill_clickhouse () { + echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S' + pkill -f "clickhouse-server" 2>/dev/null + + + for _ in {1..120} + do + if ! pkill -0 -f "clickhouse-server" ; then break ; fi + echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S' + sleep 1 + done + + if pkill -0 -f "clickhouse-server" + then + pstree -apgT + jobs + echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S' + return 1 + fi +} + +start_clickhouse () { + LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml & + counter=0 + until clickhouse-client --query "SELECT 1" + do + if [ "$counter" -gt 120 ] + then + echo "Cannot start clickhouse-server" + cat /var/log/clickhouse-server/stdout.log + tail -n1000 /var/log/clickhouse-server/stderr.log + tail -n1000 /var/log/clickhouse-server/clickhouse-server.log + break + fi + sleep 0.5 + counter=$((counter + 1)) + done +} + + +chmod 777 / + +dpkg -i package_folder/clickhouse-common-static_*.deb; \ + dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ + dpkg -i package_folder/clickhouse-server_*.deb; \ + dpkg -i package_folder/clickhouse-client_*.deb; \ + dpkg -i package_folder/clickhouse-test_*.deb + +mkdir -p /var/lib/clickhouse +mkdir -p /var/log/clickhouse-server +chmod 777 -R /var/log/clickhouse-server/ + +# install test configs +/usr/share/clickhouse-test/config/install.sh + +start_clickhouse + +# shellcheck disable=SC2086 # No quotes because I want to split it into words. +if ! /s3downloader --dataset-names $DATASETS; then + echo "Cannot download datatsets" + exit 1 +fi + + +chmod 777 -R /var/lib/clickhouse + + +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW DATABASES" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "CREATE DATABASE test" + +kill_clickhouse +start_clickhouse + +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM datasets" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test" + +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time --use-skip-list 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt + +readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt") + +kill_clickhouse + +sleep 3 + +if [[ -n "${FAILED_TESTS[*]}" ]] +then + # Clean the data so that there is no interference from the previous test run. + rm -rf /var/lib/clickhouse/{{meta,}data,user_files} ||: + + start_clickhouse + + echo "Going to run again: ${FAILED_TESTS[*]}" + + LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper --use-skip-list "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt +else + echo "No failed tests" +fi + +mkdir -p $COVERAGE_DIR +mv /*.profraw $COVERAGE_DIR + +mkdir -p $SOURCE_DIR/obj-x86_64-linux-gnu +cd $SOURCE_DIR/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd / +llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata +llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov +genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR} diff --git a/docker/test/stateful_with_coverage/s3downloader b/docker/test/coverage/s3downloader similarity index 100% rename from docker/test/stateful_with_coverage/s3downloader rename to docker/test/coverage/s3downloader diff --git a/docker/test/stateful_with_coverage/Dockerfile b/docker/test/stateful_with_coverage/Dockerfile deleted file mode 100644 index 07acf4ed4e6..00000000000 --- a/docker/test/stateful_with_coverage/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# docker build -t yandex/clickhouse-stateful-test-with-coverage . -FROM yandex/clickhouse-stateless-test-with-coverage - -RUN apt-get update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ - python3-requests procps psmisc - -COPY s3downloader /s3downloader -COPY run.sh /run.sh - -ENV DATASETS="hits visits" - -CMD ["/bin/bash", "/run.sh"] - diff --git a/docker/test/stateful_with_coverage/run.sh b/docker/test/stateful_with_coverage/run.sh deleted file mode 100755 index 5fc6350fad8..00000000000 --- a/docker/test/stateful_with_coverage/run.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -kill_clickhouse () { - echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S' - pkill -f "clickhouse-server" 2>/dev/null - - - for _ in {1..120} - do - if ! pkill -0 -f "clickhouse-server" ; then break ; fi - echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S' - sleep 1 - done - - if pkill -0 -f "clickhouse-server" - then - pstree -apgT - jobs - echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S' - return 1 - fi -} - -start_clickhouse () { - LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml & - counter=0 - until clickhouse-client --query "SELECT 1" - do - if [ "$counter" -gt 120 ] - then - echo "Cannot start clickhouse-server" - cat /var/log/clickhouse-server/stdout.log - tail -n1000 /var/log/clickhouse-server/stderr.log - tail -n1000 /var/log/clickhouse-server/clickhouse-server.log - break - fi - sleep 0.5 - counter=$((counter + 1)) - done -} - - -chmod 777 / - -dpkg -i package_folder/clickhouse-common-static_*.deb; \ - dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ - dpkg -i package_folder/clickhouse-server_*.deb; \ - dpkg -i package_folder/clickhouse-client_*.deb; \ - dpkg -i package_folder/clickhouse-test_*.deb - -mkdir -p /var/lib/clickhouse -mkdir -p /var/log/clickhouse-server -chmod 777 -R /var/log/clickhouse-server/ - -# install test configs -/usr/share/clickhouse-test/config/install.sh - -start_clickhouse - -# shellcheck disable=SC2086 # No quotes because I want to split it into words. -if ! /s3downloader --dataset-names $DATASETS; then - echo "Cannot download datatsets" - exit 1 -fi - - -chmod 777 -R /var/lib/clickhouse - - -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW DATABASES" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "CREATE DATABASE test" - -kill_clickhouse -start_clickhouse - -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM datasets" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test" - - -if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then - SKIP_LIST_OPT="--use-skip-list" -fi - -# We can have several additional options so we path them as array because it's -# more idiologically correct. -read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" - -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt - -kill_clickhouse - -sleep 3 - -cp /*.profraw /profraw ||: diff --git a/docker/test/stateless_with_coverage/Dockerfile b/docker/test/stateless_with_coverage/Dockerfile deleted file mode 100644 index e31d2d1e009..00000000000 --- a/docker/test/stateless_with_coverage/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# docker build -t yandex/clickhouse-stateless-test-with-coverage . -# TODO: that can be based on yandex/clickhouse-stateless-test (llvm version and CMD differs) -FROM yandex/clickhouse-test-base - -ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" - -RUN apt-get update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ - bash \ - tzdata \ - fakeroot \ - debhelper \ - expect \ - python3 \ - python3-lxml \ - python3-termcolor \ - python3-requests \ - sudo \ - openssl \ - ncdu \ - netcat-openbsd \ - telnet \ - tree \ - moreutils \ - brotli \ - zstd \ - gdb \ - lsof \ - unixodbc \ - wget \ - qemu-user-static \ - procps \ - psmisc - -RUN mkdir -p /tmp/clickhouse-odbc-tmp \ - && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ - && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \ - && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \ - && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ - && rm -rf /tmp/clickhouse-odbc-tmp - -ENV TZ=Europe/Moscow -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -COPY run.sh /run.sh - -CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless_with_coverage/run.sh b/docker/test/stateless_with_coverage/run.sh deleted file mode 100755 index 4e4d9430a11..00000000000 --- a/docker/test/stateless_with_coverage/run.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -kill_clickhouse () { - echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S' - pkill -f "clickhouse-server" 2>/dev/null - - - for _ in {1..120} - do - if ! pkill -0 -f "clickhouse-server" ; then break ; fi - echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S' - sleep 1 - done - - if pkill -0 -f "clickhouse-server" - then - pstree -apgT - jobs - echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S' - return 1 - fi -} - -start_clickhouse () { - LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml & - counter=0 - until clickhouse-client --query "SELECT 1" - do - if [ "$counter" -gt 120 ] - then - echo "Cannot start clickhouse-server" - cat /var/log/clickhouse-server/stdout.log - tail -n1000 /var/log/clickhouse-server/stderr.log - tail -n1000 /var/log/clickhouse-server/clickhouse-server.log - break - fi - sleep 0.5 - counter=$((counter + 1)) - done -} - -chmod 777 / - -dpkg -i package_folder/clickhouse-common-static_*.deb; \ - dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ - dpkg -i package_folder/clickhouse-server_*.deb; \ - dpkg -i package_folder/clickhouse-client_*.deb; \ - dpkg -i package_folder/clickhouse-test_*.deb - - -mkdir -p /var/lib/clickhouse -mkdir -p /var/log/clickhouse-server -chmod 777 -R /var/lib/clickhouse -chmod 777 -R /var/log/clickhouse-server/ - -# install test configs -/usr/share/clickhouse-test/config/install.sh - -start_clickhouse - -if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then - SKIP_LIST_OPT="--use-skip-list" -fi - -# We can have several additional options so we path them as array because it's -# more idiologically correct. -read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" - -LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt - -kill_clickhouse - -sleep 3 - -cp /*.profraw /profraw ||: diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index a65a420cd5b..d8ef6d89d44 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -195,6 +195,12 @@ class ClickHouseCluster: if tag is None: tag = self.docker_base_tag + if not env_variables: + env_variables = {} + + # Code coverage files will be placed in database directory + # (affect only WITH_COVERAGE=1 build) + env_variables['LLVM_PROFILE_FILE'] = '/var/lib/clickhouse/server_%h_%p_%m.profraw' instance = ClickHouseInstance( cluster=self, @@ -221,7 +227,7 @@ class ClickHouseCluster: clickhouse_path_dir=clickhouse_path_dir, with_odbc_drivers=with_odbc_drivers, hostname=hostname, - env_variables=env_variables or {}, + env_variables=env_variables, image=image, tag=tag, stay_alive=stay_alive, @@ -759,9 +765,11 @@ class ClickHouseCluster: if kill: try: - subprocess_check_call(self.base_cmd + ['kill']) + subprocess_check_call(self.base_cmd + ['stop', '--timeout', '20']) except Exception as e: - print("Kill command failed durung shutdown. {}".format(repr(e))) + print("Kill command failed during shutdown. {}".format(repr(e))) + print("Trying to kill forcefully") + subprocess_check_call(self.base_cmd + ['kill']) try: subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans'])