Merge pull request #18312 from ClickHouse/simplier_coverage

Make single image for building coverage report

contrib/CMakeLists.txt

@@ -14,6 +14,13 @@ unset (_current_dir_name)
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
 
+if (WITH_COVERAGE)
+  set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})
+  separate_arguments(WITHOUT_COVERAGE_LIST)
+  # disable coverage for contib files and build with optimisations
+  add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST})
+endif()
+
 if (SANITIZE STREQUAL "undefined")
     # 3rd-party libraries usually not intended to work with UBSan.
     add_compile_options(-fno-sanitize=undefined)

docker/images.json

@@ -17,10 +17,6 @@
         "name": "yandex/clickhouse-unbundled-builder",
         "dependent": []
     },
-    "docker/test/coverage": {
-        "name": "yandex/clickhouse-coverage",
-        "dependent": []
-    },
     "docker/test/compatibility/centos": {
         "name": "yandex/clickhouse-test-old-centos",
         "dependent": []
@@ -48,27 +44,22 @@
     "docker/test/stateless": {
         "name": "yandex/clickhouse-stateless-test",
         "dependent": [
-            "docker/test/stateful"
+            "docker/test/stateful",
+            "docker/test/coverage"
         ]
     },
     "docker/test/stateless_pytest": {
         "name": "yandex/clickhouse-stateless-pytest",
         "dependent": []
     },
-    "docker/test/stateless_with_coverage": {
-        "name": "yandex/clickhouse-stateless-test-with-coverage",
-        "dependent": [
-            "docker/test/stateful_with_coverage"
-        ]
-    },
     "docker/test/stateful": {
         "name": "yandex/clickhouse-stateful-test",
         "dependent": [
             "docker/test/stress"
         ]
     },
-    "docker/test/stateful_with_coverage": {
+    "docker/test/coverage": {
-        "name": "yandex/clickhouse-stateful-test-with-coverage",
+        "name": "yandex/clickhouse-test-coverage",
         "dependent": []
     },
     "docker/test/unit": {
@@ -143,9 +134,7 @@
          "name": "yandex/clickhouse-test-base",
          "dependent": [
             "docker/test/stateless",
-            "docker/test/stateless_with_coverage",
+            "docker/test/stateless_pytest"
-            "docker/test/stateless_pytest",
-            "docker/test/coverage"
          ]
     },
     "docker/packager/unbundled": {

docker/test/coverage/Dockerfile

@@ -1,16 +1,18 @@
-# docker build -t yandex/clickhouse-coverage .
+# docker build -t yandex/clickhouse-test-coverage .
-ARG PARENT_TAG=latest
+FROM yandex/clickhouse-stateless-test
-FROM yandex/clickhouse-test-base:${PARENT_TAG}
 
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            cmake
+
+COPY s3downloader /s3downloader
+COPY run.sh /run.sh
+
+ENV DATASETS="hits visits"
 ENV COVERAGE_DIR=/coverage_reports
 ENV SOURCE_DIR=/build
 ENV OUTPUT_DIR=/output
 ENV IGNORE='.*contrib.*'
 
-RUN apt-get update && apt-get install cmake --yes --no-install-recommends
+CMD ["/bin/bash", "/run.sh"]
-
-CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /; \
-    dpkg -i /package_folder/clickhouse-common-static_*.deb; \
-    llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
-    llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
-    genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}

docker/test/coverage/run.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+kill_clickhouse () {
+    echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
+    pkill -f "clickhouse-server" 2>/dev/null
+
+
+    for _ in {1..120}
+    do
+        if ! pkill -0 -f "clickhouse-server" ; then break ; fi
+        echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
+        sleep 1
+    done
+
+    if pkill -0 -f "clickhouse-server"
+    then
+        pstree -apgT
+        jobs
+        echo "Failed to kill the ClickHouse server"  | ts '%Y-%m-%d %H:%M:%S'
+        return 1
+    fi
+}
+
+start_clickhouse () {
+    LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
+    counter=0
+    until clickhouse-client --query "SELECT 1"
+    do
+        if [ "$counter" -gt 120 ]
+        then
+            echo "Cannot start clickhouse-server"
+            cat /var/log/clickhouse-server/stdout.log
+            tail -n1000 /var/log/clickhouse-server/stderr.log
+            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
+            break
+        fi
+        sleep 0.5
+        counter=$((counter + 1))
+    done
+}
+
+
+chmod 777 /
+
+dpkg -i package_folder/clickhouse-common-static_*.deb; \
+    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
+    dpkg -i package_folder/clickhouse-server_*.deb;  \
+    dpkg -i package_folder/clickhouse-client_*.deb; \
+    dpkg -i package_folder/clickhouse-test_*.deb
+
+mkdir -p /var/lib/clickhouse
+mkdir -p /var/log/clickhouse-server
+chmod 777 -R /var/log/clickhouse-server/
+
+# install test configs
+/usr/share/clickhouse-test/config/install.sh
+
+start_clickhouse
+
+# shellcheck disable=SC2086 # No quotes because I want to split it into words.
+if ! /s3downloader --dataset-names $DATASETS; then
+    echo "Cannot download datatsets"
+    exit 1
+fi
+
+
+chmod 777 -R /var/lib/clickhouse
+
+
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW DATABASES"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "CREATE DATABASE test"
+
+kill_clickhouse
+start_clickhouse
+
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+
+LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time --use-skip-list 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt
+
+readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt")
+
+kill_clickhouse
+
+sleep 3
+
+if [[ -n "${FAILED_TESTS[*]}" ]]
+then
+    # Clean the data so that there is no interference from the previous test run.
+    rm -rf /var/lib/clickhouse/{{meta,}data,user_files} ||:
+
+    start_clickhouse
+
+    echo "Going to run again: ${FAILED_TESTS[*]}"
+
+    LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper --use-skip-list "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt
+else
+    echo "No failed tests"
+fi
+
+mkdir -p $COVERAGE_DIR
+mv /*.profraw $COVERAGE_DIR
+
+mkdir -p $SOURCE_DIR/obj-x86_64-linux-gnu
+cd $SOURCE_DIR/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
+llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata
+llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov
+genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}

docker/test/stateful_with_coverage/Dockerfile

@@ -1,15 +0,0 @@
-# docker build -t yandex/clickhouse-stateful-test-with-coverage .
-FROM yandex/clickhouse-stateless-test-with-coverage
-
-RUN apt-get update -y \
-    && env DEBIAN_FRONTEND=noninteractive \
-        apt-get install --yes --no-install-recommends \
-        python3-requests procps psmisc
-
-COPY s3downloader /s3downloader
-COPY run.sh /run.sh
-
-ENV DATASETS="hits visits"
-
-CMD ["/bin/bash", "/run.sh"]
-

docker/test/stateful_with_coverage/run.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-kill_clickhouse () {
-    echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
-    pkill -f "clickhouse-server" 2>/dev/null
-
-
-    for _ in {1..120}
-    do
-        if ! pkill -0 -f "clickhouse-server" ; then break ; fi
-        echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
-        sleep 1
-    done
-
-    if pkill -0 -f "clickhouse-server"
-    then
-        pstree -apgT
-        jobs
-        echo "Failed to kill the ClickHouse server"  | ts '%Y-%m-%d %H:%M:%S'
-        return 1
-    fi
-}
-
-start_clickhouse () {
-    LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
-    counter=0
-    until clickhouse-client --query "SELECT 1"
-    do
-        if [ "$counter" -gt 120 ]
-        then
-            echo "Cannot start clickhouse-server"
-            cat /var/log/clickhouse-server/stdout.log
-            tail -n1000 /var/log/clickhouse-server/stderr.log
-            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
-            break
-        fi
-        sleep 0.5
-        counter=$((counter + 1))
-    done
-}
-
-
-chmod 777 /
-
-dpkg -i package_folder/clickhouse-common-static_*.deb; \
-    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
-    dpkg -i package_folder/clickhouse-server_*.deb;  \
-    dpkg -i package_folder/clickhouse-client_*.deb; \
-    dpkg -i package_folder/clickhouse-test_*.deb
-
-mkdir -p /var/lib/clickhouse
-mkdir -p /var/log/clickhouse-server
-chmod 777 -R /var/log/clickhouse-server/
-
-# install test configs
-/usr/share/clickhouse-test/config/install.sh
-
-start_clickhouse
-
-# shellcheck disable=SC2086 # No quotes because I want to split it into words.
-if ! /s3downloader --dataset-names $DATASETS; then
-    echo "Cannot download datatsets"
-    exit 1
-fi
-
-
-chmod 777 -R /var/lib/clickhouse
-
-
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW DATABASES"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "CREATE DATABASE test"
-
-kill_clickhouse
-start_clickhouse
-
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
-
-
-if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
-    SKIP_LIST_OPT="--use-skip-list"
-fi
-
-# We can have several additional options so we path them as array because it's
-# more idiologically correct.
-read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
-
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
-
-kill_clickhouse
-
-sleep 3
-
-cp /*.profraw /profraw ||:

docker/test/stateless_with_coverage/Dockerfile

@@ -1,47 +0,0 @@
-# docker build -t yandex/clickhouse-stateless-test-with-coverage .
-# TODO: that can be based on yandex/clickhouse-stateless-test (llvm version and CMD differs)
-FROM yandex/clickhouse-test-base
-
-ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
-
-RUN apt-get update -y \
-    && env DEBIAN_FRONTEND=noninteractive \
-        apt-get install --yes --no-install-recommends \
-            bash \
-            tzdata \
-            fakeroot \
-            debhelper \
-            expect \
-            python3 \
-            python3-lxml \
-            python3-termcolor \
-            python3-requests \
-            sudo \
-            openssl \
-            ncdu \
-            netcat-openbsd \
-            telnet \
-            tree \
-            moreutils \
-            brotli \
-            zstd \
-            gdb \
-            lsof \
-            unixodbc \
-            wget \
-            qemu-user-static \
-            procps \
-            psmisc
-
-RUN mkdir -p /tmp/clickhouse-odbc-tmp \
-   && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \
-   && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \
-   && odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
-   && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
-   && rm -rf /tmp/clickhouse-odbc-tmp
-
-ENV TZ=Europe/Moscow
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-COPY run.sh /run.sh
-
-CMD ["/bin/bash", "/run.sh"]

docker/test/stateless_with_coverage/run.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-kill_clickhouse () {
-    echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
-    pkill -f "clickhouse-server" 2>/dev/null
-
-
-    for _ in {1..120}
-    do
-        if ! pkill -0 -f "clickhouse-server" ; then break ; fi
-        echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
-        sleep 1
-    done
-
-    if pkill -0 -f "clickhouse-server"
-    then
-        pstree -apgT
-        jobs
-        echo "Failed to kill the ClickHouse server"  | ts '%Y-%m-%d %H:%M:%S'
-        return 1
-    fi
-}
-
-start_clickhouse () {
-    LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
-    counter=0
-    until clickhouse-client --query "SELECT 1"
-    do
-        if [ "$counter" -gt 120 ]
-        then
-            echo "Cannot start clickhouse-server"
-            cat /var/log/clickhouse-server/stdout.log
-            tail -n1000 /var/log/clickhouse-server/stderr.log
-            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
-            break
-        fi
-        sleep 0.5
-        counter=$((counter + 1))
-    done
-}
-
-chmod 777 /
-
-dpkg -i package_folder/clickhouse-common-static_*.deb; \
-    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
-    dpkg -i package_folder/clickhouse-server_*.deb;  \
-    dpkg -i package_folder/clickhouse-client_*.deb; \
-    dpkg -i package_folder/clickhouse-test_*.deb
-
-
-mkdir -p /var/lib/clickhouse
-mkdir -p /var/log/clickhouse-server
-chmod 777 -R /var/lib/clickhouse
-chmod 777 -R /var/log/clickhouse-server/
-
-# install test configs
-/usr/share/clickhouse-test/config/install.sh
-
-start_clickhouse
-
-if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
-    SKIP_LIST_OPT="--use-skip-list"
-fi
-
-# We can have several additional options so we path them as array because it's
-# more idiologically correct.
-read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
-
-LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard --zookeeper --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
-
-kill_clickhouse
-
-sleep 3
-
-cp /*.profraw /profraw ||:

tests/integration/helpers/cluster.py

@@ -195,6 +195,12 @@ class ClickHouseCluster:
 
         if tag is None:
             tag = self.docker_base_tag
+        if not env_variables:
+            env_variables = {}
+
+        # Code coverage files will be placed in database directory
+        # (affect only WITH_COVERAGE=1 build)
+        env_variables['LLVM_PROFILE_FILE'] = '/var/lib/clickhouse/server_%h_%p_%m.profraw'
 
         instance = ClickHouseInstance(
             cluster=self,
@@ -221,7 +227,7 @@ class ClickHouseCluster:
             clickhouse_path_dir=clickhouse_path_dir,
             with_odbc_drivers=with_odbc_drivers,
             hostname=hostname,
-            env_variables=env_variables or {},
+            env_variables=env_variables,
             image=image,
             tag=tag,
             stay_alive=stay_alive,
@@ -759,9 +765,11 @@ class ClickHouseCluster:
 
         if kill:
             try:
-                subprocess_check_call(self.base_cmd + ['kill'])
+                subprocess_check_call(self.base_cmd + ['stop', '--timeout', '20'])
             except Exception as e:
-                print("Kill command failed durung shutdown. {}".format(repr(e)))
+                print("Kill command failed during shutdown. {}".format(repr(e)))
+                print("Trying to kill forcefully")
+                subprocess_check_call(self.base_cmd + ['kill'])
 
         try:
             subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans'])