Improvements in coverage images

docker/test/coverage/Dockerfile

@@ -7,8 +7,10 @@ ENV SOURCE_DIR=/build
 ENV OUTPUT_DIR=/output
 ENV IGNORE='.*contrib.*'
 
-CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-10 CXX=clang++-10 cmake .. && cd /; \
+RUN apt-get update && apt-get install cmake --yes --no-install-recommends
+
+CMD mkdir -p /build/obj-x86_64-linux-gnu && cd /build/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /; \
     dpkg -i /package_folder/clickhouse-common-static_*.deb; \
-    llvm-profdata-10 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
-    llvm-cov-10 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
+    llvm-profdata-11 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
+    llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
     genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}

docker/test/stateful_with_coverage/Dockerfile

@@ -1,12 +1,12 @@
 # docker build -t yandex/clickhouse-stateful-test-with-coverage .
-FROM yandex/clickhouse-stateless-test
+FROM yandex/clickhouse-stateless-test-with-coverage
 
 RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list
 
 RUN apt-get update -y \
     && env DEBIAN_FRONTEND=noninteractive \
         apt-get install --yes --no-install-recommends \
-        python3-requests
+        python3-requests procps psmisc
 
 COPY s3downloader /s3downloader
 COPY run.sh /run.sh

docker/test/stateful_with_coverage/run.sh

@@ -1,40 +1,44 @@
 #!/bin/bash
 
 kill_clickhouse () {
-    kill "$(pgrep -u clickhouse)" 2>/dev/null
+    echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
+    pkill -f "clickhouse-server" 2>/dev/null
 
-    for _ in {1..10}
+
+    for _ in {1..120}
     do
-        if ! kill -0 "$(pgrep -u clickhouse)"; then
-            echo "No clickhouse process"
-            break
-        else
-            echo "Process $(pgrep -u clickhouse) still alive"
-            sleep 10
-        fi
+        if ! pkill -0 -f "clickhouse-server" ; then break ; fi
+        echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
+        sleep 1
     done
+
+    if pkill -0 -f "clickhouse-server"
+    then
+        pstree -apgT
+        jobs
+        echo "Failed to kill the ClickHouse server"  | ts '%Y-%m-%d %H:%M:%S'
+        return 1
+    fi
 }
 
 start_clickhouse () {
     LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
-}
-
-wait_llvm_profdata () {
-    while kill -0 "$(pgrep llvm-profdata-10)"
+    counter=0
+    until clickhouse-client --query "SELECT 1"
     do
-        echo "Waiting for profdata $(pgrep llvm-profdata-10) still alive"
-        sleep 3
+        if [ "$counter" -gt 120 ]
+        then
+            echo "Cannot start clickhouse-server"
+            cat /var/log/clickhouse-server/stdout.log
+            tail -n1000 /var/log/clickhouse-server/stderr.log
+            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
+            break
+        fi
+        sleep 0.5
+        counter=$((counter + 1))
     done
 }
 
-merge_client_files_in_background () {
-    client_files=$(ls /client_*profraw 2>/dev/null)
-    if [ -n "$client_files" ]
-    then
-        llvm-profdata-10 merge -sparse "$client_files" -o "merged_client_$(date +%s).profraw"
-        rm "$client_files"
-    fi
-}
 
 chmod 777 /
 
@@ -51,26 +55,7 @@ chmod 777 -R /var/log/clickhouse-server/
 # install test configs
 /usr/share/clickhouse-test/config/install.sh
 
-function start()
-{
-    counter=0
-    until clickhouse-client --query "SELECT 1"
-    do
-        if [ "$counter" -gt 120 ]
-        then
-            echo "Cannot start clickhouse-server"
-            cat /var/log/clickhouse-server/stdout.log
-            tail -n1000 /var/log/clickhouse-server/stderr.log
-            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
-            break
-        fi
-        timeout 120 service clickhouse-server start
-        sleep 0.5
-        counter=$((counter + 1))
-    done
-}
-
-start
+start_clickhouse
 
 # shellcheck disable=SC2086 # No quotes because I want to split it into words.
 if ! /s3downloader --dataset-names $DATASETS; then
@@ -81,10 +66,6 @@ fi
 
 chmod 777 -R /var/lib/clickhouse
 
-while /bin/true; do
-    merge_client_files_in_background
-    sleep 2
-done &
 
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW DATABASES"
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
@@ -93,14 +74,13 @@ LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "CREATE DA
 kill_clickhouse
 start_clickhouse
 
-sleep 10
-
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
 LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-client --query "SHOW TABLES FROM test"
 
+
 if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
     SKIP_LIST_OPT="--use-skip-list"
 fi
@@ -113,11 +93,6 @@ LLVM_PROFILE_FILE='client_coverage.profraw' clickhouse-test --testname --shard -
 
 kill_clickhouse
 
-wait_llvm_profdata
-
 sleep 3
 
-wait_llvm_profdata # 100% merged all parts
-
-
 cp /*.profraw /profraw ||:

docker/test/stateless_with_coverage/Dockerfile

@@ -1,4 +1,4 @@
-# docker build -t yandex/clickhouse-stateless-with-coverage-test .
+# docker build -t yandex/clickhouse-stateless-test-with-coverage .
 # TODO: that can be based on yandex/clickhouse-stateless-test (llvm version and CMD differs)
 FROM yandex/clickhouse-test-base
 
@@ -28,7 +28,9 @@ RUN apt-get update -y \
             lsof \
             unixodbc \
             wget \
-            qemu-user-static
+            qemu-user-static \
+            procps \
+            psmisc
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
    && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \

docker/test/stateless_with_coverage/run.sh

@@ -2,27 +2,41 @@
 
 kill_clickhouse () {
     echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
-    kill "$(pgrep -u clickhouse)" 2>/dev/null
+    pkill -f "clickhouse-server" 2>/dev/null
 
-    for _ in {1..10}
+
+    for _ in {1..120}
     do
-        if ! kill -0 "$(pgrep -u clickhouse)"; then
-            echo "No clickhouse process" | ts '%Y-%m-%d %H:%M:%S'
-            break
-        else
-            echo "Process $(pgrep -u clickhouse) still alive" | ts '%Y-%m-%d %H:%M:%S'
-            sleep 10
-        fi
+        if ! pkill -0 -f "clickhouse-server" ; then break ; fi
+        echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
+        sleep 1
     done
 
-    echo "Will try to send second kill signal for sure"
-    kill "$(pgrep -u clickhouse)" 2>/dev/null
-    sleep 5
-    echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
+    if pkill -0 -f "clickhouse-server"
+    then
+        pstree -apgT
+        jobs
+        echo "Failed to kill the ClickHouse server"  | ts '%Y-%m-%d %H:%M:%S'
+        return 1
+    fi
 }
 
 start_clickhouse () {
     LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
+    counter=0
+    until clickhouse-client --query "SELECT 1"
+    do
+        if [ "$counter" -gt 120 ]
+        then
+            echo "Cannot start clickhouse-server"
+            cat /var/log/clickhouse-server/stdout.log
+            tail -n1000 /var/log/clickhouse-server/stderr.log
+            tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
+            break
+        fi
+        sleep 0.5
+        counter=$((counter + 1))
+    done
 }
 
 chmod 777 /
@@ -44,9 +58,6 @@ chmod 777 -R /var/log/clickhouse-server/
 
 start_clickhouse
 
-sleep 10
-
-
 if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
     SKIP_LIST_OPT="--use-skip-list"
 fi