Add two new images for coverage tests run

docker/images.json

@@ -7,7 +7,9 @@
     "docker/test/performance": "yandex/clickhouse-performance-test",
     "docker/test/pvs": "yandex/clickhouse-pvs-test",
     "docker/test/stateful": "yandex/clickhouse-stateful-test",
+    "docker/test/stateful_with_coverage": "yandex/clickhouse-stateful-with-coverage-test",
     "docker/test/stateless": "yandex/clickhouse-stateless-test",
+    "docker/test/stateless_with_coverage": "yandex/clickhouse-stateless-with-coverage-test",
     "docker/test/unit": "yandex/clickhouse-unit-test",
     "docker/test/stress": "yandex/clickhouse-stress-test",
     "dbms/tests/integration/image": "yandex/clickhouse-integration-tests-runner"

docker/test/coverage/Dockerfile

@@ -31,5 +31,5 @@ ENV IGNORE='.*contrib.*'
 
 CMD dpkg -i /package_folder/clickhouse-common-static_*.deb; \
     llvm-profdata-9 merge -sparse ${COVERAGE_DIR}/* -o clickhouse.profdata && \
-    llvm-cov-9 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -arch=x86_64 -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
+    llvm-cov-9 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex $IGNORE > output.lcov && \
     genhtml output.lcov --ignore-errors source --output-directory ${OUTPUT_DIR}

docker/test/stateful/Dockerfile

@@ -10,10 +10,8 @@ RUN apt-get update -y \
 COPY s3downloader /s3downloader
 
 ENV DATASETS="hits visits"
-ENV WITH_COVERAGE=0
 
-CMD chmod 777 /; \
+CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
-    dpkg -i package_folder/clickhouse-common-static_*.deb; \
     dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
     dpkg -i package_folder/clickhouse-server_*.deb;  \
     dpkg -i package_folder/clickhouse-client_*.deb; \
@@ -46,5 +44,4 @@ CMD chmod 777 /; \
     && clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \
     && clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \
     && clickhouse-client --query "SHOW TABLES FROM test" \
-    && clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt; \
+    && clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt;
-    if [ $WITH_COVERAGE -eq 1 ]; then cp /*.profraw /profraw ||:; fi

docker/test/stateful_with_coverage/Dockerfile

@@ -0,0 +1,16 @@
+# docker build -t yandex/clickhouse-stateful-test .
+FROM yandex/clickhouse-stateless-test
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+        python-requests \
+        llvm-8
+
+COPY s3downloader /s3downloader
+COPY run.sh /run.sh
+
+ENV DATASETS="hits visits"
+
+CMD ["/bin/bash", "/run.sh"]
+

docker/test/stateful_with_coverage/run.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -x
+
+kill_clickhouse () {
+    while kill -0 `pgrep -u clickhouse`;
+    do
+        kill `pgrep -u clickhouse` 2>/dev/null
+        echo "Process" `pgrep -u clickhouse` "still alive"
+        sleep 10
+    done
+}
+
+start_clickhouse () {
+    LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
+}
+
+chmod 777 /
+
+dpkg -i package_folder/clickhouse-common-static_*.deb; \
+    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
+    dpkg -i package_folder/clickhouse-server_*.deb;  \
+    dpkg -i package_folder/clickhouse-client_*.deb; \
+    dpkg -i package_folder/clickhouse-test_*.deb
+
+ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer
+
+service zookeeper start
+
+sleep 5
+
+start_clickhouse
+
+sleep 5
+
+/s3downloader --dataset-names $DATASETS
+chmod 777 -R /var/lib/clickhouse
+
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW DATABASES"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "CREATE DATABASE datasets"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "CREATE DATABASE test"
+
+kill_clickhouse
+start_clickhouse
+
+sleep 10
+
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-client --query "SHOW TABLES FROM test"
+LLVM_PROFILE_FILE='client.profraw' clickhouse-test --shard --zookeeper --no-stateless 00146 $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+
+kill_clickhouse
+
+cp /*.profraw /profraw ||:

docker/test/stateful_with_coverage/s3downloader

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import sys
+import tarfile
+import logging
+import argparse
+import requests
+import tempfile
+
+
+DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net'
+
+AVAILABLE_DATASETS = {
+    'hits': 'hits_v1.tar',
+    'visits': 'visits_v1.tar',
+}
+
+def _get_temp_file_name():
+    return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
+
+def build_url(base_url, dataset):
+    return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
+
+def dowload_with_progress(url, path):
+    logging.info("Downloading from %s to temp path %s", url, path)
+    with open(path, 'w') as f:
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+        total_length = response.headers.get('content-length')
+        if total_length is None or int(total_length) == 0:
+            logging.info("No content-length, will download file without progress")
+            f.write(response.content)
+        else:
+            dl = 0
+            total_length = int(total_length)
+            logging.info("Content length is %ld bytes", total_length)
+            for data in response.iter_content(chunk_size=4096):
+                dl += len(data)
+                f.write(data)
+                if sys.stdout.isatty():
+                    done = int(50 * dl / total_length)
+                    percent = int(100 * float(dl) / total_length)
+                    sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
+                    sys.stdout.flush()
+    sys.stdout.write("\n")
+    logging.info("Downloading finished")
+
+def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
+    logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
+    with tarfile.open(tar_path, 'r') as comp_file:
+        comp_file.extractall(path=clickhouse_path)
+    logging.info("Unpack finished")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser(
+        description="Simple tool for dowloading datasets for clickhouse from S3")
+
+    parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys())
+    parser.add_argument('--url-prefix', default=DEFAULT_URL)
+    parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
+
+    args = parser.parse_args()
+    datasets = args.dataset_names
+    logging.info("Will fetch following datasets: %s", ', '.join(datasets))
+    for dataset in datasets:
+        logging.info("Processing %s", dataset)
+        temp_archive_path = _get_temp_file_name()
+        try:
+            download_url_for_dataset = build_url(args.url_prefix, dataset)
+            dowload_with_progress(download_url_for_dataset, temp_archive_path)
+            unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
+        except Exception as ex:
+            logging.info("Some exception occured %s", str(ex))
+            raise
+        finally:
+            logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
+            if os.path.exists(temp_archive_path):
+                os.remove(temp_archive_path)
+        logging.info("Processing of %s finished", dataset)
+    logging.info("Fetch finished, enjoy your tables!")
+
+

docker/test/stateless/Dockerfile

@@ -29,11 +29,9 @@ RUN apt-get update -y \
 
 ENV TZ=Europe/Moscow
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-ENV WITH_COVERAGE=0
 
 
-CMD chmod 777 /; \
+CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
-    dpkg -i package_folder/clickhouse-common-static_*.deb; \
     dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
     dpkg -i package_folder/clickhouse-server_*.deb;  \
     dpkg -i package_folder/clickhouse-client_*.deb; \
@@ -54,6 +52,4 @@ CMD chmod 777 /; \
     echo "TSAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer" >> /etc/environment; \
     echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer" >> /etc/environment; \
     service zookeeper start; sleep 5; \
-    service clickhouse-server start && sleep 5 && \
+    service clickhouse-server start && sleep 5 && clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
-    clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt; \
-    if [ $WITH_COVERAGE -eq 1 ]; then cp /*.profraw /profraw ||:; fi

docker/test/stateless_with_coverage/Dockerfile

@@ -0,0 +1,55 @@
+# docker build -t yandex/clickhouse-stateless-with-coverage-test .
+FROM yandex/clickhouse-deb-builder
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            bash \
+            tzdata \
+            fakeroot \
+            debhelper \
+            zookeeper \
+            zookeeperd \
+            expect \
+            python \
+            python-lxml \
+            python-termcolor \
+            python-requests \
+            curl \
+            sudo \
+            openssl \
+            netcat-openbsd \
+            telnet \
+            moreutils \
+            brotli \
+            gdb \
+            lsof
+
+
+ENV TZ=Europe/Moscow
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+ENV LLVM_PROFILE_FILE='code_%h_%p_%m.profraw'
+
+
+CMD chmod 777 /; \
+    echo "LLVM_PROFILE_FILE='code_%h_%p_%m.profraw'" >> /etc/environment; \
+    dpkg -i package_folder/clickhouse-common-static_*.deb; \
+    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
+    dpkg -i package_folder/clickhouse-server_*.deb;  \
+    dpkg -i package_folder/clickhouse-client_*.deb; \
+    dpkg -i package_folder/clickhouse-test_*.deb; \
+    ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; \
+    service zookeeper start; sleep 5; \
+    LLVM_PROFILE_FILE='code_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml 1>/dev/null 2>/dev/null & \
+    sleep 10; \
+    LLVM_PROFILE_FILE='client.profraw' clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt; \
+    while kill -0 `pgrep -u clickhouse`; do kill `pgrep -u clickhouse` 2>/dev/null; echo "Process" `pgrep -u clickhouse` "still alive"; sleep 10; done; \
+    cp /*.profraw /profraw ||:

docker/test/stateless_with_coverage/run.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -x
+
+kill_clickhouse () {
+    while kill -0 `pgrep -u clickhouse`;
+    do
+        kill `pgrep -u clickhouse` 2>/dev/null
+        echo "Process" `pgrep -u clickhouse` "still alive"
+        sleep 10
+    done
+}
+
+start_clickhouse () {
+    LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
+}
+
+chmod 777 /
+dpkg -i package_folder/clickhouse-common-static_*.deb; \
+    dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
+    dpkg -i package_folder/clickhouse-server_*.deb;  \
+    dpkg -i package_folder/clickhouse-client_*.deb; \
+    dpkg -i package_folder/clickhouse-test_*.deb
+
+ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/; \
+    ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \
+    ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \
+    ln -s /usr/lib/llvm-8/bin/llvm-symbolizer /usr/bin/llvm-symbolizer
+
+service zookeeper start
+sleep 5
+
+start_clickhouse
+
+sleep 10
+
+LLVM_PROFILE_FILE='client.profraw' clickhouse-test --shard --zookeeper $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
+
+kill_clickhouse
+
+cp /*.profraw /profraw ||: