Merge pull request #3438 from yandex/CLICKHOUSE-4080

CLICKHOUSE-4080: Add simple stress-test script and docker image for it

docker/test/stress/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y \
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get install --yes --no-install-recommends \
+            bash \
+            tzdata \
+            fakeroot \
+            debhelper \
+            parallel \
+            expect \
+            python \
+            python-lxml \
+            python-termcolor \
+            python-requests \
+            curl \
+            sudo \
+            openssl \
+            netcat-openbsd \
+            telnet
+
+COPY ./stress /stress
+
+CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
+    dpkg -i package_folder/clickhouse-server_*.deb;  \
+    dpkg -i package_folder/clickhouse-client_*.deb; \
+    dpkg -i package_folder/clickhouse-test_*.deb; \
+    service clickhouse-server start && sleep 1 && ./stress --output-folder test_output

docker/test/stress/README.md

@@ -0,0 +1,31 @@
+Allow to run simple ClickHouse stress test in Docker from debian packages.
+Actually it runs single copy of clickhouse-performance-test and multiple copies
+of clickhouse-test (functional tests). This allows to find problems like
+segmentation fault which cause shutdown of server.
+
+Usage:
+```
+$ ls $HOME/someclickhouse
+clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb
+$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output yandex/clickhouse-stress-test
+Selecting previously unselected package clickhouse-common-static.
+(Reading database ... 14442 files and directories currently installed.)
+...
+Start clickhouse-server service: Path to data directory in /etc/clickhouse-server/config.xml: /var/lib/clickhouse/
+DONE
+2018-10-22 13:40:35,744 Will wait functests to finish
+2018-10-22 13:40:40,747 Finished 0 from 16 processes
+2018-10-22 13:40:45,751 Finished 0 from 16 processes
+...
+2018-10-22 13:49:11,165 Finished 15 from 16 processes
+2018-10-22 13:49:16,171 Checking ClickHouse still alive
+Still alive
+2018-10-22 13:49:16,195 Stress is ok
+2018-10-22 13:49:16,195 Copying server log files
+$ ls $HOME/test_result
+clickhouse-server.err.log clickhouse-server.log.0.gz stderr.log stress_test_run_0.txt  stress_test_run_11.txt stress_test_run_13.txt
+stress_test_run_15.txt stress_test_run_2.txt stress_test_run_4.txt stress_test_run_6.txt stress_test_run_8.txt clickhouse-server.log
+perf_stress_run.txt stdout.log stress_test_run_10.txt stress_test_run_12.txt
+stress_test_run_14.txt stress_test_run_1.txt
+stress_test_run_3.txt stress_test_run_5.txt stress_test_run_7.txt stress_test_run_9.txt
+```

docker/test/stress/stress

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+from multiprocessing import cpu_count
+from subprocess import Popen, check_call
+import os
+import shutil
+import argparse
+import logging
+import time
+
+def run_perf_test(cmd, xmls_path, output_folder):
+    output_path = os.path.join(output_folder, "perf_stress_run.txt")
+    f = open(output_path, 'w')
+    p = Popen("{} --skip-tags=long --r {}".format(cmd, xmls_path), shell=True, stdout=f, stderr=f)
+    return p
+
+def run_func_test(cmd, output_prefix, num_processes):
+    output_paths = [os.path.join(output_prefix, "stress_test_run_{}.txt".format(i)) for i in range(num_processes)]
+    f = open(output_paths[0], 'w')
+    pipes = [Popen("{}".format(cmd), shell=True, stdout=f, stderr=f)]
+    for output_path in output_paths[1:]:
+        time.sleep(0.5)
+        f = open(output_path, 'w')
+        p = Popen("{} --order=random".format(cmd), shell=True, stdout=f, stderr=f)
+        pipes.append(p)
+    return pipes
+
+def check_clickhouse_alive(cmd):
+    try:
+        logging.info("Checking ClickHouse still alive")
+        check_call("{} --query \"select 'Still alive'\"".format(cmd), shell=True)
+        return True
+    except:
+        return False
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    parser = argparse.ArgumentParser(description="ClickHouse script for running stresstest")
+    parser.add_argument("--test-cmd", default='clickhouse-test')
+    parser.add_argument("--client-cmd", default='clickhouse-client')
+    parser.add_argument("--perf-test-cmd", default='clickhouse-performance-test')
+    parser.add_argument("--perf-test-xml-path", default='/usr/share/clickhouse-test/performance/')
+    parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
+    parser.add_argument("--output-folder")
+    parser.add_argument("--num-parallel", default=cpu_count() // 3);
+
+    args = parser.parse_args()
+    func_pipes = []
+    perf_process = None
+    try:
+        perf_process = run_perf_test(args.perf_test_cmd, args.perf_test_xml_path, args.output_folder)
+        func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel)
+
+        logging.info("Will wait functests to finish")
+        while True:
+            retcodes = []
+            for p in func_pipes:
+                if p.poll():
+                    retcodes.append(p.returncode)
+            if len(retcodes) == len(func_pipes):
+                break
+            logging.info("Finished %s from %s processes", len(retcodes), len(func_pipes))
+            time.sleep(5)
+
+        if not check_clickhouse_alive(args.client_cmd):
+            raise Exception("Stress failed, results in logs")
+        else:
+            logging.info("Stress is ok")
+    except Exception as ex:
+        raise ex
+    finally:
+        if os.path.exists(args.server_log_folder):
+            logging.info("Copying server log files")
+            for log_file in os.listdir(args.server_log_folder):
+                shutil.copy(os.path.join(args.server_log_folder, log_file), os.path.join(args.output_folder, log_file))