Add runner for testflows

docker/test/testflows/runner/Dockerfile

@@ -0,0 +1,76 @@
+# docker build -t yandex/clickhouse-testflows-runner .
+FROM ubuntu:20.04
+
+RUN apt-get update \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+    ca-certificates \
+    bash \
+    btrfs-progs \
+    e2fsprogs \
+    iptables \
+    xfsprogs \
+    tar \
+    pigz \
+    wget \
+    git \
+    iproute2 \
+    cgroupfs-mount \
+    python3-pip \
+    tzdata \
+    libreadline-dev \
+    libicu-dev \
+    bsdutils \
+    curl \
+    liblua5.1-dev \
+    luajit \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    gdb \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /var/cache/debconf \
+        /tmp/* \
+    && apt-get clean
+
+ENV TZ=Europe/Moscow
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN pip3 install urllib3 testflows==1.6.24 docker-compose docker dicttoxml kazoo tzlocal
+
+ENV DOCKER_CHANNEL stable
+ENV DOCKER_VERSION 17.09.1-ce
+
+RUN set -eux; \
+  \
+# this "case" statement is generated via "update.sh"
+  \
+  if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \
+    echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \
+    exit 1; \
+  fi; \
+  \
+  tar --extract \
+    --file docker.tgz \
+    --strip-components 1 \
+    --directory /usr/local/bin/ \
+  ; \
+  rm docker.tgz; \
+  \
+  dockerd --version; \
+  docker --version
+
+COPY modprobe.sh /usr/local/bin/modprobe
+COPY dockerd-entrypoint.sh /usr/local/bin/
+
+RUN set -x \
+  && addgroup --system dockremap \
+    && adduser --system dockremap \
+  && adduser dockremap dockremap \
+  && echo 'dockremap:165536:65536' >> /etc/subuid \
+    && echo 'dockremap:165536:65536' >> /etc/subgid
+
+VOLUME /var/lib/docker
+EXPOSE 2375
+ENTRYPOINT ["dockerd-entrypoint.sh"]
+CMD ["sh", "-c", "python3 regression.py --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS} && cat test.log | tfs report results --format json > results.json"]
+

docker/test/testflows/runner/dockerd-entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &>/var/log/somefile &
+
+set +e
+reties=0
+while true; do
+    docker info &>/dev/null && break
+    reties=$[$reties+1]
+    if [[ $reties -ge 100 ]]; then # 10 sec max
+        echo "Can't start docker daemon, timeout exceeded." >&2
+        exit 1;
+    fi
+    sleep 0.1
+done
+set -e
+
+echo "Start tests"
+export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse
+export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse
+export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
+export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
+
+cd /ClickHouse/tests/testflows
+exec "$@"

docker/test/testflows/runner/modprobe.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+    if [ "${module#-}" = "$module" ]; then
+        ip link show "$module" || true
+        lsmod | grep "$module" || true
+    fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

tests/testflows/runner

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+import subprocess
+import os
+import getpass
+import argparse
+import logging
+import signal
+import subprocess
+import sys
+
+CUR_FILE_DIR = os.path.dirname(os.path.realpath(__file__))
+DEFAULT_CLICKHOUSE_ROOT = os.path.abspath(os.path.join(CUR_FILE_DIR, "../../"))
+CURRENT_WORK_DIR = os.getcwd()
+CONTAINER_NAME = "clickhouse_testflows_tests"
+
+DIND_TESTFLOWS_TESTS_IMAGE_NAME = "yandex/clickhouse-testflows-runner"
+
+def check_args_and_update_paths(args):
+    if not os.path.isabs(args.binary):
+        args.binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.binary))
+
+    if not args.bridge_binary:
+        args.bridge_binary = os.path.join(os.path.dirname(args.binary), 'clickhouse-odbc-bridge')
+    elif not os.path.isabs(args.bridge_binary):
+        args.bridge_binary = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.bridge_binary))
+
+    if not os.path.isabs(args.configs_dir):
+        args.configs_dir = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.configs_dir))
+
+    if not os.path.isabs(args.clickhouse_root):
+        args.clickhouse_root = os.path.abspath(os.path.join(CURRENT_WORK_DIR, args.clickhouse_root))
+
+    for path in [args.binary, args.configs_dir, args.clickhouse_root]:
+        if not os.path.exists(path):
+            raise Exception("Path {} doesn't exists".format(path))
+
+def docker_kill_handler_handler(signum, frame):
+    subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True)
+    raise KeyboardInterrupt("Killed by Ctrl+C")
+
+signal.signal(signal.SIGINT, docker_kill_handler_handler)
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    parser = argparse.ArgumentParser(description="ClickHouse testflows runner")
+
+    parser.add_argument(
+        "--binary",
+        default=os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH", os.environ.get("CLICKHOUSE_TESTS_CLIENT_BIN_PATH", "/usr/bin/clickhouse")),
+        help="Path to clickhouse binary")
+
+    parser.add_argument(
+        "--bridge-binary",
+        default=os.environ.get("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH", ""),
+        help="Path to clickhouse-odbc-bridge binary. Defaults to clickhouse-odbc-bridge in the same dir as clickhouse.")
+
+    parser.add_argument(
+        "--configs-dir",
+        default=os.environ.get("CLICKHOUSE_TESTS_BASE_CONFIG_DIR", os.path.join(DEFAULT_CLICKHOUSE_ROOT, "programs/server")),
+        help="Path to clickhouse configs directory")
+
+    parser.add_argument(
+        "--clickhouse-root",
+        default=DEFAULT_CLICKHOUSE_ROOT,
+        help="Path to repository root folder")
+
+    parser.add_argument(
+        "--command",
+        default='',
+        help="Set it to run some other command in container (for example bash)")
+
+    parser.add_argument(
+        "--disable-net-host",
+        action='store_true',
+        default=False,
+        help="Don't use net host in parent docker container")
+
+    parser.add_argument(
+        "--docker-image-version",
+        default="latest",
+        help="Version of docker image which runner will use to run tests")
+
+
+    parser.add_argument('testflows_args', nargs='*', help="args for testflows command")
+
+    args = parser.parse_args()
+
+    check_args_and_update_paths(args)
+
+    net = ""
+    if not args.disable_net_host:
+        net = "--net=host"
+
+    # create named volume which will be used inside to store images and other docker related files,
+    # to avoid redownloading it every time
+    #
+    # should be removed manually when not needed
+    subprocess.check_call('docker volume create {name}_volume'.format(name=CONTAINER_NAME), shell=True)
+
+    # enable tty mode & interactive for docker if we have real tty
+    tty = ""
+    if sys.stdout.isatty() and sys.stdin.isatty():
+        tty = "-it"
+
+    cmd = "docker run {net} {tty} --rm --name {name} --privileged --volume={bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \
+        --volume={cfg}:/clickhouse-config --volume={pth}:/ClickHouse --volume={name}_volume:/var/lib/docker -e TESTFLOWS_OPTS='{opts}' {img} {command}".format(
+        net=net,
+        tty=tty,
+        bin=args.binary,
+        bridge_bin=args.bridge_binary,
+        cfg=args.configs_dir,
+        pth=args.clickhouse_root,
+        opts=' '.join(args.testflows_args),
+        img=DIND_TESTFLOWS_TESTS_IMAGE_NAME + ":" + args.docker_image_version,
+        name=CONTAINER_NAME,
+        command=args.command
+    )
+
+    print("Running testflows container as: '" + cmd + "'.")
+    subprocess.check_call(cmd, shell=True)