From a0ce15befdb3f76ca58c28522559209990298f43 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 17 Apr 2020 19:01:16 +0300 Subject: [PATCH] Move integration tests docker files to docker/ directory. Rename image -> runner. --- docker/README.md | 2 +- docker/images.json | 2 +- docker/test/integration/README.md | 6 +++ docker/test/integration/{ => base}/Dockerfile | 0 .../compose}/docker_compose_hdfs.yml | 0 .../compose}/docker_compose_kafka.yml | 0 .../compose}/docker_compose_minio.yml | 0 .../compose}/docker_compose_mongo.yml | 0 .../compose}/docker_compose_mysql.yml | 0 .../compose}/docker_compose_net.yml | 0 .../compose}/docker_compose_postgres.yml | 0 .../compose}/docker_compose_redis.yml | 0 .../compose}/docker_compose_zookeeper.yml | 0 .../integration}/helper_container/Dockerfile | 0 .../test/integration/runner}/Dockerfile | 38 ++++++++-------- .../integration/runner}/dockerd-entrypoint.sh | 0 .../test/integration/runner}/modprobe.sh | 8 ++-- tests/integration/README.md | 2 +- tests/integration/helpers/cluster.py | 44 ++++++++++--------- tests/integration/helpers/network.py | 4 +- 20 files changed, 57 insertions(+), 49 deletions(-) create mode 100644 docker/test/integration/README.md rename docker/test/integration/{ => base}/Dockerfile (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_hdfs.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_kafka.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_minio.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_mongo.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_mysql.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_net.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_postgres.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_redis.yml (100%) rename {tests/integration/helpers => docker/test/integration/compose}/docker_compose_zookeeper.yml (100%) rename {tests/integration/helpers => docker/test/integration}/helper_container/Dockerfile (100%) rename {tests/integration/image => docker/test/integration/runner}/Dockerfile (70%) rename {tests/integration/image => docker/test/integration/runner}/dockerd-entrypoint.sh (100%) rename {tests/integration/image => docker/test/integration/runner}/modprobe.sh (80%) diff --git a/docker/README.md b/docker/README.md index 083c6a1dc28..ec52ddd143e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,5 @@ ## ClickHouse Dockerfiles -This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They updated each release. +This directory contain Dockerfiles for `clickhouse-client` and `clickhouse-server`. They are updated in each release. Also there is bunch of images for testing and CI. They are listed in `images.json` file and updated on each commit to master. If you need to add another image, place information about it into `images.json`. diff --git a/docker/images.json b/docker/images.json index 434a3c7af10..0ff38981242 100644 --- a/docker/images.json +++ b/docker/images.json @@ -15,5 +15,5 @@ "docker/test/stress": "yandex/clickhouse-stress-test", "docker/test/split_build_smoke_test": "yandex/clickhouse-split-build-smoke-test", "docker/test/codebrowser": "yandex/clickhouse-codebrowser", - "tests/integration/image": "yandex/clickhouse-integration-tests-runner" + "docker/test/integration/runner": "yandex/clickhouse-integration-tests-runner" } diff --git a/docker/test/integration/README.md b/docker/test/integration/README.md new file mode 100644 index 00000000000..4aa10d6db80 --- /dev/null +++ b/docker/test/integration/README.md @@ -0,0 +1,6 @@ +## Docker containers for integration tests +- `base` container with required packages +- `runner` container with that runs integration tests in docker +- `compose` contains docker_compose YaML files that are used in tests + +How to run integration tests is described in tests/integration/README.md \ No newline at end of file diff --git a/docker/test/integration/Dockerfile b/docker/test/integration/base/Dockerfile similarity index 100% rename from docker/test/integration/Dockerfile rename to docker/test/integration/base/Dockerfile diff --git a/tests/integration/helpers/docker_compose_hdfs.yml b/docker/test/integration/compose/docker_compose_hdfs.yml similarity index 100% rename from tests/integration/helpers/docker_compose_hdfs.yml rename to docker/test/integration/compose/docker_compose_hdfs.yml diff --git a/tests/integration/helpers/docker_compose_kafka.yml b/docker/test/integration/compose/docker_compose_kafka.yml similarity index 100% rename from tests/integration/helpers/docker_compose_kafka.yml rename to docker/test/integration/compose/docker_compose_kafka.yml diff --git a/tests/integration/helpers/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml similarity index 100% rename from tests/integration/helpers/docker_compose_minio.yml rename to docker/test/integration/compose/docker_compose_minio.yml diff --git a/tests/integration/helpers/docker_compose_mongo.yml b/docker/test/integration/compose/docker_compose_mongo.yml similarity index 100% rename from tests/integration/helpers/docker_compose_mongo.yml rename to docker/test/integration/compose/docker_compose_mongo.yml diff --git a/tests/integration/helpers/docker_compose_mysql.yml b/docker/test/integration/compose/docker_compose_mysql.yml similarity index 100% rename from tests/integration/helpers/docker_compose_mysql.yml rename to docker/test/integration/compose/docker_compose_mysql.yml diff --git a/tests/integration/helpers/docker_compose_net.yml b/docker/test/integration/compose/docker_compose_net.yml similarity index 100% rename from tests/integration/helpers/docker_compose_net.yml rename to docker/test/integration/compose/docker_compose_net.yml diff --git a/tests/integration/helpers/docker_compose_postgres.yml b/docker/test/integration/compose/docker_compose_postgres.yml similarity index 100% rename from tests/integration/helpers/docker_compose_postgres.yml rename to docker/test/integration/compose/docker_compose_postgres.yml diff --git a/tests/integration/helpers/docker_compose_redis.yml b/docker/test/integration/compose/docker_compose_redis.yml similarity index 100% rename from tests/integration/helpers/docker_compose_redis.yml rename to docker/test/integration/compose/docker_compose_redis.yml diff --git a/tests/integration/helpers/docker_compose_zookeeper.yml b/docker/test/integration/compose/docker_compose_zookeeper.yml similarity index 100% rename from tests/integration/helpers/docker_compose_zookeeper.yml rename to docker/test/integration/compose/docker_compose_zookeeper.yml diff --git a/tests/integration/helpers/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile similarity index 100% rename from tests/integration/helpers/helper_container/Dockerfile rename to docker/test/integration/helper_container/Dockerfile diff --git a/tests/integration/image/Dockerfile b/docker/test/integration/runner/Dockerfile similarity index 70% rename from tests/integration/image/Dockerfile rename to docker/test/integration/runner/Dockerfile index 173551eb14c..648b98fa60d 100644 --- a/tests/integration/image/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -41,32 +41,32 @@ ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce RUN set -eux; \ - \ + \ # this "case" statement is generated via "update.sh" - \ - if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ - exit 1; \ - fi; \ - \ - tar --extract \ - --file docker.tgz \ - --strip-components 1 \ - --directory /usr/local/bin/ \ - ; \ - rm docker.tgz; \ - \ - dockerd --version; \ - docker --version + \ + if ! wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ + echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ + exit 1; \ + fi; \ + \ + tar --extract \ + --file docker.tgz \ + --strip-components 1 \ + --directory /usr/local/bin/ \ + ; \ + rm docker.tgz; \ + \ + dockerd --version; \ + docker --version COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ RUN set -x \ - && addgroup --system dockremap \ + && addgroup --system dockremap \ && adduser --system dockremap \ - && adduser dockremap dockremap \ - && echo 'dockremap:165536:65536' >> /etc/subuid \ + && adduser dockremap dockremap \ + && echo 'dockremap:165536:65536' >> /etc/subuid \ && echo 'dockremap:165536:65536' >> /etc/subgid VOLUME /var/lib/docker diff --git a/tests/integration/image/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh similarity index 100% rename from tests/integration/image/dockerd-entrypoint.sh rename to docker/test/integration/runner/dockerd-entrypoint.sh diff --git a/tests/integration/image/modprobe.sh b/docker/test/integration/runner/modprobe.sh similarity index 80% rename from tests/integration/image/modprobe.sh rename to docker/test/integration/runner/modprobe.sh index b357d893fda..141e45d0ec8 100755 --- a/tests/integration/image/modprobe.sh +++ b/docker/test/integration/runner/modprobe.sh @@ -9,10 +9,10 @@ set -eu # Docker often uses "modprobe -va foo bar baz" # so we ignore modules that start with "-" for module; do - if [ "${module#-}" = "$module" ]; then - ip link show "$module" || true - lsmod | grep "$module" || true - fi + if [ "${module#-}" = "$module" ]; then + ip link show "$module" || true + lsmod | grep "$module" || true + fi done # remove /usr/local/... from PATH so we can exec the real modprobe as a last resort diff --git a/tests/integration/README.md b/tests/integration/README.md index e067b385577..c72c009a0d6 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -94,7 +94,7 @@ cd docker/test/integration docker build -t yandex/clickhouse-integration-test . ``` -The helper container used by the `runner` script is in `tests/integration/image/Dockerfile`. +The helper container used by the `runner` script is in `docker/test/integration/runner/Dockerfile`. ### Adding new tests diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index a9fd572a8b0..39c6322acb1 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -28,6 +28,8 @@ from .client import Client from .hdfs_api import HDFSApi HELPERS_DIR = p.dirname(__file__) +CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") +DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "docker/test/integration/compose/") DEFAULT_ENV_NAME = 'env_file' SANITIZER_SIGN = "==================" @@ -174,14 +176,14 @@ class ClickHouseCluster: self.instances[name] = instance if ipv4_address is not None or ipv6_address is not None: self.with_net_trics = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_net.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_net.yml')]) self.base_cmd.extend(['--file', instance.docker_compose_path]) cmds = [] if with_zookeeper and not self.with_zookeeper: if not zookeeper_docker_compose_path: - zookeeper_docker_compose_path = p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml') + zookeeper_docker_compose_path = p.join(DOCKER_COMPOSE_DIR, 'docker_compose_zookeeper.yml') self.with_zookeeper = True self.base_cmd.extend(['--file', zookeeper_docker_compose_path]) @@ -191,72 +193,72 @@ class ClickHouseCluster: if with_mysql and not self.with_mysql: self.with_mysql = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) if with_postgres and not self.with_postgres: self.with_postgres = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')]) self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')] cmds.append(self.base_postgres_cmd) if with_odbc_drivers and not self.with_odbc_drivers: self.with_odbc_drivers = True if not self.with_mysql: self.with_mysql = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')]) self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mysql.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mysql.yml')] cmds.append(self.base_mysql_cmd) if not self.with_postgres: self.with_postgres = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')]) self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', - p.join(HELPERS_DIR, 'docker_compose_postgres.yml')] + p.join(DOCKER_COMPOSE_DIR, 'docker_compose_postgres.yml')] cmds.append(self.base_postgres_cmd) if with_kafka and not self.with_kafka: self.with_kafka = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_kafka.yml')]) self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_kafka.yml')] cmds.append(self.base_kafka_cmd) if with_hdfs and not self.with_hdfs: self.with_hdfs = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_hdfs.yml')]) self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_hdfs.yml')] cmds.append(self.base_hdfs_cmd) if with_mongo and not self.with_mongo: self.with_mongo = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_mongo.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mongo.yml')]) self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_mongo.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_mongo.yml')] cmds.append(self.base_mongo_cmd) if self.with_net_trics: for cmd in cmds: - cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_net.yml')]) + cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_net.yml')]) if with_redis and not self.with_redis: self.with_redis = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_redis.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_redis.yml')]) self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_redis.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_redis.yml')] if with_minio and not self.with_minio: self.with_minio = True - self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_minio.yml')]) + self.base_cmd.extend(['--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_minio.yml')]) self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', - self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_minio.yml')] + self.project_name, '--file', p.join(DOCKER_COMPOSE_DIR, 'docker_compose_minio.yml')] cmds.append(self.base_minio_cmd) return instance diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 82384c6e183..3ba8ae3f9fd 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -5,7 +5,7 @@ import os import docker -from .cluster import HELPERS_DIR +from .cluster import CLICKHOUSE_ROOT_DIR class PartitionManager: @@ -156,7 +156,7 @@ class _NetworkManager: def __init__( self, image_name='clickhouse_tests_helper', - image_path=p.join(HELPERS_DIR, 'helper_container'), + image_path=p.join(CLICKHOUSE_ROOT_DIR, 'docker', 'test', 'integration', 'helper_container'), container_expire_timeout=50, container_exit_timeout=60): self.container_expire_timeout = container_expire_timeout