From 6cb893f0736ba04196eb43c0da566baa90d6c36e Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 1 Sep 2020 09:38:23 +0300 Subject: [PATCH 1/5] Draft. --- docker/test/integration/base/Dockerfile | 1 + docker/test/stateless_unbundled/Dockerfile | 68 ++++++++++++++++--- tests/integration/helpers/cluster.py | 14 ++-- tests/integration/runner | 2 + .../configs/dns_update_short.xml | 2 +- tests/integration/test_host_ip_change/test.py | 7 +- 6 files changed, 80 insertions(+), 14 deletions(-) diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 53627c78208..2b8877437f8 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ odbc-postgresql \ sqlite3 \ curl \ + bind9-host \ tar RUN rm -rf \ /var/lib/apt/lists/* \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 7de29fede72..4978252d556 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -1,12 +1,56 @@ # docker build -t yandex/clickhouse-stateless-unbundled-test . -FROM yandex/clickhouse-test-base +FROM ubuntu:20.04 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - alien \ +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 + +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list + +# initial packages +RUN apt-get update \ + && apt-get install \ + bash \ + fakeroot \ + ccache \ + curl \ + software-properties-common \ + --yes --no-install-recommends + +# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able +# to compress files using pigz (https://zlib.net/pigz/) instead of gzip. +# Significantly increase deb packaging speed and compatible with old systems +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ + && chmod +x dpkg-deb \ + && cp dpkg-deb /usr/bin + + +RUN apt-get update \ + && apt-get install \ + clang-${LLVM_VERSION} \ + debhelper \ + devscripts \ + gdb \ + git \ + gperf \ + lcov \ + llvm-${LLVM_VERSION} \ + moreutils \ + perl \ + perl \ + pigz \ + pkg-config \ + tzdata \ + alien \ brotli \ cmake \ devscripts \ @@ -56,8 +100,6 @@ RUN apt-get --allow-unauthenticated update -y \ pkg-config \ python \ python-lxml \ - python-requests \ - python-termcolor \ qemu-user-static \ sudo \ telnet \ @@ -68,7 +110,10 @@ RUN apt-get --allow-unauthenticated update -y \ wget \ zlib1g-dev \ zookeeper \ - zookeeperd + zookeeperd \ + --yes --no-install-recommends + + RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ @@ -77,6 +122,13 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ && rm -rf /tmp/clickhouse-odbc-tmp +# Sanitizer options +RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ + echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ + echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \ + ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; + + ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index f421f979947..88a2611774a 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -111,6 +111,7 @@ class ClickHouseCluster: custom_dockerd_host = custom_dockerd_host or os.environ.get('CLICKHOUSE_TESTS_DOCKERD_HOST') self.docker_api_version = os.environ.get("DOCKER_API_VERSION") + self.docker_base_tag = os.environ.get("DOCKER_BASE_TAG") self.base_cmd = ['docker-compose'] if custom_dockerd_host: @@ -165,7 +166,7 @@ class ClickHouseCluster: with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_redis=False, with_minio=False, with_cassandra=False, - hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", + hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None): """Add an instance to the cluster. @@ -183,13 +184,16 @@ class ClickHouseCluster: if name in self.instances: raise Exception("Can\'t add instance `%s': there is already an instance with the same name!" % name) + if tag is None: + tag = self.docker_base_tag + instance = ClickHouseInstance( self, self.base_dir, name, config_dir, main_configs or [], user_configs or [], macros or {}, with_zookeeper, self.zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra, self.base_configs_dir, self.server_bin_path, self.odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=hostname, - env_variables=env_variables or {}, image=image, stay_alive=stay_alive, ipv4_address=ipv4_address, + env_variables=env_variables or {}, image=image, tag=tag, stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address, with_installed_binary=with_installed_binary, tmpfs=tmpfs or []) @@ -704,7 +708,7 @@ DOCKER_COMPOSE_TEMPLATE = ''' version: '2.3' services: {name}: - image: {image} + image: {image}:{tag} hostname: {hostname} volumes: - {configs_dir}:/etc/clickhouse-server/ @@ -739,7 +743,7 @@ class ClickHouseInstance: with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra, base_configs_dir, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, - image="yandex/clickhouse-integration-test", + image="yandex/clickhouse-integration-test", tag="latest", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): self.name = name @@ -783,6 +787,7 @@ class ClickHouseInstance: self.client = None self.default_timeout = 20.0 # 20 sec self.image = image + self.tag = tag self.stay_alive = stay_alive self.ipv4_address = ipv4_address self.ipv6_address = ipv6_address @@ -1160,6 +1165,7 @@ class ClickHouseInstance: with open(self.docker_compose_path, 'w') as docker_compose: docker_compose.write(DOCKER_COMPOSE_TEMPLATE.format( image=self.image, + tag=self.tag, name=self.name, hostname=self.hostname, binary_volume=binary_volume, diff --git a/tests/integration/runner b/tests/integration/runner index e5d6eabe794..67a174eccfa 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -154,6 +154,8 @@ if __name__ == "__main__": env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag) elif image == "yandex/clickhouse-postgresql-java-client": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) + elif image == "yandex/clickhouse-integration-test": + env_tags += "-e {}={}".format("DOCKER_BASE_TAG", tag) else: raise Exception("Unknown image {}".format(image)) diff --git a/tests/integration/test_host_ip_change/configs/dns_update_short.xml b/tests/integration/test_host_ip_change/configs/dns_update_short.xml index 2bfafe2ef21..3317f709b4a 100644 --- a/tests/integration/test_host_ip_change/configs/dns_update_short.xml +++ b/tests/integration/test_host_ip_change/configs/dns_update_short.xml @@ -1,3 +1,3 @@ - 2 + 1 diff --git a/tests/integration/test_host_ip_change/test.py b/tests/integration/test_host_ip_change/test.py index ac35478277c..e3e8c08c848 100644 --- a/tests/integration/test_host_ip_change/test.py +++ b/tests/integration/test_host_ip_change/test.py @@ -106,11 +106,16 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): # Put some data to source node3 node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)") + + # Check that data is placed on node3 assert node3.query("SELECT count(*) from test_table_update") == "6\n" + result = node4.exec_in_container(["bash", "-c", "/usr/bin/host node3"]) + print("HOST RESULT %s", result) + # Because of DNS cache update, ip of node3 would be updated - assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6") + assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6", sleep_time=3) # Just to be sure check one more time node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 8)") From 45e54f81c7d0fe7f8f440a1a9253b64a3911a5f8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 2 Sep 2020 11:07:46 +0300 Subject: [PATCH 2/5] better resolv.conf, add DNSCacheUpdater logs --- programs/server/Server.cpp | 1 + src/Interpreters/DNSCacheUpdater.cpp | 1 + tests/integration/helpers/cluster.py | 9 +++++++-- tests/integration/test_host_ip_change/test.py | 9 +++++++-- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3a975325851..f24ba444203 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -716,6 +716,7 @@ int Server::main(const std::vector & /*args*/) { /// Disable DNS caching at all DNSResolver::instance().setDisableCacheFlag(); + LOG_DEBUG(log, "DNS caching disabled"); } else { diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index e5a97dc76d9..248c0ffa4dd 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -42,6 +42,7 @@ void DNSCacheUpdater::run() void DNSCacheUpdater::start() { + LOG_INFO(&Poco::Logger::get("DNSCacheUpdater"), "Update period {} seconds", update_period_seconds); task_handle->activateAndSchedule(); } diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 88a2611774a..11c08312c72 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -727,6 +727,11 @@ services: - {env_file} security_opt: - label:disable + dns_opt: + - attempts:2 + - timeout:1 + - inet6 + - rotate {networks} {app_net} {ipv4_address} @@ -740,8 +745,8 @@ class ClickHouseInstance: def __init__( self, cluster, base_path, name, custom_config_dir, custom_main_configs, custom_user_configs, macros, - with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, with_cassandra, - base_configs_dir, server_bin_path, odbc_bridge_bin_path, + with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, + with_redis, with_minio, with_cassandra, base_configs_dir, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag="latest", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): diff --git a/tests/integration/test_host_ip_change/test.py b/tests/integration/test_host_ip_change/test.py index e3e8c08c848..4bc07e0bf63 100644 --- a/tests/integration/test_host_ip_change/test.py +++ b/tests/integration/test_host_ip_change/test.py @@ -111,8 +111,13 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): # Check that data is placed on node3 assert node3.query("SELECT count(*) from test_table_update") == "6\n" - result = node4.exec_in_container(["bash", "-c", "/usr/bin/host node3"]) - print("HOST RESULT %s", result) + curl_result = node4.exec_in_container(["bash", "-c", "curl -s 'node3:8123'"]) + assert curl_result == 'Ok.\n' + cat_resolv = node4.exec_in_container(["bash", "-c", "cat /etc/resolv.conf"]) + print("RESOLV {}".format(cat_resolv)) + + + assert node4.query("SELECT * FROM remote('node3', 'system', 'one')") == "0\n" # Because of DNS cache update, ip of node3 would be updated assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6", sleep_time=3) From 5d39b8ce0478e637ad5ac34b8f1c05b5f1aab3d8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 2 Sep 2020 11:39:16 +0300 Subject: [PATCH 3/5] unnecessary change --- docker/test/stateless_unbundled/Dockerfile | 68 +++------------------- 1 file changed, 8 insertions(+), 60 deletions(-) diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 4978252d556..7de29fede72 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -1,56 +1,12 @@ # docker build -t yandex/clickhouse-stateless-unbundled-test . -FROM ubuntu:20.04 +FROM yandex/clickhouse-test-base ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 - -RUN apt-get update \ - && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ - --yes --no-install-recommends --verbose-versions \ - && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ - && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ - && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ - && apt-key add /tmp/llvm-snapshot.gpg.key \ - && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ - /etc/apt/sources.list - -# initial packages -RUN apt-get update \ - && apt-get install \ - bash \ - fakeroot \ - ccache \ - curl \ - software-properties-common \ - --yes --no-install-recommends - -# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able -# to compress files using pigz (https://zlib.net/pigz/) instead of gzip. -# Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ - && chmod +x dpkg-deb \ - && cp dpkg-deb /usr/bin - - -RUN apt-get update \ - && apt-get install \ - clang-${LLVM_VERSION} \ - debhelper \ - devscripts \ - gdb \ - git \ - gperf \ - lcov \ - llvm-${LLVM_VERSION} \ - moreutils \ - perl \ - perl \ - pigz \ - pkg-config \ - tzdata \ - alien \ +RUN apt-get --allow-unauthenticated update -y \ + && env DEBIAN_FRONTEND=noninteractive \ + apt-get --allow-unauthenticated install --yes --no-install-recommends \ + alien \ brotli \ cmake \ devscripts \ @@ -100,6 +56,8 @@ RUN apt-get update \ pkg-config \ python \ python-lxml \ + python-requests \ + python-termcolor \ qemu-user-static \ sudo \ telnet \ @@ -110,10 +68,7 @@ RUN apt-get update \ wget \ zlib1g-dev \ zookeeper \ - zookeeperd \ - --yes --no-install-recommends - - + zookeeperd RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ @@ -122,13 +77,6 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ && rm -rf /tmp/clickhouse-odbc-tmp -# Sanitizer options -RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ - echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ - echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \ - ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; - - ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone From 2c7d7f9f49bfa33382c9ae7ed81c875aeaeee0ba Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 2 Sep 2020 11:40:31 +0300 Subject: [PATCH 4/5] Update Dockerfile --- docker/test/integration/base/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 2b8877437f8..53627c78208 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -17,7 +17,6 @@ RUN apt-get update \ odbc-postgresql \ sqlite3 \ curl \ - bind9-host \ tar RUN rm -rf \ /var/lib/apt/lists/* \ From c2ab85f214e2cee931d0311a1f1d80bafe5d4911 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 2 Sep 2020 17:22:54 +0300 Subject: [PATCH 5/5] Split tag part from image --- .../test_adaptive_granularity/test.py | 16 ++++++++-------- .../test.py | 2 +- .../test_adaptive_granularity_replicated/test.py | 2 +- .../test_backup_with_other_granularity/test.py | 6 +++--- .../test_backward_compatibility/test.py | 2 +- .../test_aggregate_function_state_avg.py | 4 ++-- .../test_short_strings_aggregation.py | 4 ++-- .../test_default_compression_codec/test.py | 1 - .../test.py | 2 +- tests/integration/test_odbc_interaction/test.py | 2 +- tests/integration/test_old_versions/test.py | 14 +++++++------- tests/integration/test_polymorphic_parts/test.py | 2 +- .../test_replicating_constants/test.py | 2 +- .../test_version_update_after_mutation/test.py | 6 +++--- 14 files changed, 32 insertions(+), 33 deletions(-) diff --git a/tests/integration/test_adaptive_granularity/test.py b/tests/integration/test_adaptive_granularity/test.py index f60a932b7e8..6dc960ed428 100644 --- a/tests/integration/test_adaptive_granularity/test.py +++ b/tests/integration/test_adaptive_granularity/test.py @@ -12,20 +12,20 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', with_installed_binary=True) +node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', with_installed_binary=True) node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', with_installed_binary=True) +node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', with_installed_binary=True) node6 = cluster.add_instance('node6', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) -node7 = cluster.add_instance('node7', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', stay_alive=True, with_installed_binary=True) -node8 = cluster.add_instance('node8', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True) +node7 = cluster.add_instance('node7', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) +node8 = cluster.add_instance('node8', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) -node9 = cluster.add_instance('node9', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True) -node10 = cluster.add_instance('node10', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.6.3.18', stay_alive=True, with_installed_binary=True) +node9 = cluster.add_instance('node9', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) +node10 = cluster.add_instance('node10', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) -node11 = cluster.add_instance('node11', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True) -node12 = cluster.add_instance('node12', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server:19.1.15', stay_alive=True, with_installed_binary=True) +node11 = cluster.add_instance('node11', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) +node12 = cluster.add_instance('node12', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) def prepare_single_pair_with_setting(first_node, second_node, group): diff --git a/tests/integration/test_adaptive_granularity_different_settings/test.py b/tests/integration/test_adaptive_granularity_different_settings/test.py index 23c95ef1701..d84b438f77f 100644 --- a/tests/integration/test_adaptive_granularity_different_settings/test.py +++ b/tests/integration/test_adaptive_granularity_different_settings/test.py @@ -7,7 +7,7 @@ node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) # no adaptive granularity by default -node3 = cluster.add_instance('node3', image='yandex/clickhouse-server:19.9.5.36', with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True, stay_alive=True) @pytest.fixture(scope="module") def start_cluster(): diff --git a/tests/integration/test_adaptive_granularity_replicated/test.py b/tests/integration/test_adaptive_granularity_replicated/test.py index 52978ade685..87956c82661 100644 --- a/tests/integration/test_adaptive_granularity_replicated/test.py +++ b/tests/integration/test_adaptive_granularity_replicated/test.py @@ -11,7 +11,7 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:19.1.14', with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True) @pytest.fixture(scope="module") def start_cluster(): diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index d4ca9bd1bac..c27cd732a05 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -5,9 +5,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:19.4.5.35', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:19.4.5.35', stay_alive=True, with_installed_binary=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:19.4.5.35', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) node4 = cluster.add_instance('node4') diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 914153342f8..5b51823d361 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -4,7 +4,7 @@ import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:19.17.8.54', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node2 = cluster.add_instance('node2', with_zookeeper=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py index c9f3acc2e2e..b1b9fecf54e 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py @@ -5,9 +5,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', - with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) + with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) node2 = cluster.add_instance('node2', - with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) + with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) node3 = cluster.add_instance('node3', with_zookeeper=False) node4 = cluster.add_instance('node4', with_zookeeper=False) diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 1c264d1e636..5cf78b481b9 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -4,8 +4,8 @@ import helpers.client as client from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) node3 = cluster.add_instance('node3', with_zookeeper=False) @pytest.fixture(scope="module") diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 4de4aba63f6..d5cd43434eb 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -9,7 +9,6 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True) - @pytest.fixture(scope="module") def start_cluster(): try: diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index a0362cea49e..7ce7edb2860 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -8,7 +8,7 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server:19.17.8.54', stay_alive=True, with_installed_binary=True) +node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) @pytest.fixture(scope="module") diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 46845802083..cf0d2bee410 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -10,7 +10,7 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) -node1 = cluster.add_instance('node1', with_odbc_drivers=True, with_mysql=True, image='yandex/clickhouse-integration-test', main_configs=['configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml', 'configs/dictionaries/sqlite3_odbc_cached_dictionary.xml', 'configs/dictionaries/postgres_odbc_hashed_dictionary.xml'], stay_alive=True) +node1 = cluster.add_instance('node1', with_odbc_drivers=True, with_mysql=True, main_configs=['configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml', 'configs/dictionaries/sqlite3_odbc_cached_dictionary.xml', 'configs/dictionaries/postgres_odbc_hashed_dictionary.xml'], stay_alive=True) create_table_sql_template = """ CREATE TABLE `clickhouse`.`{}` ( diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index d77b4af016a..80d81c127e1 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -9,13 +9,13 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server:18.14.19', with_installed_binary=True, config_dir="configs") -node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server:19.1.16', with_installed_binary=True, config_dir="configs") -node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server:19.4.5.35', with_installed_binary=True, config_dir="configs") -node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server:19.8.3.8', with_installed_binary=True, config_dir="configs") -node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server:19.11.13.74', with_installed_binary=True, config_dir="configs") -node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server:19.13.7.57', with_installed_binary=True, config_dir="configs") -node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server:19.16.2.2', with_installed_binary=True, config_dir="configs") +node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server', tag='18.14.19', with_installed_binary=True, config_dir="configs") +node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server', tag='19.1.16', with_installed_binary=True, config_dir="configs") +node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server', tag='19.4.5.35', with_installed_binary=True, config_dir="configs") +node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server', tag='19.8.3.8', with_installed_binary=True, config_dir="configs") +node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server', tag='19.11.13.74', with_installed_binary=True, config_dir="configs") +node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server', tag='19.13.7.57', with_installed_binary=True, config_dir="configs") +node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server', tag='19.16.2.2', with_installed_binary=True, config_dir="configs") old_nodes = [node18_14, node19_1, node19_4, node19_8, node19_11, node19_13, node19_16] new_node = cluster.add_instance('node_new') diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index d3ebbd8c7a8..db1121a5404 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -213,7 +213,7 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format(table))) == TSV(expected) -node7 = cluster.add_instance('node7', config_dir="configs_old", with_zookeeper=True, image='yandex/clickhouse-server:19.17.8.54', stay_alive=True, with_installed_binary=True) +node7 = cluster.add_instance('node7', config_dir="configs_old", with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node8 = cluster.add_instance('node8', config_dir="configs", with_zookeeper=True) settings7 = {'index_granularity_bytes' : 10485760} diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index f340817b584..b72b9089f65 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -5,7 +5,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:19.1.14', with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True) @pytest.fixture(scope="module") def start_cluster(): diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 839664638ff..f78dbf18c0d 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -5,9 +5,9 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server:20.1.10.70', with_installed_binary=True, stay_alive=True) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', with_installed_binary=True, stay_alive=True) @pytest.fixture(scope="module") def start_cluster():