Merge pull request #14864 from bharatnc/ncb/format-integration-tests

Format and cleanup imports form all *.py integration test files

tests/integration/conftest.py

@@ -1,5 +1,6 @@
 from helpers.test_tools import TSV
 
+
 def pytest_assertrepr_compare(op, left, right):
     if isinstance(left, TSV) and isinstance(right, TSV) and op == '==':
         return ['TabSeparated values differ: '] + left.diff(right)

tests/integration/helpers/client.py

@@ -1,8 +1,7 @@
-import errno
-import subprocess as sp
-from threading import Timer
-import tempfile
 import os
+import subprocess as sp
+import tempfile
+from threading import Timer
 
 
 class Client:
@@ -16,12 +15,13 @@ class Client:
 
         self.command += ['--host', self.host, '--port', str(self.port), '--stacktrace']
 
+    def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
+              ignore_error=False):
+        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
+                                      password=password, database=database, ignore_error=ignore_error).get_answer()
 
-    def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False):
-        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error).get_answer()
-
-
-    def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False):
+    def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
+                          ignore_error=False):
         command = self.command[:]
 
         if stdin is None:
@@ -45,14 +45,17 @@ class Client:
 
         return CommandRequest(command, stdin, timeout, ignore_error)
 
+    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
+                            database=None):
+        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
+                                      password=password, database=database).get_error()
 
-    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None):
-        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_error()
+    def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
+                                        database=None):
+        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
+                                      password=password, database=database).get_answer_and_error()
 
 
-    def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None):
-        return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_answer_and_error()
-
 class QueryTimeoutExceedException(Exception):
     pass
 
@@ -90,7 +93,6 @@ class CommandRequest:
             self.timer = Timer(timeout, kill_process)
             self.timer.start()
 
-
     def get_answer(self):
         self.process.wait()
         self.stdout_file.seek(0)
@@ -103,11 +105,11 @@ class CommandRequest:
             raise QueryTimeoutExceedException('Client timed out!')
 
         if (self.process.returncode != 0 or stderr) and not self.ignore_error:
-            raise QueryRuntimeException('Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr))
+            raise QueryRuntimeException(
+                'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr))
 
         return stdout
 
-
     def get_error(self):
         self.process.wait()
         self.stdout_file.seek(0)
@@ -124,7 +126,6 @@ class CommandRequest:
 
         return stderr
 
-
     def get_answer_and_error(self):
         self.process.wait()
         self.stdout_file.seek(0)

tests/integration/helpers/cluster.py

@@ -1,30 +1,31 @@
 import base64
-import cassandra.cluster
-import docker
 import errno
 import httplib
 import logging
 import os
 import os.path as p
 import pprint
-import psycopg2
 import pwd
-import pymongo
-import pymysql
 import re
-import requests
 import shutil
 import socket
 import subprocess
 import time
-import urllib
 import traceback
+import urllib
+
+import cassandra.cluster
+import docker
+import psycopg2
+import pymongo
+import pymysql
+import requests
 import xml.dom.minidom
+from confluent.schemaregistry.client import CachedSchemaRegistryClient
 from dicttoxml import dicttoxml
 from kazoo.client import KazooClient
 from kazoo.exceptions import KazooException
 from minio import Minio
-from confluent.schemaregistry.client import CachedSchemaRegistryClient
 
 from .client import Client
 from .hdfs_api import HDFSApi
@@ -67,13 +68,14 @@ def get_odbc_bridge_path():
             return '/usr/bin/clickhouse-odbc-bridge'
     return path
 
+
 def get_docker_compose_path():
     compose_path = os.environ.get('DOCKER_COMPOSE_DIR')
     if compose_path is not None:
         return os.path.dirname(compose_path)
     else:
         if os.path.exists(os.path.dirname('/compose/')):
-            return os.path.dirname('/compose/') #default in docker runner container
+            return os.path.dirname('/compose/')  # default in docker runner container
         else:
             print("Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {}".format(LOCAL_DOCKER_COMPOSE_DIR))
             return LOCAL_DOCKER_COMPOSE_DIR
@@ -91,12 +93,12 @@ class ClickHouseCluster:
     def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None,
                  odbc_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None):
         for param in os.environ.keys():
-            print "ENV %40s %s" % (param,os.environ[param])
+            print "ENV %40s %s" % (param, os.environ[param])
         self.base_dir = p.dirname(base_path)
         self.name = name if name is not None else ''
 
         self.base_config_dir = base_config_dir or os.environ.get('CLICKHOUSE_TESTS_BASE_CONFIG_DIR',
-                                                                   '/etc/clickhouse-server/')
+                                                                 '/etc/clickhouse-server/')
         self.server_bin_path = p.realpath(
             server_bin_path or os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH', '/usr/bin/clickhouse'))
         self.odbc_bridge_bin_path = p.realpath(odbc_bridge_bin_path or get_odbc_bridge_path())
@@ -165,8 +167,10 @@ class ClickHouseCluster:
             cmd += " client"
         return cmd
 
-    def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries = None, macros=None,
-                     with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None,
+    def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None,
+                     macros=None,
+                     with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False,
+                     clickhouse_path_dir=None,
                      with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False,
                      with_redis=False, with_minio=False, with_cassandra=False,
                      hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None,
@@ -247,7 +251,8 @@ class ClickHouseCluster:
             self.with_mysql = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')])
             self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                   self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
+                                   self.project_name, '--file',
+                                   p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
 
             cmds.append(self.base_mysql_cmd)
 
@@ -255,7 +260,8 @@ class ClickHouseCluster:
             self.with_postgres = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')])
             self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                      self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]
+                                      self.project_name, '--file',
+                                      p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]
             cmds.append(self.base_postgres_cmd)
 
         if with_odbc_drivers and not self.with_odbc_drivers:
@@ -264,7 +270,8 @@ class ClickHouseCluster:
                 self.with_mysql = True
                 self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')])
                 self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                       self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
+                                       self.project_name, '--file',
+                                       p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
                 cmds.append(self.base_mysql_cmd)
 
             if not self.with_postgres:
@@ -279,28 +286,32 @@ class ClickHouseCluster:
             self.with_kafka = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')])
             self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                   self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]
+                                   self.project_name, '--file',
+                                   p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]
             cmds.append(self.base_kafka_cmd)
 
         if with_rabbitmq and not self.with_rabbitmq:
             self.with_rabbitmq = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')])
             self.base_rabbitmq_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                      self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]
+                                      self.project_name, '--file',
+                                      p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]
             cmds.append(self.base_rabbitmq_cmd)
 
         if with_hdfs and not self.with_hdfs:
             self.with_hdfs = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')])
             self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                  self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]
+                                  self.project_name, '--file',
+                                  p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]
             cmds.append(self.base_hdfs_cmd)
 
         if with_mongo and not self.with_mongo:
             self.with_mongo = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')])
             self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                   self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]
+                                   self.project_name, '--file',
+                                   p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]
             cmds.append(self.base_mongo_cmd)
 
         if self.with_net_trics:
@@ -311,21 +322,24 @@ class ClickHouseCluster:
             self.with_redis = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')])
             self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                   self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]
+                                   self.project_name, '--file',
+                                   p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]
 
         if with_minio and not self.with_minio:
             self.with_minio = True
             self.minio_certs_dir = minio_certs_dir
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')])
             self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                   self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]
+                                   self.project_name, '--file',
+                                   p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]
             cmds.append(self.base_minio_cmd)
 
         if with_cassandra and not self.with_cassandra:
             self.with_cassandra = True
             self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')])
             self.base_cassandra_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                       self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]
+                                       self.project_name, '--file',
+                                       p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]
 
         return instance
 
@@ -390,7 +404,8 @@ class ClickHouseCluster:
             print("Container {} uses image {}: ".format(container_id, image_id))
             pprint.pprint(image_info)
             print("")
-            message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, exit_code, output)
+            message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id,
+                                                                                           exit_code, output)
             if nothrow:
                 print(message)
             else:
@@ -401,7 +416,8 @@ class ClickHouseCluster:
         with open(local_path, 'r') as fdata:
             data = fdata.read()
             encoded_data = base64.b64encode(data)
-            self.exec_in_container(container_id, ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)],
+            self.exec_in_container(container_id,
+                                   ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)],
                                    user='root')
 
     def wait_mysql_to_start(self, timeout=60):
@@ -650,7 +666,6 @@ class ClickHouseCluster:
             subprocess_check_call(clickhouse_start_cmd)
             print("ClickHouse instance created")
 
-
             start_deadline = time.time() + 20.0  # seconds
             for instance in self.instances.itervalues():
                 instance.docker_client = self.docker_client
@@ -692,7 +707,7 @@ class ClickHouseCluster:
         try:
             subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans'])
         except Exception as e:
-                print "Down + remove orphans failed durung shutdown. {}".format(repr(e))
+            print "Down + remove orphans failed durung shutdown. {}".format(repr(e))
 
         self.is_up = False
 
@@ -704,23 +719,26 @@ class ClickHouseCluster:
             instance.client = None
 
         if not self.zookeeper_use_tmpfs:
-             for i in range(1, 4):
-                 zk_data_path = self.instances_dir + '/zkdata' + str(i)
-                 zk_log_data_path = self.instances_dir + '/zklog' + str(i)
-                 if os.path.exists(zk_data_path):
-                     shutil.rmtree(zk_data_path)
-                 if os.path.exists(zk_log_data_path):
-                     shutil.rmtree(zk_log_data_path)
+            for i in range(1, 4):
+                zk_data_path = self.instances_dir + '/zkdata' + str(i)
+                zk_log_data_path = self.instances_dir + '/zklog' + str(i)
+                if os.path.exists(zk_data_path):
+                    shutil.rmtree(zk_data_path)
+                if os.path.exists(zk_log_data_path):
+                    shutil.rmtree(zk_log_data_path)
 
         if sanitizer_assert_instance is not None:
-            raise Exception("Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance))
+            raise Exception(
+                "Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance))
 
     def pause_container(self, instance_name):
         subprocess_check_call(self.base_cmd + ['pause', instance_name])
+
     #    subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name])
 
     def unpause_container(self, instance_name):
         subprocess_check_call(self.base_cmd + ['unpause', instance_name])
+
     #    subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name])
 
     def open_bash_shell(self, instance_name):
@@ -790,9 +808,12 @@ services:
 class ClickHouseInstance:
 
     def __init__(
-            self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, custom_dictionaries,
-            macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio,
-            with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None,
+            self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs,
+            custom_dictionaries,
+            macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo,
+            with_redis, with_minio,
+            with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers,
+            hostname=None, env_variables=None,
             image="yandex/clickhouse-integration-test", tag="latest",
             stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
 
@@ -848,15 +869,19 @@ class ClickHouseInstance:
         return "-fsanitize=thread" in build_opts
 
     # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer
-    def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False):
-        return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error)
+    def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
+              ignore_error=False):
+        return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password,
+                                 database=database, ignore_error=ignore_error)
 
-    def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False,
+    def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
+                         ignore_error=False,
                          retry_count=20, sleep_time=0.5, check_callback=lambda x: True):
         result = None
         for i in range(retry_count):
             try:
-                result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error)
+                result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password,
+                                    database=database, ignore_error=ignore_error)
                 if check_callback(result):
                     return result
                 time.sleep(sleep_time)
@@ -873,12 +898,16 @@ class ClickHouseInstance:
         return self.client.get_query_request(*args, **kwargs)
 
     # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code
-    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None):
-        return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database)
+    def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
+                            database=None):
+        return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
+                                               password=password, database=database)
 
     # The same as query_and_get_error but ignores successful query.
-    def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None):
-        return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database)
+    def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
+                                        database=None):
+        return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings,
+                                                           user=user, password=password, database=database)
 
     # Connects to the instance via HTTP interface, sends a query and returns the answer
     def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False):
@@ -900,7 +929,8 @@ class ClickHouseInstance:
         open_result = urllib.urlopen(url, data)
 
         def http_code_and_message():
-            return str(open_result.getcode()) + " " + httplib.responses[open_result.getcode()] + ": " + open_result.read()
+            return str(open_result.getcode()) + " " + httplib.responses[
+                open_result.getcode()] + ": " + open_result.read()
 
         if expect_fail_and_get_error:
             if open_result.getcode() == 200:
@@ -913,18 +943,19 @@ class ClickHouseInstance:
 
     # Connects to the instance via HTTP interface, sends a query and returns the answer
     def http_request(self, url, method='GET', params=None, data=None, headers=None):
-        url = "http://" + self.ip_address + ":8123/"+url
+        url = "http://" + self.ip_address + ":8123/" + url
         return requests.request(method=method, url=url, params=params, data=data, headers=headers)
 
     # Connects to the instance via HTTP interface, sends a query, expects an error and return the error message
     def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None):
-        return self.http_query(sql=sql, data=data, params=params, user=user, password=password, expect_fail_and_get_error=True)
+        return self.http_query(sql=sql, data=data, params=params, user=user, password=password,
+                               expect_fail_and_get_error=True)
 
     def kill_clickhouse(self, stop_start_wait_sec=5):
         pid = self.get_process_pid("clickhouse")
         if not pid:
             raise Exception("No clickhouse found")
-        self.exec_in_container(["bash",  "-c", "kill -9 {}".format(pid)], user='root')
+        self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root')
         time.sleep(stop_start_wait_sec)
 
     def restore_clickhouse(self, retries=100):
@@ -1030,7 +1061,8 @@ class ClickHouseInstance:
             time_left = deadline - current_time
             if deadline is not None and current_time >= deadline:
                 raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. "
-                                "Container status: {}, logs: {}".format(self.name, self.ip_address, status, handle.logs()))
+                                "Container status: {}, logs: {}".format(self.name, self.ip_address, status,
+                                                                        handle.logs()))
 
             # Repeatedly poll the instance address until there is something that listens there.
             # Usually it means that ClickHouse is ready to accept queries.

tests/integration/helpers/dictionary.py

@@ -59,7 +59,8 @@ class Row(object):
 
 
 class Field(object):
-    def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False, range_hash_type=None, default_value_for_get=None):
+    def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False,
+                 range_hash_type=None, default_value_for_get=None):
         self.name = name
         self.field_type = field_type
         self.is_key = is_key
@@ -123,7 +124,8 @@ class DictionaryStructure(object):
                 self.range_key = field
 
         if not self.layout.is_complex and len(self.keys) > 1:
-            raise Exception("More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name))
+            raise Exception(
+                "More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name))
 
         if self.layout.is_ranged and (not self.range_key or len(self.range_fields) != 2):
             raise Exception("Inconsistent configuration of ranged dictionary")
@@ -213,7 +215,8 @@ class DictionaryStructure(object):
         if or_default:
             or_default_expr = 'OrDefault'
             if field.default_value_for_get is None:
-                raise Exception("Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name))
+                raise Exception(
+                    "Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name))
 
             val = field.default_value_for_get
             if isinstance(val, str):
@@ -259,15 +262,16 @@ class DictionaryStructure(object):
     def get_get_or_default_expressions(self, dict_name, field, row):
         if not self.layout.is_ranged:
             return [
-                self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False, has=False),
+                self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False,
+                                                     has=False),
                 self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=True, has=False),
             ]
         return []
 
-
     def get_has_expressions(self, dict_name, field, row):
         if not self.layout.is_ranged:
-            return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, has=True)]
+            return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False,
+                                                         has=True)]
         return []
 
     def get_hierarchical_expressions(self, dict_name, row):
@@ -290,7 +294,7 @@ class DictionaryStructure(object):
                 "dictIsIn('{dict_name}', {child_key}, {parent_key})".format(
                     dict_name=dict_name,
                     child_key=child_key_expr,
-                    parent_key=parent_key_expr,)
+                    parent_key=parent_key_expr, )
             ]
 
         return []
@@ -364,7 +368,8 @@ class Dictionary(object):
         return ['select {}'.format(expr) for expr in self.structure.get_get_expressions(self.name, field, row)]
 
     def get_select_get_or_default_queries(self, field, row):
-        return ['select {}'.format(expr) for expr in self.structure.get_get_or_default_expressions(self.name, field, row)]
+        return ['select {}'.format(expr) for expr in
+                self.structure.get_get_or_default_expressions(self.name, field, row)]
 
     def get_select_has_queries(self, field, row):
         return ['select {}'.format(expr) for expr in self.structure.get_has_expressions(self.name, field, row)]

tests/integration/helpers/external_sources.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
-import warnings
-import pymysql.cursors
-import pymongo
-import cassandra.cluster
-import redis
-import aerospike
-from tzlocal import get_localzone
 import datetime
 import os
 import uuid
+import warnings
+
+import aerospike
+import cassandra.cluster
+import pymongo
+import pymysql.cursors
+import redis
+from tzlocal import get_localzone
 
 
 class ExternalSource(object):
@@ -89,12 +90,12 @@ class SourceMySQL(ExternalSource):
                     <db>test</db>
                     <table>{tbl}</table>
                 </mysql>'''.format(
-                hostname=self.docker_hostname,
-                port=self.docker_port,
-                user=self.user,
-                password=self.password,
-                tbl=table_name,
-            )
+            hostname=self.docker_hostname,
+            port=self.docker_port,
+            user=self.user,
+            password=self.password,
+            tbl=table_name,
+        )
 
     def prepare(self, structure, table_name, cluster):
         self.create_mysql_conn()
@@ -160,7 +161,8 @@ class SourceMongo(ExternalSource):
             if field.field_type == "Date":
                 self.converters[field.name] = lambda x: datetime.datetime.strptime(x, "%Y-%m-%d")
             elif field.field_type == "DateTime":
-                self.converters[field.name] = lambda x: get_localzone().localize(datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
+                self.converters[field.name] = lambda x: get_localzone().localize(
+                    datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
             else:
                 self.converters[field.name] = lambda x: x
 
@@ -180,6 +182,7 @@ class SourceMongo(ExternalSource):
 
         result = tbl.insert_many(to_insert)
 
+
 class SourceMongoURI(SourceMongo):
     def compatible_with_layout(self, layout):
         # It is enough to test one layout for this dictionary, since we're
@@ -200,6 +203,7 @@ class SourceMongoURI(SourceMongo):
             tbl=table_name,
         )
 
+
 class SourceClickHouse(ExternalSource):
 
     def get_source_str(self, table_name):
@@ -284,7 +288,8 @@ class SourceFile(ExternalSource):
                 sorted_row.append(str(row.data[name]))
 
             str_data = '\t'.join(sorted_row)
-            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user="root")
+            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
+                                        user="root")
 
     def compatible_with_layout(self, layout):
         return 'cache' not in layout.name and 'direct' not in layout.name
@@ -324,7 +329,8 @@ class _SourceExecutableBase(ExternalSource):
                 sorted_row.append(str(row.data[name]))
 
             str_data = '\t'.join(sorted_row)
-            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root')
+            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
+                                        user='root')
 
 
 class SourceExecutableCache(_SourceExecutableBase):
@@ -344,12 +350,14 @@ class SourceExecutableHashed(_SourceExecutableBase):
     def compatible_with_layout(self, layout):
         return 'cache' in layout.name
 
-class SourceHTTPBase(ExternalSource):
 
+class SourceHTTPBase(ExternalSource):
     PORT_COUNTER = 5555
+
     def get_source_str(self, table_name):
         self.http_port = SourceHTTPBase.PORT_COUNTER
-        url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname, port=self.http_port)
+        url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname,
+                                                 port=self.http_port)
         SourceHTTPBase.PORT_COUNTER += 1
         return '''
             <http>
@@ -395,7 +403,8 @@ class SourceHTTPBase(ExternalSource):
                 sorted_row.append(str(row.data[name]))
 
             str_data = '\t'.join(sorted_row)
-            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root')
+            self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
+                                        user='root')
 
 
 class SourceHTTP(SourceHTTPBase):
@@ -407,6 +416,7 @@ class SourceHTTPS(SourceHTTPBase):
     def _get_schema(self):
         return "https"
 
+
 class SourceCassandra(ExternalSource):
     TYPE_MAPPING = {
         'UInt8': 'tinyint',
@@ -426,7 +436,8 @@ class SourceCassandra(ExternalSource):
     }
 
     def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password):
-        ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password)
+        ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user,
+                                password)
         self.structure = dict()
 
     def get_source_str(self, table_name):
@@ -448,13 +459,14 @@ class SourceCassandra(ExternalSource):
     def prepare(self, structure, table_name, cluster):
         self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port)
         self.session = self.client.connect()
-        self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};")
+        self.session.execute(
+            "create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};")
         self.session.execute('drop table if exists test."{}"'.format(table_name))
         self.structure[table_name] = structure
         columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()]
         keys = ['"' + col.name + '"' for col in structure.keys]
         query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format(
-                name=table_name, columns=', '.join(columns),  pk=', '.join(keys))
+            name=table_name, columns=', '.join(columns), pk=', '.join(keys))
         self.session.execute(query)
         self.prepared = True
 
@@ -470,14 +482,16 @@ class SourceCassandra(ExternalSource):
         names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()]
         columns = ['"' + col[0] + '"' for col in names_and_types]
         insert = 'insert into test."{table}" ({columns}) values ({args})'.format(
-                table=table_name, columns=','.join(columns), args=','.join(['%s']*len(columns)))
+            table=table_name, columns=','.join(columns), args=','.join(['%s'] * len(columns)))
         for row in data:
             values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types]
             self.session.execute(insert, values)
 
+
 class SourceRedis(ExternalSource):
     def __init__(
-            self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index, storage_type
+            self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index,
+            storage_type
     ):
         super(SourceRedis, self).__init__(
             name, internal_hostname, internal_port, docker_hostname, docker_port, user, password
@@ -503,7 +517,8 @@ class SourceRedis(ExternalSource):
         )
 
     def prepare(self, structure, table_name, cluster):
-        self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index, password=self.password or None)
+        self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index,
+                                        password=self.password or None)
         self.prepared = True
         self.ordered_names = structure.get_ordered_names()
 
@@ -521,11 +536,12 @@ class SourceRedis(ExternalSource):
     def compatible_with_layout(self, layout):
         return layout.is_simple and self.storage_type == "simple" or layout.is_complex and self.storage_type == "hash_map"
 
+
 class SourceAerospike(ExternalSource):
     def __init__(self, name, internal_hostname, internal_port,
                  docker_hostname, docker_port, user, password):
         ExternalSource.__init__(self, name, internal_hostname, internal_port,
-                 docker_hostname, docker_port, user, password)
+                                docker_hostname, docker_port, user, password)
         self.namespace = "test"
         self.set = "test_set"
 
@@ -543,7 +559,7 @@ class SourceAerospike(ExternalSource):
 
     def prepare(self, structure, table_name, cluster):
         config = {
-            'hosts': [ (self.internal_hostname, self.internal_port) ]
+            'hosts': [(self.internal_hostname, self.internal_port)]
         }
         self.client = aerospike.client(config).connect()
         self.prepared = True
@@ -580,7 +596,7 @@ class SourceAerospike(ExternalSource):
                 self.client.put(key, {"bin_value": value[1]}, policy={"key": aerospike.POLICY_KEY_SEND})
                 assert self.client.exists(key)
         else:
-            assert("VALUES SIZE != 2")
+            assert ("VALUES SIZE != 2")
 
         # print(values)
 

tests/integration/helpers/hdfs_api.py

@@ -1,10 +1,12 @@
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 import StringIO
 import gzip
-import requests
 import subprocess
 from tempfile import NamedTemporaryFile
 
+import requests
+
+
 class HDFSApi(object):
     def __init__(self, user):
         self.host = "localhost"
@@ -13,11 +15,15 @@ class HDFSApi(object):
         self.user = user
 
     def read_data(self, path):
-        response = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port, path=path), allow_redirects=False)
+        response = requests.get(
+            "http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port,
+                                                                   path=path), allow_redirects=False)
         if response.status_code != 307:
             response.raise_for_status()
         additional_params = '&'.join(response.headers['Location'].split('&')[1:2])
-        response_data = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port, path=path, params=additional_params))
+        response_data = requests.get(
+            "http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port,
+                                                                            path=path, params=additional_params))
         if response_data.status_code != 200:
             response_data.raise_for_status()
 
@@ -25,7 +31,9 @@ class HDFSApi(object):
 
     # Requests can't put file
     def _curl_to_put(self, filename, path, params):
-        url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host, port=self.http_data_port, path=path, params=params)
+        url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host,
+                                                                                port=self.http_data_port, path=path,
+                                                                                params=params)
         cmd = "curl -s -i -X PUT -T {fname} '{url}'".format(fname=filename, url=url)
         output = subprocess.check_output(cmd, shell=True)
         return output
@@ -36,13 +44,15 @@ class HDFSApi(object):
         named_file.write(content)
         named_file.flush()
         response = requests.put(
-            "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port, path=path, user=self.user),
+            "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port,
+                                                                     path=path, user=self.user),
             allow_redirects=False
         )
         if response.status_code != 307:
             response.raise_for_status()
 
-        additional_params = '&'.join(response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"])
+        additional_params = '&'.join(
+            response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"])
         output = self._curl_to_put(fpath, path, additional_params)
         if "201 Created" not in output:
             raise Exception("Can't create file on hdfs:\n {}".format(output))

tests/integration/helpers/http_server.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import argparse
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import csv
 import socket
 import ssl
-import csv
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 
 
 # Decorator used to see if authentication works for external dictionary who use a HTTP source.
@@ -15,6 +15,7 @@ def check_auth(fn):
             req.send_response(401)
         else:
             fn(req)
+
     return wrapper
 
 
@@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family):
             self.send_header('Content-type', 'text/tsv')
             self.end_headers()
 
-        def __send_data(self, only_ids = None):
+        def __send_data(self, only_ids=None):
             with open(data_path, 'r') as fl:
                 reader = csv.reader(fl, delimiter='\t')
                 for row in reader:

tests/integration/helpers/network.py

@@ -1,12 +1,9 @@
-import os.path as p
+import os
 import subprocess
 import time
-import os
 
 import docker
 
-from .cluster import CLICKHOUSE_ROOT_DIR
-
 
 class PartitionManager:
     """Allows introducing failures in the network between docker containers.
@@ -23,21 +20,18 @@ class PartitionManager:
     def __init__(self):
         self._iptables_rules = []
 
-
     def drop_instance_zk_connections(self, instance, action='DROP'):
         self._check_instance(instance)
 
         self._add_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action})
         self._add_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action})
 
-
     def restore_instance_zk_connections(self, instance, action='DROP'):
         self._check_instance(instance)
 
         self._delete_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action})
         self._delete_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action})
 
-
     def partition_instances(self, left, right, port=None, action='DROP'):
         self._check_instance(left)
         self._check_instance(right)
@@ -51,7 +45,6 @@ class PartitionManager:
         self._add_rule(create_rule(left, right))
         self._add_rule(create_rule(right, left))
 
-
     def heal_all(self):
         while self._iptables_rules:
             rule = self._iptables_rules.pop()
@@ -66,7 +59,6 @@ class PartitionManager:
         for rule in rules:
             self._add_rule(rule)
 
-
     @staticmethod
     def _check_instance(instance):
         if instance.ip_address is None:
@@ -152,7 +144,6 @@ class _NetworkManager:
             ret.extend(['-j'] + action.split())
         return ret
 
-
     def __init__(
             self,
             container_expire_timeout=50, container_exit_timeout=60):
@@ -175,7 +166,8 @@ class _NetworkManager:
                 except docker.errors.NotFound:
                     pass
 
-            self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper', auto_remove=True,
+            self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper',
+                                                                 auto_remove=True,
                                                                  command=('sleep %s' % self.container_exit_timeout),
                                                                  detach=True, network_mode='host')
             container_id = self._container.id

tests/integration/helpers/test_tools.py

@@ -1,6 +1,7 @@
 import difflib
 import time
 
+
 class TSV:
     """Helper to get pretty diffs between expected and actual tab-separated value files"""
 
@@ -40,17 +41,22 @@ class TSV:
     def toMat(contents):
         return [line.split("\t") for line in contents.split("\n") if line.strip()]
 
-def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None, settings=None, user=None, ignore_error=False):
+
+def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None,
+                         settings=None, user=None, ignore_error=False):
     expectation_tsv = TSV(expectation)
     for i in xrange(retry_count):
         try:
-            if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) == expectation_tsv:
+            if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings,
+                                  ignore_error=ignore_error)) == expectation_tsv:
                 break
             time.sleep(sleep_time)
         except Exception as ex:
             print "assert_eq_with_retry retry {} exception {}".format(i + 1, ex)
             time.sleep(sleep_time)
     else:
-        val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error))
+        val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings,
+                                 ignore_error=ignore_error))
         if expectation_tsv != val:
-            raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(expectation_tsv.diff(val, n1="expectation", n2="query"))))
+            raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(
+                expectation_tsv.diff(val, n1="expectation", n2="query"))))

tests/integration/helpers/uclient.py

@@ -11,9 +11,10 @@ import uexpect
 prompt = ':\) '
 end_of_block = r'.*\r\n.*\r\n'
 
+
 class client(object):
     def __init__(self, command=None, name='', log=None):
-        self.client = uexpect.spawn(['/bin/bash','--noediting'])
+        self.client = uexpect.spawn(['/bin/bash', '--noediting'])
         if command is None:
             command = '/usr/bin/clickhouse-client'
         self.client.command = command

tests/integration/helpers/uexpect.py

@@ -13,13 +13,12 @@
 # limitations under the License.
 import os
 import pty
-import time
-import sys
 import re
-
-from threading import Thread, Event
-from subprocess import Popen
+import time
 from Queue import Queue, Empty
+from subprocess import Popen
+from threading import Thread, Event
+
 
 class TimeoutError(Exception):
     def __init__(self, timeout):
@@ -28,6 +27,7 @@ class TimeoutError(Exception):
     def __str__(self):
         return 'Timeout %.3fs' % float(self.timeout)
 
+
 class ExpectTimeoutError(Exception):
     def __init__(self, pattern, timeout, buffer):
         self.pattern = pattern
@@ -43,6 +43,7 @@ class ExpectTimeoutError(Exception):
             s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]])
         return s
 
+
 class IO(object):
     class EOF(object):
         pass
@@ -59,7 +60,7 @@ class IO(object):
             self._prefix = prefix
 
         def write(self, data):
-            self._logger.write(('\n' + data).replace('\n','\n' + self._prefix))
+            self._logger.write(('\n' + data).replace('\n', '\n' + self._prefix))
 
         def flush(self):
             self._logger.flush()
@@ -165,7 +166,7 @@ class IO(object):
         data = ''
         timeleft = timeout
         try:
-            while timeleft >= 0 :
+            while timeleft >= 0:
                 start_time = time.time()
                 data += self.queue.get(timeout=timeleft)
                 if data:
@@ -182,6 +183,7 @@ class IO(object):
 
         return data
 
+
 def spawn(command):
     master, slave = pty.openpty()
     process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1)
@@ -193,7 +195,8 @@ def spawn(command):
     thread.daemon = True
     thread.start()
 
-    return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event})
+    return IO(process, master, queue, reader={'thread': thread, 'kill_event': reader_kill_event})
+
 
 def reader(process, out, queue, kill_event):
     while True:

tests/integration/test_SYSTEM_FLUSH_LOGS/test.py

@@ -24,6 +24,7 @@ system_logs = [
 # decrease timeout for the test to show possible issues.
 timeout = pytest.mark.timeout(30)
 
+
 @pytest.fixture(scope='module', autouse=True)
 def start_cluster():
     try:
@@ -32,10 +33,12 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 @pytest.fixture(scope='function')
 def flush_logs():
     node.query('SYSTEM FLUSH LOGS')
 
+
 @timeout
 @pytest.mark.parametrize('table,exists', system_logs)
 def test_system_logs(flush_logs, table, exists):
@@ -45,6 +48,7 @@ def test_system_logs(flush_logs, table, exists):
     else:
         assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q)
 
+
 # Logic is tricky, let's check that there is no hang in case of message queue
 # is not empty (this is another code path in the code).
 @timeout

tests/integration/test_access_control_on_cluster/test.py

@@ -1,13 +1,12 @@
-import time
 import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.client import QueryRuntimeException
 
 cluster = ClickHouseCluster(__file__)
 ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
 ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
 ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
 
+
 @pytest.fixture(scope="module", autouse=True)
 def started_cluster():
     try:

tests/integration/test_adaptive_granularity/test.py

@@ -1,31 +1,50 @@
 import time
+
 import pytest
-
-from helpers.cluster import ClickHouseCluster
-from multiprocessing.dummy import Pool
 from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
-
+from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
 
-
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
-node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
+node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True)
+node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True)
 
-node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', with_installed_binary=True)
-node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
+node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18',
+                             with_installed_binary=True)
+node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True)
 
-node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', with_installed_binary=True)
-node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
+node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15',
+                             with_installed_binary=True)
+node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True)
 
-node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True)
-node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True)
+node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
+                             with_installed_binary=True)
+node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                             with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
+                             with_installed_binary=True)
 
-node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True)
-node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True)
+node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
+                                                    'configs/merge_tree_settings.xml'], with_zookeeper=True,
+                             image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
+                             with_installed_binary=True)
+node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
+                                                      'configs/merge_tree_settings.xml'], with_zookeeper=True,
+                              image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
+                              with_installed_binary=True)
 
-node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True)
-node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True)
+node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                              with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
+                              with_installed_binary=True)
+node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
+                              with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
+                              with_installed_binary=True)
 
 
 def prepare_single_pair_with_setting(first_node, second_node, group):
@@ -34,80 +53,80 @@ def prepare_single_pair_with_setting(first_node, second_node, group):
 
     # Two tables with adaptive granularity
     first_node.query(
-    '''
-        CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-        SETTINGS index_granularity_bytes = 10485760
-    '''.format(g=group))
-
-    second_node.query(
-    '''
-        CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-        SETTINGS index_granularity_bytes = 10485760
-    '''.format(g=group))
-
-    # Two tables with fixed granularity
-    first_node.query(
-    '''
-        CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-        SETTINGS index_granularity_bytes = 0
-    '''.format(g=group))
-
-    second_node.query(
-    '''
-        CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-        SETTINGS index_granularity_bytes = 0
-    '''.format(g=group))
-
-    # Two tables with different granularity
-    with pytest.raises(QueryRuntimeException):
-        first_node.query(
         '''
-            CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1')
+            CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
             PARTITION BY toYYYYMM(date)
             ORDER BY id
             SETTINGS index_granularity_bytes = 10485760
         '''.format(g=group))
 
-        second_node.query(
+    second_node.query(
         '''
-            CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2')
+            CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
+            PARTITION BY toYYYYMM(date)
+            ORDER BY id
+            SETTINGS index_granularity_bytes = 10485760
+        '''.format(g=group))
+
+    # Two tables with fixed granularity
+    first_node.query(
+        '''
+            CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
             PARTITION BY toYYYYMM(date)
             ORDER BY id
             SETTINGS index_granularity_bytes = 0
         '''.format(g=group))
 
-        # Two tables with different granularity, but enabled mixed parts
-        first_node.query(
+    second_node.query(
         '''
-            CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1')
+            CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
             PARTITION BY toYYYYMM(date)
             ORDER BY id
-            SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1
+            SETTINGS index_granularity_bytes = 0
         '''.format(g=group))
 
+    # Two tables with different granularity
+    with pytest.raises(QueryRuntimeException):
+        first_node.query(
+            '''
+                CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+                SETTINGS index_granularity_bytes = 10485760
+            '''.format(g=group))
+
         second_node.query(
-        '''
-            CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-            SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1
-        '''.format(g=group))
+            '''
+                CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+                SETTINGS index_granularity_bytes = 0
+            '''.format(g=group))
+
+        # Two tables with different granularity, but enabled mixed parts
+        first_node.query(
+            '''
+                CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+                SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1
+            '''.format(g=group))
+
+        second_node.query(
+            '''
+                CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+                SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1
+            '''.format(g=group))
 
 
 def prepare_single_pair_without_setting(first_node, second_node, group):
@@ -116,21 +135,21 @@ def prepare_single_pair_without_setting(first_node, second_node, group):
 
     # Two tables with fixed granularity
     first_node.query(
-    '''
-        CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-    '''.format(g=group))
+        '''
+            CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
+            PARTITION BY toYYYYMM(date)
+            ORDER BY id
+        '''.format(g=group))
 
     second_node.query(
-    '''
-        CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-        SETTINGS index_granularity_bytes = 0
-    '''.format(g=group))
+        '''
+            CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
+            PARTITION BY toYYYYMM(date)
+            ORDER BY id
+            SETTINGS index_granularity_bytes = 0
+        '''.format(g=group))
 
 
 @pytest.fixture(scope="module")
@@ -160,7 +179,8 @@ def start_static_cluster():
 def test_different_versions_cluster(start_static_cluster, first_node, second_node, table):
     counter = 1
     for n1, n2 in ((first_node, second_node), (second_node, first_node)):
-        n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(tbl=table, c1=counter * 2, c2=counter * 2 + 1))
+        n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(
+            tbl=table, c1=counter * 2, c2=counter * 2 + 1))
         n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
         assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
         n1.query("DETACH TABLE {tbl}".format(tbl=table))
@@ -175,73 +195,74 @@ def test_different_versions_cluster(start_static_cluster, first_node, second_nod
         assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
         counter += 1
 
+
 @pytest.fixture(scope="module")
 def start_dynamic_cluster():
     try:
         cluster.start()
         node7.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         node7.query(
-        '''
-            CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-            SETTINGS index_granularity_bytes=10485760
-        ''')
+            '''
+                CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+                SETTINGS index_granularity_bytes=10485760
+            ''')
 
         node8.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         node9.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         node10.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         node11.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         node12.query(
-        '''
-            CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id
-        ''')
-
+            '''
+                CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id
+            ''')
 
         yield cluster
 
     finally:
         cluster.shutdown()
 
+
 @pytest.mark.parametrize(
     ('n', 'tables'),
     [
@@ -251,13 +272,16 @@ def start_dynamic_cluster():
 )
 def test_version_single_node_update(start_dynamic_cluster, n, tables):
     for table in tables:
-        n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table))
+        n.query(
+            "INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table))
     n.restart_with_latest_version()
     for table in tables:
         assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n'
-        n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table))
+        n.query(
+            "INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table))
         assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n'
 
+
 @pytest.mark.parametrize(
     ('node',),
     [
@@ -266,27 +290,38 @@ def test_version_single_node_update(start_dynamic_cluster, n, tables):
     ]
 )
 def test_mixed_granularity_single_node(start_dynamic_cluster, node):
-    node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
-    node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)")
+    node.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
+    node.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)")
 
     def callback(n):
-        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
-        n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
+        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
+                         "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
+        n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
+                         "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
 
     node.restart_with_latest_version(callback_onstop=callback)
     node.query("SYSTEM RELOAD CONFIG")
-    assert_eq_with_retry(node, "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'", '1')
+    assert_eq_with_retry(node,
+                         "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'",
+                         '1')
     assert node.query("SELECT count() from table_with_default_granularity") == '4\n'
-    node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
+    node.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
     assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
     node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL")
     assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
-    path_to_merged_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
-    node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_to_merged_part)]) # check that we have adaptive files
+    path_to_merged_part = node.query(
+        "SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
+    node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(
+        p=path_to_merged_part)])  # check that we have adaptive files
 
-    path_to_old_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip()
+    path_to_old_part = node.query(
+        "SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip()
 
-    node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(p=path_to_old_part)]) # check that we have non adaptive files
+    node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(
+        p=path_to_old_part)])  # check that we have non adaptive files
 
     node.query("ALTER TABLE table_with_default_granularity UPDATE dummy = dummy + 1 WHERE 1")
     # still works
@@ -295,46 +330,54 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node):
     node.query("ALTER TABLE table_with_default_granularity MODIFY COLUMN dummy String")
     node.query("ALTER TABLE table_with_default_granularity ADD COLUMN dummy2 Float64")
 
-    #still works
+    # still works
     assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
 
+
 @pytest.mark.skip(reason="flaky")
 def test_version_update_two_nodes(start_dynamic_cluster):
-    node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
+    node11.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
     node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
     assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n'
+
     def callback(n):
-        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
-        n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
+        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
+                         "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
+        n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
+                         "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
 
     node12.restart_with_latest_version(callback_onstop=callback)
 
-    node12.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
+    node12.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
     node11.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
     assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n'
 
     node12.query(
-    '''
-        CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-    ''')
+        '''
+            CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2')
+            PARTITION BY toYYYYMM(date)
+            ORDER BY id
+        ''')
 
     node11.query(
-    '''
-        CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1')
-        PARTITION BY toYYYYMM(date)
-        ORDER BY id
-    ''')
+        '''
+            CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1')
+            PARTITION BY toYYYYMM(date)
+            ORDER BY id
+        ''')
 
-    node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
+    node12.query(
+        "INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
     with pytest.raises(QueryTimeoutExceedException):
         node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=20)
-    node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
+    node12.query(
+        "INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
 
-    node11.restart_with_latest_version(callback_onstop=callback) # just to be sure
+    node11.restart_with_latest_version(callback_onstop=callback)  # just to be sure
 
     for i in range(3):
         try:
@@ -350,7 +393,8 @@ def test_version_update_two_nodes(start_dynamic_cluster):
     assert node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
     assert node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
 
-    node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)")
+    node11.query(
+        "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)")
     for i in range(3):
         try:
             node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=120)

tests/integration/test_adaptive_granularity_different_settings/test.py

@@ -7,7 +7,9 @@ node1 = cluster.add_instance('node1', with_zookeeper=True)
 node2 = cluster.add_instance('node2', with_zookeeper=True)
 
 # no adaptive granularity by default
-node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True, stay_alive=True)
+node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True,
+                             stay_alive=True)
+
 
 @pytest.fixture(scope="module")
 def start_cluster():
@@ -20,7 +22,6 @@ def start_cluster():
 
 
 def test_attach_detach(start_cluster):
-
     node1.query("""
         CREATE TABLE test (key UInt64)
         ENGINE = ReplicatedMergeTree('/clickhouse/test', '1')
@@ -58,7 +59,8 @@ def test_mutate_with_mixed_granularity(start_cluster):
         ENGINE = MergeTree
         ORDER BY key PARTITION BY date""")
 
-    node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)")
+    node3.query(
+        "INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)")
 
     assert node3.query("SELECT COUNT() FROM test") == "500\n"
 
@@ -68,7 +70,8 @@ def test_mutate_with_mixed_granularity(start_cluster):
 
     node3.query("ALTER TABLE test MODIFY SETTING enable_mixed_granularity_parts = 1")
 
-    node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)")
+    node3.query(
+        "INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)")
 
     assert node3.query("SELECT COUNT() FROM test") == "1000\n"
     assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "10\n"

tests/integration/test_adaptive_granularity_replicated/test.py

@@ -1,17 +1,15 @@
 import time
+
 import pytest
-
 from helpers.cluster import ClickHouseCluster
-from multiprocessing.dummy import Pool
-from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
-
-from helpers.test_tools import assert_eq_with_retry
 
 cluster = ClickHouseCluster(__file__)
 
 node1 = cluster.add_instance('node1', with_zookeeper=True)
 node2 = cluster.add_instance('node2', with_zookeeper=True)
-node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True)
+node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14',
+                             with_installed_binary=True)
+
 
 @pytest.fixture(scope="module")
 def start_cluster():
@@ -23,11 +21,14 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_creating_table_different_setting(start_cluster):
-    node1.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0")
+    node1.query(
+        "CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0")
     node1.query("INSERT INTO t1 VALUES('x', 'y')")
 
-    node2.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
+    node2.query(
+        "CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
 
     node1.query("INSERT INTO t1 VALUES('a', 'b')")
     node2.query("SYSTEM SYNC REPLICA t1", timeout=5)
@@ -49,22 +50,26 @@ def test_creating_table_different_setting(start_cluster):
     node1.query("SELECT count() FROM t1") == "3\n"
     node2.query("SELECT count() FROM t1") == "2\n"
 
-    path_part = node1.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
+    path_part = node1.query(
+        "SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
 
-    with pytest.raises(Exception): # check that we have no adaptive files
+    with pytest.raises(Exception):  # check that we have no adaptive files
         node1.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
 
-    path_part = node2.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
+    path_part = node2.query(
+        "SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
 
-    with pytest.raises(Exception): # check that we have no adaptive files
+    with pytest.raises(Exception):  # check that we have no adaptive files
         node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
 
 
 def test_old_node_with_new_node(start_cluster):
-    node3.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)")
+    node3.query(
+        "CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)")
     node3.query("INSERT INTO t2 VALUES('x', 'y')")
 
-    node2.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
+    node2.query(
+        "CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
 
     node3.query("INSERT INTO t2 VALUES('a', 'b')")
     node2.query("SYSTEM SYNC REPLICA t2", timeout=5)
@@ -86,12 +91,14 @@ def test_old_node_with_new_node(start_cluster):
     node3.query("SELECT count() FROM t2") == "3\n"
     node2.query("SELECT count() FROM t2") == "2\n"
 
-    path_part = node3.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
+    path_part = node3.query(
+        "SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
 
-    with pytest.raises(Exception): # check that we have no adaptive files
+    with pytest.raises(Exception):  # check that we have no adaptive files
         node3.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
 
-    path_part = node2.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
+    path_part = node2.query(
+        "SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
 
-    with pytest.raises(Exception): # check that we have no adaptive files
+    with pytest.raises(Exception):  # check that we have no adaptive files
         node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])

tests/integration/test_aggregation_memory_efficient/test.py

@@ -1,22 +1,20 @@
-import time
 import pytest
 
 from helpers.cluster import ClickHouseCluster
-from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
-
 
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1')
 node2 = cluster.add_instance('node2')
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
         cluster.start()
 
         for node in [node1, node2]:
-            node.query("create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;")
-
+            node.query(
+                "create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;")
 
         node1.query("insert into da_memory_efficient_shard select number, number from numbers(100000);")
         node2.query("insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);")
@@ -28,19 +26,24 @@ def start_cluster():
 
 
 def test_remote(start_cluster):
-
-    node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
-    res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
+    node1.query(
+        "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
+    res = node1.query(
+        "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
     assert res == '200000\n'
 
     node1.query("set distributed_aggregation_memory_efficient = 0")
-    res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
+    res = node1.query(
+        "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
     assert res == '200000\n'
 
-    node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
-    res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
+    node1.query(
+        "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
+    res = node1.query(
+        "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
     assert res == 'node1\t100000\nnode2\t100000\n'
 
     node1.query("set distributed_aggregation_memory_efficient = 0")
-    res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
+    res = node1.query(
+        "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
     assert res == 'node1\t100000\nnode2\t100000\n'

tests/integration/test_allowed_client_hosts/test.py

@@ -1,32 +1,32 @@
-import os
 import pytest
 from helpers.cluster import ClickHouseCluster
 
-
 cluster = ClickHouseCluster(__file__)
 server = cluster.add_instance('server', user_configs=["configs/users.d/network.xml"])
 
-clientA1 = cluster.add_instance('clientA1', hostname = 'clientA1.com')
-clientA2 = cluster.add_instance('clientA2', hostname = 'clientA2.com')
-clientA3 = cluster.add_instance('clientA3', hostname = 'clientA3.com')
-clientB1 = cluster.add_instance('clientB1', hostname = 'clientB001.ru')
-clientB2 = cluster.add_instance('clientB2', hostname = 'clientB002.ru')
-clientB3 = cluster.add_instance('clientB3', hostname = 'xxx.clientB003.rutracker.com')
-clientC1 = cluster.add_instance('clientC1', hostname = 'clientC01.ru')
-clientC2 = cluster.add_instance('clientC2', hostname = 'xxx.clientC02.ru')
-clientC3 = cluster.add_instance('clientC3', hostname = 'xxx.clientC03.rutracker.com')
-clientD1 = cluster.add_instance('clientD1', hostname = 'clientD0001.ru')
-clientD2 = cluster.add_instance('clientD2', hostname = 'xxx.clientD0002.ru')
-clientD3 = cluster.add_instance('clientD3', hostname = 'clientD0003.ru')
+clientA1 = cluster.add_instance('clientA1', hostname='clientA1.com')
+clientA2 = cluster.add_instance('clientA2', hostname='clientA2.com')
+clientA3 = cluster.add_instance('clientA3', hostname='clientA3.com')
+clientB1 = cluster.add_instance('clientB1', hostname='clientB001.ru')
+clientB2 = cluster.add_instance('clientB2', hostname='clientB002.ru')
+clientB3 = cluster.add_instance('clientB3', hostname='xxx.clientB003.rutracker.com')
+clientC1 = cluster.add_instance('clientC1', hostname='clientC01.ru')
+clientC2 = cluster.add_instance('clientC2', hostname='xxx.clientC02.ru')
+clientC3 = cluster.add_instance('clientC3', hostname='xxx.clientC03.rutracker.com')
+clientD1 = cluster.add_instance('clientD1', hostname='clientD0001.ru')
+clientD2 = cluster.add_instance('clientD2', hostname='xxx.clientD0002.ru')
+clientD3 = cluster.add_instance('clientD3', hostname='clientD0003.ru')
 
 
 def check_clickhouse_is_ok(client_node, server_node):
-    assert client_node.exec_in_container(["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n"
+    assert client_node.exec_in_container(
+        ["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n"
 
 
 def query_from_one_node_to_another(client_node, server_node, query):
     check_clickhouse_is_ok(client_node, server_node)
-    return client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)])
+    return client_node.exec_in_container(
+        ["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)])
 
 
 def query(node, query):
@@ -53,8 +53,8 @@ def test_allowed_host():
     # Reverse DNS lookup currently isn't working as expected in this test.
     # For example, it gives something like "vitbartestallowedclienthosts_clientB1_1.vitbartestallowedclienthosts_default" instead of "clientB001.ru".
     # Maybe we should setup the test network better.
-    #expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3])
-    #expected_to_fail.extend([clientC3, clientD2])
+    # expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3])
+    # expected_to_fail.extend([clientC3, clientD2])
 
     for client_node in expected_to_pass:
         assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") == "5\n"

tests/integration/test_allowed_url_from_config/test.py

@@ -1,8 +1,6 @@
-import time
 import pytest
-
-from helpers.hdfs_api import HDFSApi
 from helpers.cluster import ClickHouseCluster
+from helpers.hdfs_api import HDFSApi
 
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml'])
@@ -21,26 +19,37 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_config_with_hosts(start_cluster):
     assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == ""
     assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
-    assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)")
-    assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)")
+    assert "not allowed" in node1.query_and_get_error(
+        "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)")
+    assert "not allowed" in node1.query_and_get_error(
+        "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)")
+
 
 def test_config_with_only_primary_hosts(start_cluster):
     assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == ""
     assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == ""
     assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
     assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == ""
-    assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)")
-    assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)")
-    assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)")
+    assert "not allowed" in node2.query_and_get_error(
+        "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)")
+    assert "not allowed" in node2.query_and_get_error(
+        "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)")
+    assert "not allowed" in node2.query_and_get_error(
+        "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)")
+
 
 def test_config_with_only_regexp_hosts(start_cluster):
     assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == ""
     assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
-    assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)")
-    assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)")
+    assert "not allowed" in node3.query_and_get_error(
+        "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)")
+    assert "not allowed" in node3.query_and_get_error(
+        "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)")
+
 
 def test_config_without_allowed_hosts(start_cluster):
     assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == ""
@@ -48,27 +57,60 @@ def test_config_without_allowed_hosts(start_cluster):
     assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
     assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == ""
 
+
 def test_table_function_remote(start_cluster):
-    assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-1', system, events", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
-    assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1})
+    assert "not allowed in config.xml" not in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed in config.xml" not in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed in config.xml" not in node6.query_and_get_error(
+        "SELECT * FROM remote('example01-0{1,2}-1', system, events",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed in config.xml" not in node6.query_and_get_error(
+        "SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed in config.xml" not in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed" in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)",
+                                                      settings={"connections_with_failover_max_tries": 1,
+                                                                "connect_timeout_with_failover_ms": 1000,
+                                                                "connect_timeout_with_failover_secure_ms": 1000,
+                                                                "connect_timeout": 1, "send_timeout": 1})
+    assert "not allowed" in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)",
+        settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
+                  "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
     assert node6.query("SELECT * FROM remote('localhost', system, events)") != ""
     assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != ""
-    assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remoteSecure('localhost:800', system, events)")
-    assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remote('localhost:800', system, metrics)")
+    assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error(
+        "SELECT * FROM remoteSecure('localhost:800', system, events)")
+    assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error(
+        "SELECT * FROM remote('localhost:800', system, metrics)")
+
 
 def test_redirect(start_cluster):
     hdfs_api = HDFSApi("root")
     hdfs_api.write_data("/simple_storage", "1\t\n")
     assert hdfs_api.read_data("/simple_storage") == "1\t\n"
-    node7.query("CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)")
+    node7.query(
+        "CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)")
     assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1")
 
+
 def test_HDFS(start_cluster):
-    assert "not allowed" in node7.query_and_get_error("CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')")
-    assert "not allowed" in node7.query_and_get_error("SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')")
+    assert "not allowed" in node7.query_and_get_error(
+        "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')")
+    assert "not allowed" in node7.query_and_get_error(
+        "SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')")

tests/integration/test_alter_codec/test.py

@@ -2,14 +2,13 @@ import pytest
 from helpers.client import QueryRuntimeException
 from helpers.cluster import ClickHouseCluster
 
-
 cluster = ClickHouseCluster(__file__)
 
 node1 = cluster.add_instance('node1',
-            main_configs=['configs/logs_config.xml'])
+                             main_configs=['configs/logs_config.xml'])
 
 node2 = cluster.add_instance('node2',
-            main_configs=['configs/logs_config.xml'])
+                             main_configs=['configs/logs_config.xml'])
 
 
 @pytest.fixture(scope="module")
@@ -39,7 +38,6 @@ def test_alter_codec_pk(started_cluster):
         with pytest.raises(QueryRuntimeException):
             node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name))
 
-
         node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name))
 
         node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name))

tests/integration/test_always_fetch_merged/test.py

@@ -1,14 +1,15 @@
-import pytest
 import time
+
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
 
-
 cluster = ClickHouseCluster(__file__)
 
 node1 = cluster.add_instance('node1', with_zookeeper=True)
 node2 = cluster.add_instance('node2', with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:

tests/integration/test_asynchronous_metric_log_table/test.py

@@ -39,7 +39,7 @@ def test_event_time_microseconds_field(started_cluster):
         node1.query(query_create)
         node1.query('''INSERT INTO replica.test VALUES (1, now())''')
         node1.query("SYSTEM FLUSH LOGS;")
-        #query assumes that the event_time field is accurate
+        # query assumes that the event_time field is accurate
         equals_query = '''WITH (
                             (
                                 SELECT event_time_microseconds

tests/integration/test_atomic_drop_table/test.py

@@ -1,12 +1,12 @@
 import time
+
 import pytest
-
-from helpers.network import PartitionManager
 from helpers.cluster import ClickHouseCluster
-
+from helpers.network import PartitionManager
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml", "configs/remote_servers.xml"], with_zookeeper=True)
+node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml",
+                                                    "configs/remote_servers.xml"], with_zookeeper=True)
 
 
 @pytest.fixture(scope="module")
@@ -25,12 +25,13 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_atomic_delete_with_stopped_zookeeper(start_cluster):
     node1.query("insert into zktest.atomic_drop_table values (8192)")
 
     with PartitionManager() as pm:
         pm.drop_instance_zk_connections(node1)
-        error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") #Table won't drop
+        error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table")  # Table won't drop
         assert error != ""
 
     time.sleep(5)

tests/integration/test_attach_without_checksums/test.py

@@ -17,7 +17,8 @@ def start_cluster():
 
 
 def test_attach_without_checksums(start_cluster):
-    node1.query("CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date")
+    node1.query(
+        "CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date")
 
     node1.query("INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)")
 
@@ -29,9 +30,13 @@ def test_attach_without_checksums(start_cluster):
     assert node1.query("SELECT COUNT() FROM test") == "0\n"
 
     # to be sure output not empty
-    node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '], privileged=True, user='root')
+    node1.exec_in_container(
+        ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '],
+        privileged=True, user='root')
 
-    node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'], privileged=True, user='root')
+    node1.exec_in_container(
+        ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'],
+        privileged=True, user='root')
 
     node1.query("ALTER TABLE test ATTACH PARTITION '2019-10-01'")
 

tests/integration/test_authentication/test.py

@@ -30,7 +30,8 @@ def test_authentication_pass():
 
 def test_authentication_fail():
     # User doesn't exist.
-    assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'vasya')
-    
+    assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='vasya')
+
     # Wrong password.
-    assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'masha', password = '123')
+    assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='masha',
+                                                                          password='123')

tests/integration/test_backup_restore/test.py

@@ -1,10 +1,9 @@
 import os.path
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
-
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance')
 q = instance.query
@@ -47,7 +46,7 @@ def backup_restore(started_cluster):
 
     expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
     res = q("SELECT * FROM test.tbl ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("ALTER TABLE test.tbl FREEZE")
 
@@ -69,7 +68,7 @@ def test_restore(backup_restore):
     # Validate the attached parts are identical to the backup.
     expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
     res = q("SELECT * FROM test.tbl1 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1")
     q("SELECT sleep(2)")
@@ -77,7 +76,7 @@ def test_restore(backup_restore):
     # Validate mutation has been applied to all attached parts.
     expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10')
     res = q("SELECT * FROM test.tbl1 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("DROP TABLE IF EXISTS test.tbl1")
 
@@ -91,7 +90,7 @@ def test_attach_partition(backup_restore):
 
     expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34')
     res = q("SELECT * FROM test.tbl2 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     copy_backup_to_detached('test', 'tbl', 'tbl2')
 
@@ -102,17 +101,19 @@ def test_attach_partition(backup_restore):
     q("ALTER TABLE test.tbl2 ATTACH PARTITION 197002")
     q("SELECT sleep(2)")
 
-    expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34')
+    expected = TSV(
+        '1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34')
     res = q("SELECT * FROM test.tbl2 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1")
     q("SELECT sleep(2)")
 
     # Validate mutation has been applied to all attached parts.
-    expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10')
+    expected = TSV(
+        '1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10')
     res = q("SELECT * FROM test.tbl2 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("DROP TABLE IF EXISTS test.tbl2")
 
@@ -126,7 +127,7 @@ def test_replace_partition(backup_restore):
 
     expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34')
     res = q("SELECT * FROM test.tbl3 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     copy_backup_to_detached('test', 'tbl', 'tbl3')
 
@@ -138,7 +139,7 @@ def test_replace_partition(backup_restore):
 
     expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
     res = q("SELECT * FROM test.tbl3 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1")
     q("SELECT sleep(2)")
@@ -146,6 +147,6 @@ def test_replace_partition(backup_restore):
     # Validate mutation has been applied to all copied parts.
     expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10')
     res = q("SELECT * FROM test.tbl3 ORDER BY p")
-    assert(TSV(res) == expected)
+    assert (TSV(res) == expected)
 
     q("DROP TABLE IF EXISTS test.tbl3")

tests/integration/test_backup_with_other_granularity/test.py

@@ -1,13 +1,15 @@
 import pytest
 
-
 from helpers.cluster import ClickHouseCluster
+
 cluster = ClickHouseCluster(__file__)
 
-
-node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True)
-node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True)
-node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True)
+node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
+                             stay_alive=True, with_installed_binary=True)
+node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
+                             stay_alive=True, with_installed_binary=True)
+node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
+                             stay_alive=True, with_installed_binary=True)
 node4 = cluster.add_instance('node4')
 
 
@@ -33,13 +35,15 @@ def test_backup_from_old_version(started_cluster):
 
     node1.restart_with_latest_version()
 
-    node1.query("CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1')  ORDER BY tuple()")
+    node1.query(
+        "CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1')  ORDER BY tuple()")
 
     node1.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
 
     assert node1.query("SELECT COUNT() FROM dest_table") == "1\n"
 
-    node1.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
+    node1.exec_in_container(['bash', '-c',
+                             'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
 
     assert node1.query("SELECT COUNT() FROM dest_table") == "1\n"
 
@@ -69,13 +73,15 @@ def test_backup_from_old_version_setting(started_cluster):
 
     node2.restart_with_latest_version()
 
-    node2.query("CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1')  ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
+    node2.query(
+        "CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1')  ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
 
     node2.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
 
     assert node2.query("SELECT COUNT() FROM dest_table") == "1\n"
 
-    node2.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
+    node2.exec_in_container(['bash', '-c',
+                             'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
 
     assert node2.query("SELECT COUNT() FROM dest_table") == "1\n"
 
@@ -104,17 +110,20 @@ def test_backup_from_old_version_config(started_cluster):
     node3.query("ALTER TABLE source_table FREEZE PARTITION tuple();")
 
     def callback(n):
-        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
+        n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
+                         "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
 
     node3.restart_with_latest_version(callback_onstop=callback)
 
-    node3.query("CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1')  ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
+    node3.query(
+        "CREATE TABLE dest_table (A Int64,  B String,  Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1')  ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
 
     node3.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
 
     assert node3.query("SELECT COUNT() FROM dest_table") == "1\n"
 
-    node3.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
+    node3.exec_in_container(['bash', '-c',
+                             'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
 
     assert node3.query("SELECT COUNT() FROM dest_table") == "1\n"
 
@@ -144,7 +153,8 @@ def test_backup_and_alter(started_cluster):
 
     node4.query("ALTER TABLE backup_table DROP PARTITION tuple()")
 
-    node4.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached'])
+    node4.exec_in_container(['bash', '-c',
+                             'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached'])
 
     node4.query("ALTER TABLE backup_table ATTACH PARTITION tuple()")
 

tests/integration/test_backward_compatibility/test.py

@@ -1,22 +1,23 @@
 import pytest
 
-import helpers.client as client
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True)
+node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54',
+                             stay_alive=True, with_installed_binary=True)
 node2 = cluster.add_instance('node2', with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
         cluster.start()
         for i, node in enumerate([node1, node2]):
             node.query(
-            '''CREATE TABLE t(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}')
-            PARTITION BY toYYYYMM(date)
-            ORDER BY id'''.format(i))
+                '''CREATE TABLE t(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}')
+                PARTITION BY toYYYYMM(date)
+                ORDER BY id'''.format(i))
 
         yield cluster
 

tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py

@@ -1,16 +1,18 @@
 import pytest
 
-import helpers.client as client
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1',
-    with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True)
+                             with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True,
+                             with_installed_binary=True)
 node2 = cluster.add_instance('node2',
-    with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True)
+                             with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True,
+                             with_installed_binary=True)
 node3 = cluster.add_instance('node3', with_zookeeper=False)
 node4 = cluster.add_instance('node4', with_zookeeper=False)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
@@ -20,6 +22,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 # We will test that serialization of internal state of "avg" function is compatible between different versions.
 # TODO Implement versioning of serialization format for aggregate function states.
 # NOTE This test is too ad-hoc.
@@ -35,18 +38,18 @@ def test_backward_compatability(start_cluster):
     node3.query("INSERT INTO tab VALUES (3)")
     node4.query("INSERT INTO tab VALUES (4)")
 
-    assert(node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
-    assert(node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
-    assert(node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
-    assert(node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
+    assert (node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
+    assert (node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
+    assert (node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
+    assert (node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
 
     # Also check with persisted aggregate function state
 
     node1.query("create table state (x AggregateFunction(avg, UInt64)) engine = Log")
     node1.query("INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))")
 
-    assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')
+    assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')
 
     node1.restart_with_latest_version()
 
-    assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')
+    assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')

tests/integration/test_backward_compatibility/test_short_strings_aggregation.py

@@ -1,13 +1,15 @@
 import pytest
 
-import helpers.client as client
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True)
-node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True)
+node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37',
+                             stay_alive=True, with_installed_binary=True)
+node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37',
+                             stay_alive=True, with_installed_binary=True)
 node3 = cluster.add_instance('node3', with_zookeeper=False)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
@@ -23,6 +25,7 @@ def test_backward_compatability(start_cluster):
     node2.query("create table tab (s String) engine = MergeTree order by s")
     node1.query("insert into tab select number from numbers(50)")
     node2.query("insert into tab select number from numbers(1000000)")
-    res = node3.query("select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50")
+    res = node3.query(
+        "select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50")
     print(res)
     assert res == ''.join('{}\t2\n'.format(i) for i in range(50))

tests/integration/test_block_structure_mismatch/test.py

@@ -1,7 +1,5 @@
-import time
 import pytest
 
-from contextlib import contextmanager
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
@@ -9,7 +7,8 @@ cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'])
 node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'])
 
-#test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162
+
+# test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -44,7 +43,9 @@ CREATE TABLE dist_test (
     finally:
         cluster.shutdown()
 
+
 def test(started_cluster):
     node1.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')")
     node2.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')")
-    assert node1.query("SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n"
+    assert node1.query(
+        "SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n"

tests/integration/test_check_table/test.py

@@ -1,9 +1,6 @@
-import time
-
 import pytest
 
 from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import assert_eq_with_retry
 
 cluster = ClickHouseCluster(__file__)
 
@@ -34,15 +31,22 @@ def started_cluster():
 
 
 def corrupt_data_part_on_disk(node, table, part_name):
-    part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
-    node.exec_in_container(['bash', '-c', 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(p=part_path)], privileged=True)
+    part_path = node.query(
+        "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
+    node.exec_in_container(['bash', '-c',
+                            'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(
+                                p=part_path)], privileged=True)
+
 
 def remove_checksums_on_disk(node, table, part_name):
-    part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
+    part_path = node.query(
+        "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
     node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True)
 
+
 def remove_part_from_disk(node, table, part_name):
-    part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
+    part_path = node.query(
+        "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
     if not part_path:
         raise Exception("Part " + part_name + "doesn't exist")
     node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True)
@@ -50,35 +54,42 @@ def remove_part_from_disk(node, table, part_name):
 
 def test_check_normal_table_corruption(started_cluster):
     node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)")
-    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n"
+    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902",
+                       settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n"
 
     remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
 
-    assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
+    assert node1.query("CHECK TABLE non_replicated_mt", settings={
+        "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
 
     assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"
 
     remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
 
-    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
+    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={
+        "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
 
     assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"
 
     corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
 
-    assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
+    assert node1.query("CHECK TABLE non_replicated_mt", settings={
+        "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
 
-    assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
+    assert node1.query("CHECK TABLE non_replicated_mt", settings={
+        "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
 
     node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)")
 
-    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n"
+    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901",
+                       settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n"
 
     corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0")
 
     remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0")
 
-    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n"
+    assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={
+        "check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n"
 
 
 def test_check_replicated_table_simple(started_cluster):
@@ -90,16 +101,20 @@ def test_check_replicated_table_simple(started_cluster):
     assert node1.query("SELECT count() from replicated_mt") == "2\n"
     assert node2.query("SELECT count() from replicated_mt") == "2\n"
 
-    assert node1.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
-    assert node2.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
+    assert node1.query("CHECK TABLE replicated_mt",
+                       settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
+    assert node2.query("CHECK TABLE replicated_mt",
+                       settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
 
     node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)")
     node1.query("SYSTEM SYNC REPLICA replicated_mt")
     assert node1.query("SELECT count() from replicated_mt") == "4\n"
     assert node2.query("SELECT count() from replicated_mt") == "4\n"
 
-    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
-    assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
+    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
+                       settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
+    assert node2.query("CHECK TABLE replicated_mt PARTITION 201901",
+                       settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
 
 
 def test_check_replicated_table_corruption(started_cluster):
@@ -112,18 +127,25 @@ def test_check_replicated_table_corruption(started_cluster):
     assert node1.query("SELECT count() from replicated_mt") == "4\n"
     assert node2.query("SELECT count() from replicated_mt") == "4\n"
 
-    part_name = node1.query("SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip()
+    part_name = node1.query(
+        "SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip()
 
     corrupt_data_part_on_disk(node1, "replicated_mt", part_name)
-    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name)
+    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={
+        "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(
+        p=part_name)
 
     node1.query("SYSTEM SYNC REPLICA replicated_mt")
-    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
+    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
+                       settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
     assert node1.query("SELECT count() from replicated_mt") == "4\n"
 
     remove_part_from_disk(node2, "replicated_mt", part_name)
-    assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name)
+    assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={
+        "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(
+        p=part_name)
 
     node1.query("SYSTEM SYNC REPLICA replicated_mt")
-    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
+    assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
+                       settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
     assert node1.query("SELECT count() from replicated_mt") == "4\n"

tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py

@@ -1,13 +1,13 @@
 import time
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.network import PartitionManager
 
-
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
@@ -16,6 +16,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 # This tests if the data directory for a table is cleaned up if there is a Zookeeper
 # connection exception during a CreateQuery operation involving ReplicatedMergeTree tables.
 # Test flow is as follows:
@@ -48,20 +49,30 @@ def test_cleanup_dir_after_bad_zk_conn(start_cluster):
     node1.query('''INSERT INTO replica.test VALUES (1, now())''')
     assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''')
 
+
 def test_cleanup_dir_after_wrong_replica_name(start_cluster):
-    node1.query("CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
-    error = node1.query_and_get_error("CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
+    node1.query(
+        "CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
+    error = node1.query_and_get_error(
+        "CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
     assert "already exists" in error
-    node1.query("CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n")
+    node1.query(
+        "CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n")
+
 
 def test_cleanup_dir_after_wrong_zk_path(start_cluster):
-    node1.query("CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n")
-    error = node1.query_and_get_error("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n")
+    node1.query(
+        "CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n")
+    error = node1.query_and_get_error(
+        "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n")
     assert "Cannot create" in error
-    node1.query("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n")
+    node1.query(
+        "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n")
+
 
 def test_attach_without_zk(start_cluster):
-    node1.query("CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n")
+    node1.query(
+        "CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n")
     node1.query("DETACH TABLE test4_r1")
     with PartitionManager() as pm:
         pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'})

tests/integration/test_cluster_all_replicas/test.py

@@ -7,6 +7,7 @@ cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:

tests/integration/test_cluster_copier/test.py

@@ -1,15 +1,15 @@
 import os
+import random
 import sys
 import time
+from contextlib import contextmanager
+
+import docker
 import kazoo
 import pytest
-import docker
-import random
-from contextlib import contextmanager
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
-
 CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR))
 
@@ -18,9 +18,10 @@ MOVING_FAIL_PROBABILITY = 0.2
 
 cluster = ClickHouseCluster(__file__)
 
+
 def check_all_hosts_sucesfully_executed(tsv_content, num_hosts):
     M = TSV.toMat(tsv_content)
-    hosts = [(l[0], l[1]) for l in M] # (host, port)
+    hosts = [(l[0], l[1]) for l in M]  # (host, port)
     codes = [l[2] for l in M]
 
     assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content
@@ -39,14 +40,14 @@ def started_cluster():
     global cluster
     try:
         clusters_schema = {
-         "0" : {
-            "0" : ["0", "1"],
-            "1" : ["0"]
-         },
-         "1" : {
-            "0" : ["0", "1"],
-            "1" : ["0"]
-         }
+            "0": {
+                "0": ["0", "1"],
+                "1": ["0"]
+            },
+            "1": {
+                "0": ["0", "1"],
+                "1": ["0"]
+            }
         }
 
         for cluster_name, shards in clusters_schema.iteritems():
@@ -54,10 +55,11 @@ def started_cluster():
                 for replica_name in replicas:
                     name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name)
                     cluster.add_instance(name,
-                        main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", "configs/conf.d/clusters.xml"],
-                        user_configs=["configs/users.xml"],
-                        macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name},
-                        with_zookeeper=True)
+                                         main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml",
+                                                       "configs/conf.d/clusters.xml"],
+                                         user_configs=["configs/users.xml"],
+                                         macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name},
+                                         with_zookeeper=True)
 
         cluster.start()
         yield cluster
@@ -70,24 +72,27 @@ class Task1:
 
     def __init__(self, cluster):
         self.cluster = cluster
-        self.zk_task_path="/clickhouse-copier/task_simple"
+        self.zk_task_path = "/clickhouse-copier/task_simple"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task0_description.xml'), 'r').read()
 
-
     def start(self):
         instance = cluster.instances['s0_0_0']
 
         for cluster_num in ["0", "1"]:
             ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num))
-            ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num))
+            ddl_check_query(instance,
+                            "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(
+                                cluster_num))
 
         ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " +
-                                  "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " +
-                                  "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16")
-        ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)")
-        ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)")
-        instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1})
-
+                        "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " +
+                        "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16")
+        ddl_check_query(instance,
+                        "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)")
+        ddl_check_query(instance,
+                        "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)")
+        instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002",
+                       settings={"insert_distributed_sync": 1})
 
     def check(self):
         assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM hits_all")) == TSV("1002\n")
@@ -107,31 +112,44 @@ class Task2:
 
     def __init__(self, cluster):
         self.cluster = cluster
-        self.zk_task_path="/clickhouse-copier/task_month_to_week_partition"
+        self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read()
 
-
     def start(self):
         instance = cluster.instances['s0_0_0']
 
         for cluster_num in ["0", "1"]:
             ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num))
-            ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num))
+            ddl_check_query(instance,
+                            "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(
+                                cluster_num))
 
-        ddl_check_query(instance, "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)")
-        ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)")
+        ddl_check_query(instance,
+                        "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)")
+        ddl_check_query(instance,
+                        "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)")
 
-        instance.query("INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85", settings={"insert_distributed_sync": 1})
+        instance.query(
+            "INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85",
+            settings={"insert_distributed_sync": 1})
 
     def check(self):
-        assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV("85\n")
-        assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n")
+        assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV(
+            "85\n")
+        assert TSV(self.cluster.instances['s1_0_0'].query(
+            "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n")
 
-        assert TSV(self.cluster.instances['s1_0_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n")
-        assert TSV(self.cluster.instances['s1_1_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n")
+        assert TSV(self.cluster.instances['s1_0_0'].query(
+            "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n")
+        assert TSV(self.cluster.instances['s1_1_0'].query(
+            "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n")
 
-        assert TSV(self.cluster.instances['s1_0_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n")
-        assert TSV(self.cluster.instances['s1_1_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n")
+        assert TSV(self.cluster.instances['s1_0_0'].query(
+            "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV(
+            "1\n")
+        assert TSV(self.cluster.instances['s1_1_0'].query(
+            "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV(
+            "1\n")
 
         instance = cluster.instances['s0_0_0']
         ddl_check_query(instance, "DROP TABLE a ON CLUSTER cluster0")
@@ -142,11 +160,10 @@ class Task_test_block_size:
 
     def __init__(self, cluster):
         self.cluster = cluster
-        self.zk_task_path="/clickhouse-copier/task_test_block_size"
+        self.zk_task_path = "/clickhouse-copier/task_test_block_size"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_test_block_size.xml'), 'r').read()
         self.rows = 1000000
 
-
     def start(self):
         instance = cluster.instances['s0_0_0']
 
@@ -155,11 +172,13 @@ class Task_test_block_size:
             ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}')
             ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2)
 
-        instance.query("INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(self.rows))
-
+        instance.query(
+            "INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(
+                self.rows))
 
     def check(self):
-        assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows))
+        assert TSV(self.cluster.instances['s1_0_0'].query(
+            "SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows))
 
         instance = cluster.instances['s0_0_0']
         ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER shard_0_0", 2)
@@ -170,17 +189,15 @@ class Task_no_index:
 
     def __init__(self, cluster):
         self.cluster = cluster
-        self.zk_task_path="/clickhouse-copier/task_no_index"
+        self.zk_task_path = "/clickhouse-copier/task_no_index"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_index.xml'), 'r').read()
         self.rows = 1000000
 
-
     def start(self):
         instance = cluster.instances['s0_0_0']
         instance.query("create table ontime (Year UInt16, FlightDate String) ENGINE = Memory")
         instance.query("insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')")
 
-
     def check(self):
         assert TSV(self.cluster.instances['s1_1_0'].query("SELECT Year FROM ontime22")) == TSV("2017\n")
         instance = cluster.instances['s0_0_0']
@@ -193,17 +210,16 @@ class Task_no_arg:
 
     def __init__(self, cluster):
         self.cluster = cluster
-        self.zk_task_path="/clickhouse-copier/task_no_arg"
+        self.zk_task_path = "/clickhouse-copier/task_no_arg"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_arg.xml'), 'r').read()
         self.rows = 1000000
 
-
     def start(self):
         instance = cluster.instances['s0_0_0']
-        instance.query("create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192")
+        instance.query(
+            "create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192")
         instance.query("insert into copier_test1 values ('2016-01-01', 10);")
 
-
     def check(self):
         assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n")
         instance = cluster.instances['s0_0_0']
@@ -227,15 +243,14 @@ def execute_task(task, cmd_options):
     zk.ensure_path(zk_task_path)
     zk.create(zk_task_path + "/description", task.copier_task_config)
 
-
     # Run cluster-copier processes on each node
     docker_api = docker.from_env().api
     copiers_exec_ids = []
 
     cmd = ['/usr/bin/clickhouse', 'copier',
-        '--config', '/etc/clickhouse-server/config-copier.xml',
-        '--task-path', zk_task_path,
-        '--base-dir', '/var/log/clickhouse-server/copier']
+           '--config', '/etc/clickhouse-server/config-copier.xml',
+           '--task-path', zk_task_path,
+           '--base-dir', '/var/log/clickhouse-server/copier']
     cmd += cmd_options
 
     copiers = random.sample(cluster.instances.keys(), 3)
@@ -243,7 +258,8 @@ def execute_task(task, cmd_options):
     for instance_name in copiers:
         instance = cluster.instances[instance_name]
         container = instance.get_docker_handle()
-        instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml")
+        instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"),
+                                        "/etc/clickhouse-server/config-copier.xml")
         print "Copied copier config to {}".format(instance.name)
         exec_id = docker_api.exec_create(container.id, cmd, stderr=True)
         output = docker_api.exec_start(exec_id).decode('utf8')
@@ -277,7 +293,6 @@ def execute_task(task, cmd_options):
         True
     ]
 )
-
 def test_copy_simple(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(Task1(started_cluster), ['--experimental-use-sample-offset', '1'])
@@ -292,7 +307,6 @@ def test_copy_simple(started_cluster, use_sample_offset):
         True
     ]
 )
-
 def test_copy_with_recovering(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY),
@@ -300,6 +314,7 @@ def test_copy_with_recovering(started_cluster, use_sample_offset):
     else:
         execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
 
+
 @pytest.mark.parametrize(
     ('use_sample_offset'),
     [
@@ -307,7 +322,6 @@ def test_copy_with_recovering(started_cluster, use_sample_offset):
         True
     ]
 )
-
 def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY),
@@ -315,29 +329,36 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs
     else:
         execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)])
 
+
 @pytest.mark.timeout(600)
 def test_copy_month_to_week_partition(started_cluster):
     execute_task(Task2(started_cluster), [])
 
+
 @pytest.mark.timeout(600)
 def test_copy_month_to_week_partition_with_recovering(started_cluster):
     execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
 
+
 @pytest.mark.timeout(600)
 def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster):
     execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)])
 
+
 def test_block_size(started_cluster):
     execute_task(Task_test_block_size(started_cluster), [])
 
+
 def test_no_index(started_cluster):
     execute_task(Task_no_index(started_cluster), [])
 
+
 def test_no_arg(started_cluster):
     execute_task(Task_no_arg(started_cluster), [])
 
+
 if __name__ == '__main__':
     with contextmanager(started_cluster)() as cluster:
-       for name, instance in cluster.instances.items():
-           print name, instance.ip_address
-       raw_input("Cluster created, press any key to destroy...")
+        for name, instance in cluster.instances.items():
+            print name, instance.ip_address
+        raw_input("Cluster created, press any key to destroy...")

tests/integration/test_cluster_copier/trivial_test.py

@@ -1,13 +1,10 @@
 import os
-import os.path as p
 import sys
 import time
-import datetime
-import pytest
 from contextlib import contextmanager
-import docker
-from kazoo.client import KazooClient
 
+import docker
+import pytest
 
 CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR))
@@ -18,13 +15,14 @@ COPYING_FAIL_PROBABILITY = 0.33
 MOVING_FAIL_PROBABILITY = 0.1
 cluster = None
 
+
 @pytest.fixture(scope="function")
 def started_cluster():
     global cluster
     try:
         clusters_schema = {
-            "0" : {"0" : ["0"]},
-            "1" : {"0" : ["0"]}
+            "0": {"0": ["0"]},
+            "1": {"0": ["0"]}
         }
 
         cluster = ClickHouseCluster(__file__)
@@ -50,12 +48,11 @@ class TaskTrivial:
     def __init__(self, cluster, use_sample_offset):
         self.cluster = cluster
         if use_sample_offset:
-            self.zk_task_path="/clickhouse-copier/task_trivial_use_sample_offset"
+            self.zk_task_path = "/clickhouse-copier/task_trivial_use_sample_offset"
         else:
-            self.zk_task_path="/clickhouse-copier/task_trivial"
+            self.zk_task_path = "/clickhouse-copier/task_trivial"
         self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read()
 
-
     def start(self):
         source = cluster.instances['s0_0_0']
         destination = cluster.instances['s1_0_0']
@@ -68,8 +65,8 @@ class TaskTrivial:
                      "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') "
                      "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16")
 
-        source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1})
-
+        source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002",
+                     settings={"insert_distributed_sync": 1})
 
     def check(self):
         source = cluster.instances['s0_0_0']
@@ -138,7 +135,6 @@ def execute_task(task, cmd_options):
         True
     ]
 )
-
 def test_trivial_copy(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--experimental-use-sample-offset', '1'])
@@ -146,6 +142,7 @@ def test_trivial_copy(started_cluster, use_sample_offset):
         print("AAAAA")
         execute_task(TaskTrivial(started_cluster, use_sample_offset), [])
 
+
 @pytest.mark.parametrize(
     ('use_sample_offset'),
     [
@@ -153,7 +150,6 @@ def test_trivial_copy(started_cluster, use_sample_offset):
         True
     ]
 )
-
 def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY),
@@ -161,6 +157,7 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
     else:
         execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
 
+
 @pytest.mark.parametrize(
     ('use_sample_offset'),
     [
@@ -168,7 +165,6 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
         True
     ]
 )
-
 def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset):
     if use_sample_offset:
         execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY),

tests/integration/test_concurrent_queries_for_user_restriction/test.py

@@ -1,8 +1,7 @@
 import time
+from multiprocessing.dummy import Pool
 
 import pytest
-
-from multiprocessing.dummy import Pool
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
@@ -10,6 +9,7 @@ cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml'])
 node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml'])
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -21,6 +21,7 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_exception_message(started_cluster):
     assert node1.query("select number from nums order by number") == "0\n1\n"
 
@@ -30,7 +31,7 @@ def test_exception_message(started_cluster):
 
     busy_pool = Pool(3)
     busy_pool.map_async(node_busy, xrange(3))
-    time.sleep(1) # wait a little until polling starts
+    time.sleep(1)  # wait a little until polling starts
     try:
         assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n"
     except Exception as ex:

tests/integration/test_concurrent_ttl_merges/test.py

@@ -1,12 +1,9 @@
 import time
+
 import pytest
-
-import helpers.client as client
 from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import TSV
 from helpers.test_tools import assert_eq_with_retry
 
-
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True)
 node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True)
@@ -24,14 +21,16 @@ def started_cluster():
 
 
 def count_ttl_merges_in_queue(node, table):
-    result = node.query("SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
+    result = node.query(
+        "SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
     if not result:
         return 0
     return int(result.strip())
 
 
 def count_ttl_merges_in_background_pool(node, table):
-    result = node.query("SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
+    result = node.query(
+        "SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
     if not result:
         return 0
     return int(result.strip())
@@ -55,12 +54,14 @@ def count_running_mutations(node, table):
 # but it revealed a bug when we assign different merges to the same part
 # on the borders of partitions.
 def test_no_ttl_merges_in_busy_pool(started_cluster):
-    node1.query("CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0")
+    node1.query(
+        "CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0")
 
     node1.query("SYSTEM STOP TTL MERGES")
 
     for i in range(1, 7):
-        node1.query("INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i))
+        node1.query(
+            "INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i))
 
     node1.query("ALTER TABLE test_ttl UPDATE data = data + 1 WHERE sleepEachRow(1) = 0")
 
@@ -85,7 +86,8 @@ def test_no_ttl_merges_in_busy_pool(started_cluster):
 
 
 def test_limited_ttl_merges_in_empty_pool(started_cluster):
-    node1.query("CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
+    node1.query(
+        "CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
 
     node1.query("SYSTEM STOP TTL MERGES")
 
@@ -107,7 +109,8 @@ def test_limited_ttl_merges_in_empty_pool(started_cluster):
 
 
 def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster):
-    node1.query("CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
+    node1.query(
+        "CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
 
     node1.query("SYSTEM STOP TTL MERGES")
 
@@ -134,14 +137,17 @@ def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster):
 def test_limited_ttl_merges_two_replicas(started_cluster):
     # Actually this test quite fast and often we cannot catch any merges.
     # To check for sure just add some sleeps in mergePartsToTemporaryPart
-    node1.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
-    node2.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
+    node1.query(
+        "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
+    node2.query(
+        "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
 
     node1.query("SYSTEM STOP TTL MERGES")
     node2.query("SYSTEM STOP TTL MERGES")
 
     for i in range(100):
-        node1.query("INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i))
+        node1.query(
+            "INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i))
 
     node2.query("SYSTEM SYNC REPLICA replicated_ttl_2", timeout=10)
     assert node1.query("SELECT COUNT() FROM replicated_ttl_2") == "1000000\n"
@@ -155,7 +161,8 @@ def test_limited_ttl_merges_two_replicas(started_cluster):
     while True:
         merges_with_ttl_count_node1.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl_2"))
         merges_with_ttl_count_node2.add(count_ttl_merges_in_background_pool(node2, "replicated_ttl_2"))
-        if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n":
+        if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query(
+                "SELECT COUNT() FROM replicated_ttl_2") == "0\n":
             break
 
     # Both replicas can assign merges with TTL. If one will perform better than

tests/integration/test_config_corresponding_root/test.py

@@ -1,6 +1,6 @@
 import os
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -9,6 +9,7 @@ cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance('node', main_configs=["configs/config.d/bad.xml"])
 caught_exception = ""
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     global caught_exception
@@ -17,6 +18,7 @@ def start_cluster():
     except Exception as e:
         caught_exception = str(e)
 
+
 def test_work(start_cluster):
     print(caught_exception)
     assert caught_exception.find("Root element doesn't have the corresponding root element as the config file.") != -1

tests/integration/test_config_substitutions/test.py

@@ -1,21 +1,26 @@
-import time
 import pytest
 
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333
-node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], env_variables={"MAX_QUERY_SIZE": "55555"})
+node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml'])  # hardcoded value 33333
+node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'],
+                             env_variables={"MAX_QUERY_SIZE": "55555"})
 node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True)
-node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], main_configs=['configs/max_query_size.xml']) # include value 77777
+node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'],
+                             main_configs=['configs/max_query_size.xml'])  # include value 77777
 node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml'])
-node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, main_configs=['configs/max_query_size.xml'])
+node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'],
+                             env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"},
+                             main_configs=['configs/max_query_size.xml'])
+
 
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
         def create_zk_roots(zk):
             zk.create(path="/setting/max_query_size", value="77777", makepath=True)
+
         cluster.add_zookeeper_startup_command(create_zk_roots)
 
         cluster.start()
@@ -23,25 +28,36 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_config(start_cluster):
-   assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n"
-   assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n"
-   assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n"
-   assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
-   assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
+    assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n"
+    assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n"
+    assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n"
+    assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
+    assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
+
 
 def test_allow_databases(start_cluster):
     node5.query("CREATE DATABASE db1")
-    node5.query("CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)")
+    node5.query(
+        "CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)")
     node5.query("INSERT INTO db1.test_table VALUES('2000-01-01', 'test_key', 1)")
     assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n"
-    assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n"
-    assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n"
-    assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n"
-    assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n"
+    assert node5.query(
+        "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n"
+    assert node5.query(
+        "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n"
+    assert node5.query(
+        "SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n"
+    assert node5.query(
+        "SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n"
 
     assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == ""
-    assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", user="test_allow").strip() == ""
-    assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == ""
-    assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == ""
-    assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == ""
+    assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'",
+                       user="test_allow").strip() == ""
+    assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'",
+                       user="test_allow").strip() == ""
+    assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'",
+                       user="test_allow").strip() == ""
+    assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'",
+                       user="test_allow").strip() == ""

tests/integration/test_consistant_parts_after_move_partition/test.py

@@ -1,12 +1,8 @@
-import os
-
 import pytest
 
 from helpers.cluster import ClickHouseCluster
-from helpers.network import PartitionManager
 from helpers.test_tools import assert_eq_with_retry
 
-
 CLICKHOUSE_DATABASE = 'test'
 
 
@@ -46,13 +42,14 @@ def test_consistent_part_after_move_partition(start_cluster):
     # insert into all replicas
     for i in range(100):
         node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE,
-                                                                                       value=i))
+                                                                                        value=i))
     query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE)
     query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)
     assert_eq_with_retry(node2, query_source, node1.query(query_source))
     assert_eq_with_retry(node2, query_dest, node1.query(query_dest))
 
-    node1.query('ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE))
+    node1.query(
+        'ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE))
 
     assert_eq_with_retry(node2, query_source, node1.query(query_source))
     assert_eq_with_retry(node2, query_dest, node1.query(query_dest))

tests/integration/test_consistent_parts_after_clone_replica/test.py

@@ -8,19 +8,20 @@ from helpers.test_tools import assert_eq_with_retry
 def fill_nodes(nodes, shard):
     for node in nodes:
         node.query(
-        '''
-        CREATE DATABASE test;
-        CREATE TABLE test_table(date Date, id UInt32)
-        ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
-        ORDER BY id PARTITION BY toYYYYMM(date) 
-        SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
+            '''
+            CREATE DATABASE test;
+            CREATE TABLE test_table(date Date, id UInt32)
+            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
+            ORDER BY id PARTITION BY toYYYYMM(date) 
+            SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
 
 cluster = ClickHouseCluster(__file__)
 node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def start_cluster():
     try:
@@ -56,5 +57,3 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster):
         # the first replica will be cloned from the second
         pm.heal_all()
         assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table"))
-
-

tests/integration/test_cross_replication/test.py

@@ -2,18 +2,17 @@ import time
 from contextlib import contextmanager
 
 import pytest
-
 from helpers.cluster import ClickHouseCluster
 from helpers.network import PartitionManager
 from helpers.test_tools import assert_eq_with_retry
 
-
 cluster = ClickHouseCluster(__file__)
 
 node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:

tests/integration/test_custom_settings/test.py

@@ -2,7 +2,8 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"], user_configs=["configs/users.d/custom_settings.xml"])
+node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"],
+                            user_configs=["configs/users.d/custom_settings.xml"])
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -22,9 +23,10 @@ def test():
     assert node.query("SELECT getSetting('custom_d')") == "some text\n"
 
     assert "custom_a = -5, custom_b = 10000000000, custom_c = -4.325, custom_d = \\'some text\\'" \
-        in node.query("SHOW CREATE SETTINGS PROFILE default")
+           in node.query("SHOW CREATE SETTINGS PROFILE default")
 
-    assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting")
+    assert "no settings profile" in node.query_and_get_error(
+        "SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting")
     assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_illformed_setting")
 
 
@@ -33,9 +35,9 @@ def test_invalid_settings():
     node.query("SYSTEM FLUSH LOGS")
 
     assert node.query("SELECT COUNT() FROM system.text_log WHERE"
-        " message LIKE '%Could not parse profile `profile_illformed_setting`%'"
-        " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n"
+                      " message LIKE '%Could not parse profile `profile_illformed_setting`%'"
+                      " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n"
 
     assert node.query("SELECT COUNT() FROM system.text_log WHERE"
-        " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'"
-        " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n"
+                      " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'"
+                      " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n"

tests/integration/test_ddl_alter_query/test.py

@@ -10,7 +10,6 @@ node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'
 node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
 
 
-
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -18,10 +17,14 @@ def started_cluster():
 
         for i, node in enumerate([node1, node2]):
             node.query("CREATE DATABASE testdb")
-            node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(i))
+            node.query(
+                '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(
+                    i))
         for i, node in enumerate([node3, node4]):
             node.query("CREATE DATABASE testdb")
-            node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(i))
+            node.query(
+                '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(
+                    i))
         yield cluster
 
     finally:
@@ -34,7 +37,8 @@ def test_alter(started_cluster):
     node2.query("SYSTEM SYNC REPLICA testdb.test_table")
     node4.query("SYSTEM SYNC REPLICA testdb.test_table")
 
-    node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", settings={"replication_alter_partitions_sync": "2"})
+    node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val",
+                settings={"replication_alter_partitions_sync": "2"})
 
     node1.query("SYSTEM SYNC REPLICA testdb.test_table")
     node2.query("SYSTEM SYNC REPLICA testdb.test_table")

tests/integration/test_default_compression_codec/test.py

@@ -1,14 +1,17 @@
-import string
 import random
-import pytest
+import string
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
 node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True)
 node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True)
-node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True)
+node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'],
+                             image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True,
+                             with_installed_binary=True)
+
 
 @pytest.fixture(scope="module")
 def start_cluster():
@@ -21,12 +24,14 @@ def start_cluster():
 
 
 def get_compression_codec_byte(node, table_name, part_name):
-    cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name)
+    cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format(
+        table_name, part_name)
     return node.exec_in_container(["bash", "-c", cmd]).strip()
 
 
 def get_second_multiple_codec_byte(node, table_name, part_name):
-    cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name)
+    cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format(
+        table_name, part_name)
     return node.exec_in_container(["bash", "-c", cmd]).strip()
 
 
@@ -74,16 +79,22 @@ def test_default_codec_single(start_cluster):
 
     # Same codec for all
     assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table", "2_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table", "3_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
 
     # just to be sure that replication works
     node1.query("OPTIMIZE TABLE compression_table FINAL")
@@ -101,16 +112,22 @@ def test_default_codec_single(start_cluster):
     node2.query("SYSTEM FLUSH LOGS")
 
     assert get_compression_codec_byte(node1, "compression_table", "1_0_0_1") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
 
     assert get_compression_codec_byte(node1, "compression_table", "3_0_0_1") == CODECS_MAPPING['LZ4']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
 
     assert node1.query("SELECT COUNT() FROM compression_table") == "3\n"
     assert node2.query("SELECT COUNT() FROM compression_table") == "3\n"
@@ -137,18 +154,24 @@ def test_default_codec_multiple(start_cluster):
     # Same codec for all
     assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
 
     node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15)
 
@@ -156,18 +179,24 @@ def test_default_codec_multiple(start_cluster):
 
     assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['ZSTD']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
 
     assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
 
     assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['LZ4']
-    assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
-    assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
+    assert node1.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
+    assert node2.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
 
     assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n"
     assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n"
@@ -187,15 +216,21 @@ def test_default_codec_version_update(start_cluster):
 
     node3.restart_with_latest_version()
 
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n"
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n"
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n"
 
     node3.query("OPTIMIZE TABLE compression_table FINAL")
 
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n"
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n"
-    assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n"
+    assert node3.query(
+        "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n"
     assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple']
     assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
     assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple']

tests/integration/test_default_database_on_cluster/test.py

@@ -1,12 +1,20 @@
-import time
 import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True)
-ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True)
-ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True)
-ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True)
+ch1 = cluster.add_instance('ch1',
+                           main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
+                           with_zookeeper=True)
+ch2 = cluster.add_instance('ch2',
+                           main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
+                           with_zookeeper=True)
+ch3 = cluster.add_instance('ch3',
+                           main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
+                           with_zookeeper=True)
+ch4 = cluster.add_instance('ch4',
+                           main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
+                           with_zookeeper=True)
+
 
 @pytest.fixture(scope="module")
 def started_cluster():
@@ -20,13 +28,17 @@ def started_cluster():
 
 
 def test_default_database_on_cluster(started_cluster):
-    ch1.query(database='test_default_database', sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;")
+    ch1.query(database='test_default_database',
+              sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;")
 
     for node in [ch1, ch2, ch3, ch4]:
         assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_local_table\n"
 
-    ch1.query(database='test_default_database', sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');")
+    ch1.query(database='test_default_database',
+              sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');")
 
     for node in [ch1, ch2, ch3, ch4]:
-        assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n"
-        assert node.query("SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n    `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n"
+        assert node.query(
+            "SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n"
+        assert node.query(
+            "SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n    `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n"

tests/integration/test_default_role/test.py

@@ -1,7 +1,6 @@
 import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
-import re
 
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance')
@@ -11,7 +10,7 @@ instance = cluster.add_instance('instance')
 def started_cluster():
     try:
         cluster.start()
-        
+
         instance.query("CREATE USER john")
         instance.query("CREATE ROLE rx")
         instance.query("CREATE ROLE ry")
@@ -32,41 +31,41 @@ def test_set_default_roles():
     assert instance.query("SHOW CURRENT ROLES", user="john") == ""
 
     instance.query("GRANT rx, ry TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
 
     instance.query("SET DEFAULT ROLE NONE TO john")
     assert instance.query("SHOW CURRENT ROLES", user="john") == ""
 
     instance.query("SET DEFAULT ROLE rx TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]])
 
     instance.query("SET DEFAULT ROLE ry TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
 
     instance.query("SET DEFAULT ROLE ALL TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
 
     instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
 
 
 def test_alter_user():
     assert instance.query("SHOW CURRENT ROLES", user="john") == ""
 
     instance.query("GRANT rx, ry TO john")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
 
     instance.query("ALTER USER john DEFAULT ROLE NONE")
     assert instance.query("SHOW CURRENT ROLES", user="john") == ""
 
     instance.query("ALTER USER john DEFAULT ROLE rx")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]])
 
     instance.query("ALTER USER john DEFAULT ROLE ALL")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
 
     instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx")
-    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] )
+    assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
 
 
 def test_wrong_set_default_role():

tests/integration/test_delayed_replica_failover/test.py

@@ -1,14 +1,14 @@
-import pytest
+import os
+import sys
 import time
-import os, sys
+
+import pytest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import helpers
 
 from helpers.cluster import ClickHouseCluster
 from helpers.network import PartitionManager
 
-
 cluster = ClickHouseCluster(__file__)
 
 # Cluster with 2 shards of 2 replicas each. node_1_1 is the instance with Distributed table.
@@ -19,6 +19,7 @@ node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True)
 node_2_1 = cluster.add_instance('node_2_1', with_zookeeper=True)
 node_2_2 = cluster.add_instance('node_2_2', with_zookeeper=True)
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -30,7 +31,7 @@ def started_cluster():
                 node.query('''
 CREATE TABLE replicated (d Date, x UInt32) ENGINE =
     ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)'''
-                    .format(shard=shard, instance=node.name))
+                           .format(shard=shard, instance=node.name))
 
         node_1_1.query(
             "CREATE TABLE distributed (d Date, x UInt32) ENGINE = "
@@ -51,7 +52,7 @@ def test(started_cluster):
         node_1_2.query("INSERT INTO replicated VALUES ('2017-05-08', 1)")
         node_2_2.query("INSERT INTO replicated VALUES ('2017-05-08', 2)")
 
-        time.sleep(1) # accrue replica delay
+        time.sleep(1)  # accrue replica delay
 
         assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == '0'
         assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == '1'
@@ -78,7 +79,7 @@ SELECT sum(x) FROM distributed WITH TOTALS SETTINGS
         pm.drop_instance_zk_connections(node_1_2)
         pm.drop_instance_zk_connections(node_2_2)
 
-        time.sleep(4) # allow pings to zookeeper to timeout (must be greater than ZK session timeout).
+        time.sleep(4)  # allow pings to zookeeper to timeout (must be greater than ZK session timeout).
 
         # At this point all replicas are stale, but the query must still go to second replicas which are the least stale ones.
         assert instance_with_dist_table.query('''

tests/integration/test_dictionaries_access/test.py

@@ -9,7 +9,7 @@ instance = cluster.add_instance('instance')
 def started_cluster():
     try:
         cluster.start()
-        
+
         instance.query("CREATE USER mira")
         instance.query("CREATE TABLE test_table(x Int32, y Int32) ENGINE=Log")
         instance.query("INSERT INTO test_table VALUES (5,6)")

tests/integration/test_dictionaries_all_layouts_and_sources/test.py

@@ -1,11 +1,12 @@
-import pytest
+import math
 import os
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
-from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed
-from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra
-import math
+from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceCassandra
+from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, \
+    SourceExecutableHashed
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries')
@@ -103,8 +104,6 @@ VALUES = {
     ]
 }
 
-
-
 LAYOUTS = [
     Layout("flat"),
     Layout("hashed"),
@@ -135,6 +134,7 @@ DICTIONARIES = []
 cluster = None
 node = None
 
+
 def get_dict(source, layout, fields, suffix_name=''):
     global dict_configs_path
 
@@ -173,7 +173,8 @@ def setup_module(module):
     for fname in os.listdir(dict_configs_path):
         dictionaries.append(os.path.join(dict_configs_path, fname))
 
-    node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True, with_mongo=True, with_redis=True, with_cassandra=True)
+    node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True,
+                                with_mongo=True, with_redis=True, with_cassandra=True)
 
 
 @pytest.fixture(scope="module")
@@ -195,7 +196,7 @@ def get_dictionaries(fold, total_folds, all_dicts):
     chunk_len = int(math.ceil(len(all_dicts) / float(total_folds)))
     if chunk_len * fold >= len(all_dicts):
         return []
-    return all_dicts[fold * chunk_len : (fold + 1) * chunk_len]
+    return all_dicts[fold * chunk_len: (fold + 1) * chunk_len]
 
 
 def remove_mysql_dicts():
@@ -225,8 +226,8 @@ def remove_mysql_dicts():
     TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed
     """
 
-    #global DICTIONARIES
-    #DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")]
+    # global DICTIONARIES
+    # DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")]
 
 
 @pytest.mark.parametrize("fold", list(range(10)))
@@ -281,7 +282,6 @@ def test_simple_dictionaries(started_cluster, fold):
 
 @pytest.mark.parametrize("fold", list(range(10)))
 def test_complex_dictionaries(started_cluster, fold):
-
     if node.is_built_with_thread_sanitizer():
         remove_mysql_dicts()
 

tests/integration/test_dictionaries_complex_key_cache_string/test.py

@@ -1,30 +1,40 @@
-import pytest
 import os
+
+import pytest
 from helpers.cluster import ClickHouseCluster
 
+
 @pytest.fixture(scope="function")
 def cluster(request):
     SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
     cluster = ClickHouseCluster(__file__)
     try:
         if request.param == "memory":
-            node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/complex_key_cache_string.xml'])
+            node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml',
+                                                              'configs/dictionaries/complex_key_cache_string.xml'])
         if request.param == "ssd":
-            node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/ssd_complex_key_cache_string.xml'])
+            node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml',
+                                                              'configs/dictionaries/ssd_complex_key_cache_string.xml'])
         cluster.start()
-        node.query("create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id")
+        node.query(
+            "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id")
 
         yield cluster
     finally:
         cluster.shutdown()
 
+
 @pytest.mark.parametrize("cluster", ["memory", "ssd"], indirect=True)
 def test_memory_consumption(cluster):
     node = cluster.instances['node']
-    node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8))
-    node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16))
-    node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32))
-    node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64))
+    node.query(
+        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8))
+    node.query(
+        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16))
+    node.query(
+        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32))
+    node.query(
+        "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64))
 
     # Fill dictionary
     node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)")

tests/integration/test_dictionaries_ddl/test.py

@@ -1,17 +1,24 @@
-import pytest
 import os
-from helpers.cluster import ClickHouseCluster
-from helpers.client import QueryRuntimeException
-import pymysql
 import warnings
 import time
 
+import pymysql
+import pytest
+from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
+
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
-node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
-node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'], dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml', 'configs/dictionaries/conflict_name_dictionary.xml'], user_configs=['configs/user_admin.xml'])
+node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'],
+                             user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
+node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'],
+                             main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'],
+                             user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
+node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'],
+                             dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml',
+                                           'configs/dictionaries/conflict_name_dictionary.xml'],
+                             user_configs=['configs/user_admin.xml'])
 node4 = cluster.add_instance('node4', user_configs=['configs/user_admin.xml', 'configs/config_password.xml'])
 
 
@@ -22,6 +29,7 @@ def create_mysql_conn(user, password, hostname, port):
         host=hostname,
         port=port)
 
+
 def execute_mysql_query(connection, query):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -36,15 +44,18 @@ def started_cluster():
         cluster.start()
         for clickhouse in [node1, node2, node3, node4]:
             clickhouse.query("CREATE DATABASE test", user="admin")
-            clickhouse.query("CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", user="admin")
-            clickhouse.query("INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", user="admin")
+            clickhouse.query(
+                "CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id",
+                user="admin")
+            clickhouse.query(
+                "INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)",
+                user="admin")
         yield cluster
 
     finally:
         cluster.shutdown()
 
 
-
 @pytest.mark.parametrize("clickhouse,name,layout", [
     (node1, 'complex_node1_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())'),
     (node1, 'complex_node1_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))'),
@@ -54,7 +65,9 @@ def started_cluster():
 def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
     mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308)
     execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse")
-    execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format(name))
+    execute_mysql_query(mysql_conn,
+                        "CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format(
+                            name))
     values = []
     for i in range(1000):
         values.append('(' + ','.join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ')')
@@ -81,12 +94,16 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
     """.format(name, name, layout))
 
     for i in range(172, 200):
-        assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 5 + '\n'
-        stroka = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip()
+        assert clickhouse.query(
+            "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i,
+                                                                                                   i * i)) == str(
+            i) * 5 + '\n'
+        stroka = clickhouse.query(
+            "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i,
+                                                                                                    i * i)).strip()
         value = float(stroka)
         assert int(value) == int(i * 3.14)
 
-
     for i in range(1000):
         values.append('(' + ','.join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ')')
     execute_mysql_query(mysql_conn, "REPLACE INTO clickhouse.{} VALUES ".format(name) + ','.join(values))
@@ -94,8 +111,13 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
     clickhouse.query("SYSTEM RELOAD DICTIONARY 'default.{}'".format(name))
 
     for i in range(172, 200):
-        assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 3 + '\n'
-        string = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip()
+        assert clickhouse.query(
+            "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i,
+                                                                                                   i * i)) == str(
+            i) * 3 + '\n'
+        string = clickhouse.query(
+            "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i,
+                                                                                                    i * i)).strip()
         value = float(string)
         assert int(value) == int(i * 2.718)
 
@@ -183,6 +205,7 @@ def test_conflicting_name(started_cluster):
     # old version still works
     node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n'
 
+
 def test_http_dictionary_restrictions(started_cluster):
     try:
         node3.query("""
@@ -199,6 +222,7 @@ def test_http_dictionary_restrictions(started_cluster):
     except QueryRuntimeException as ex:
         assert 'is not allowed in config.xml' in str(ex)
 
+
 def test_file_dictionary_restrictions(started_cluster):
     try:
         node3.query("""
@@ -219,7 +243,8 @@ def test_file_dictionary_restrictions(started_cluster):
 def test_dictionary_with_where(started_cluster):
     mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308)
     execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse")
-    execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))")
+    execute_mysql_query(mysql_conn,
+                        "CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))")
     execute_mysql_query(mysql_conn, "INSERT INTO clickhouse.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')")
 
     node1.query("""

tests/integration/test_dictionaries_dependency/test.py

@@ -17,10 +17,10 @@ def start_cluster():
             node.query("CREATE DATABASE IF NOT EXISTS ztest")
             node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log")
             node.query("INSERT INTO test.source VALUES (5,6)")
-            
-            node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x "\
-                        "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) "\
-                        "LAYOUT(FLAT()) LIFETIME(0)")
+
+            node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x " \
+                       "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \
+                       "LAYOUT(FLAT()) LIFETIME(0)")
         yield cluster
 
     finally:
@@ -48,14 +48,14 @@ def cleanup_after_test():
 def test_dependency_via_implicit_table(node):
     d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"]
     for d_name in d_names:
-        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\
-                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) "\
+        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
+                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) " \
                    "LAYOUT(FLAT()) LIFETIME(0)".format(d_name))
-    
+
     def check():
         for d_name in d_names:
             assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n"
-    
+
     check()
 
     # Restart must not break anything.
@@ -72,14 +72,14 @@ def test_dependency_via_explicit_table(node):
         tbl_database, tbl_shortname = tbl_name.split('.')
         d_name = d_names[i]
         node.query("CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format(tbl_name))
-        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\
-                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) "\
+        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
+                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) " \
                    "LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database))
-    
+
     def check():
         for d_name in d_names:
             assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n"
-    
+
     check()
 
     # Restart must not break anything.
@@ -93,14 +93,14 @@ def test_dependency_via_dictionary_database(node):
 
     d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"]
     for d_name in d_names:
-        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\
-                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) "\
+        node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
+                   "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) " \
                    "LAYOUT(FLAT()) LIFETIME(0)".format(d_name))
-    
+
     def check():
         for d_name in d_names:
             assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n"
-    
+
     check()
 
     # Restart must not break anything.

tests/integration/test_dictionaries_dependency_xml/test.py

@@ -1,13 +1,13 @@
 import pytest
-import os
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
 
 ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
-DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', 'configs/dictionaries/dep_z.xml']
+DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml',
+                    'configs/dictionaries/dep_z.xml']
 
 cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES,)
+instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES, )
 
 
 @pytest.fixture(scope="module")
@@ -60,14 +60,14 @@ def test_get_data(started_cluster):
     query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)")
 
     # Wait for dictionaries to be reloaded.
-    assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10)
+    assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time=2, retry_count=10)
     assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n"
     assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n"
     assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n"
 
     # dep_x and dep_z are updated only when there `intDiv(count(), 4)`  is changed.
     query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)")
-    assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10)
+    assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time=2, retry_count=10)
     assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n"
     assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n"
     assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n"

tests/integration/test_dictionaries_mysql/test.py

@@ -1,17 +1,13 @@
-import pytest
-import os
-import time
-
 ## sudo -H pip install PyMySQL
 import pymysql.cursors
-
+import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import assert_eq_with_retry
 
-CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml', 'configs/remote_servers.xml']
+CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml',
+                'configs/remote_servers.xml']
 CONFIG_FILES += ['configs/enable_dictionaries.xml']
 cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql = True)
+instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql=True)
 
 create_table_mysql_template = """
     CREATE TABLE IF NOT EXISTS `test`.`{}` (
@@ -25,10 +21,11 @@ create_clickhouse_dictionary_table_template = """
     CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` UInt64, `value` String) ENGINE = Dictionary({})
     """
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
-        #time.sleep(30)
+        # time.sleep(30)
         cluster.start()
 
         # Create a MySQL database
@@ -66,10 +63,12 @@ def test_load_mysql_dictionaries(started_cluster):
         # Check number of row
         assert query("SELECT count() FROM `test`.`dict_table_{}`".format('test' + str(n % 5))).rstrip() == '10000'
 
+
 def create_mysql_db(mysql_connection, name):
     with mysql_connection.cursor() as cursor:
         cursor.execute("CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET 'utf8'".format(name))
 
+
 def prepare_mysql_table(table_name, index):
     mysql_connection = get_mysql_conn()
 
@@ -78,17 +77,21 @@ def prepare_mysql_table(table_name, index):
 
     # Insert rows using CH
     query = instance.query
-    query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name + str(index), table_name + str(index)))
+    query(
+        "INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(
+            table_name + str(index), table_name + str(index)))
     assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000'
     mysql_connection.close()
 
-    #Create CH Dictionary tables based on MySQL tables
+    # Create CH Dictionary tables based on MySQL tables
     query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index)))
 
+
 def get_mysql_conn():
     conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.10', port=3308)
     return conn
 
+
 def create_mysql_table(conn, table_name):
     with conn.cursor() as cursor:
         cursor.execute(create_table_mysql_template.format(table_name))

tests/integration/test_dictionaries_null_value/test.py

@@ -1,13 +1,11 @@
 import pytest
-import os
 from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import TSV, assert_eq_with_retry
 
 ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
 DICTIONARY_FILES = ['configs/dictionaries/cache.xml']
 
 cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES)
+instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
 
 
 @pytest.fixture(scope="module")
@@ -42,4 +40,5 @@ def test_null_value(started_cluster):
 
     # Check, that empty null_value interprets as default value
     assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n"
-    assert query("select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n"
+    assert query(
+        "select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n"

tests/integration/test_dictionaries_redis/test.py

@@ -1,7 +1,6 @@
 import os
-import pytest
-import redis
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
 from helpers.external_sources import SourceRedis
@@ -22,10 +21,10 @@ KEY_FIELDS = {
 }
 
 KEY_VALUES = {
-    "simple" : [
+    "simple": [
         [1], [2]
     ],
-    "complex" : [
+    "complex": [
         [1, 'world'], [2, 'qwerty2']
     ]
 }
@@ -76,6 +75,7 @@ LAYOUTS = [
 
 DICTIONARIES = []
 
+
 def get_dict(source, layout, fields, suffix_name=''):
     global dict_configs_path
 
@@ -99,8 +99,10 @@ def setup_module(module):
     for i, field in enumerate(FIELDS):
         DICTIONARIES.append([])
         sources = []
-        sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2, storage_type="simple"))
-        sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1, storage_type="hash_map"))
+        sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2,
+                                   storage_type="simple"))
+        sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1,
+                                   storage_type="hash_map"))
         for source in sources:
             for layout in LAYOUTS:
                 if not source.compatible_with_layout(layout):
@@ -118,6 +120,7 @@ def setup_module(module):
     cluster = ClickHouseCluster(__file__)
     node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_redis=True)
 
+
 @pytest.fixture(scope="module", autouse=True)
 def started_cluster():
     try:
@@ -134,6 +137,7 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 @pytest.mark.parametrize("id", range(len(FIELDS)))
 def test_redis_dictionaries(started_cluster, id):
     print 'id:', id

tests/integration/test_dictionaries_select_all/generate_dictionaries.py

@@ -1,6 +1,5 @@
-import os
-import glob
 import difflib
+import os
 
 files = ['key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv']
 
@@ -12,7 +11,6 @@ types = [
     'Date', 'DateTime'
 ]
 
-
 implicit_defaults = [
     '1', '1', '1', '',
     '-1', '-1', '-1', '-1',

tests/integration/test_dictionaries_select_all/test.py

@@ -1,7 +1,9 @@
-import pytest
 import os
+
+import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import TSV, assert_eq_with_retry
+from helpers.test_tools import TSV
+
 from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -20,7 +22,7 @@ def setup_module(module):
     dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure)
 
     cluster = ClickHouseCluster(__file__)
-    instance = cluster.add_instance('instance', main_configs=dictionary_files+['configs/enable_dictionaries.xml'])
+    instance = cluster.add_instance('instance', main_configs=dictionary_files + ['configs/enable_dictionaries.xml'])
     test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv'))
 
 

tests/integration/test_dictionaries_update_and_reload/test.py

@@ -1,16 +1,18 @@
-import pytest
 import os
 import time
-from helpers.cluster import ClickHouseCluster
+
+import pytest
 from helpers.client import QueryTimeoutExceedException
+from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
-DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml']
+DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml',
+                    'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml']
 
 cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES)
+instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
 
 
 @pytest.fixture(scope="module")
@@ -30,24 +32,31 @@ def get_status(dictionary_name):
 
 
 def get_last_exception(dictionary_name):
-    return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'")
+    return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
+        "\n").replace("\\'", "'")
 
 
 def get_loading_start_time(dictionary_name):
-    s = instance.query("SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n")
+    s = instance.query(
+        "SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
+        "\n")
     if s == "1970-01-01 00:00:00":
         return None
     return time.strptime(s, "%Y-%m-%d %H:%M:%S")
 
+
 def get_last_successful_update_time(dictionary_name):
-    s = instance.query("SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n")
+    s = instance.query(
+        "SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
+        "\n")
     if s == "1970-01-01 00:00:00":
         return None
     return time.strptime(s, "%Y-%m-%d %H:%M:%S")
 
 
 def get_loading_duration(dictionary_name):
-    return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'"))
+    return float(
+        instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'"))
 
 
 def replace_in_file_in_container(file_name, what, replace_with):
@@ -63,14 +72,14 @@ def test_reload_while_loading(started_cluster):
 
     # It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout.
     with pytest.raises(QueryTimeoutExceedException):
-        query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 0.5)
+        query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=0.5)
 
     # The dictionary is now loading.
     assert get_status('slow') == "LOADING"
     start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
     assert duration > 0
 
-    time.sleep(0.5) # Still loading.
+    time.sleep(0.5)  # Still loading.
     assert get_status('slow') == "LOADING"
     prev_start_time, prev_duration = start_time, duration
     start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
@@ -79,14 +88,14 @@ def test_reload_while_loading(started_cluster):
 
     # SYSTEM RELOAD DICTIONARY should restart loading.
     with pytest.raises(QueryTimeoutExceedException):
-        query("SYSTEM RELOAD DICTIONARY 'slow'", timeout = 0.5)
+        query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=0.5)
     assert get_status('slow') == "LOADING"
     prev_start_time, prev_duration = start_time, duration
     start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
     assert start_time > prev_start_time
     assert duration < prev_duration
 
-    time.sleep(0.5) # Still loading.
+    time.sleep(0.5)  # Still loading.
     assert get_status('slow') == "LOADING"
     prev_start_time, prev_duration = start_time, duration
     start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
@@ -95,7 +104,7 @@ def test_reload_while_loading(started_cluster):
 
     # Changing the configuration file should restart loading again.
     replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0')
-    time.sleep(5) # Configuration files are reloaded once in 5 seconds.
+    time.sleep(5)  # Configuration files are reloaded once in 5 seconds.
 
     # This time loading should finish quickly.
     assert get_status('slow') == "LOADED"
@@ -129,7 +138,7 @@ def test_reload_after_loading(started_cluster):
     assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n"
 
     # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries.
-    time.sleep(1) # see the comment above
+    time.sleep(1)  # see the comment above
     replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '81', '82')
     replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '101', '102')
     query("SYSTEM RELOAD DICTIONARIES")
@@ -138,7 +147,7 @@ def test_reload_after_loading(started_cluster):
 
     # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds.
     # Wait slightly more, to be sure it did reload.
-    time.sleep(1) # see the comment above
+    time.sleep(1)  # see the comment above
     replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '82', '83')
     replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '102', '103')
     time.sleep(7)
@@ -163,7 +172,8 @@ def test_reload_after_fail_by_system_reload(started_cluster):
     assert get_status("no_file") == "FAILED"
 
     # Creating the file source makes the dictionary able to load.
-    instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file.txt")
+    instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"),
+                                    "/etc/clickhouse-server/config.d/no_file.txt")
     query("SYSTEM RELOAD DICTIONARY 'no_file'")
     query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n"
     assert get_status("no_file") == "LOADED"
@@ -192,7 +202,8 @@ def test_reload_after_fail_by_timer(started_cluster):
     assert get_status("no_file_2") == "FAILED"
 
     # Creating the file source makes the dictionary able to load.
-    instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file_2.txt")
+    instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"),
+                                    "/etc/clickhouse-server/config.d/no_file_2.txt")
     time.sleep(6);
     query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n"
     assert get_status("no_file_2") == "LOADED"

tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py

@@ -1,9 +1,8 @@
 from __future__ import print_function
-import pytest
-import time
-import os
-from contextlib import contextmanager
 
+import time
+
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.cluster import ClickHouseKiller
 from helpers.network import PartitionManager
@@ -14,6 +13,7 @@ dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
 main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
                                                             'configs/dictionaries/cache_ints_dictionary.xml'])
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -32,6 +32,7 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 # @pytest.mark.skip(reason="debugging")
 def test_default_reading(started_cluster):
     assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"
@@ -39,14 +40,22 @@ def test_default_reading(started_cluster):
     # Key 0 is not in dictionary, so default value will be returned
 
     def test_helper():
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i8',  toUInt64(13),  toInt8(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(13),  toInt16(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(13),  toInt32(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(13),  toInt64(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u8',  toUInt64(13),  toUInt8(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(13),  toUInt16(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(13),  toUInt32(42));").rstrip()
-        assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(13),  toUInt64(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i8',  toUInt64(13),  toInt8(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i16', toUInt64(13),  toInt16(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i32', toUInt64(13),  toInt32(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i64', toUInt64(13),  toInt64(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u8',  toUInt64(13),  toUInt8(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u16', toUInt64(13),  toUInt16(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u32', toUInt64(13),  toUInt32(42));").rstrip()
+        assert '42' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u64', toUInt64(13),  toUInt64(42));").rstrip()
 
     test_helper()
 
@@ -61,4 +70,3 @@ def test_default_reading(started_cluster):
         time.sleep(3)
 
         test_helper()
-

tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py

@@ -1,10 +1,11 @@
 from __future__ import print_function
-import pytest
+
 import os
 import random
 import string
 import time
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
@@ -12,13 +13,16 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 cluster = ClickHouseCluster(__file__)
 
 dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
-main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml','configs/dictionaries/cache_strings_default_settings.xml'])
+main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
+                                                            'configs/dictionaries/cache_ints_dictionary.xml',
+                                                            'configs/dictionaries/cache_strings_default_settings.xml'])
 
 
 def get_random_string(string_length=8):
     alphabet = string.ascii_letters + string.digits
     return ''.join((random.choice(alphabet) for _ in range(string_length)))
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -31,13 +35,15 @@ def started_cluster():
                              ENGINE = Memory;
                              """)
 
-        values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)])
+        values_to_insert = ", ".join(
+            ["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)])
         dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert))
 
         yield cluster
     finally:
         cluster.shutdown()
 
+
 # @pytest.mark.skip(reason="debugging")
 def test_return_real_values(started_cluster):
     assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"

tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py

@@ -1,18 +1,18 @@
 from __future__ import print_function
-import pytest
-import time
-import os
-from contextlib import contextmanager
 
+import time
+
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.cluster import ClickHouseKiller
 from helpers.network import PartitionManager
-from helpers.network import PartitionManagerDisabler
 
 cluster = ClickHouseCluster(__file__)
 
 dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
-main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/cache_ints_dictionary.xml'])
+main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
+                                                            'configs/dictionaries/cache_ints_dictionary.xml'])
+
 
 @pytest.fixture(scope="module")
 def started_cluster():

tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py

@@ -1,9 +1,8 @@
 from __future__ import print_function
-import pytest
-import time
-import os
-from contextlib import contextmanager
 
+import time
+
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.cluster import ClickHouseKiller
 from helpers.network import PartitionManager
@@ -11,7 +10,9 @@ from helpers.network import PartitionManager
 cluster = ClickHouseCluster(__file__)
 
 dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
-main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml'])
+main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
+                                                            'configs/dictionaries/cache_ints_dictionary.xml'])
+
 
 @pytest.fixture(scope="module")
 def started_cluster():
@@ -31,19 +32,28 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 # @pytest.mark.skip(reason="debugging")
 def test_simple_dict_get_or_default(started_cluster):
     assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"
 
     def test_helper():
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i8',  toUInt64(5),  toInt8(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(5),  toInt16(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(5),  toInt32(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(5),  toInt64(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u8',  toUInt64(5),  toUInt8(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(5),  toUInt16(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(5),  toUInt32(42));").rstrip()
-        assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(5),  toUInt64(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i8',  toUInt64(5),  toInt8(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i16', toUInt64(5),  toInt16(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i32', toUInt64(5),  toInt32(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'i64', toUInt64(5),  toInt64(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u8',  toUInt64(5),  toUInt8(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u16', toUInt64(5),  toUInt16(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u32', toUInt64(5),  toUInt32(42));").rstrip()
+        assert '5' == main_node.query(
+            "select dictGetOrDefault('anime_dict', 'u64', toUInt64(5),  toUInt64(42));").rstrip()
 
     test_helper()
 

tests/integration/test_dictionary_custom_settings/http_server.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import argparse
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import csv
 import socket
 import ssl
-import csv
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 
 
 # Decorator used to see if authentication works for external dictionary who use a HTTP source.
@@ -15,6 +15,7 @@ def check_auth(fn):
             req.send_response(401)
         else:
             fn(req)
+
     return wrapper
 
 
@@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family):
             self.send_header('Content-type', 'text/csv')
             self.end_headers()
 
-        def __send_data(self, only_ids = None):
+        def __send_data(self, only_ids=None):
             with open(data_path, 'r') as fl:
                 reader = csv.reader(fl, delimiter='\t')
                 for row in reader:

tests/integration/test_dictionary_custom_settings/test.py

@@ -1,6 +1,6 @@
 import os
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 
 ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
@@ -13,7 +13,8 @@ DICTIONARY_FILES = [
 ]
 
 cluster = ClickHouseCluster(__file__)
-instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES)
+instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
+
 
 def prepare():
     node = instance

tests/integration/test_dictionary_ddl_on_cluster/test.py

@@ -1,19 +1,24 @@
-import time
 import pytest
-from helpers.cluster import ClickHouseCluster
 from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True)
-ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True)
-ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True)
-ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True)
+ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
+                           with_zookeeper=True)
+ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
+                           with_zookeeper=True)
+ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
+                           with_zookeeper=True)
+ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
+                           with_zookeeper=True)
+
 
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
         cluster.start()
-        ch1.query("CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key")
+        ch1.query(
+            "CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key")
         yield cluster
 
     finally:
@@ -27,7 +32,6 @@ def test_dictionary_ddl_on_cluster(started_cluster):
     for num, node in enumerate([ch1, ch2, ch3, ch4]):
         node.query("insert into sometbl values ({}, '{}')".format(num, node.name))
 
-
     ch1.query(
         """
         CREATE DICTIONARY somedict ON CLUSTER 'cluster' (
@@ -42,7 +46,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
 
     for num, node in enumerate([ch1, ch2, ch3, ch4]):
         assert node.query("SELECT count() from sometbl") == "1\n"
-        assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
+        assert node.query(
+            "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
 
     ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'")
 
@@ -54,8 +59,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
 
     for num, node in enumerate([ch1, ch2, ch3, ch4]):
         assert node.query("SELECT count() from sometbl") == "1\n"
-        assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
-
+        assert node.query(
+            "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
 
     for num, node in enumerate([ch1, ch2, ch3, ch4]):
         node.query("ALTER TABLE sometbl UPDATE value = 'new_key' WHERE 1")
@@ -63,7 +68,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
     ch1.query("SYSTEM RELOAD DICTIONARY ON CLUSTER 'cluster' `default.somedict`")
 
     for num, node in enumerate([ch1, ch2, ch3, ch4]):
-        assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n'
+        assert node.query(
+            "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n'
 
     ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'")
 

tests/integration/test_disk_access_storage/test.py

@@ -27,27 +27,32 @@ def create_entities():
 
 @pytest.fixture(autouse=True)
 def drop_entities():
-    instance.query("DROP USER IF EXISTS u1, u2")    
-    instance.query("DROP ROLE IF EXISTS rx, ry")    
+    instance.query("DROP USER IF EXISTS u1, u2")
+    instance.query("DROP ROLE IF EXISTS rx, ry")
     instance.query("DROP ROW POLICY IF EXISTS p ON mydb.mytable")
     instance.query("DROP QUOTA IF EXISTS q")
     instance.query("DROP SETTINGS PROFILE IF EXISTS s1, s2")
-    
+
 
 def test_create():
     create_entities()
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
-        assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
-        assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n"
+        assert instance.query(
+            "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
+        assert instance.query(
+            "SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
+        assert instance.query(
+            "SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n"
         assert instance.query("SHOW GRANTS FOR u1") == ""
         assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n"
         assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n"
         assert instance.query("SHOW GRANTS FOR rx") == ""
-        assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n"
-        assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
+        assert instance.query(
+            "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n"
+        assert instance.query(
+            "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
 
     check()
     instance.restart_clickhouse()  # Check persistency
@@ -69,15 +74,18 @@ def test_alter():
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
+        assert instance.query(
+            "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
         assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n"
         assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n"
         assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n"
         assert instance.query("SHOW CREATE ROLE ry") == "CREATE ROLE ry\n"
         assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n"
         assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n"
-        assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n"
-        assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
+        assert instance.query(
+            "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n"
+        assert instance.query(
+            "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
 
     check()
     instance.restart_clickhouse()  # Check persistency
@@ -98,7 +106,8 @@ def test_drop():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n"
         assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n"
         assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2")
-        assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable")
+        assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error(
+            "SHOW CREATE ROW POLICY p ON mydb.mytable")
         assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q")
 
     check()

tests/integration/test_disk_types/test.py

@@ -1,15 +1,14 @@
-
 import pytest
 from helpers.cluster import ClickHouseCluster
 
 disk_types = {
-    "default" : "local",
-    "disk_s3" : "s3",
-    "disk_memory" : "memory",
+    "default": "local",
+    "disk_s3": "s3",
+    "disk_memory": "memory",
 }
 
-@pytest.fixture(scope="module")
 
+@pytest.fixture(scope="module")
 def cluster():
     try:
         cluster = ClickHouseCluster(__file__)
@@ -19,19 +18,20 @@ def cluster():
     finally:
         cluster.shutdown()
 
+
 def test_different_types(cluster):
     node = cluster.instances["node"]
     responce = node.query("SELECT * FROM system.disks")
     disks = responce.split("\n")
     for disk in disks:
-        if disk == '': # skip empty line (after split at last position)
+        if disk == '':  # skip empty line (after split at last position)
             continue
         fields = disk.split("\t")
         assert len(fields) >= 6
         assert disk_types.get(fields[0], "UNKNOWN") == fields[5]
 
+
 def test_select_by_type(cluster):
     node = cluster.instances["node"]
     for name, disk_type in disk_types.items():
         assert node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") == name + "\n"
-

tests/integration/test_distributed_backward_compatability/test.py

@@ -1,16 +1,14 @@
 import pytest
-import time
 
 from helpers.cluster import ClickHouseCluster
-from helpers.network import PartitionManager
-from helpers.test_tools import TSV
-
 
 cluster = ClickHouseCluster(__file__)
 
-node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True)
+node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server',
+                                tag='19.17.8.54', stay_alive=True, with_installed_binary=True)
 node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'])
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -22,14 +20,17 @@ def started_cluster():
         node_old.query("INSERT INTO local_table VALUES (1, 'node1')")
         node_new.query("INSERT INTO local_table VALUES (2, 'node2')")
 
-        node_old.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
-        node_new.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
+        node_old.query(
+            "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
+        node_new.query(
+            "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
 
         yield cluster
 
     finally:
         cluster.shutdown()
 
+
 def test_distributed_in_tuple(started_cluster):
     query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))"
     query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed"

tests/integration/test_distributed_ddl/cluster.py

@@ -17,18 +17,21 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
 
     def prepare(self, replace_hostnames_with_ips=True):
         try:
-            main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml","ddl.xml"]
+            main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml",
+                                  "ddl.xml"]
             main_configs = [os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files]
-            user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in ["restricted_user.xml", "query_log.xml"]]
+            user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in
+                            ["restricted_user.xml", "query_log.xml"]]
             if self.test_config_dir == "configs_secure":
-                main_configs += [os.path.join(self.test_config_dir, f) for f in ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]]
+                main_configs += [os.path.join(self.test_config_dir, f) for f in
+                                 ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]]
 
             for i in xrange(4):
                 self.add_instance(
-                    'ch{}'.format(i+1),
+                    'ch{}'.format(i + 1),
                     main_configs=main_configs,
                     user_configs=user_configs,
-                    macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1},
+                    macros={"layer": 0, "shard": i / 2 + 1, "replica": i % 2 + 1},
                     with_zookeeper=True)
 
             self.start()
@@ -40,8 +43,12 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
             # Select sacrifice instance to test CONNECTION_LOSS and server fail on it
             sacrifice = self.instances['ch4']
             self.pm_random_drops = PartitionManager()
-            self.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'})
-            self.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'})
+            self.pm_random_drops._add_rule(
+                {'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181,
+                 'action': 'REJECT --reject-with tcp-reset'})
+            self.pm_random_drops._add_rule(
+                {'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181,
+                 'action': 'REJECT --reject-with tcp-reset'})
 
             # Initialize databases and service tables
             instance = self.instances['ch1']
@@ -67,7 +74,7 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
             num_hosts = len(self.instances)
 
         M = TSV.toMat(tsv_content)
-        hosts = [(l[0], l[1]) for l in M] # (host, port)
+        hosts = [(l[0], l[1]) for l in M]  # (host, port)
         codes = [l[2] for l in M]
         messages = [l[3] for l in M]
 
@@ -88,14 +95,17 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
 
         for inst_name in instances_to_replace:
             inst = self.instances[inst_name]
-            self.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True)
+            self.instances[inst_name].exec_in_container(
+                ['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'],
+                environment={"NEW_CONFIG": clusters_config}, privileged=True)
             # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"])
 
     @staticmethod
     def ddl_check_there_are_no_dublicates(instance):
         query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)"
         rows = instance.query(query)
-        assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address, query)
+        assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name,
+                                                                                           instance.ip_address, query)
 
     @staticmethod
     def insert_reliable(instance, query_insert):

tests/integration/test_distributed_ddl/test.py

@@ -41,7 +41,8 @@ def test_default_database(test_cluster):
 
     test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV")
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null")
 
     contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h")
     assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n")
@@ -52,13 +53,18 @@ def test_default_database(test_cluster):
 
 def test_create_view(test_cluster):
     instance = test_cluster.instances['ch3']
-    test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV")
-    test_cluster.ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV")
     test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV")
-    test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV")
+    test_cluster.ddl_check_query(instance,
+                                 "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV")
 
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory")
-    test_cluster.ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory")
+    test_cluster.ddl_check_query(instance,
+                                 "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV")
     test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'")
 
 
@@ -69,7 +75,8 @@ def test_on_server_fail(test_cluster):
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'")
 
     kill_instance.get_docker_handle().stop()
-    request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", timeout=30)
+    request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null",
+                                         timeout=30)
     kill_instance.get_docker_handle().start()
 
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'")
@@ -78,7 +85,8 @@ def test_on_server_fail(test_cluster):
     test_cluster.check_all_hosts_successfully_executed(request.get_answer())
 
     # And check query artefacts
-    contents = instance.query("SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h")
+    contents = instance.query(
+        "SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h")
     assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n")
 
     test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'")
@@ -98,11 +106,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout):
 
 
 def test_on_connection_loss(test_cluster):
-    _test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config)
+    _test_on_connection_losses(test_cluster, 1.5)  # connection loss will occur only (3 sec ZK timeout in config)
 
 
 def test_on_session_expired(test_cluster):
-    _test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config)
+    _test_on_connection_losses(test_cluster, 4)  # session should be expired (3 sec ZK timeout in config)
 
 
 def test_simple_alters(test_cluster):
@@ -127,28 +135,31 @@ ENGINE = Distributed('{cluster}', default, merge, i)
 
     for i in xrange(0, 4, 2):
         k = (i / 2) * 2
-        test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1))
-
-    assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)]))
+        test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1))
 
+    assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(
+        ''.join(['{}\n'.format(x) for x in xrange(4)]))
 
     time.sleep(5)
     test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64")
     time.sleep(5)
-    test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV")
-
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
+    test_cluster.ddl_check_query(instance,
+                                 "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV")
 
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
 
     for i in xrange(0, 4, 2):
         k = (i / 2) * 2 + 4
-        test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1))
-
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)]))
+        test_cluster.instances['ch{}'.format(i + 1)].query(
+            "INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k + 1))
 
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)]))
 
     test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002")
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
 
     test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'")
     test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'")
@@ -160,9 +171,11 @@ def test_macro(test_cluster):
     test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory")
 
     for i in xrange(4):
-        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i))
+        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
+                                     "INSERT INTO tab VALUES ({})".format(i))
 
-    test_cluster.ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)")
 
     assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n')
     assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n')
@@ -197,22 +210,27 @@ def test_allowed_databases(test_cluster):
     instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster")
     instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster")
 
-    instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"})
+    instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory",
+                   settings={"user": "restricted_user"})
 
     with pytest.raises(Exception):
-        instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"})
+        instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory",
+                       settings={"user": "restricted_user"})
     with pytest.raises(Exception):
-        instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"})
+        instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory",
+                       settings={"user": "restricted_user"})
     with pytest.raises(Exception):
-        instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user" : "restricted_user"})
+        instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user": "restricted_user"})
+
+    instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user": "restricted_user"})
 
-    instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user" : "restricted_user"})
 
 def test_kill_query(test_cluster):
     instance = test_cluster.instances['ch3']
 
     test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV")
 
+
 def test_detach_query(test_cluster):
     instance = test_cluster.instances['ch3']
 
@@ -226,21 +244,25 @@ def test_optimize_query(test_cluster):
     instance = test_cluster.instances['ch3']
 
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)")
     test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV")
 
 
 def test_create_as_select(test_cluster):
     instance = test_cluster.instances['ch2']
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)")
     assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n")
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster")
 
 
 def test_create_reserved(test_cluster):
     instance = test_cluster.instances['ch2']
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)")
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster")
     test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster")
 
@@ -248,11 +270,12 @@ def test_create_reserved(test_cluster):
 def test_rename(test_cluster):
     instance = test_cluster.instances['ch1']
     rules = test_cluster.pm_random_drops.pop_rules()
-    test_cluster.ddl_check_query(instance, "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
     test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;")
 
-
     for i in range(10):
         instance.query("insert into rename (id) values ({})".format(i))
 
@@ -261,10 +284,13 @@ def test_rename(test_cluster):
     # so ddl query will always fail on some replicas even if query was actually executed by leader
     # Also such inconsistency in cluster configuration may lead to query duplication if leader suddenly changed
     # because path of lock in zk contains shard name, which is list of host names of replicas
-    instance.query("ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", ignore_error=True)
+    instance.query(
+        "ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))",
+        ignore_error=True)
     time.sleep(1)
 
-    test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
 
     instance.query("system stop distributed sends rename")
 
@@ -283,36 +309,43 @@ def test_rename(test_cluster):
     # system stop distributed sends does not affect inserts into local shard,
     # so some ids in range (10, 20) will be inserted into rename_shard
     assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360"
-    #assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290"
+    # assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290"
     assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115"
-    #assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45"
+    # assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45"
     assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245"
     test_cluster.pm_random_drops.push_rules(rules)
 
+
 def test_socket_timeout(test_cluster):
     instance = test_cluster.instances['ch1']
     # queries should not fail with "Timeout exceeded while reading from socket" in case of EINTR caused by query profiler
     for i in range(0, 100):
         instance.query("select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host")
 
+
 def test_replicated_without_arguments(test_cluster):
     rules = test_cluster.pm_random_drops.pop_rules()
     instance = test_cluster.instances['ch1']
     test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic",
                                  settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
                                  settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
     test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
                                  settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
     test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster")
-    test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
+    test_cluster.ddl_check_query(instance,
+                                 "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
                                  settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
-    test_cluster.ddl_check_query(instance, "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster")
+    test_cluster.ddl_check_query(instance,
+                                 "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster")
     test_cluster.pm_random_drops.push_rules(rules)
 
+
 if __name__ == '__main__':
     with contextmanager(test_cluster)() as ctx_cluster:
-       for name, instance in ctx_cluster.instances.items():
-           print name, instance.ip_address
-       raw_input("Cluster created, press any key to destroy...")
+        for name, instance in ctx_cluster.instances.items():
+            print name, instance.ip_address
+        raw_input("Cluster created, press any key to destroy...")

tests/integration/test_distributed_ddl/test_replicated_alter.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import time
+
 import pytest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -60,29 +61,34 @@ ENGINE = Distributed(cluster, default, merge_for_alter, i)
 
     for i in xrange(4):
         k = (i / 2) * 2
-        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1))
+        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
+                                     "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k + 1))
 
     test_cluster.sync_replicas("merge_for_alter")
 
-    assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)]))
-
+    assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(
+        ''.join(['{}\n'.format(x) for x in xrange(4)]))
 
     test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64")
-    test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)")
-
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
+    test_cluster.ddl_check_query(instance,
+                                 "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)")
 
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
 
     for i in xrange(4):
         k = (i / 2) * 2 + 4
-        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1))
+        test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
+                                     "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k + 1))
 
     test_cluster.sync_replicas("merge_for_alter")
 
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)]))
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)]))
 
     test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002")
-    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
+    assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
+        ''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
 
     test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster")
 

tests/integration/test_distributed_ddl_on_cross_replication/test.py

@@ -1,4 +1,3 @@
-import time
 import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
@@ -12,6 +11,7 @@ node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'
 node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True,
                              macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2})
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -49,7 +49,8 @@ def started_cluster():
 2017-06-17	31	2
 '''
 
-        node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert, settings={"insert_distributed_sync" : 1})
+        node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert,
+                    settings={"insert_distributed_sync": 1})
         yield cluster
 
     finally:
@@ -64,7 +65,8 @@ def test_alter_ddl(started_cluster):
                 WHERE part_key='2017-06-16'")
 
     node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5)
-    assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3')
+    assert_eq_with_retry(node1,
+                         "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3')
 
     node1.query("ALTER TABLE replica_1.replicated_local  \
                 ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;")
@@ -75,4 +77,3 @@ def test_alter_ddl(started_cluster):
 
     node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5)
     assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0')
-

tests/integration/test_distributed_ddl_password/test.py

@@ -1,17 +1,21 @@
-import time
 import pytest
+from helpers.client import QueryRuntimeException
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry
 
-from helpers.client import QueryRuntimeException
-
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
-node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
-node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
-node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
-node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
-node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
+node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"],
+                             user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
 
 
 @pytest.fixture(scope="module")
@@ -21,27 +25,29 @@ def start_cluster():
 
         for node, shard in [(node1, 1), (node2, 1), (node3, 2), (node4, 2), (node5, 3), (node6, 3)]:
             node.query(
-            '''
-                CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
-                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
-                PARTITION BY date
-                ORDER BY id
-            '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"})
+                '''
+                    CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
+                    ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
+                    PARTITION BY date
+                    ORDER BY id
+                '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"})
 
         yield cluster
 
     finally:
         cluster.shutdown()
 
+
 def test_truncate(start_cluster):
-    node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"})
+    node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)",
+                settings={"password": "clickhouse"})
 
     assert node1.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
     node2.query("system sync replica test_table", settings={"password": "clickhouse"})
     assert node2.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
 
-
-    node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", settings={"password": "clickhouse"})
+    node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)",
+                settings={"password": "clickhouse"})
 
     assert node3.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
     node4.query("system sync replica test_table", settings={"password": "clickhouse"})
@@ -55,11 +61,15 @@ def test_truncate(start_cluster):
     node2.query("drop table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"})
 
     for node in [node1, node2, node3, node4]:
-        assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0", settings={"password": "clickhouse"})
+        assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0",
+                             settings={"password": "clickhouse"})
+
 
 def test_alter(start_cluster):
-    node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"})
-    node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", settings={"password": "clickhouse"})
+    node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)",
+                settings={"password": "clickhouse"})
+    node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)",
+                settings={"password": "clickhouse"})
 
     node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"})
     node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"})
@@ -75,24 +85,30 @@ def test_alter(start_cluster):
     assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
     assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
 
-    node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", settings={"password": "clickhouse"})
+    node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'",
+                settings={"password": "clickhouse"})
     assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
     assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
 
     with pytest.raises(QueryRuntimeException):
-        node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"})
+        node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'",
+                    settings={"password": "clickhouse"})
 
     node5.query("ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"})
 
     assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
     assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
 
-    node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", settings={"password": "clickhouse"})
+    node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String",
+                settings={"password": "clickhouse"})
 
-    assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"})
-    assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"})
+    assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1",
+                         settings={"password": "clickhouse"})
+    assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1",
+                         settings={"password": "clickhouse"})
 
-    node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", settings={"password": "clickhouse"})
+    node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'",
+                settings={"password": "clickhouse"})
 
     assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
     assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"})

tests/integration/test_distributed_format/test.py

@@ -1,12 +1,6 @@
-import time
 import pytest
 
 from helpers.cluster import ClickHouseCluster
-from multiprocessing.dummy import Pool
-from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
-
-from helpers.test_tools import assert_eq_with_retry
-
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml'])
@@ -27,10 +21,13 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 @cluster_param
 def test_single_file(started_cluster, cluster):
-    node.query("create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
-    node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"})
+    node.query(
+        "create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
+    node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')",
+               settings={"use_compact_format_in_distributed_parts_names": "1"})
 
     query = "select * from file('/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin', 'Distributed')"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
@@ -48,9 +45,12 @@ def test_single_file(started_cluster, cluster):
 
 @cluster_param
 def test_two_files(started_cluster, cluster):
-    node.query("create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
-    node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", settings={"use_compact_format_in_distributed_parts_names": "1"})
-    node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"})
+    node.query(
+        "create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
+    node.query("insert into test.distr_2 values (0, '_'), (1, 'a')",
+               settings={"use_compact_format_in_distributed_parts_names": "1"})
+    node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')",
+               settings={"use_compact_format_in_distributed_parts_names": "1"})
 
     query = "select * from file('/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
@@ -68,7 +68,8 @@ def test_two_files(started_cluster, cluster):
 
 @cluster_param
 def test_single_file_old(started_cluster, cluster):
-    node.query("create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
+    node.query(
+        "create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
     node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')")
 
     query = "select * from file('/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin', 'Distributed')"

tests/integration/test_distributed_load_balancing/test.py

@@ -3,8 +3,8 @@
 # pylint: disable=line-too-long
 
 import uuid
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
@@ -14,7 +14,8 @@ n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml'])
 n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml'])
 
 nodes = len(cluster.instances)
-queries = nodes*5
+queries = nodes * 5
+
 
 def bootstrap():
     for n in cluster.instances.values():
@@ -58,9 +59,11 @@ def bootstrap():
             data)
         """.format())
 
+
 def make_uuid():
     return uuid.uuid4().hex
 
+
 @pytest.fixture(scope='module', autouse=True)
 def start_cluster():
     try:
@@ -70,6 +73,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def get_node(query_node, table='dist', *args, **kwargs):
     query_id = make_uuid()
 
@@ -106,6 +110,7 @@ def get_node(query_node, table='dist', *args, **kwargs):
     """.format(query_id=query_id))
     return rows.strip()
 
+
 # TODO: right now random distribution looks bad, but works
 def test_load_balancing_default():
     unique_nodes = set()
@@ -113,6 +118,7 @@ def test_load_balancing_default():
         unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'}))
     assert len(unique_nodes) == nodes, unique_nodes
 
+
 def test_load_balancing_nearest_hostname():
     unique_nodes = set()
     for _ in range(0, queries):
@@ -120,6 +126,7 @@ def test_load_balancing_nearest_hostname():
     assert len(unique_nodes) == 1, unique_nodes
     assert unique_nodes == set(['n1'])
 
+
 def test_load_balancing_in_order():
     unique_nodes = set()
     for _ in range(0, queries):
@@ -127,6 +134,7 @@ def test_load_balancing_in_order():
     assert len(unique_nodes) == 1, unique_nodes
     assert unique_nodes == set(['n1'])
 
+
 def test_load_balancing_first_or_random():
     unique_nodes = set()
     for _ in range(0, queries):
@@ -134,6 +142,7 @@ def test_load_balancing_first_or_random():
     assert len(unique_nodes) == 1, unique_nodes
     assert unique_nodes == set(['n1'])
 
+
 def test_load_balancing_round_robin():
     unique_nodes = set()
     for _ in range(0, nodes):
@@ -141,6 +150,7 @@ def test_load_balancing_round_robin():
     assert len(unique_nodes) == nodes, unique_nodes
     assert unique_nodes == set(['n1', 'n2', 'n3'])
 
+
 @pytest.mark.parametrize('dist_table', [
     ('dist_priority'),
     ('dist_priority_negative'),
@@ -153,6 +163,7 @@ def test_load_balancing_priority_round_robin(dist_table):
     # n2 has bigger priority in config
     assert unique_nodes == set(['n1', 'n3'])
 
+
 def test_distributed_replica_max_ignored_errors():
     settings = {
         'load_balancing': 'in_order',

tests/integration/test_distributed_over_distributed/test.py

@@ -3,16 +3,8 @@
 
 from __future__ import print_function
 
-import itertools
-import timeit
-import logging
-
 import pytest
-
 from helpers.cluster import ClickHouseCluster
-from helpers.network import PartitionManager
-from helpers.test_tools import TSV
-
 
 cluster = ClickHouseCluster(__file__)
 
@@ -44,6 +36,7 @@ ENGINE = Distributed('test_cluster', default, distributed_table);
 
 INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})"
 
+
 @pytest.fixture(scope="session")
 def started_cluster():
     try:
@@ -59,11 +52,12 @@ def started_cluster():
 
 
 @pytest.mark.parametrize("node", NODES.values())
-@pytest.mark.parametrize("source", ["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"])
+@pytest.mark.parametrize("source",
+                         ["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"])
 class TestDistributedOverDistributedSuite:
     def test_select_with_order_by_node(self, started_cluster, node, source):
         assert node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) \
-            == """node1	0	0
+               == """node1	0	0
 node1	0	0
 node1	1	1
 node1	1	1
@@ -75,7 +69,7 @@ node2	1	11
 
     def test_select_with_order_by_key(self, started_cluster, node, source):
         assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \
-            == """node1	0	0
+               == """node1	0	0
 node1	0	0
 node2	0	10
 node2	0	10
@@ -87,12 +81,12 @@ node2	1	11
 
     def test_select_with_group_by_node(self, started_cluster, node, source):
         assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \
-            == "node1	2\nnode2	42\n"
+               == "node1	2\nnode2	42\n"
 
     def test_select_with_group_by_key(self, started_cluster, node, source):
         assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \
-            == "0	20\n1	24\n"
+               == "0	20\n1	24\n"
 
     def test_select_sum(self, started_cluster, node, source):
         assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \
-            == "44\n"
+               == "44\n"

tests/integration/test_distributed_over_live_view/test.py

@@ -2,16 +2,10 @@ from __future__ import print_function
 
 import sys
 import time
-import itertools
-import timeit
-import logging
 
 import pytest
-
-from helpers.uclient import client, prompt, end_of_block
 from helpers.cluster import ClickHouseCluster
-from helpers.network import PartitionManager
-from helpers.test_tools import TSV
+from helpers.uclient import client, prompt, end_of_block
 
 cluster = ClickHouseCluster(__file__)
 
@@ -46,6 +40,7 @@ ENGINE = Distributed(test_cluster, default, base_table, rand());
 
 INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})"
 
+
 @pytest.fixture(scope="function")
 def started_cluster():
     try:
@@ -77,7 +72,8 @@ class TestLiveViewOverDistributedSuite:
 
             client1.send("DROP TABLE IF EXISTS distributed_over_lv")
             client1.expect(prompt)
-            client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
+            client1.send(
+                "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
             client1.expect(prompt)
 
             client1.send(select_query)
@@ -115,7 +111,8 @@ class TestLiveViewOverDistributedSuite:
 
             client1.send("DROP TABLE IF EXISTS distributed_over_lv")
             client1.expect(prompt)
-            client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
+            client1.send(
+                "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
             client1.expect(prompt)
 
             client1.send(select_query)
@@ -153,7 +150,8 @@ class TestLiveViewOverDistributedSuite:
 
             client1.send("DROP TABLE IF EXISTS distributed_over_lv")
             client1.expect(prompt)
-            client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
+            client1.send(
+                "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
             client1.expect(prompt)
 
             client1.send(select_query)
@@ -192,7 +190,8 @@ class TestLiveViewOverDistributedSuite:
 
             client1.send("DROP TABLE IF EXISTS distributed_over_lv")
             client1.expect(prompt)
-            client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
+            client1.send(
+                "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
             client1.expect(prompt)
 
             client1.send(select_query)
@@ -230,7 +229,8 @@ class TestLiveViewOverDistributedSuite:
 
             client1.send("DROP TABLE IF EXISTS distributed_over_lv")
             client1.expect(prompt)
-            client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
+            client1.send(
+                "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
             client1.expect(prompt)
 
             client1.send("SELECT sum(value) FROM distributed_over_lv")
@@ -254,4 +254,3 @@ class TestLiveViewOverDistributedSuite:
             client1.send("SELECT sum(value) FROM distributed_over_lv")
             client1.expect(r"31" + end_of_block)
             client1.expect(prompt)
-

tests/integration/test_distributed_respect_user_timeouts/test.py

@@ -1,13 +1,12 @@
 import itertools
-import timeit
 import os.path
-import pytest
+import timeit
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.network import PartitionManager
 from helpers.test_tools import TSV
 
-
 cluster = ClickHouseCluster(__file__)
 
 NODES = {'node' + str(i): None for i in (1, 2)}
@@ -62,6 +61,7 @@ TIMEOUT_DIFF_UPPER_BOUND = {
     },
 }
 
+
 def _check_exception(exception, expected_tries=3):
     lines = exception.split('\n')
 
@@ -88,7 +88,6 @@ def _check_exception(exception, expected_tries=3):
 
 @pytest.fixture(scope="module", params=["configs", "configs_secure"])
 def started_cluster(request):
-
     cluster = ClickHouseCluster(__file__)
     cluster.__with_ssl_config = request.param == "configs_secure"
     main_configs = []

tests/integration/test_distributed_storage_configuration/test.py

@@ -9,8 +9,9 @@ from helpers.cluster import ClickHouseCluster
 cluster = ClickHouseCluster(__file__)
 
 node = cluster.add_instance('node',
-            main_configs=["configs/config.d/storage_configuration.xml"],
-            tmpfs=['/disk1:size=100M', '/disk2:size=100M'])
+                            main_configs=["configs/config.d/storage_configuration.xml"],
+                            tmpfs=['/disk1:size=100M', '/disk2:size=100M'])
+
 
 @pytest.fixture(scope='module')
 def start_cluster():
@@ -21,14 +22,17 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def _files_in_dist_mon(node, root, table):
     return int(node.exec_in_container([
         'bash',
         '-c',
         # `-maxdepth 1` to avoid /tmp/ subdirectory
-        'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format(root=root, table=table)
+        'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format(
+            root=root, table=table)
     ]).split('\n')[0])
 
+
 def test_insert(start_cluster):
     node.query('CREATE TABLE test.foo (key Int) Engine=Memory()')
     node.query("""

tests/integration/test_distributed_system_query/test.py

@@ -1,5 +1,3 @@
-from contextlib import contextmanager
-
 import pytest
 
 from helpers.cluster import ClickHouseCluster
@@ -18,7 +16,8 @@ def started_cluster():
         for node in (node1, node2):
             node.query('''CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;''')
 
-        node1.query('''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''')
+        node1.query(
+            '''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''')
 
         yield cluster
 
@@ -38,4 +37,3 @@ def test_start_and_stop_replica_send(started_cluster):
     node1.query("SYSTEM START DISTRIBUTED SENDS distributed_table;")
     node1.query("SYSTEM FLUSH DISTRIBUTED distributed_table;")
     assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '2'
-

tests/integration/test_drop_replica/test.py

@@ -1,53 +1,52 @@
 import time
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
-from helpers.cluster import ClickHouseKiller
-from helpers.test_tools import assert_eq_with_retry
 from helpers.network import PartitionManager
 
+
 def fill_nodes(nodes, shard):
     for node in nodes:
         node.query(
-        '''
-            CREATE DATABASE test;
-
-            CREATE TABLE test.test_table(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
+            '''
+                CREATE DATABASE test;
+    
+                CREATE TABLE test.test_table(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
         node.query(
-        '''
-            CREATE DATABASE test1;
-
-            CREATE TABLE test1.test_table(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
+            '''
+                CREATE DATABASE test1;
+    
+                CREATE TABLE test1.test_table(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
         node.query(
-        '''
-            CREATE DATABASE test2;
-
-            CREATE TABLE test2.test_table(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
-
+            '''
+                CREATE DATABASE test2;
+    
+                CREATE TABLE test2.test_table(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
         node.query(
-        '''
-            CREATE DATABASE test3;
-
-            CREATE TABLE test3.test_table(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
+            '''
+                CREATE DATABASE test3;
+    
+                CREATE TABLE test3.test_table(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
         node.query(
-        '''
-            CREATE DATABASE test4;
+            '''
+                CREATE DATABASE test4;
+    
+                CREATE TABLE test4.test_table(date Date, id UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
+            '''.format(shard=shard, replica=node.name))
 
-            CREATE TABLE test4.test_table(date Date, id UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
-        '''.format(shard=shard, replica=node.name))
 
 cluster = ClickHouseCluster(__file__)
 
@@ -71,6 +70,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_drop_replica(start_cluster):
     for i in range(100):
         node_1_1.query("INSERT INTO test.test_table VALUES (1, {})".format(i))
@@ -81,17 +81,25 @@ def test_drop_replica(start_cluster):
 
     zk = cluster.get_kazoo_client('zoo1')
     assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'")
-    assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test")
-    assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
+    assert "can't drop local replica" in node_1_1.query_and_get_error(
+        "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test")
+    assert "can't drop local replica" in node_1_1.query_and_get_error(
+        "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
     assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'")
     assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test")
     assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
     assert "it's active" in \
-        node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1))
+           node_1_3.query_and_get_error(
+               "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
+                   shard=1))
     assert "There is a local table" in \
-        node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1))
+           node_1_2.query_and_get_error(
+               "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
+                   shard=1))
     assert "There is a local table" in \
-        node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1))
+           node_1_1.query_and_get_error(
+               "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
+                   shard=1))
     assert "does not look like a table path" in \
            node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'")
 
@@ -100,31 +108,48 @@ def test_drop_replica(start_cluster):
         pm.drop_instance_zk_connections(node_1_1)
         time.sleep(10)
 
-        assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
+        assert "doesn't exist" in node_1_3.query_and_get_error(
+            "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
 
         assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1")
 
         node_1_3.query("SYSTEM DROP REPLICA 'node_1_1'")
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                               replica='node_1_1'))
         assert (exists_replica_1_1 != None)
 
         ## If you want to drop a inactive/stale replicate table that does not have a local replica, you can following syntax(ZKPATH):
-        node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format(shard=1))
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        node_1_3.query(
+            "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format(
+                shard=1))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                               replica='node_1_1'))
         assert (exists_replica_1_1 == None)
 
         node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                              replica='node_1_1'))
         assert (exists_replica_1_1 == None)
 
         node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1")
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                               replica='node_1_1'))
         assert (exists_replica_1_1 == None)
 
-        node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format(shard=1))
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        node_1_3.query(
+            "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format(
+                shard=1))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                               replica='node_1_1'))
         assert (exists_replica_1_1 == None)
 
         node_1_2.query("SYSTEM DROP REPLICA 'node_1_1'")
-        exists_replica_1_1 = zk.exists("/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1'))
+        exists_replica_1_1 = zk.exists(
+            "/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
+                                                                                               replica='node_1_1'))
         assert (exists_replica_1_1 == None)

tests/integration/test_enabling_access_management/test.py

@@ -4,6 +4,7 @@ from helpers.cluster import ClickHouseCluster
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance', user_configs=["configs/users.d/extra_users.xml"])
 
+
 @pytest.fixture(scope="module", autouse=True)
 def started_cluster():
     try:

tests/integration/test_extreme_deduplication/test.py

@@ -1,21 +1,22 @@
 import time
-from contextlib import contextmanager
 
 import pytest
-
-from helpers.cluster import ClickHouseCluster
-from helpers.network import PartitionManager
-from helpers.test_tools import TSV
 from helpers.client import CommandRequest
 from helpers.client import QueryTimeoutExceedException
-
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
 
 cluster = ClickHouseCluster(__file__)
 
-node1 = cluster.add_instance('node1', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1})
-node2 = cluster.add_instance('node2', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2})
+node1 = cluster.add_instance('node1',
+                             main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"],
+                             with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1})
+node2 = cluster.add_instance('node2',
+                             main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"],
+                             with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2})
 nodes = [node1, node2]
 
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -36,15 +37,18 @@ def test_deduplication_window_in_seconds(started_cluster):
 
     node.query("INSERT INTO simple VALUES (0, 0)")
     time.sleep(1)
-    node.query("INSERT INTO simple VALUES (0, 0)") # deduplication works here
+    node.query("INSERT INTO simple VALUES (0, 0)")  # deduplication works here
     node.query("INSERT INTO simple VALUES (0, 1)")
     assert TSV(node.query("SELECT count() FROM simple")) == TSV("2\n")
 
     # wait clean thread
     time.sleep(2)
 
-    assert TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[0][0] == "1"
-    node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted
+    assert \
+        TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[
+            0][
+            0] == "1"
+    node.query("INSERT INTO simple VALUES (0, 0)")  # deduplication doesn't works here, the first hash node was deleted
     assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3"
 
     node1.query("""DROP TABLE simple ON CLUSTER test_cluster""")

tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py

@@ -1,15 +1,13 @@
 from __future__ import print_function
-from helpers.cluster import ClickHouseCluster
-from helpers.client import QueryRuntimeException
-import helpers
-import pytest
 
+import pytest
+from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True)
 
 
-
 @pytest.fixture(scope="module")
 def start_cluster():
     try:

tests/integration/test_filesystem_layout/test.py

@@ -25,4 +25,5 @@ def test_file_path_escaping(started_cluster):
     node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''')
 
     node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])
-    node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])
+    node.exec_in_container(
+        ["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])

tests/integration/test_force_deduplication/test.py

@@ -2,14 +2,14 @@
 # pylint: disable=redefined-outer-name
 
 import pytest
-
-from helpers.cluster import ClickHouseCluster
 from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
 node = cluster.add_instance('node', with_zookeeper=True)
 
+
 @pytest.fixture(scope='module')
 def start_cluster():
     try:
@@ -19,6 +19,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def get_counts():
     src = int(node.query("SELECT count() FROM test"))
     a = int(node.query("SELECT count() FROM test_mv_a"))
@@ -69,7 +70,7 @@ def test_basic(start_cluster):
     )
     src, a, b, c = get_counts()
     assert src == 11
-    assert a == old_a + 10    # first insert could be succesfull with disabled dedup
+    assert a == old_a + 10  # first insert could be succesfull with disabled dedup
     assert b == 11
     assert c == old_c + 10
 
@@ -81,7 +82,7 @@ def test_basic(start_cluster):
             INSERT INTO test SELECT number FROM numbers(100,10);
             '''
         )
-        
+
     node.query(
         '''
         SET deduplicate_blocks_in_dependent_materialized_views = 1;
@@ -94,5 +95,3 @@ def test_basic(start_cluster):
     assert a == old_a + 20
     assert b == 21
     assert c == old_c + 20
-
-

tests/integration/test_format_avro_confluent/test.py

@@ -1,20 +1,15 @@
-import json
-import logging
 import io
-
-import pytest
-
-from helpers.cluster import ClickHouseCluster, ClickHouseInstance
-
-import helpers.client
+import logging
 
 import avro.schema
-from confluent.schemaregistry.client import CachedSchemaRegistryClient
+import pytest
 from confluent.schemaregistry.serializers import MessageSerializer
+from helpers.cluster import ClickHouseCluster, ClickHouseInstance
 
 logging.getLogger().setLevel(logging.INFO)
 logging.getLogger().addHandler(logging.StreamHandler())
 
+
 @pytest.fixture(scope="module")
 def cluster():
     try:
@@ -42,7 +37,6 @@ def run_query(instance, query, data=None, settings=None):
     return result
 
 
-
 def test_select(cluster):
     # type: (ClickHouseCluster) -> None
 
@@ -55,7 +49,7 @@ def test_select(cluster):
         'fields': [
             {
                 'name': 'value',
-                'type': 'long' 
+                'type': 'long'
             }
         ]
     })
@@ -73,7 +67,7 @@ def test_select(cluster):
         cluster.schema_registry_host,
         cluster.schema_registry_port
     )
-    
+
     run_query(instance, "create table avro_data(value Int64) engine = Memory()")
     settings = {'format_avro_schema_registry_url': schema_registry_url}
     run_query(instance, "insert into avro_data format AvroConfluent", data, settings)

tests/integration/test_format_schema_on_server/test.py

@@ -36,5 +36,6 @@ def test_protobuf_format_input(started_cluster):
 def test_protobuf_format_output(started_cluster):
     create_simple_table()
     instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')");
-    assert instance.http_query("SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \
+    assert instance.http_query(
+        "SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \
            "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def"

tests/integration/test_freeze_table/test.py

@@ -39,7 +39,7 @@ def test_freeze_table(started_cluster):
         '''))
     assert 11 == len(freeze_result)
     path_col_ix = freeze_result[0].index('part_backup_path')
-    for row in freeze_result[1:]: # skip header
+    for row in freeze_result[1:]:  # skip header
         part_backup_path = row[path_col_ix]
         node.exec_in_container(
             ["bash", "-c", "test -d {}".format(part_backup_path)]
@@ -55,7 +55,7 @@ def test_freeze_table(started_cluster):
         '''))
     assert 2 == len(freeze_result)
     path_col_ix = freeze_result[0].index('part_backup_path')
-    for row in freeze_result[1:]: # skip header
+    for row in freeze_result[1:]:  # skip header
         part_backup_path = row[path_col_ix]
         assert 'test_01417_single_part' in part_backup_path
         node.exec_in_container(

tests/integration/test_globs_in_filepath/test.py

@@ -4,7 +4,8 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance('node')
-path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/"   # should be the same as in config file
+path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/"  # should be the same as in config file
+
 
 @pytest.fixture(scope="module")
 def start_cluster():
@@ -19,11 +20,13 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_strange_filenames(start_cluster):
     # 2 rows data
     some_data = "\t111.222\nData\t333.444"
 
-    node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], privileged=True, user='root')
+    node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)],
+                           privileged=True, user='root')
 
     files = ["p.o.i.n.t.s",
              "b}{ra{ces",
@@ -31,7 +34,10 @@ def test_strange_filenames(start_cluster):
 
     # filename inside testing data for debug simplicity
     for filename in files:
-        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')
+        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data,
+                                                                                        path_to_userfiles_from_defaut_config,
+                                                                                        filename)], privileged=True,
+                               user='root')
 
     test_requests = [("p.o.??n.t.s", "2"),
                      ("p.o.*t.s", "2"),
@@ -47,6 +53,7 @@ def test_strange_filenames(start_cluster):
             select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64')
         '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
 
+
 def test_linear_structure(start_cluster):
     # 2 rows data
     some_data = "\t123.456\nData\t789.012"
@@ -58,7 +65,9 @@ def test_linear_structure(start_cluster):
 
     # filename inside testing data for debug simplicity
     for filename in files:
-        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')
+        node.exec_in_container(['bash', '-c',
+                                'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
+                                                            filename)], privileged=True, user='root')
 
     test_requests = [("file{0..9}", "10"),
                      ("file?", "10"),
@@ -81,6 +90,7 @@ def test_linear_structure(start_cluster):
             select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
         '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
 
+
 def test_deep_structure(start_cluster):
     # 2 rows data
     some_data = "\t135.791\nData\t246.802"
@@ -92,7 +102,8 @@ def test_deep_structure(start_cluster):
             "we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"]
 
     for dir in dirs:
-        node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], privileged=True, user='root')
+        node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)],
+                               privileged=True, user='root')
 
     # all directories appeared in files must be listed in dirs
     files = []
@@ -102,13 +113,15 @@ def test_deep_structure(start_cluster):
                 files.append("directory1/big_dir/file" + str(i) + str(j) + str(k))
 
     for dir in dirs:
-        files.append(dir+"file")
+        files.append(dir + "file")
 
     # filename inside testing data for debug simplicity
     for filename in files:
-        node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root')
+        node.exec_in_container(['bash', '-c',
+                                'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
+                                                            filename)], privileged=True, user='root')
 
-    test_requests = [ ("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"),
+    test_requests = [("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"),
                      ("?", "0"),
                      ("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"),
                      ("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")]
@@ -121,14 +134,17 @@ def test_deep_structure(start_cluster):
             select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
         '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
 
+
 def test_table_function_and_virtual_columns(start_cluster):
     node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)])
     node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)])
-    node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)")
-    assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000'
+    node.query(
+        "insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)")
+    assert node.query(
+        "select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000'
     node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)")
-    assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1'
+    assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() == '1'
     assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
     assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
     assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
-    assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
+    assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()

tests/integration/test_grant_and_revoke/test.py

@@ -1,7 +1,6 @@
 import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
-import re
 
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance')
@@ -11,11 +10,11 @@ instance = cluster.add_instance('instance')
 def start_cluster():
     try:
         cluster.start()
-        
+
         instance.query("CREATE DATABASE test")
         instance.query("CREATE TABLE test.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()")
         instance.query("INSERT INTO test.table VALUES (1,5), (2,10)")
-        
+
         yield cluster
 
     finally:
@@ -34,7 +33,7 @@ def cleanup_after_test():
 def test_smoke():
     instance.query("CREATE USER A")
     assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test.table", user='A')
-    
+
     instance.query('GRANT SELECT ON test.table TO A')
     assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n"
 
@@ -49,7 +48,7 @@ def test_grant_option():
     instance.query('GRANT SELECT ON test.table TO A')
     assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n"
     assert "Not enough privileges" in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A')
-    
+
     instance.query('GRANT SELECT ON test.table TO A WITH GRANT OPTION')
     instance.query("GRANT SELECT ON test.table TO B", user='A')
     assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n"
@@ -60,7 +59,7 @@ def test_grant_option():
 def test_revoke_requires_grant_option():
     instance.query("CREATE USER A")
     instance.query("CREATE USER B")
-    
+
     instance.query("GRANT SELECT ON test.table TO B")
     assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n"
 
@@ -111,7 +110,8 @@ def test_grant_all_on_table():
     instance.query("CREATE USER A, B")
     instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION")
     instance.query("GRANT ALL ON test.table TO B", user='A')
-    assert instance.query("SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
+    assert instance.query(
+        "SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
     instance.query("REVOKE ALL ON test.table FROM B", user='A')
     assert instance.query("SHOW GRANTS FOR B") == ""
 
@@ -120,36 +120,42 @@ def test_implicit_show_grants():
     instance.query("CREATE USER A")
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "0\n"
 
     instance.query("GRANT SELECT(x) ON test.table TO A")
     assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT(x) ON test.table TO A\n"
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "1\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "1\n"
 
     instance.query("GRANT SELECT ON test.table TO A")
     assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.table TO A\n"
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "2\n"
 
     instance.query("GRANT SELECT ON test.* TO A")
     assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.* TO A\n"
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "2\n"
 
     instance.query("GRANT SELECT ON *.* TO A")
     assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON *.* TO A\n"
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "2\n"
 
     instance.query("REVOKE ALL ON *.* FROM A")
     assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n"
     assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n"
-    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n"
+    assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
+                          user="A") == "0\n"
 
 
 def test_implicit_create_view_grant():
@@ -184,46 +190,53 @@ def test_introspection():
     instance.query('GRANT SELECT ON test.table TO A')
     instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION')
 
-    assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ])
-    assert instance.query("SHOW CREATE USERS A") == TSV([ "CREATE USER A" ])
-    assert instance.query("SHOW CREATE USERS B") == TSV([ "CREATE USER B" ])
-    assert instance.query("SHOW CREATE USERS A,B") == TSV([ "CREATE USER A", "CREATE USER B" ])
-    assert instance.query("SHOW CREATE USERS") == TSV([ "CREATE USER A", "CREATE USER B", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" ])
+    assert instance.query("SHOW USERS") == TSV(["A", "B", "default"])
+    assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"])
+    assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"])
+    assert instance.query("SHOW CREATE USERS A,B") == TSV(["CREATE USER A", "CREATE USER B"])
+    assert instance.query("SHOW CREATE USERS") == TSV(["CREATE USER A", "CREATE USER B",
+                                                       "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"])
 
-    assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ])
-    assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ])
-    assert instance.query("SHOW GRANTS FOR A,B") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ])
-    assert instance.query("SHOW GRANTS FOR B,A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ])
-    assert instance.query("SHOW GRANTS FOR ALL") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT ALL ON *.* TO default WITH GRANT OPTION" ])
+    assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"])
+    assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
+    assert instance.query("SHOW GRANTS FOR A,B") == TSV(
+        ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
+    assert instance.query("SHOW GRANTS FOR B,A") == TSV(
+        ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
+    assert instance.query("SHOW GRANTS FOR ALL") == TSV(
+        ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION",
+         "GRANT ALL ON *.* TO default WITH GRANT OPTION"])
 
-    assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ])
-    assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ])
+    assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"])
+    assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
 
-    expected_access1 = "CREATE USER A\n"\
-                       "CREATE USER B\n"\
+    expected_access1 = "CREATE USER A\n" \
+                       "CREATE USER B\n" \
                        "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"
-    expected_access2 = "GRANT SELECT ON test.table TO A\n"\
-                       "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n"\
+    expected_access2 = "GRANT SELECT ON test.table TO A\n" \
+                       "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" \
                        "GRANT ALL ON *.* TO default WITH GRANT OPTION\n"
     assert expected_access1 in instance.query("SHOW ACCESS")
     assert expected_access2 in instance.query("SHOW ACCESS")
 
-    assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
-           TSV([[ "A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
-                [ "B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
-    
-    assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") ==\
-           TSV([[ "A",  "\N", "SELECT", "test", "table", "\N", 0, 0 ],
-                [ "B",  "\N", "CREATE", "\N",   "\N",    "\N", 0, 1 ]])
+    assert instance.query(
+        "SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") == \
+           TSV([["A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"],
+                ["B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"]])
+
+    assert instance.query(
+        "SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") == \
+           TSV([["A", "\N", "SELECT", "test", "table", "\N", 0, 0],
+                ["B", "\N", "CREATE", "\N", "\N", "\N", 0, 1]])
 
 
 def test_current_database():
     instance.query("CREATE USER A")
     instance.query("GRANT SELECT ON table TO A", database="test")
-    
-    assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ])
-    assert instance.query("SHOW GRANTS FOR A", database="test") == TSV([ "GRANT SELECT ON test.table TO A" ])
-    
+
+    assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"])
+    assert instance.query("SHOW GRANTS FOR A", database="test") == TSV(["GRANT SELECT ON test.table TO A"])
+
     assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n"
     assert instance.query("SELECT * FROM table", user='A', database='test') == "1\t5\n2\t10\n"
 

tests/integration/test_graphite_merge_tree/test.py

@@ -1,12 +1,11 @@
+import datetime
 import os.path as p
 import time
-import datetime
-import pytest
 
+import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
-
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance',
                                 main_configs=['configs/graphite_rollup.xml'])
@@ -317,20 +316,20 @@ def test_combined_rules(graphite_table):
     expected_unmerged = ''
     for i in range(384):
         to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format(
-            v=1, t=1487970000+(i*300)
+            v=1, t=1487970000 + (i * 300)
         )
         to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format(
-            v=i, t=1487970000+(i*300)
+            v=i, t=1487970000 + (i * 300)
         )
         expected_unmerged += ("five_min.count\t{v1}\t{t}\n"
                               "five_min.max\t{v2}\t{t}\n").format(
-                                  v1=1, v2=i,
-                                  t=1487970000+(i*300)
-                              )
+            v1=1, v2=i,
+            t=1487970000 + (i * 300)
+        )
 
     q(to_insert)
     assert TSV(q('SELECT metric, value, timestamp FROM test.graphite'
-               ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged)
+                 ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged)
 
     q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL')
     expected_merged = '''
@@ -370,16 +369,16 @@ CREATE TABLE test.graphite
     expected_unmerged = ''
     for i in range(100):
         to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format(
-            v=1, t=1487970000+(i*60)
+            v=1, t=1487970000 + (i * 60)
         )
         to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format(
-            v=i, t=1487970000+(i*60)
+            v=i, t=1487970000 + (i * 60)
         )
         expected_unmerged += ("top_level.count\t{v1}\t{t}\n"
                               "top_level.max\t{v2}\t{t}\n").format(
-                                  v1=1, v2=i,
-                                  t=1487970000+(i*60)
-                              )
+            v1=1, v2=i,
+            t=1487970000 + (i * 60)
+        )
 
     q(to_insert)
     assert TSV(q('SELECT metric, value, timestamp FROM test.graphite'

tests/integration/test_host_ip_change/test.py

@@ -1,15 +1,12 @@
-import time
 import pytest
-
-import subprocess
-from helpers.cluster import ClickHouseCluster
-from helpers.test_tools import assert_eq_with_retry
 from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
-
+from helpers.test_tools import assert_eq_with_retry
 
 cluster = ClickHouseCluster(__file__)
 
+
 def _fill_nodes(nodes, table_name):
     for node in nodes:
         node.query(
@@ -21,9 +18,12 @@ def _fill_nodes(nodes, table_name):
             '''.format(table_name, node.name)
         )
 
-node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True, ipv6_address='2001:3984:3989::1:1111')
+
+node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True,
+                             ipv6_address='2001:3984:3989::1:1111')
 node2 = cluster.add_instance('node2', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'],
-    with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112')
+                             with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112')
+
 
 @pytest.fixture(scope="module")
 def cluster_without_dns_cache_update():
@@ -41,6 +41,7 @@ def cluster_without_dns_cache_update():
         cluster.shutdown()
         pass
 
+
 # node1 is a source, node2 downloads data
 # node2 has long dns_cache_update_period, so dns cache update wouldn't work
 def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update):
@@ -73,9 +74,11 @@ def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update):
 
 
 node3 = cluster.add_instance('node3', main_configs=['configs/listen_host.xml'],
-    with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113')
-node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml', 'configs/dns_update_short.xml'],
-    with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114')
+                             with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113')
+node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml',
+                                                    'configs/dns_update_short.xml'],
+                             with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114')
+
 
 @pytest.fixture(scope="module")
 def cluster_with_dns_cache_update():
@@ -93,6 +96,7 @@ def cluster_with_dns_cache_update():
         cluster.shutdown()
         pass
 
+
 # node3 is a source, node4 downloads data
 # node4 has short dns_cache_update_period, so testing update of dns cache
 def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
@@ -107,7 +111,6 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
     # Put some data to source node3
     node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)")
 
-
     # Check that data is placed on node3
     assert node3.query("SELECT count(*) from test_table_update") == "6\n"
 
@@ -126,9 +129,12 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
     assert node3.query("SELECT count(*) from test_table_update") == "7\n"
     assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "7")
 
+
 def set_hosts(node, hosts):
     new_content = '\\n'.join(['127.0.0.1 localhost', '::1 localhost'] + hosts)
-    node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True, user='root')
+    node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True,
+                           user='root')
+
 
 def test_dns_cache_update(cluster_with_dns_cache_update):
     set_hosts(node4, ['127.255.255.255 lost_host'])
@@ -136,7 +142,8 @@ def test_dns_cache_update(cluster_with_dns_cache_update):
     with pytest.raises(QueryRuntimeException):
         node4.query("SELECT * FROM remote('lost_host', 'system', 'one')")
 
-    node4.query("CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')")
+    node4.query(
+        "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')")
     with pytest.raises(QueryRuntimeException):
         node4.query("SELECT * FROM distributed_lost_host")
 
@@ -146,21 +153,26 @@ def test_dns_cache_update(cluster_with_dns_cache_update):
     assert_eq_with_retry(node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0")
     assert_eq_with_retry(node4, "SELECT * FROM distributed_lost_host", "0")
 
-    assert TSV(node4.query("SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV("lost_host\t127.0.0.1\n")
+    assert TSV(node4.query(
+        "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV(
+        "lost_host\t127.0.0.1\n")
     assert TSV(node4.query("SELECT hostName()")) == TSV("node4")
 
+
 # Check SYSTEM DROP DNS CACHE on node5 and background cache update on node6
 node5 = cluster.add_instance('node5', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'],
                              user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1115')
 node6 = cluster.add_instance('node6', main_configs=['configs/listen_host.xml', 'configs/dns_update_short.xml'],
                              user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1116')
 
+
 @pytest.mark.parametrize("node", [node5, node6])
 def test_user_access_ip_change(cluster_with_dns_cache_update, node):
     node_name = node.name
     node_num = node.name[-1]
     # getaddrinfo(...) may hang for a log time without this options
-    node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'], privileged=True, user='root')
+    node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'],
+                           privileged=True, user='root')
 
     assert node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n"
     assert node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n"
@@ -180,8 +192,11 @@ def test_user_access_ip_change(cluster_with_dns_cache_update, node):
     retry_count = 60
     if node_name == 'node5':
         # client is not allowed to connect, so execute it directly in container to send query from localhost
-        node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True, user='root')
+        node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True,
+                               user='root')
         retry_count = 1
 
-    assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1)
-    assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1)
+    assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0",
+                         retry_count=retry_count, sleep_time=1)
+    assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0",
+                         retry_count=retry_count, sleep_time=1)

tests/integration/test_http_and_readonly/test.py

@@ -16,5 +16,7 @@ def setup_nodes():
 
 
 def test_http_get_is_readonly():
-    assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)")
-    assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0})
+    assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error(
+        "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)")
+    assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error(
+        "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0})

tests/integration/test_http_handlers_config/test.py

@@ -1,6 +1,6 @@
+import contextlib
 import os
 import urllib
-import contextlib
 
 from helpers.cluster import ClickHouseCluster
 
@@ -20,101 +20,145 @@ class SimpleCluster:
 
 
 def test_dynamic_query_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster:
         test_query = urllib.quote_plus('SELECT * FROM system.settings WHERE name = \'max_threads\'')
 
         assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
+        assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 200 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query,
-                method='GET', headers={'XXX': 'xxx'}).status_code
+        assert 200 == cluster.instance.http_request(
+            'test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query,
+            method='GET', headers={'XXX': 'xxx'}).status_code
 
 
 def test_predefined_query_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster:
         assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
+        assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 'max_threads\t1\n' == cluster.instance.http_request('test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET', headers={'XXX': 'xxx'}).content
+        assert 'max_threads\t1\n' == cluster.instance.http_request(
+            'test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET',
+            headers={'XXX': 'xxx'}).content
 
         assert 'max_threads\t1\nmax_alter_threads\t1\n' == cluster.instance.http_request(
-            'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1', headers={'XXX': 'max_alter_threads'}).content
+            'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1',
+            headers={'XXX': 'max_alter_threads'}).content
 
 
 def test_fixed_static_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type']
-        assert 'Test get static handler and fix content' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content
+        assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
+        assert 'text/html; charset=UTF-8' == \
+               cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
+                                             headers={'XXX': 'xxx'}).headers['Content-Type']
+        assert 'Test get static handler and fix content' == cluster.instance.http_request(
+            'test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content
 
 
 def test_config_static_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
         # check default status code
-        assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'text/plain; charset=UTF-8' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type']
-        assert 'Test get static handler and config content' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content
+        assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
+        assert 'text/plain; charset=UTF-8' == \
+               cluster.instance.http_request('test_get_config_static_handler', method='GET',
+                                             headers={'XXX': 'xxx'}).headers['Content-Type']
+        assert 'Test get static handler and config content' == cluster.instance.http_request(
+            'test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content
 
 
 def test_absolute_path_static_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
         cluster.instance.exec_in_container(
-            ['bash', '-c', 'echo "<html><body>Absolute Path File</body></html>" > /var/lib/clickhouse/user_files/absolute_path_file.html'],
+            ['bash', '-c',
+             'echo "<html><body>Absolute Path File</body></html>" > /var/lib/clickhouse/user_files/absolute_path_file.html'],
             privileged=True, user='root')
 
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
         # check default status code
-        assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type']
-        assert '<html><body>Absolute Path File</body></html>\n' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
+        assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
+        assert 'text/html; charset=UTF-8' == \
+               cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
+                                             headers={'XXX': 'xxx'}).headers['Content-Type']
+        assert '<html><body>Absolute Path File</body></html>\n' == cluster.instance.http_request(
+            'test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
 
 
 def test_relative_path_static_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
         cluster.instance.exec_in_container(
-            ['bash', '-c', 'echo "<html><body>Relative Path File</body></html>" > /var/lib/clickhouse/user_files/relative_path_file.html'],
+            ['bash', '-c',
+             'echo "<html><body>Relative Path File</body></html>" > /var/lib/clickhouse/user_files/relative_path_file.html'],
             privileged=True, user='root')
 
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
         # check default status code
-        assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type']
-        assert '<html><body>Relative Path File</body></html>\n' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
+        assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
+        assert 'text/html; charset=UTF-8' == \
+               cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
+                                             headers={'XXX': 'xxx'}).headers['Content-Type']
+        assert '<html><body>Relative Path File</body></html>\n' == cluster.instance.http_request(
+            'test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
+
 
 def test_defaults_http_handlers():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster:
         assert 200 == cluster.instance.http_request('', method='GET').status_code
         assert 'Default server response' == cluster.instance.http_request('', method='GET').content
 
@@ -130,24 +174,34 @@ def test_defaults_http_handlers():
         assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code
         assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content
 
+
 def test_prometheus_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster:
+    with contextlib.closing(
+            SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster:
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
         assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_prometheus', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_prometheus', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
         assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).content
+        assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET',
+                                                                                headers={'XXX': 'xxx'}).content
+
 
 def test_replicas_status_handler():
-    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", "test_replicas_status_handler")) as cluster:
+    with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler",
+                                          "test_replicas_status_handler")) as cluster:
         assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'bad'}).status_code
+        assert 404 == cluster.instance.http_request('test_replicas_status', method='GET',
+                                                    headers={'XXX': 'bad'}).status_code
 
-        assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code
+        assert 404 == cluster.instance.http_request('test_replicas_status', method='POST',
+                                                    headers={'XXX': 'xxx'}).status_code
 
-        assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code
-        assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content
+        assert 200 == cluster.instance.http_request('test_replicas_status', method='GET',
+                                                    headers={'XXX': 'xxx'}).status_code
+        assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET',
+                                                        headers={'XXX': 'xxx'}).content

tests/integration/test_https_replication/test.py

@@ -1,30 +1,36 @@
-import time
-import pytest
-
-from helpers.cluster import ClickHouseCluster
-
-from helpers.test_tools import assert_eq_with_retry
-from helpers.network import PartitionManager
-from multiprocessing.dummy import Pool
 import random
+import time
+from multiprocessing.dummy import Pool
+
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.network import PartitionManager
+from helpers.test_tools import assert_eq_with_retry
 
 """
 Both ssl_conf.xml and no_ssl_conf.xml have the same port
 """
 
+
 def _fill_nodes(nodes, shard):
     for node in nodes:
         node.query(
-        '''
-            CREATE DATABASE test;
+            '''
+                CREATE DATABASE test;
+    
+                CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
+                ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192);
+            '''.format(shard=shard, replica=node.name))
 
-            CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
-            ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192);
-        '''.format(shard=shard, replica=node.name))
 
 cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
-node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
+node1 = cluster.add_instance('node1',
+                             main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
+                                           "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
+node2 = cluster.add_instance('node2',
+                             main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
+                                           "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
+
 
 @pytest.fixture(scope="module")
 def both_https_cluster():
@@ -38,6 +44,7 @@ def both_https_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_both_https(both_https_cluster):
     node1.query("insert into test_table values ('2017-06-16', 111, 0)")
 
@@ -77,9 +84,11 @@ def test_replication_after_partition(both_https_cluster):
     assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100')
 
 
+node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
+                             with_zookeeper=True)
+node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
+                             with_zookeeper=True)
 
-node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True)
-node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True)
 
 @pytest.fixture(scope="module")
 def both_http_cluster():
@@ -93,6 +102,7 @@ def both_http_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_both_http(both_http_cluster):
     node3.query("insert into test_table values ('2017-06-16', 111, 0)")
 
@@ -104,8 +114,13 @@ def test_both_http(both_http_cluster):
     assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111\n222')
     assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111\n222')
 
-node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
-node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True)
+
+node5 = cluster.add_instance('node5',
+                             main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
+                                           "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
+node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
+                             with_zookeeper=True)
+
 
 @pytest.fixture(scope="module")
 def mixed_protocol_cluster():
@@ -119,6 +134,7 @@ def mixed_protocol_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_mixed_protocol(mixed_protocol_cluster):
     node5.query("insert into test_table values ('2017-06-16', 111, 0)")
 

tests/integration/test_inherit_multiple_profiles/test.py

@@ -4,7 +4,6 @@ from helpers.client import QueryRuntimeException
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import TSV
 
-
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance('instance',
                                 user_configs=['configs/combined_profile.xml'])