thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14864 from bharatnc/ncb/format-integration-tests

Browse Source 
Format and cleanup imports form all *.py integration test files
This commit is contained in:
alexey-milovidov 2020-09-16 20:40:57 +03:00 committed by GitHub
commit 84b210f93e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
185 changed files with 4660 additions and 3091 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/integration/conftest.py
View File

@ -1,5 +1,6 @@
from helpers.test_tools import TSV from helpers.test_tools import TSV
def pytest_assertrepr_compare(op, left, right): def pytest_assertrepr_compare(op, left, right):
if isinstance(left, TSV) and isinstance(right, TSV) and op == '==': if isinstance(left, TSV) and isinstance(right, TSV) and op == '==':
return ['TabSeparated values differ: '] + left.diff(right) return ['TabSeparated values differ: '] + left.diff(right)

37
tests/integration/helpers/client.py
View File

@ -1,8 +1,7 @@
import errno
import subprocess as sp
from threading import Timer
import tempfile
import os import os
import subprocess as sp
import tempfile
from threading import Timer
class Client: class Client:
@ -16,12 +15,13 @@ class Client:
self.command += ['--host', self.host, '--port', str(self.port), '--stacktrace'] self.command += ['--host', self.host, '--port', str(self.port), '--stacktrace']
def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
ignore_error=False):
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
password=password, database=database, ignore_error=ignore_error).get_answer()
def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error).get_answer() ignore_error=False):
def get_query_request(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False):
command = self.command[:] command = self.command[:]
if stdin is None: if stdin is None:
@ -45,14 +45,17 @@ class Client:
return CommandRequest(command, stdin, timeout, ignore_error) return CommandRequest(command, stdin, timeout, ignore_error)
def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
database=None):
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
password=password, database=database).get_error()
def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_error() database=None):
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
password=password, database=database).get_answer_and_error()
def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None):
return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database).get_answer_and_error()
class QueryTimeoutExceedException(Exception): class QueryTimeoutExceedException(Exception):
pass pass
@ -90,7 +93,6 @@ class CommandRequest:
self.timer = Timer(timeout, kill_process) self.timer = Timer(timeout, kill_process)
self.timer.start() self.timer.start()
def get_answer(self): def get_answer(self):
self.process.wait() self.process.wait()
self.stdout_file.seek(0) self.stdout_file.seek(0)
@ -103,11 +105,11 @@ class CommandRequest:
raise QueryTimeoutExceedException('Client timed out!') raise QueryTimeoutExceedException('Client timed out!')
if (self.process.returncode != 0 or stderr) and not self.ignore_error: if (self.process.returncode != 0 or stderr) and not self.ignore_error:
raise QueryRuntimeException('Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr)) raise QueryRuntimeException(
'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr))
return stdout return stdout
def get_error(self): def get_error(self):
self.process.wait() self.process.wait()
self.stdout_file.seek(0) self.stdout_file.seek(0)
@ -124,7 +126,6 @@ class CommandRequest:
return stderr return stderr
def get_answer_and_error(self): def get_answer_and_error(self):
self.process.wait() self.process.wait()
self.stdout_file.seek(0) self.stdout_file.seek(0)

134
tests/integration/helpers/cluster.py
View File

@ -1,30 +1,31 @@
import base64 import base64
import cassandra.cluster
import docker
import errno import errno
import httplib import httplib
import logging import logging
import os import os
import os.path as p import os.path as p
import pprint import pprint
import psycopg2
import pwd import pwd
import pymongo
import pymysql
import re import re
import requests
import shutil import shutil
import socket import socket
import subprocess import subprocess
import time import time
import urllib
import traceback import traceback
import urllib
import cassandra.cluster
import docker
import psycopg2
import pymongo
import pymysql
import requests
import xml.dom.minidom import xml.dom.minidom
from confluent.schemaregistry.client import CachedSchemaRegistryClient
from dicttoxml import dicttoxml from dicttoxml import dicttoxml
from kazoo.client import KazooClient from kazoo.client import KazooClient
from kazoo.exceptions import KazooException from kazoo.exceptions import KazooException
from minio import Minio from minio import Minio
from confluent.schemaregistry.client import CachedSchemaRegistryClient
from .client import Client from .client import Client
from .hdfs_api import HDFSApi from .hdfs_api import HDFSApi
@ -67,13 +68,14 @@ def get_odbc_bridge_path():
return '/usr/bin/clickhouse-odbc-bridge' return '/usr/bin/clickhouse-odbc-bridge'
return path return path
def get_docker_compose_path(): def get_docker_compose_path():
compose_path = os.environ.get('DOCKER_COMPOSE_DIR') compose_path = os.environ.get('DOCKER_COMPOSE_DIR')
if compose_path is not None: if compose_path is not None:
return os.path.dirname(compose_path) return os.path.dirname(compose_path)
else: else:
if os.path.exists(os.path.dirname('/compose/')): if os.path.exists(os.path.dirname('/compose/')):
return os.path.dirname('/compose/') #default in docker runner container return os.path.dirname('/compose/') # default in docker runner container
else: else:
print("Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {}".format(LOCAL_DOCKER_COMPOSE_DIR)) print("Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {}".format(LOCAL_DOCKER_COMPOSE_DIR))
return LOCAL_DOCKER_COMPOSE_DIR return LOCAL_DOCKER_COMPOSE_DIR
@ -91,12 +93,12 @@ class ClickHouseCluster:
def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None,
odbc_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None): odbc_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None):
for param in os.environ.keys(): for param in os.environ.keys():
print "ENV %40s %s" % (param,os.environ[param]) print "ENV %40s %s" % (param, os.environ[param])
self.base_dir = p.dirname(base_path) self.base_dir = p.dirname(base_path)
self.name = name if name is not None else '' self.name = name if name is not None else ''
self.base_config_dir = base_config_dir or os.environ.get('CLICKHOUSE_TESTS_BASE_CONFIG_DIR', self.base_config_dir = base_config_dir or os.environ.get('CLICKHOUSE_TESTS_BASE_CONFIG_DIR',
'/etc/clickhouse-server/') '/etc/clickhouse-server/')
self.server_bin_path = p.realpath( self.server_bin_path = p.realpath(
server_bin_path or os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH', '/usr/bin/clickhouse')) server_bin_path or os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH', '/usr/bin/clickhouse'))
self.odbc_bridge_bin_path = p.realpath(odbc_bridge_bin_path or get_odbc_bridge_path()) self.odbc_bridge_bin_path = p.realpath(odbc_bridge_bin_path or get_odbc_bridge_path())
@ -165,8 +167,10 @@ class ClickHouseCluster:
cmd += " client" cmd += " client"
return cmd return cmd
def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries = None, macros=None, def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None,
with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, macros=None,
with_zookeeper=False, with_mysql=False, with_kafka=False, with_rabbitmq=False,
clickhouse_path_dir=None,
with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False,
with_redis=False, with_minio=False, with_cassandra=False, with_redis=False, with_minio=False, with_cassandra=False,
hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None,
@ -247,7 +251,8 @@ class ClickHouseCluster:
self.with_mysql = True self.with_mysql = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')])
self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
cmds.append(self.base_mysql_cmd) cmds.append(self.base_mysql_cmd)
@ -255,7 +260,8 @@ class ClickHouseCluster:
self.with_postgres = True self.with_postgres = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')])
self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]
cmds.append(self.base_postgres_cmd) cmds.append(self.base_postgres_cmd)
if with_odbc_drivers and not self.with_odbc_drivers: if with_odbc_drivers and not self.with_odbc_drivers:
@ -264,7 +270,8 @@ class ClickHouseCluster:
self.with_mysql = True self.with_mysql = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')])
self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_mysql_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
cmds.append(self.base_mysql_cmd) cmds.append(self.base_mysql_cmd)
if not self.with_postgres: if not self.with_postgres:
@ -279,28 +286,32 @@ class ClickHouseCluster:
self.with_kafka = True self.with_kafka = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')])
self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]
cmds.append(self.base_kafka_cmd) cmds.append(self.base_kafka_cmd)
if with_rabbitmq and not self.with_rabbitmq: if with_rabbitmq and not self.with_rabbitmq:
self.with_rabbitmq = True self.with_rabbitmq = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')])
self.base_rabbitmq_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_rabbitmq_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]
cmds.append(self.base_rabbitmq_cmd) cmds.append(self.base_rabbitmq_cmd)
if with_hdfs and not self.with_hdfs: if with_hdfs and not self.with_hdfs:
self.with_hdfs = True self.with_hdfs = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')])
self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]
cmds.append(self.base_hdfs_cmd) cmds.append(self.base_hdfs_cmd)
if with_mongo and not self.with_mongo: if with_mongo and not self.with_mongo:
self.with_mongo = True self.with_mongo = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')])
self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_mongo_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]
cmds.append(self.base_mongo_cmd) cmds.append(self.base_mongo_cmd)
if self.with_net_trics: if self.with_net_trics:
@ -311,21 +322,24 @@ class ClickHouseCluster:
self.with_redis = True self.with_redis = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')])
self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_redis_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]
if with_minio and not self.with_minio: if with_minio and not self.with_minio:
self.with_minio = True self.with_minio = True
self.minio_certs_dir = minio_certs_dir self.minio_certs_dir = minio_certs_dir
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')])
self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_minio_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]
cmds.append(self.base_minio_cmd) cmds.append(self.base_minio_cmd)
if with_cassandra and not self.with_cassandra: if with_cassandra and not self.with_cassandra:
self.with_cassandra = True self.with_cassandra = True
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]) self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')])
self.base_cassandra_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.base_cassandra_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')] self.project_name, '--file',
p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]
return instance return instance
@ -390,7 +404,8 @@ class ClickHouseCluster:
print("Container {} uses image {}: ".format(container_id, image_id)) print("Container {} uses image {}: ".format(container_id, image_id))
pprint.pprint(image_info) pprint.pprint(image_info)
print("") print("")
message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, exit_code, output) message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id,
exit_code, output)
if nothrow: if nothrow:
print(message) print(message)
else: else:
@ -401,7 +416,8 @@ class ClickHouseCluster:
with open(local_path, 'r') as fdata: with open(local_path, 'r') as fdata:
data = fdata.read() data = fdata.read()
encoded_data = base64.b64encode(data) encoded_data = base64.b64encode(data)
self.exec_in_container(container_id, ["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)], self.exec_in_container(container_id,
["bash", "-c", "echo {} | base64 --decode > {}".format(encoded_data, dest_path)],
user='root') user='root')
def wait_mysql_to_start(self, timeout=60): def wait_mysql_to_start(self, timeout=60):
@ -650,7 +666,6 @@ class ClickHouseCluster:
subprocess_check_call(clickhouse_start_cmd) subprocess_check_call(clickhouse_start_cmd)
print("ClickHouse instance created") print("ClickHouse instance created")
start_deadline = time.time() + 20.0 # seconds start_deadline = time.time() + 20.0 # seconds
for instance in self.instances.itervalues(): for instance in self.instances.itervalues():
instance.docker_client = self.docker_client instance.docker_client = self.docker_client
@ -692,7 +707,7 @@ class ClickHouseCluster:
try: try:
subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans']) subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans'])
except Exception as e: except Exception as e:
print "Down + remove orphans failed durung shutdown. {}".format(repr(e)) print "Down + remove orphans failed durung shutdown. {}".format(repr(e))
self.is_up = False self.is_up = False
@ -704,23 +719,26 @@ class ClickHouseCluster:
instance.client = None instance.client = None
if not self.zookeeper_use_tmpfs: if not self.zookeeper_use_tmpfs:
for i in range(1, 4): for i in range(1, 4):
zk_data_path = self.instances_dir + '/zkdata' + str(i) zk_data_path = self.instances_dir + '/zkdata' + str(i)
zk_log_data_path = self.instances_dir + '/zklog' + str(i) zk_log_data_path = self.instances_dir + '/zklog' + str(i)
if os.path.exists(zk_data_path): if os.path.exists(zk_data_path):
shutil.rmtree(zk_data_path) shutil.rmtree(zk_data_path)
if os.path.exists(zk_log_data_path): if os.path.exists(zk_log_data_path):
shutil.rmtree(zk_log_data_path) shutil.rmtree(zk_log_data_path)
if sanitizer_assert_instance is not None: if sanitizer_assert_instance is not None:
raise Exception("Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) raise Exception(
"Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance))
def pause_container(self, instance_name): def pause_container(self, instance_name):
subprocess_check_call(self.base_cmd + ['pause', instance_name]) subprocess_check_call(self.base_cmd + ['pause', instance_name])
# subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name]) # subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name])
def unpause_container(self, instance_name): def unpause_container(self, instance_name):
subprocess_check_call(self.base_cmd + ['unpause', instance_name]) subprocess_check_call(self.base_cmd + ['unpause', instance_name])
# subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name]) # subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name])
def open_bash_shell(self, instance_name): def open_bash_shell(self, instance_name):
@ -790,9 +808,12 @@ services:
class ClickHouseInstance: class ClickHouseInstance:
def __init__( def __init__(
self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, custom_dictionaries, self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs,
macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo, with_redis, with_minio, custom_dictionaries,
with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables=None, macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, with_rabbitmq, with_mongo,
with_redis, with_minio,
with_cassandra, server_bin_path, odbc_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers,
hostname=None, env_variables=None,
image="yandex/clickhouse-integration-test", tag="latest", image="yandex/clickhouse-integration-test", tag="latest",
stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
@ -848,15 +869,19 @@ class ClickHouseInstance:
return "-fsanitize=thread" in build_opts return "-fsanitize=thread" in build_opts
# Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer
def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error) ignore_error=False):
return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password,
database=database, ignore_error=ignore_error)
def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False, def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None,
ignore_error=False,
retry_count=20, sleep_time=0.5, check_callback=lambda x: True): retry_count=20, sleep_time=0.5, check_callback=lambda x: True):
result = None result = None
for i in range(retry_count): for i in range(retry_count):
try: try:
result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error) result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password,
database=database, ignore_error=ignore_error)
if check_callback(result): if check_callback(result):
return result return result
time.sleep(sleep_time) time.sleep(sleep_time)
@ -873,12 +898,16 @@ class ClickHouseInstance:
return self.client.get_query_request(*args, **kwargs) return self.client.get_query_request(*args, **kwargs)
# Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code
def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) database=None):
return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user,
password=password, database=database)
# The same as query_and_get_error but ignores successful query. # The same as query_and_get_error but ignores successful query.
def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None,
return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) database=None):
return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings,
user=user, password=password, database=database)
# Connects to the instance via HTTP interface, sends a query and returns the answer # Connects to the instance via HTTP interface, sends a query and returns the answer
def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False): def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False):
@ -900,7 +929,8 @@ class ClickHouseInstance:
open_result = urllib.urlopen(url, data) open_result = urllib.urlopen(url, data)
def http_code_and_message(): def http_code_and_message():
return str(open_result.getcode()) + " " + httplib.responses[open_result.getcode()] + ": " + open_result.read() return str(open_result.getcode()) + " " + httplib.responses[
open_result.getcode()] + ": " + open_result.read()
if expect_fail_and_get_error: if expect_fail_and_get_error:
if open_result.getcode() == 200: if open_result.getcode() == 200:
@ -913,18 +943,19 @@ class ClickHouseInstance:
# Connects to the instance via HTTP interface, sends a query and returns the answer # Connects to the instance via HTTP interface, sends a query and returns the answer
def http_request(self, url, method='GET', params=None, data=None, headers=None): def http_request(self, url, method='GET', params=None, data=None, headers=None):
url = "http://" + self.ip_address + ":8123/"+url url = "http://" + self.ip_address + ":8123/" + url
return requests.request(method=method, url=url, params=params, data=data, headers=headers) return requests.request(method=method, url=url, params=params, data=data, headers=headers)
# Connects to the instance via HTTP interface, sends a query, expects an error and return the error message # Connects to the instance via HTTP interface, sends a query, expects an error and return the error message
def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None): def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None):
return self.http_query(sql=sql, data=data, params=params, user=user, password=password, expect_fail_and_get_error=True) return self.http_query(sql=sql, data=data, params=params, user=user, password=password,
expect_fail_and_get_error=True)
def kill_clickhouse(self, stop_start_wait_sec=5): def kill_clickhouse(self, stop_start_wait_sec=5):
pid = self.get_process_pid("clickhouse") pid = self.get_process_pid("clickhouse")
if not pid: if not pid:
raise Exception("No clickhouse found") raise Exception("No clickhouse found")
self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root') self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root')
time.sleep(stop_start_wait_sec) time.sleep(stop_start_wait_sec)
def restore_clickhouse(self, retries=100): def restore_clickhouse(self, retries=100):
@ -1030,7 +1061,8 @@ class ClickHouseInstance:
time_left = deadline - current_time time_left = deadline - current_time
if deadline is not None and current_time >= deadline: if deadline is not None and current_time >= deadline:
raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. " raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. "
"Container status: {}, logs: {}".format(self.name, self.ip_address, status, handle.logs())) "Container status: {}, logs: {}".format(self.name, self.ip_address, status,
handle.logs()))
# Repeatedly poll the instance address until there is something that listens there. # Repeatedly poll the instance address until there is something that listens there.
# Usually it means that ClickHouse is ready to accept queries. # Usually it means that ClickHouse is ready to accept queries.

21
tests/integration/helpers/dictionary.py
View File

@ -59,7 +59,8 @@ class Row(object):
class Field(object): class Field(object):
def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False, range_hash_type=None, default_value_for_get=None): def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False,
range_hash_type=None, default_value_for_get=None):
self.name = name self.name = name
self.field_type = field_type self.field_type = field_type
self.is_key = is_key self.is_key = is_key
@ -123,7 +124,8 @@ class DictionaryStructure(object):
self.range_key = field self.range_key = field
if not self.layout.is_complex and len(self.keys) > 1: if not self.layout.is_complex and len(self.keys) > 1:
raise Exception("More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name)) raise Exception(
"More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name))
if self.layout.is_ranged and (not self.range_key or len(self.range_fields) != 2): if self.layout.is_ranged and (not self.range_key or len(self.range_fields) != 2):
raise Exception("Inconsistent configuration of ranged dictionary") raise Exception("Inconsistent configuration of ranged dictionary")
@ -213,7 +215,8 @@ class DictionaryStructure(object):
if or_default: if or_default:
or_default_expr = 'OrDefault' or_default_expr = 'OrDefault'
if field.default_value_for_get is None: if field.default_value_for_get is None:
raise Exception("Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name)) raise Exception(
"Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name))
val = field.default_value_for_get val = field.default_value_for_get
if isinstance(val, str): if isinstance(val, str):
@ -259,15 +262,16 @@ class DictionaryStructure(object):
def get_get_or_default_expressions(self, dict_name, field, row): def get_get_or_default_expressions(self, dict_name, field, row):
if not self.layout.is_ranged: if not self.layout.is_ranged:
return [ return [
self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False, has=False), self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False,
has=False),
self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=True, has=False), self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=True, has=False),
] ]
return [] return []
def get_has_expressions(self, dict_name, field, row): def get_has_expressions(self, dict_name, field, row):
if not self.layout.is_ranged: if not self.layout.is_ranged:
return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, has=True)] return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False,
has=True)]
return [] return []
def get_hierarchical_expressions(self, dict_name, row): def get_hierarchical_expressions(self, dict_name, row):
@ -290,7 +294,7 @@ class DictionaryStructure(object):
"dictIsIn('{dict_name}', {child_key}, {parent_key})".format( "dictIsIn('{dict_name}', {child_key}, {parent_key})".format(
dict_name=dict_name, dict_name=dict_name,
child_key=child_key_expr, child_key=child_key_expr,
parent_key=parent_key_expr,) parent_key=parent_key_expr, )
] ]
return [] return []
@ -364,7 +368,8 @@ class Dictionary(object):
return ['select {}'.format(expr) for expr in self.structure.get_get_expressions(self.name, field, row)] return ['select {}'.format(expr) for expr in self.structure.get_get_expressions(self.name, field, row)]
def get_select_get_or_default_queries(self, field, row): def get_select_get_or_default_queries(self, field, row):
return ['select {}'.format(expr) for expr in self.structure.get_get_or_default_expressions(self.name, field, row)] return ['select {}'.format(expr) for expr in
self.structure.get_get_or_default_expressions(self.name, field, row)]
def get_select_has_queries(self, field, row): def get_select_has_queries(self, field, row):
return ['select {}'.format(expr) for expr in self.structure.get_has_expressions(self.name, field, row)] return ['select {}'.format(expr) for expr in self.structure.get_has_expressions(self.name, field, row)]

72
tests/integration/helpers/external_sources.py
View File

@ -1,14 +1,15 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import warnings
import pymysql.cursors
import pymongo
import cassandra.cluster
import redis
import aerospike
from tzlocal import get_localzone
import datetime import datetime
import os import os
import uuid import uuid
import warnings
import aerospike
import cassandra.cluster
import pymongo
import pymysql.cursors
import redis
from tzlocal import get_localzone
class ExternalSource(object): class ExternalSource(object):
@ -89,12 +90,12 @@ class SourceMySQL(ExternalSource):
<db>test</db> <db>test</db>
<table>{tbl}</table> <table>{tbl}</table>
</mysql>'''.format( </mysql>'''.format(
hostname=self.docker_hostname, hostname=self.docker_hostname,
port=self.docker_port, port=self.docker_port,
user=self.user, user=self.user,
password=self.password, password=self.password,
tbl=table_name, tbl=table_name,
) )
def prepare(self, structure, table_name, cluster): def prepare(self, structure, table_name, cluster):
self.create_mysql_conn() self.create_mysql_conn()
@ -160,7 +161,8 @@ class SourceMongo(ExternalSource):
if field.field_type == "Date": if field.field_type == "Date":
self.converters[field.name] = lambda x: datetime.datetime.strptime(x, "%Y-%m-%d") self.converters[field.name] = lambda x: datetime.datetime.strptime(x, "%Y-%m-%d")
elif field.field_type == "DateTime": elif field.field_type == "DateTime":
self.converters[field.name] = lambda x: get_localzone().localize(datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")) self.converters[field.name] = lambda x: get_localzone().localize(
datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S"))
else: else:
self.converters[field.name] = lambda x: x self.converters[field.name] = lambda x: x
@ -180,6 +182,7 @@ class SourceMongo(ExternalSource):
result = tbl.insert_many(to_insert) result = tbl.insert_many(to_insert)
class SourceMongoURI(SourceMongo): class SourceMongoURI(SourceMongo):
def compatible_with_layout(self, layout): def compatible_with_layout(self, layout):
# It is enough to test one layout for this dictionary, since we're # It is enough to test one layout for this dictionary, since we're
@ -200,6 +203,7 @@ class SourceMongoURI(SourceMongo):
tbl=table_name, tbl=table_name,
) )
class SourceClickHouse(ExternalSource): class SourceClickHouse(ExternalSource):
def get_source_str(self, table_name): def get_source_str(self, table_name):
@ -284,7 +288,8 @@ class SourceFile(ExternalSource):
sorted_row.append(str(row.data[name])) sorted_row.append(str(row.data[name]))
str_data = '\t'.join(sorted_row) str_data = '\t'.join(sorted_row)
self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user="root") self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
user="root")
def compatible_with_layout(self, layout): def compatible_with_layout(self, layout):
return 'cache' not in layout.name and 'direct' not in layout.name return 'cache' not in layout.name and 'direct' not in layout.name
@ -324,7 +329,8 @@ class _SourceExecutableBase(ExternalSource):
sorted_row.append(str(row.data[name])) sorted_row.append(str(row.data[name]))
str_data = '\t'.join(sorted_row) str_data = '\t'.join(sorted_row)
self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root') self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
user='root')
class SourceExecutableCache(_SourceExecutableBase): class SourceExecutableCache(_SourceExecutableBase):
@ -344,12 +350,14 @@ class SourceExecutableHashed(_SourceExecutableBase):
def compatible_with_layout(self, layout): def compatible_with_layout(self, layout):
return 'cache' in layout.name return 'cache' in layout.name
class SourceHTTPBase(ExternalSource):
class SourceHTTPBase(ExternalSource):
PORT_COUNTER = 5555 PORT_COUNTER = 5555
def get_source_str(self, table_name): def get_source_str(self, table_name):
self.http_port = SourceHTTPBase.PORT_COUNTER self.http_port = SourceHTTPBase.PORT_COUNTER
url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname, port=self.http_port) url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname,
port=self.http_port)
SourceHTTPBase.PORT_COUNTER += 1 SourceHTTPBase.PORT_COUNTER += 1
return ''' return '''
<http> <http>
@ -395,7 +403,8 @@ class SourceHTTPBase(ExternalSource):
sorted_row.append(str(row.data[name])) sorted_row.append(str(row.data[name]))
str_data = '\t'.join(sorted_row) str_data = '\t'.join(sorted_row)
self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], user='root') self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)],
user='root')
class SourceHTTP(SourceHTTPBase): class SourceHTTP(SourceHTTPBase):
@ -407,6 +416,7 @@ class SourceHTTPS(SourceHTTPBase):
def _get_schema(self): def _get_schema(self):
return "https" return "https"
class SourceCassandra(ExternalSource): class SourceCassandra(ExternalSource):
TYPE_MAPPING = { TYPE_MAPPING = {
'UInt8': 'tinyint', 'UInt8': 'tinyint',
@ -426,7 +436,8 @@ class SourceCassandra(ExternalSource):
} }
def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password): def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password):
ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password) ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user,
password)
self.structure = dict() self.structure = dict()
def get_source_str(self, table_name): def get_source_str(self, table_name):
@ -448,13 +459,14 @@ class SourceCassandra(ExternalSource):
def prepare(self, structure, table_name, cluster): def prepare(self, structure, table_name, cluster):
self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port)
self.session = self.client.connect() self.session = self.client.connect()
self.session.execute("create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") self.session.execute(
"create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};")
self.session.execute('drop table if exists test."{}"'.format(table_name)) self.session.execute('drop table if exists test."{}"'.format(table_name))
self.structure[table_name] = structure self.structure[table_name] = structure
columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()]
keys = ['"' + col.name + '"' for col in structure.keys] keys = ['"' + col.name + '"' for col in structure.keys]
query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format( query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format(
name=table_name, columns=', '.join(columns), pk=', '.join(keys)) name=table_name, columns=', '.join(columns), pk=', '.join(keys))
self.session.execute(query) self.session.execute(query)
self.prepared = True self.prepared = True
@ -470,14 +482,16 @@ class SourceCassandra(ExternalSource):
names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()] names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()]
columns = ['"' + col[0] + '"' for col in names_and_types] columns = ['"' + col[0] + '"' for col in names_and_types]
insert = 'insert into test."{table}" ({columns}) values ({args})'.format( insert = 'insert into test."{table}" ({columns}) values ({args})'.format(
table=table_name, columns=','.join(columns), args=','.join(['%s']*len(columns))) table=table_name, columns=','.join(columns), args=','.join(['%s'] * len(columns)))
for row in data: for row in data:
values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types] values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types]
self.session.execute(insert, values) self.session.execute(insert, values)
class SourceRedis(ExternalSource): class SourceRedis(ExternalSource):
def __init__( def __init__(
self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index, storage_type self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index,
storage_type
): ):
super(SourceRedis, self).__init__( super(SourceRedis, self).__init__(
name, internal_hostname, internal_port, docker_hostname, docker_port, user, password name, internal_hostname, internal_port, docker_hostname, docker_port, user, password
@ -503,7 +517,8 @@ class SourceRedis(ExternalSource):
) )
def prepare(self, structure, table_name, cluster): def prepare(self, structure, table_name, cluster):
self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index, password=self.password or None) self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index,
password=self.password or None)
self.prepared = True self.prepared = True
self.ordered_names = structure.get_ordered_names() self.ordered_names = structure.get_ordered_names()
@ -521,11 +536,12 @@ class SourceRedis(ExternalSource):
def compatible_with_layout(self, layout): def compatible_with_layout(self, layout):
return layout.is_simple and self.storage_type == "simple" or layout.is_complex and self.storage_type == "hash_map" return layout.is_simple and self.storage_type == "simple" or layout.is_complex and self.storage_type == "hash_map"
class SourceAerospike(ExternalSource): class SourceAerospike(ExternalSource):
def __init__(self, name, internal_hostname, internal_port, def __init__(self, name, internal_hostname, internal_port,
docker_hostname, docker_port, user, password): docker_hostname, docker_port, user, password):
ExternalSource.__init__(self, name, internal_hostname, internal_port, ExternalSource.__init__(self, name, internal_hostname, internal_port,
docker_hostname, docker_port, user, password) docker_hostname, docker_port, user, password)
self.namespace = "test" self.namespace = "test"
self.set = "test_set" self.set = "test_set"
@ -543,7 +559,7 @@ class SourceAerospike(ExternalSource):
def prepare(self, structure, table_name, cluster): def prepare(self, structure, table_name, cluster):
config = { config = {
'hosts': [ (self.internal_hostname, self.internal_port) ] 'hosts': [(self.internal_hostname, self.internal_port)]
} }
self.client = aerospike.client(config).connect() self.client = aerospike.client(config).connect()
self.prepared = True self.prepared = True
@ -580,7 +596,7 @@ class SourceAerospike(ExternalSource):
self.client.put(key, {"bin_value": value[1]}, policy={"key": aerospike.POLICY_KEY_SEND}) self.client.put(key, {"bin_value": value[1]}, policy={"key": aerospike.POLICY_KEY_SEND})
assert self.client.exists(key) assert self.client.exists(key)
else: else:
assert("VALUES SIZE != 2") assert ("VALUES SIZE != 2")
# print(values) # print(values)

24
tests/integration/helpers/hdfs_api.py
View File

@ -1,10 +1,12 @@
#-*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import StringIO import StringIO
import gzip import gzip
import requests
import subprocess import subprocess
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
import requests
class HDFSApi(object): class HDFSApi(object):
def __init__(self, user): def __init__(self, user):
self.host = "localhost" self.host = "localhost"
@ -13,11 +15,15 @@ class HDFSApi(object):
self.user = user self.user = user
def read_data(self, path): def read_data(self, path):
response = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port, path=path), allow_redirects=False) response = requests.get(
"http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port,
path=path), allow_redirects=False)
if response.status_code != 307: if response.status_code != 307:
response.raise_for_status() response.raise_for_status()
additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) additional_params = '&'.join(response.headers['Location'].split('&')[1:2])
response_data = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port, path=path, params=additional_params)) response_data = requests.get(
"http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port,
path=path, params=additional_params))
if response_data.status_code != 200: if response_data.status_code != 200:
response_data.raise_for_status() response_data.raise_for_status()
@ -25,7 +31,9 @@ class HDFSApi(object):
# Requests can't put file # Requests can't put file
def _curl_to_put(self, filename, path, params): def _curl_to_put(self, filename, path, params):
url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host, port=self.http_data_port, path=path, params=params) url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host,
port=self.http_data_port, path=path,
params=params)
cmd = "curl -s -i -X PUT -T {fname} '{url}'".format(fname=filename, url=url) cmd = "curl -s -i -X PUT -T {fname} '{url}'".format(fname=filename, url=url)
output = subprocess.check_output(cmd, shell=True) output = subprocess.check_output(cmd, shell=True)
return output return output
@ -36,13 +44,15 @@ class HDFSApi(object):
named_file.write(content) named_file.write(content)
named_file.flush() named_file.flush()
response = requests.put( response = requests.put(
"http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port, path=path, user=self.user), "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port,
path=path, user=self.user),
allow_redirects=False allow_redirects=False
) )
if response.status_code != 307: if response.status_code != 307:
response.raise_for_status() response.raise_for_status()
additional_params = '&'.join(response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) additional_params = '&'.join(
response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"])
output = self._curl_to_put(fpath, path, additional_params) output = self._curl_to_put(fpath, path, additional_params)
if "201 Created" not in output: if "201 Created" not in output:
raise Exception("Can't create file on hdfs:\n {}".format(output)) raise Exception("Can't create file on hdfs:\n {}".format(output))

7
tests/integration/helpers/http_server.py
View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import argparse import argparse
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer import csv
import socket import socket
import ssl import ssl
import csv from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
# Decorator used to see if authentication works for external dictionary who use a HTTP source. # Decorator used to see if authentication works for external dictionary who use a HTTP source.
@ -15,6 +15,7 @@ def check_auth(fn):
req.send_response(401) req.send_response(401)
else: else:
fn(req) fn(req)
return wrapper return wrapper
@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family):
self.send_header('Content-type', 'text/tsv') self.send_header('Content-type', 'text/tsv')
self.end_headers() self.end_headers()
def __send_data(self, only_ids = None): def __send_data(self, only_ids=None):
with open(data_path, 'r') as fl: with open(data_path, 'r') as fl:
reader = csv.reader(fl, delimiter='\t') reader = csv.reader(fl, delimiter='\t')
for row in reader: for row in reader:

14
tests/integration/helpers/network.py
View File

@ -1,12 +1,9 @@
import os.path as p import os
import subprocess import subprocess
import time import time
import os
import docker import docker
from .cluster import CLICKHOUSE_ROOT_DIR
class PartitionManager: class PartitionManager:
"""Allows introducing failures in the network between docker containers. """Allows introducing failures in the network between docker containers.
@ -23,21 +20,18 @@ class PartitionManager:
def __init__(self): def __init__(self):
self._iptables_rules = [] self._iptables_rules = []
def drop_instance_zk_connections(self, instance, action='DROP'): def drop_instance_zk_connections(self, instance, action='DROP'):
self._check_instance(instance) self._check_instance(instance)
self._add_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) self._add_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action})
self._add_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) self._add_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action})
def restore_instance_zk_connections(self, instance, action='DROP'): def restore_instance_zk_connections(self, instance, action='DROP'):
self._check_instance(instance) self._check_instance(instance)
self._delete_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) self._delete_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action})
self._delete_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) self._delete_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action})
def partition_instances(self, left, right, port=None, action='DROP'): def partition_instances(self, left, right, port=None, action='DROP'):
self._check_instance(left) self._check_instance(left)
self._check_instance(right) self._check_instance(right)
@ -51,7 +45,6 @@ class PartitionManager:
self._add_rule(create_rule(left, right)) self._add_rule(create_rule(left, right))
self._add_rule(create_rule(right, left)) self._add_rule(create_rule(right, left))
def heal_all(self): def heal_all(self):
while self._iptables_rules: while self._iptables_rules:
rule = self._iptables_rules.pop() rule = self._iptables_rules.pop()
@ -66,7 +59,6 @@ class PartitionManager:
for rule in rules: for rule in rules:
self._add_rule(rule) self._add_rule(rule)
@staticmethod @staticmethod
def _check_instance(instance): def _check_instance(instance):
if instance.ip_address is None: if instance.ip_address is None:
@ -152,7 +144,6 @@ class _NetworkManager:
ret.extend(['-j'] + action.split()) ret.extend(['-j'] + action.split())
return ret return ret
def __init__( def __init__(
self, self,
container_expire_timeout=50, container_exit_timeout=60): container_expire_timeout=50, container_exit_timeout=60):
@ -175,7 +166,8 @@ class _NetworkManager:
except docker.errors.NotFound: except docker.errors.NotFound:
pass pass
self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper', auto_remove=True, self._container = self._docker_client.containers.run('yandex/clickhouse-integration-helper',
auto_remove=True,
command=('sleep %s' % self.container_exit_timeout), command=('sleep %s' % self.container_exit_timeout),
detach=True, network_mode='host') detach=True, network_mode='host')
container_id = self._container.id container_id = self._container.id

14
tests/integration/helpers/test_tools.py
View File

@ -1,6 +1,7 @@
import difflib import difflib
import time import time
class TSV: class TSV:
"""Helper to get pretty diffs between expected and actual tab-separated value files""" """Helper to get pretty diffs between expected and actual tab-separated value files"""
@ -40,17 +41,22 @@ class TSV:
def toMat(contents): def toMat(contents):
return [line.split("\t") for line in contents.split("\n") if line.strip()] return [line.split("\t") for line in contents.split("\n") if line.strip()]
def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None, settings=None, user=None, ignore_error=False):
def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None,
settings=None, user=None, ignore_error=False):
expectation_tsv = TSV(expectation) expectation_tsv = TSV(expectation)
for i in xrange(retry_count): for i in xrange(retry_count):
try: try:
if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) == expectation_tsv: if TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings,
ignore_error=ignore_error)) == expectation_tsv:
break break
time.sleep(sleep_time) time.sleep(sleep_time)
except Exception as ex: except Exception as ex:
print "assert_eq_with_retry retry {} exception {}".format(i + 1, ex) print "assert_eq_with_retry retry {} exception {}".format(i + 1, ex)
time.sleep(sleep_time) time.sleep(sleep_time)
else: else:
val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, ignore_error=ignore_error)) val = TSV(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings,
ignore_error=ignore_error))
if expectation_tsv != val: if expectation_tsv != val:
raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(expectation_tsv.diff(val, n1="expectation", n2="query")))) raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join(
expectation_tsv.diff(val, n1="expectation", n2="query"))))

3
tests/integration/helpers/uclient.py
View File

@ -11,9 +11,10 @@ import uexpect
prompt = ':\) ' prompt = ':\) '
end_of_block = r'.*\r\n.*\r\n' end_of_block = r'.*\r\n.*\r\n'
class client(object): class client(object):
def __init__(self, command=None, name='', log=None): def __init__(self, command=None, name='', log=None):
self.client = uexpect.spawn(['/bin/bash','--noediting']) self.client = uexpect.spawn(['/bin/bash', '--noediting'])
if command is None: if command is None:
command = '/usr/bin/clickhouse-client' command = '/usr/bin/clickhouse-client'
self.client.command = command self.client.command = command

19
tests/integration/helpers/uexpect.py
View File

@ -13,13 +13,12 @@
# limitations under the License. # limitations under the License.
import os import os
import pty import pty
import time
import sys
import re import re
import time
from threading import Thread, Event
from subprocess import Popen
from Queue import Queue, Empty from Queue import Queue, Empty
from subprocess import Popen
from threading import Thread, Event
class TimeoutError(Exception): class TimeoutError(Exception):
def __init__(self, timeout): def __init__(self, timeout):
@ -28,6 +27,7 @@ class TimeoutError(Exception):
def __str__(self): def __str__(self):
return 'Timeout %.3fs' % float(self.timeout) return 'Timeout %.3fs' % float(self.timeout)
class ExpectTimeoutError(Exception): class ExpectTimeoutError(Exception):
def __init__(self, pattern, timeout, buffer): def __init__(self, pattern, timeout, buffer):
self.pattern = pattern self.pattern = pattern
@ -43,6 +43,7 @@ class ExpectTimeoutError(Exception):
s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]])
return s return s
class IO(object): class IO(object):
class EOF(object): class EOF(object):
pass pass
@ -59,7 +60,7 @@ class IO(object):
self._prefix = prefix self._prefix = prefix
def write(self, data): def write(self, data):
self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) self._logger.write(('\n' + data).replace('\n', '\n' + self._prefix))
def flush(self): def flush(self):
self._logger.flush() self._logger.flush()
@ -165,7 +166,7 @@ class IO(object):
data = '' data = ''
timeleft = timeout timeleft = timeout
try: try:
while timeleft >= 0 : while timeleft >= 0:
start_time = time.time() start_time = time.time()
data += self.queue.get(timeout=timeleft) data += self.queue.get(timeout=timeleft)
if data: if data:
@ -182,6 +183,7 @@ class IO(object):
return data return data
def spawn(command): def spawn(command):
master, slave = pty.openpty() master, slave = pty.openpty()
process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1)
@ -193,7 +195,8 @@ def spawn(command):
thread.daemon = True thread.daemon = True
thread.start() thread.start()
return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) return IO(process, master, queue, reader={'thread': thread, 'kill_event': reader_kill_event})
def reader(process, out, queue, kill_event): def reader(process, out, queue, kill_event):
while True: while True:

4
tests/integration/test_SYSTEM_FLUSH_LOGS/test.py
View File

@ -24,6 +24,7 @@ system_logs = [
# decrease timeout for the test to show possible issues. # decrease timeout for the test to show possible issues.
timeout = pytest.mark.timeout(30) timeout = pytest.mark.timeout(30)
@pytest.fixture(scope='module', autouse=True) @pytest.fixture(scope='module', autouse=True)
def start_cluster(): def start_cluster():
try: try:
@ -32,10 +33,12 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def flush_logs(): def flush_logs():
node.query('SYSTEM FLUSH LOGS') node.query('SYSTEM FLUSH LOGS')
@timeout @timeout
@pytest.mark.parametrize('table,exists', system_logs) @pytest.mark.parametrize('table,exists', system_logs)
def test_system_logs(flush_logs, table, exists): def test_system_logs(flush_logs, table, exists):
@ -45,6 +48,7 @@ def test_system_logs(flush_logs, table, exists):
else: else:
assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q) assert "Table {} doesn't exist".format(table) in node.query_and_get_error(q)
# Logic is tricky, let's check that there is no hang in case of message queue # Logic is tricky, let's check that there is no hang in case of message queue
# is not empty (this is another code path in the code). # is not empty (this is another code path in the code).
@timeout @timeout

3
tests/integration/test_access_control_on_cluster/test.py
View File

@ -1,13 +1,12 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True)
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def started_cluster(): def started_cluster():
try: try:

372
tests/integration/test_adaptive_granularity/test.py
View File

@ -1,31 +1,50 @@
import time import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster
from multiprocessing.dummy import Pool
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
with_zookeeper=True)
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', with_installed_binary=True) node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18',
with_installed_binary=True)
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
with_zookeeper=True)
node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', with_installed_binary=True) node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True) with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15',
with_installed_binary=True)
node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
with_zookeeper=True)
node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
with_installed_binary=True)
node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
with_installed_binary=True)
node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml', 'configs/merge_tree_settings.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True, with_installed_binary=True) 'configs/merge_tree_settings.xml'], with_zookeeper=True,
image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
with_installed_binary=True)
node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
'configs/merge_tree_settings.xml'], with_zookeeper=True,
image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
with_installed_binary=True)
node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True, with_installed_binary=True) with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
with_installed_binary=True)
node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
with_installed_binary=True)
def prepare_single_pair_with_setting(first_node, second_node, group): def prepare_single_pair_with_setting(first_node, second_node, group):
@ -34,80 +53,80 @@ def prepare_single_pair_with_setting(first_node, second_node, group):
# Two tables with adaptive granularity # Two tables with adaptive granularity
first_node.query( first_node.query(
'''
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 10485760
'''.format(g=group))
second_node.query(
'''
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 10485760
'''.format(g=group))
# Two tables with fixed granularity
first_node.query(
'''
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 0
'''.format(g=group))
second_node.query(
'''
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 0
'''.format(g=group))
# Two tables with different granularity
with pytest.raises(QueryRuntimeException):
first_node.query(
''' '''
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes = 10485760 SETTINGS index_granularity_bytes = 10485760
'''.format(g=group)) '''.format(g=group))
second_node.query( second_node.query(
''' '''
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 10485760
'''.format(g=group))
# Two tables with fixed granularity
first_node.query(
'''
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes = 0 SETTINGS index_granularity_bytes = 0
'''.format(g=group)) '''.format(g=group))
# Two tables with different granularity, but enabled mixed parts second_node.query(
first_node.query(
''' '''
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1 SETTINGS index_granularity_bytes = 0
'''.format(g=group)) '''.format(g=group))
# Two tables with different granularity
with pytest.raises(QueryRuntimeException):
first_node.query(
'''
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 10485760
'''.format(g=group))
second_node.query( second_node.query(
''' '''
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1 SETTINGS index_granularity_bytes = 0
'''.format(g=group)) '''.format(g=group))
# Two tables with different granularity, but enabled mixed parts
first_node.query(
'''
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1
'''.format(g=group))
second_node.query(
'''
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2')
PARTITION BY toYYYYMM(date)
ORDER BY id
SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1
'''.format(g=group))
def prepare_single_pair_without_setting(first_node, second_node, group): def prepare_single_pair_without_setting(first_node, second_node, group):
@ -116,21 +135,21 @@ def prepare_single_pair_without_setting(first_node, second_node, group):
# Two tables with fixed granularity # Two tables with fixed granularity
first_node.query( first_node.query(
''' '''
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
'''.format(g=group)) '''.format(g=group))
second_node.query( second_node.query(
''' '''
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes = 0 SETTINGS index_granularity_bytes = 0
'''.format(g=group)) '''.format(g=group))
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -160,7 +179,8 @@ def start_static_cluster():
def test_different_versions_cluster(start_static_cluster, first_node, second_node, table): def test_different_versions_cluster(start_static_cluster, first_node, second_node, table):
counter = 1 counter = 1
for n1, n2 in ((first_node, second_node), (second_node, first_node)): for n1, n2 in ((first_node, second_node), (second_node, first_node)):
n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(tbl=table, c1=counter * 2, c2=counter * 2 + 1)) n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(
tbl=table, c1=counter * 2, c2=counter * 2 + 1))
n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table)) n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2)) assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
n1.query("DETACH TABLE {tbl}".format(tbl=table)) n1.query("DETACH TABLE {tbl}".format(tbl=table))
@ -175,73 +195,74 @@ def test_different_versions_cluster(start_static_cluster, first_node, second_nod
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2)) assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
counter += 1 counter += 1
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_dynamic_cluster(): def start_dynamic_cluster():
try: try:
cluster.start() cluster.start()
node7.query( node7.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node7.query( node7.query(
''' '''
CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
SETTINGS index_granularity_bytes=10485760 SETTINGS index_granularity_bytes=10485760
''') ''')
node8.query( node8.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node9.query( node9.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node10.query( node10.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node11.query( node11.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node12.query( node12.query(
''' '''
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
@pytest.mark.parametrize( @pytest.mark.parametrize(
('n', 'tables'), ('n', 'tables'),
[ [
@ -251,13 +272,16 @@ def start_dynamic_cluster():
) )
def test_version_single_node_update(start_dynamic_cluster, n, tables): def test_version_single_node_update(start_dynamic_cluster, n, tables):
for table in tables: for table in tables:
n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table)) n.query(
"INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table))
n.restart_with_latest_version() n.restart_with_latest_version()
for table in tables: for table in tables:
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n' assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n'
n.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table)) n.query(
"INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table))
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n' assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n'
@pytest.mark.parametrize( @pytest.mark.parametrize(
('node',), ('node',),
[ [
@ -266,27 +290,38 @@ def test_version_single_node_update(start_dynamic_cluster, n, tables):
] ]
) )
def test_mixed_granularity_single_node(start_dynamic_cluster, node): def test_mixed_granularity_single_node(start_dynamic_cluster, node):
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") node.query(
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)") "INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
node.query(
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)")
def callback(n): def callback(n):
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>") n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>") "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
"<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
node.restart_with_latest_version(callback_onstop=callback) node.restart_with_latest_version(callback_onstop=callback)
node.query("SYSTEM RELOAD CONFIG") node.query("SYSTEM RELOAD CONFIG")
assert_eq_with_retry(node, "SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'", '1') assert_eq_with_retry(node,
"SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'",
'1')
assert node.query("SELECT count() from table_with_default_granularity") == '4\n' assert node.query("SELECT count() from table_with_default_granularity") == '4\n'
node.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") node.query(
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
assert node.query("SELECT count() from table_with_default_granularity") == '6\n' assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL") node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL")
assert node.query("SELECT count() from table_with_default_granularity") == '6\n' assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
path_to_merged_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip() path_to_merged_part = node.query(
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_to_merged_part)]) # check that we have adaptive files "SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(
p=path_to_merged_part)]) # check that we have adaptive files
path_to_old_part = node.query("SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip() path_to_old_part = node.query(
"SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip()
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(p=path_to_old_part)]) # check that we have non adaptive files node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(
p=path_to_old_part)]) # check that we have non adaptive files
node.query("ALTER TABLE table_with_default_granularity UPDATE dummy = dummy + 1 WHERE 1") node.query("ALTER TABLE table_with_default_granularity UPDATE dummy = dummy + 1 WHERE 1")
# still works # still works
@ -295,46 +330,54 @@ def test_mixed_granularity_single_node(start_dynamic_cluster, node):
node.query("ALTER TABLE table_with_default_granularity MODIFY COLUMN dummy String") node.query("ALTER TABLE table_with_default_granularity MODIFY COLUMN dummy String")
node.query("ALTER TABLE table_with_default_granularity ADD COLUMN dummy2 Float64") node.query("ALTER TABLE table_with_default_granularity ADD COLUMN dummy2 Float64")
#still works # still works
assert node.query("SELECT count() from table_with_default_granularity") == '6\n' assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
@pytest.mark.skip(reason="flaky") @pytest.mark.skip(reason="flaky")
def test_version_update_two_nodes(start_dynamic_cluster): def test_version_update_two_nodes(start_dynamic_cluster):
node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") node11.query(
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20) node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n' assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n'
def callback(n): def callback(n):
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>") n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>") "<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
"<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
node12.restart_with_latest_version(callback_onstop=callback) node12.restart_with_latest_version(callback_onstop=callback)
node12.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") node12.query(
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20) node11.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n' assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n'
node12.query( node12.query(
''' '''
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node11.query( node11.query(
''' '''
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32) CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id ORDER BY id
''') ''')
node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)") node12.query(
"INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
with pytest.raises(QueryTimeoutExceedException): with pytest.raises(QueryTimeoutExceedException):
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=20) node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=20)
node12.query("INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)") node12.query(
"INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
node11.restart_with_latest_version(callback_onstop=callback) # just to be sure node11.restart_with_latest_version(callback_onstop=callback) # just to be sure
for i in range(3): for i in range(3):
try: try:
@ -350,7 +393,8 @@ def test_version_update_two_nodes(start_dynamic_cluster):
assert node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n" assert node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
assert node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n" assert node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
node11.query("INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)") node11.query(
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)")
for i in range(3): for i in range(3):
try: try:
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=120) node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=120)

11
tests/integration/test_adaptive_granularity_different_settings/test.py
View File

@ -7,7 +7,9 @@ node1 = cluster.add_instance('node1', with_zookeeper=True)
node2 = cluster.add_instance('node2', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True)
# no adaptive granularity by default # no adaptive granularity by default
node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True, stay_alive=True) node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='19.9.5.36', with_installed_binary=True,
stay_alive=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
@ -20,7 +22,6 @@ def start_cluster():
def test_attach_detach(start_cluster): def test_attach_detach(start_cluster):
node1.query(""" node1.query("""
CREATE TABLE test (key UInt64) CREATE TABLE test (key UInt64)
ENGINE = ReplicatedMergeTree('/clickhouse/test', '1') ENGINE = ReplicatedMergeTree('/clickhouse/test', '1')
@ -58,7 +59,8 @@ def test_mutate_with_mixed_granularity(start_cluster):
ENGINE = MergeTree ENGINE = MergeTree
ORDER BY key PARTITION BY date""") ORDER BY key PARTITION BY date""")
node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)") node3.query(
"INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500)")
assert node3.query("SELECT COUNT() FROM test") == "500\n" assert node3.query("SELECT COUNT() FROM test") == "500\n"
@ -68,7 +70,8 @@ def test_mutate_with_mixed_granularity(start_cluster):
node3.query("ALTER TABLE test MODIFY SETTING enable_mixed_granularity_parts = 1") node3.query("ALTER TABLE test MODIFY SETTING enable_mixed_granularity_parts = 1")
node3.query("INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)") node3.query(
"INSERT INTO test SELECT toDate('2019-10-01') + number % 5, number, toString(number), toString(number * number) FROM numbers(500, 500)")
assert node3.query("SELECT COUNT() FROM test") == "1000\n" assert node3.query("SELECT COUNT() FROM test") == "1000\n"
assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "10\n" assert node3.query("SELECT COUNT() FROM test WHERE key % 100 == 0") == "10\n"

43
tests/integration/test_adaptive_granularity_replicated/test.py
View File

@ -1,17 +1,15 @@
import time import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from multiprocessing.dummy import Pool
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True) node1 = cluster.add_instance('node1', with_zookeeper=True)
node2 = cluster.add_instance('node2', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True)
node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', with_installed_binary=True) node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14',
with_installed_binary=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
@ -23,11 +21,14 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_creating_table_different_setting(start_cluster): def test_creating_table_different_setting(start_cluster):
node1.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0") node1.query(
"CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '1') ORDER BY tuple(c1) SETTINGS index_granularity_bytes = 0")
node1.query("INSERT INTO t1 VALUES('x', 'y')") node1.query("INSERT INTO t1 VALUES('x', 'y')")
node2.query("CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") node2.query(
"CREATE TABLE t1 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t1', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
node1.query("INSERT INTO t1 VALUES('a', 'b')") node1.query("INSERT INTO t1 VALUES('a', 'b')")
node2.query("SYSTEM SYNC REPLICA t1", timeout=5) node2.query("SYSTEM SYNC REPLICA t1", timeout=5)
@ -49,22 +50,26 @@ def test_creating_table_different_setting(start_cluster):
node1.query("SELECT count() FROM t1") == "3\n" node1.query("SELECT count() FROM t1") == "3\n"
node2.query("SELECT count() FROM t1") == "2\n" node2.query("SELECT count() FROM t1") == "2\n"
path_part = node1.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() path_part = node1.query(
"SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
with pytest.raises(Exception): # check that we have no adaptive files with pytest.raises(Exception): # check that we have no adaptive files
node1.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) node1.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
path_part = node2.query("SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip() path_part = node2.query(
"SELECT path FROM system.parts WHERE table = 't1' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
with pytest.raises(Exception): # check that we have no adaptive files with pytest.raises(Exception): # check that we have no adaptive files
node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
def test_old_node_with_new_node(start_cluster): def test_old_node_with_new_node(start_cluster):
node3.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)") node3.query(
"CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '3') ORDER BY tuple(c1)")
node3.query("INSERT INTO t2 VALUES('x', 'y')") node3.query("INSERT INTO t2 VALUES('x', 'y')")
node2.query("CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0") node2.query(
"CREATE TABLE t2 (c1 String, c2 String) ENGINE=ReplicatedMergeTree('/clickhouse/t2', '2') ORDER BY tuple(c1) SETTINGS enable_mixed_granularity_parts = 0")
node3.query("INSERT INTO t2 VALUES('a', 'b')") node3.query("INSERT INTO t2 VALUES('a', 'b')")
node2.query("SYSTEM SYNC REPLICA t2", timeout=5) node2.query("SYSTEM SYNC REPLICA t2", timeout=5)
@ -86,12 +91,14 @@ def test_old_node_with_new_node(start_cluster):
node3.query("SELECT count() FROM t2") == "3\n" node3.query("SELECT count() FROM t2") == "3\n"
node2.query("SELECT count() FROM t2") == "2\n" node2.query("SELECT count() FROM t2") == "2\n"
path_part = node3.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() path_part = node3.query(
"SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
with pytest.raises(Exception): # check that we have no adaptive files with pytest.raises(Exception): # check that we have no adaptive files
node3.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) node3.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])
path_part = node2.query("SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip() path_part = node2.query(
"SELECT path FROM system.parts WHERE table = 't2' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
with pytest.raises(Exception): # check that we have no adaptive files with pytest.raises(Exception): # check that we have no adaptive files
node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)]) node2.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(p=path_part)])

27
tests/integration/test_aggregation_memory_efficient/test.py
View File

@ -1,22 +1,20 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1') node1 = cluster.add_instance('node1')
node2 = cluster.add_instance('node2') node2 = cluster.add_instance('node2')
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
cluster.start() cluster.start()
for node in [node1, node2]: for node in [node1, node2]:
node.query("create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;") node.query(
"create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;")
node1.query("insert into da_memory_efficient_shard select number, number from numbers(100000);") node1.query("insert into da_memory_efficient_shard select number, number from numbers(100000);")
node2.query("insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);") node2.query("insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);")
@ -28,19 +26,24 @@ def start_cluster():
def test_remote(start_cluster): def test_remote(start_cluster):
node1.query(
node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") res = node1.query(
"select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
assert res == '200000\n' assert res == '200000\n'
node1.query("set distributed_aggregation_memory_efficient = 0") node1.query("set distributed_aggregation_memory_efficient = 0")
res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") res = node1.query(
"select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)")
assert res == '200000\n' assert res == '200000\n'
node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") node1.query(
res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1")
res = node1.query(
"SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
assert res == 'node1\t100000\nnode2\t100000\n' assert res == 'node1\t100000\nnode2\t100000\n'
node1.query("set distributed_aggregation_memory_efficient = 0") node1.query("set distributed_aggregation_memory_efficient = 0")
res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") res = node1.query(
"SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;")
assert res == 'node1\t100000\nnode2\t100000\n' assert res == 'node1\t100000\nnode2\t100000\n'

36
tests/integration/test_allowed_client_hosts/test.py
View File

@ -1,32 +1,32 @@
import os
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
server = cluster.add_instance('server', user_configs=["configs/users.d/network.xml"]) server = cluster.add_instance('server', user_configs=["configs/users.d/network.xml"])
clientA1 = cluster.add_instance('clientA1', hostname = 'clientA1.com') clientA1 = cluster.add_instance('clientA1', hostname='clientA1.com')
clientA2 = cluster.add_instance('clientA2', hostname = 'clientA2.com') clientA2 = cluster.add_instance('clientA2', hostname='clientA2.com')
clientA3 = cluster.add_instance('clientA3', hostname = 'clientA3.com') clientA3 = cluster.add_instance('clientA3', hostname='clientA3.com')
clientB1 = cluster.add_instance('clientB1', hostname = 'clientB001.ru') clientB1 = cluster.add_instance('clientB1', hostname='clientB001.ru')
clientB2 = cluster.add_instance('clientB2', hostname = 'clientB002.ru') clientB2 = cluster.add_instance('clientB2', hostname='clientB002.ru')
clientB3 = cluster.add_instance('clientB3', hostname = 'xxx.clientB003.rutracker.com') clientB3 = cluster.add_instance('clientB3', hostname='xxx.clientB003.rutracker.com')
clientC1 = cluster.add_instance('clientC1', hostname = 'clientC01.ru') clientC1 = cluster.add_instance('clientC1', hostname='clientC01.ru')
clientC2 = cluster.add_instance('clientC2', hostname = 'xxx.clientC02.ru') clientC2 = cluster.add_instance('clientC2', hostname='xxx.clientC02.ru')
clientC3 = cluster.add_instance('clientC3', hostname = 'xxx.clientC03.rutracker.com') clientC3 = cluster.add_instance('clientC3', hostname='xxx.clientC03.rutracker.com')
clientD1 = cluster.add_instance('clientD1', hostname = 'clientD0001.ru') clientD1 = cluster.add_instance('clientD1', hostname='clientD0001.ru')
clientD2 = cluster.add_instance('clientD2', hostname = 'xxx.clientD0002.ru') clientD2 = cluster.add_instance('clientD2', hostname='xxx.clientD0002.ru')
clientD3 = cluster.add_instance('clientD3', hostname = 'clientD0003.ru') clientD3 = cluster.add_instance('clientD3', hostname='clientD0003.ru')
def check_clickhouse_is_ok(client_node, server_node): def check_clickhouse_is_ok(client_node, server_node):
assert client_node.exec_in_container(["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n" assert client_node.exec_in_container(
["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n"
def query_from_one_node_to_another(client_node, server_node, query): def query_from_one_node_to_another(client_node, server_node, query):
check_clickhouse_is_ok(client_node, server_node) check_clickhouse_is_ok(client_node, server_node)
return client_node.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)]) return client_node.exec_in_container(
["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)])
def query(node, query): def query(node, query):
@ -53,8 +53,8 @@ def test_allowed_host():
# Reverse DNS lookup currently isn't working as expected in this test. # Reverse DNS lookup currently isn't working as expected in this test.
# For example, it gives something like "vitbartestallowedclienthosts_clientB1_1.vitbartestallowedclienthosts_default" instead of "clientB001.ru". # For example, it gives something like "vitbartestallowedclienthosts_clientB1_1.vitbartestallowedclienthosts_default" instead of "clientB001.ru".
# Maybe we should setup the test network better. # Maybe we should setup the test network better.
#expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3]) # expected_to_pass.extend([clientB1, clientB2, clientB3, clientC1, clientC2, clientD1, clientD3])
#expected_to_fail.extend([clientC3, clientD2]) # expected_to_fail.extend([clientC3, clientD2])
for client_node in expected_to_pass: for client_node in expected_to_pass:
assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") == "5\n" assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_table") == "5\n"

88
tests/integration/test_allowed_url_from_config/test.py
View File

@ -1,8 +1,6 @@
import time
import pytest import pytest
from helpers.hdfs_api import HDFSApi
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.hdfs_api import HDFSApi
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml']) node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml'])
@ -21,26 +19,37 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_config_with_hosts(start_cluster): def test_config_with_hosts(start_cluster):
assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == "" assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == ""
assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)") assert "not allowed" in node1.query_and_get_error(
assert "not allowed" in node1.query_and_get_error("CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)") "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)")
assert "not allowed" in node1.query_and_get_error(
"CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)")
def test_config_with_only_primary_hosts(start_cluster): def test_config_with_only_primary_hosts(start_cluster):
assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == "" assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == ""
assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == "" assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == ""
assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == "" assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == ""
assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)") assert "not allowed" in node2.query_and_get_error(
assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)") "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)")
assert "not allowed" in node2.query_and_get_error("CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)") assert "not allowed" in node2.query_and_get_error(
"CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)")
assert "not allowed" in node2.query_and_get_error(
"CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)")
def test_config_with_only_regexp_hosts(start_cluster): def test_config_with_only_regexp_hosts(start_cluster):
assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == "" assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == ""
assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)") assert "not allowed" in node3.query_and_get_error(
assert "not allowed" in node3.query_and_get_error("CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)") "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)")
assert "not allowed" in node3.query_and_get_error(
"CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)")
def test_config_without_allowed_hosts(start_cluster): def test_config_without_allowed_hosts(start_cluster):
assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == ""
@ -48,27 +57,60 @@ def test_config_without_allowed_hosts(start_cluster):
assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == ""
assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == "" assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == ""
def test_table_function_remote(start_cluster): def test_table_function_remote(start_cluster):
assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) assert "not allowed in config.xml" not in node6.query_and_get_error(
assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) "SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)",
assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-1', system, events", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed in config.xml" not in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) assert "not allowed in config.xml" not in node6.query_and_get_error(
assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) "SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)",
assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
assert "not allowed" in node6.query_and_get_error("SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)", settings={"connections_with_failover_max_tries":1, "connect_timeout_with_failover_ms": 1000, "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout":1}) "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed in config.xml" not in node6.query_and_get_error(
"SELECT * FROM remote('example01-0{1,2}-1', system, events",
settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed in config.xml" not in node6.query_and_get_error(
"SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)",
settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed in config.xml" not in node6.query_and_get_error(
"SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)",
settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed" in node6.query_and_get_error(
"SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)",
settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)",
settings={"connections_with_failover_max_tries": 1,
"connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000,
"connect_timeout": 1, "send_timeout": 1})
assert "not allowed" in node6.query_and_get_error(
"SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)",
settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000,
"connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1})
assert node6.query("SELECT * FROM remote('localhost', system, events)") != "" assert node6.query("SELECT * FROM remote('localhost', system, events)") != ""
assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != "" assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != ""
assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remoteSecure('localhost:800', system, events)") assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error(
assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error("SELECT * FROM remote('localhost:800', system, metrics)") "SELECT * FROM remoteSecure('localhost:800', system, events)")
assert "URL \"localhost:800\" is not allowed in config.xml" in node6.query_and_get_error(
"SELECT * FROM remote('localhost:800', system, metrics)")
def test_redirect(start_cluster): def test_redirect(start_cluster):
hdfs_api = HDFSApi("root") hdfs_api = HDFSApi("root")
hdfs_api.write_data("/simple_storage", "1\t\n") hdfs_api.write_data("/simple_storage", "1\t\n")
assert hdfs_api.read_data("/simple_storage") == "1\t\n" assert hdfs_api.read_data("/simple_storage") == "1\t\n"
node7.query("CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)") node7.query(
"CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)")
assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1") assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1")
def test_HDFS(start_cluster): def test_HDFS(start_cluster):
assert "not allowed" in node7.query_and_get_error("CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')") assert "not allowed" in node7.query_and_get_error(
assert "not allowed" in node7.query_and_get_error("SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')") "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')")
assert "not allowed" in node7.query_and_get_error(
"SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')")

6
tests/integration/test_alter_codec/test.py
View File

@ -2,14 +2,13 @@ import pytest
from helpers.client import QueryRuntimeException from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', node1 = cluster.add_instance('node1',
main_configs=['configs/logs_config.xml']) main_configs=['configs/logs_config.xml'])
node2 = cluster.add_instance('node2', node2 = cluster.add_instance('node2',
main_configs=['configs/logs_config.xml']) main_configs=['configs/logs_config.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -39,7 +38,6 @@ def test_alter_codec_pk(started_cluster):
with pytest.raises(QueryRuntimeException): with pytest.raises(QueryRuntimeException):
node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name))
node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name))
node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name))

5
tests/integration/test_always_fetch_merged/test.py
View File

@ -1,14 +1,15 @@
import pytest
import time import time
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True) node1 = cluster.add_instance('node1', with_zookeeper=True)
node2 = cluster.add_instance('node2', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:

2
tests/integration/test_asynchronous_metric_log_table/test.py
View File

@ -39,7 +39,7 @@ def test_event_time_microseconds_field(started_cluster):
node1.query(query_create) node1.query(query_create)
node1.query('''INSERT INTO replica.test VALUES (1, now())''') node1.query('''INSERT INTO replica.test VALUES (1, now())''')
node1.query("SYSTEM FLUSH LOGS;") node1.query("SYSTEM FLUSH LOGS;")
#query assumes that the event_time field is accurate # query assumes that the event_time field is accurate
equals_query = '''WITH ( equals_query = '''WITH (
( (
SELECT event_time_microseconds SELECT event_time_microseconds

11
tests/integration/test_atomic_drop_table/test.py
View File

@ -1,12 +1,12 @@
import time import time
import pytest import pytest
from helpers.network import PartitionManager
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml", "configs/remote_servers.xml"], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml",
"configs/remote_servers.xml"], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -25,12 +25,13 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_atomic_delete_with_stopped_zookeeper(start_cluster): def test_atomic_delete_with_stopped_zookeeper(start_cluster):
node1.query("insert into zktest.atomic_drop_table values (8192)") node1.query("insert into zktest.atomic_drop_table values (8192)")
with PartitionManager() as pm: with PartitionManager() as pm:
pm.drop_instance_zk_connections(node1) pm.drop_instance_zk_connections(node1)
error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") #Table won't drop error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") # Table won't drop
assert error != "" assert error != ""
time.sleep(5) time.sleep(5)

11
tests/integration/test_attach_without_checksums/test.py
View File

@ -17,7 +17,8 @@ def start_cluster():
def test_attach_without_checksums(start_cluster): def test_attach_without_checksums(start_cluster):
node1.query("CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date") node1.query(
"CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date")
node1.query("INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)") node1.query("INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)")
@ -29,9 +30,13 @@ def test_attach_without_checksums(start_cluster):
assert node1.query("SELECT COUNT() FROM test") == "0\n" assert node1.query("SELECT COUNT() FROM test") == "0\n"
# to be sure output not empty # to be sure output not empty
node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '], privileged=True, user='root') node1.exec_in_container(
['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '],
privileged=True, user='root')
node1.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'], privileged=True, user='root') node1.exec_in_container(
['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'],
privileged=True, user='root')
node1.query("ALTER TABLE test ATTACH PARTITION '2019-10-01'") node1.query("ALTER TABLE test ATTACH PARTITION '2019-10-01'")

5
tests/integration/test_authentication/test.py
View File

@ -30,7 +30,8 @@ def test_authentication_pass():
def test_authentication_fail(): def test_authentication_fail():
# User doesn't exist. # User doesn't exist.
assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'vasya') assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='vasya')
# Wrong password. # Wrong password.
assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user = 'masha', password = '123') assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='masha',
password='123')

27
tests/integration/test_backup_restore/test.py
View File

@ -1,10 +1,9 @@
import os.path import os.path
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance') instance = cluster.add_instance('instance')
q = instance.query q = instance.query
@ -47,7 +46,7 @@ def backup_restore(started_cluster):
expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
res = q("SELECT * FROM test.tbl ORDER BY p") res = q("SELECT * FROM test.tbl ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("ALTER TABLE test.tbl FREEZE") q("ALTER TABLE test.tbl FREEZE")
@ -69,7 +68,7 @@ def test_restore(backup_restore):
# Validate the attached parts are identical to the backup. # Validate the attached parts are identical to the backup.
expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
res = q("SELECT * FROM test.tbl1 ORDER BY p") res = q("SELECT * FROM test.tbl1 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1") q("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1")
q("SELECT sleep(2)") q("SELECT sleep(2)")
@ -77,7 +76,7 @@ def test_restore(backup_restore):
# Validate mutation has been applied to all attached parts. # Validate mutation has been applied to all attached parts.
expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10')
res = q("SELECT * FROM test.tbl1 ORDER BY p") res = q("SELECT * FROM test.tbl1 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("DROP TABLE IF EXISTS test.tbl1") q("DROP TABLE IF EXISTS test.tbl1")
@ -91,7 +90,7 @@ def test_attach_partition(backup_restore):
expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34')
res = q("SELECT * FROM test.tbl2 ORDER BY p") res = q("SELECT * FROM test.tbl2 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
copy_backup_to_detached('test', 'tbl', 'tbl2') copy_backup_to_detached('test', 'tbl', 'tbl2')
@ -102,17 +101,19 @@ def test_attach_partition(backup_restore):
q("ALTER TABLE test.tbl2 ATTACH PARTITION 197002") q("ALTER TABLE test.tbl2 ATTACH PARTITION 197002")
q("SELECT sleep(2)") q("SELECT sleep(2)")
expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34') expected = TSV(
'1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34')
res = q("SELECT * FROM test.tbl2 ORDER BY p") res = q("SELECT * FROM test.tbl2 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1") q("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1")
q("SELECT sleep(2)") q("SELECT sleep(2)")
# Validate mutation has been applied to all attached parts. # Validate mutation has been applied to all attached parts.
expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10') expected = TSV(
'1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10')
res = q("SELECT * FROM test.tbl2 ORDER BY p") res = q("SELECT * FROM test.tbl2 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("DROP TABLE IF EXISTS test.tbl2") q("DROP TABLE IF EXISTS test.tbl2")
@ -126,7 +127,7 @@ def test_replace_partition(backup_restore):
expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34')
res = q("SELECT * FROM test.tbl3 ORDER BY p") res = q("SELECT * FROM test.tbl3 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
copy_backup_to_detached('test', 'tbl', 'tbl3') copy_backup_to_detached('test', 'tbl', 'tbl3')
@ -138,7 +139,7 @@ def test_replace_partition(backup_restore):
expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33')
res = q("SELECT * FROM test.tbl3 ORDER BY p") res = q("SELECT * FROM test.tbl3 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1") q("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1")
q("SELECT sleep(2)") q("SELECT sleep(2)")
@ -146,6 +147,6 @@ def test_replace_partition(backup_restore):
# Validate mutation has been applied to all copied parts. # Validate mutation has been applied to all copied parts.
expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10')
res = q("SELECT * FROM test.tbl3 ORDER BY p") res = q("SELECT * FROM test.tbl3 ORDER BY p")
assert(TSV(res) == expected) assert (TSV(res) == expected)
q("DROP TABLE IF EXISTS test.tbl3") q("DROP TABLE IF EXISTS test.tbl3")

36
tests/integration/test_backup_with_other_granularity/test.py
View File

@ -1,13 +1,15 @@
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) stay_alive=True, with_installed_binary=True)
node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', stay_alive=True, with_installed_binary=True) stay_alive=True, with_installed_binary=True)
node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35',
stay_alive=True, with_installed_binary=True)
node4 = cluster.add_instance('node4') node4 = cluster.add_instance('node4')
@ -33,13 +35,15 @@ def test_backup_from_old_version(started_cluster):
node1.restart_with_latest_version() node1.restart_with_latest_version()
node1.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()") node1.query(
"CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()")
node1.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") node1.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" assert node1.query("SELECT COUNT() FROM dest_table") == "1\n"
node1.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) node1.exec_in_container(['bash', '-c',
'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" assert node1.query("SELECT COUNT() FROM dest_table") == "1\n"
@ -69,13 +73,15 @@ def test_backup_from_old_version_setting(started_cluster):
node2.restart_with_latest_version() node2.restart_with_latest_version()
node2.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") node2.query(
"CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
node2.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") node2.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" assert node2.query("SELECT COUNT() FROM dest_table") == "1\n"
node2.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) node2.exec_in_container(['bash', '-c',
'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" assert node2.query("SELECT COUNT() FROM dest_table") == "1\n"
@ -104,17 +110,20 @@ def test_backup_from_old_version_config(started_cluster):
node3.query("ALTER TABLE source_table FREEZE PARTITION tuple();") node3.query("ALTER TABLE source_table FREEZE PARTITION tuple();")
def callback(n): def callback(n):
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", "<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>") n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
"<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
node3.restart_with_latest_version(callback_onstop=callback) node3.restart_with_latest_version(callback_onstop=callback)
node3.query("CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") node3.query(
"CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1")
node3.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") node3.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')")
assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" assert node3.query("SELECT COUNT() FROM dest_table") == "1\n"
node3.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) node3.exec_in_container(['bash', '-c',
'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached'])
assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" assert node3.query("SELECT COUNT() FROM dest_table") == "1\n"
@ -144,7 +153,8 @@ def test_backup_and_alter(started_cluster):
node4.query("ALTER TABLE backup_table DROP PARTITION tuple()") node4.query("ALTER TABLE backup_table DROP PARTITION tuple()")
node4.exec_in_container(['bash', '-c', 'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached']) node4.exec_in_container(['bash', '-c',
'cp -r /var/lib/clickhouse/shadow/1/data/default/backup_table/all_1_1_0/ /var/lib/clickhouse/data/default/backup_table/detached'])
node4.query("ALTER TABLE backup_table ATTACH PARTITION tuple()") node4.query("ALTER TABLE backup_table ATTACH PARTITION tuple()")

13
tests/integration/test_backward_compatibility/test.py
View File

@ -1,22 +1,23 @@
import pytest import pytest
import helpers.client as client
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54',
stay_alive=True, with_installed_binary=True)
node2 = cluster.add_instance('node2', with_zookeeper=True) node2 = cluster.add_instance('node2', with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
cluster.start() cluster.start()
for i, node in enumerate([node1, node2]): for i, node in enumerate([node1, node2]):
node.query( node.query(
'''CREATE TABLE t(date Date, id UInt32) '''CREATE TABLE t(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}')
PARTITION BY toYYYYMM(date) PARTITION BY toYYYYMM(date)
ORDER BY id'''.format(i)) ORDER BY id'''.format(i))
yield cluster yield cluster

21
tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py
View File

@ -1,16 +1,18 @@
import pytest import pytest
import helpers.client as client
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', node1 = cluster.add_instance('node1',
with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True,
with_installed_binary=True)
node2 = cluster.add_instance('node2', node2 = cluster.add_instance('node2',
with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True,
with_installed_binary=True)
node3 = cluster.add_instance('node3', with_zookeeper=False) node3 = cluster.add_instance('node3', with_zookeeper=False)
node4 = cluster.add_instance('node4', with_zookeeper=False) node4 = cluster.add_instance('node4', with_zookeeper=False)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
@ -20,6 +22,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
# We will test that serialization of internal state of "avg" function is compatible between different versions. # We will test that serialization of internal state of "avg" function is compatible between different versions.
# TODO Implement versioning of serialization format for aggregate function states. # TODO Implement versioning of serialization format for aggregate function states.
# NOTE This test is too ad-hoc. # NOTE This test is too ad-hoc.
@ -35,18 +38,18 @@ def test_backward_compatability(start_cluster):
node3.query("INSERT INTO tab VALUES (3)") node3.query("INSERT INTO tab VALUES (3)")
node4.query("INSERT INTO tab VALUES (4)") node4.query("INSERT INTO tab VALUES (4)")
assert(node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') assert (node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
assert(node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') assert (node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
assert(node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') assert (node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
assert(node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') assert (node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n')
# Also check with persisted aggregate function state # Also check with persisted aggregate function state
node1.query("create table state (x AggregateFunction(avg, UInt64)) engine = Log") node1.query("create table state (x AggregateFunction(avg, UInt64)) engine = Log")
node1.query("INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))") node1.query("INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))")
assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')
node1.restart_with_latest_version() node1.restart_with_latest_version()
assert(node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n')

11
tests/integration/test_backward_compatibility/test_short_strings_aggregation.py
View File

@ -1,13 +1,15 @@
import pytest import pytest
import helpers.client as client
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37',
node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, with_installed_binary=True) stay_alive=True, with_installed_binary=True)
node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37',
stay_alive=True, with_installed_binary=True)
node3 = cluster.add_instance('node3', with_zookeeper=False) node3 = cluster.add_instance('node3', with_zookeeper=False)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
@ -23,6 +25,7 @@ def test_backward_compatability(start_cluster):
node2.query("create table tab (s String) engine = MergeTree order by s") node2.query("create table tab (s String) engine = MergeTree order by s")
node1.query("insert into tab select number from numbers(50)") node1.query("insert into tab select number from numbers(50)")
node2.query("insert into tab select number from numbers(1000000)") node2.query("insert into tab select number from numbers(1000000)")
res = node3.query("select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") res = node3.query(
"select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50")
print(res) print(res)
assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) assert res == ''.join('{}\t2\n'.format(i) for i in range(50))

9
tests/integration/test_block_structure_mismatch/test.py
View File

@ -1,7 +1,5 @@
import time
import pytest import pytest
from contextlib import contextmanager
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -9,7 +7,8 @@ cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'])
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'])
#test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162
# test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -44,7 +43,9 @@ CREATE TABLE dist_test (
finally: finally:
cluster.shutdown() cluster.shutdown()
def test(started_cluster): def test(started_cluster):
node1.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')") node1.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')")
node2.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')") node2.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')")
assert node1.query("SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n" assert node1.query(
"SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n"

68
tests/integration/test_check_table/test.py
View File

@ -1,9 +1,6 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -34,15 +31,22 @@ def started_cluster():
def corrupt_data_part_on_disk(node, table, part_name): def corrupt_data_part_on_disk(node, table, part_name):
part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() part_path = node.query(
node.exec_in_container(['bash', '-c', 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(p=part_path)], privileged=True) "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
node.exec_in_container(['bash', '-c',
'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format(
p=part_path)], privileged=True)
def remove_checksums_on_disk(node, table, part_name): def remove_checksums_on_disk(node, table, part_name):
part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() part_path = node.query(
"SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True) node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True)
def remove_part_from_disk(node, table, part_name): def remove_part_from_disk(node, table, part_name):
part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() part_path = node.query(
"SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip()
if not part_path: if not part_path:
raise Exception("Part " + part_name + "doesn't exist") raise Exception("Part " + part_name + "doesn't exist")
node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True)
@ -50,35 +54,42 @@ def remove_part_from_disk(node, table, part_name):
def test_check_normal_table_corruption(started_cluster): def test_check_normal_table_corruption(started_cluster):
node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)")
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n" assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902",
settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n"
remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." assert node1.query("CHECK TABLE non_replicated_mt", settings={
"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"
remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={
"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."
assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"
corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0") corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0")
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." assert node1.query("CHECK TABLE non_replicated_mt", settings={
"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16." assert node1.query("CHECK TABLE non_replicated_mt", settings={
"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)") node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)")
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n" assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901",
settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n"
corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0") corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0")
remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0") remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0")
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n" assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={
"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n"
def test_check_replicated_table_simple(started_cluster): def test_check_replicated_table_simple(started_cluster):
@ -90,16 +101,20 @@ def test_check_replicated_table_simple(started_cluster):
assert node1.query("SELECT count() from replicated_mt") == "2\n" assert node1.query("SELECT count() from replicated_mt") == "2\n"
assert node2.query("SELECT count() from replicated_mt") == "2\n" assert node2.query("SELECT count() from replicated_mt") == "2\n"
assert node1.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" assert node1.query("CHECK TABLE replicated_mt",
assert node2.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
assert node2.query("CHECK TABLE replicated_mt",
settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n"
node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)")
node1.query("SYSTEM SYNC REPLICA replicated_mt") node1.query("SYSTEM SYNC REPLICA replicated_mt")
assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node1.query("SELECT count() from replicated_mt") == "4\n"
assert node2.query("SELECT count() from replicated_mt") == "4\n" assert node2.query("SELECT count() from replicated_mt") == "4\n"
assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
assert node2.query("CHECK TABLE replicated_mt PARTITION 201901",
settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n"
def test_check_replicated_table_corruption(started_cluster): def test_check_replicated_table_corruption(started_cluster):
@ -112,18 +127,25 @@ def test_check_replicated_table_corruption(started_cluster):
assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node1.query("SELECT count() from replicated_mt") == "4\n"
assert node2.query("SELECT count() from replicated_mt") == "4\n" assert node2.query("SELECT count() from replicated_mt") == "4\n"
part_name = node1.query("SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip() part_name = node1.query(
"SELECT name from system.parts where table = 'replicated_mt' and partition_id = '201901' and active = 1").strip()
corrupt_data_part_on_disk(node1, "replicated_mt", part_name) corrupt_data_part_on_disk(node1, "replicated_mt", part_name)
assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={
"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(
p=part_name)
node1.query("SYSTEM SYNC REPLICA replicated_mt") node1.query("SYSTEM SYNC REPLICA replicated_mt")
assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node1.query("SELECT count() from replicated_mt") == "4\n"
remove_part_from_disk(node2, "replicated_mt", part_name) remove_part_from_disk(node2, "replicated_mt", part_name)
assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(p=part_name) assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", settings={
"check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and queueing a fetch.\n".format(
p=part_name)
node1.query("SYSTEM SYNC REPLICA replicated_mt") node1.query("SYSTEM SYNC REPLICA replicated_mt")
assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) assert node1.query("CHECK TABLE replicated_mt PARTITION 201901",
settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name)
assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node1.query("SELECT count() from replicated_mt") == "4\n"

29
tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py
View File

@ -1,13 +1,13 @@
import time import time
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager from helpers.network import PartitionManager
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True) node1 = cluster.add_instance('node1', with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
@ -16,6 +16,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
# This tests if the data directory for a table is cleaned up if there is a Zookeeper # This tests if the data directory for a table is cleaned up if there is a Zookeeper
# connection exception during a CreateQuery operation involving ReplicatedMergeTree tables. # connection exception during a CreateQuery operation involving ReplicatedMergeTree tables.
# Test flow is as follows: # Test flow is as follows:
@ -48,20 +49,30 @@ def test_cleanup_dir_after_bad_zk_conn(start_cluster):
node1.query('''INSERT INTO replica.test VALUES (1, now())''') node1.query('''INSERT INTO replica.test VALUES (1, now())''')
assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''') assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''')
def test_cleanup_dir_after_wrong_replica_name(start_cluster): def test_cleanup_dir_after_wrong_replica_name(start_cluster):
node1.query("CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") node1.query(
error = node1.query_and_get_error("CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") "CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
error = node1.query_and_get_error(
"CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
assert "already exists" in error assert "already exists" in error
node1.query("CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n") node1.query(
"CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n")
def test_cleanup_dir_after_wrong_zk_path(start_cluster): def test_cleanup_dir_after_wrong_zk_path(start_cluster):
node1.query("CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n") node1.query(
error = node1.query_and_get_error("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n") "CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n")
error = node1.query_and_get_error(
"CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n")
assert "Cannot create" in error assert "Cannot create" in error
node1.query("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n") node1.query(
"CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n")
def test_attach_without_zk(start_cluster): def test_attach_without_zk(start_cluster):
node1.query("CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n") node1.query(
"CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n")
node1.query("DETACH TABLE test4_r1") node1.query("DETACH TABLE test4_r1")
with PartitionManager() as pm: with PartitionManager() as pm:
pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'}) pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'})

1
tests/integration/test_cluster_all_replicas/test.py
View File

@ -7,6 +7,7 @@ cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:

143
tests/integration/test_cluster_copier/test.py
View File

@ -1,15 +1,15 @@
import os import os
import random
import sys import sys
import time import time
from contextlib import contextmanager
import docker
import kazoo import kazoo
import pytest import pytest
import docker
import random
from contextlib import contextmanager
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR))
@ -18,9 +18,10 @@ MOVING_FAIL_PROBABILITY = 0.2
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
def check_all_hosts_sucesfully_executed(tsv_content, num_hosts): def check_all_hosts_sucesfully_executed(tsv_content, num_hosts):
M = TSV.toMat(tsv_content) M = TSV.toMat(tsv_content)
hosts = [(l[0], l[1]) for l in M] # (host, port) hosts = [(l[0], l[1]) for l in M] # (host, port)
codes = [l[2] for l in M] codes = [l[2] for l in M]
assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content
@ -39,14 +40,14 @@ def started_cluster():
global cluster global cluster
try: try:
clusters_schema = { clusters_schema = {
"0" : { "0": {
"0" : ["0", "1"], "0": ["0", "1"],
"1" : ["0"] "1": ["0"]
}, },
"1" : { "1": {
"0" : ["0", "1"], "0": ["0", "1"],
"1" : ["0"] "1": ["0"]
} }
} }
for cluster_name, shards in clusters_schema.iteritems(): for cluster_name, shards in clusters_schema.iteritems():
@ -54,10 +55,11 @@ def started_cluster():
for replica_name in replicas: for replica_name in replicas:
name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name) name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name)
cluster.add_instance(name, cluster.add_instance(name,
main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", "configs/conf.d/clusters.xml"], main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml",
user_configs=["configs/users.xml"], "configs/conf.d/clusters.xml"],
macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name}, user_configs=["configs/users.xml"],
with_zookeeper=True) macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name},
with_zookeeper=True)
cluster.start() cluster.start()
yield cluster yield cluster
@ -70,24 +72,27 @@ class Task1:
def __init__(self, cluster): def __init__(self, cluster):
self.cluster = cluster self.cluster = cluster
self.zk_task_path="/clickhouse-copier/task_simple" self.zk_task_path = "/clickhouse-copier/task_simple"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task0_description.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task0_description.xml'), 'r').read()
def start(self): def start(self):
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
for cluster_num in ["0", "1"]: for cluster_num in ["0", "1"]:
ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num))
ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num)) ddl_check_query(instance,
"CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(
cluster_num))
ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " +
"ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/hits', '{replica}') " +
"PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16")
ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") ddl_check_query(instance,
ddl_check_query(instance, "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)") "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)")
instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) ddl_check_query(instance,
"CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)")
instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002",
settings={"insert_distributed_sync": 1})
def check(self): def check(self):
assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM hits_all")) == TSV("1002\n") assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM hits_all")) == TSV("1002\n")
@ -107,31 +112,44 @@ class Task2:
def __init__(self, cluster): def __init__(self, cluster):
self.cluster = cluster self.cluster = cluster
self.zk_task_path="/clickhouse-copier/task_month_to_week_partition" self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_month_to_week_description.xml'), 'r').read()
def start(self): def start(self):
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
for cluster_num in ["0", "1"]: for cluster_num in ["0", "1"]:
ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num))
ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(cluster_num)) ddl_check_query(instance,
"CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{} ENGINE=Ordinary".format(
cluster_num))
ddl_check_query(instance, "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") ddl_check_query(instance,
ddl_check_query(instance, "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/a', '{replica}', date, intHash64(d), (date, intHash64(d)), 8192)")
ddl_check_query(instance,
"CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)")
instance.query("INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85", settings={"insert_distributed_sync": 1}) instance.query(
"INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85",
settings={"insert_distributed_sync": 1})
def check(self): def check(self):
assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV("85\n") assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV(
assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n") "85\n")
assert TSV(self.cluster.instances['s1_0_0'].query(
"SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n")
assert TSV(self.cluster.instances['s1_0_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n") assert TSV(self.cluster.instances['s1_0_0'].query(
assert TSV(self.cluster.instances['s1_1_0'].query("SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n") "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n")
assert TSV(self.cluster.instances['s1_1_0'].query(
"SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n")
assert TSV(self.cluster.instances['s1_0_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n") assert TSV(self.cluster.instances['s1_0_0'].query(
assert TSV(self.cluster.instances['s1_1_0'].query("SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV("1\n") "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV(
"1\n")
assert TSV(self.cluster.instances['s1_1_0'].query(
"SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV(
"1\n")
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
ddl_check_query(instance, "DROP TABLE a ON CLUSTER cluster0") ddl_check_query(instance, "DROP TABLE a ON CLUSTER cluster0")
@ -142,11 +160,10 @@ class Task_test_block_size:
def __init__(self, cluster): def __init__(self, cluster):
self.cluster = cluster self.cluster = cluster
self.zk_task_path="/clickhouse-copier/task_test_block_size" self.zk_task_path = "/clickhouse-copier/task_test_block_size"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_test_block_size.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_test_block_size.xml'), 'r').read()
self.rows = 1000000 self.rows = 1000000
def start(self): def start(self):
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
@ -155,11 +172,13 @@ class Task_test_block_size:
ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}') ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/test_block_size', '{replica}')
ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2)
instance.query("INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(self.rows)) instance.query(
"INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format(
self.rows))
def check(self): def check(self):
assert TSV(self.cluster.instances['s1_0_0'].query("SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows)) assert TSV(self.cluster.instances['s1_0_0'].query(
"SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows))
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER shard_0_0", 2) ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER shard_0_0", 2)
@ -170,17 +189,15 @@ class Task_no_index:
def __init__(self, cluster): def __init__(self, cluster):
self.cluster = cluster self.cluster = cluster
self.zk_task_path="/clickhouse-copier/task_no_index" self.zk_task_path = "/clickhouse-copier/task_no_index"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_index.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_index.xml'), 'r').read()
self.rows = 1000000 self.rows = 1000000
def start(self): def start(self):
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
instance.query("create table ontime (Year UInt16, FlightDate String) ENGINE = Memory") instance.query("create table ontime (Year UInt16, FlightDate String) ENGINE = Memory")
instance.query("insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')") instance.query("insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')")
def check(self): def check(self):
assert TSV(self.cluster.instances['s1_1_0'].query("SELECT Year FROM ontime22")) == TSV("2017\n") assert TSV(self.cluster.instances['s1_1_0'].query("SELECT Year FROM ontime22")) == TSV("2017\n")
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
@ -193,17 +210,16 @@ class Task_no_arg:
def __init__(self, cluster): def __init__(self, cluster):
self.cluster = cluster self.cluster = cluster
self.zk_task_path="/clickhouse-copier/task_no_arg" self.zk_task_path = "/clickhouse-copier/task_no_arg"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_arg.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_no_arg.xml'), 'r').read()
self.rows = 1000000 self.rows = 1000000
def start(self): def start(self):
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
instance.query("create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192") instance.query(
"create table copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192")
instance.query("insert into copier_test1 values ('2016-01-01', 10);") instance.query("insert into copier_test1 values ('2016-01-01', 10);")
def check(self): def check(self):
assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n") assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n")
instance = cluster.instances['s0_0_0'] instance = cluster.instances['s0_0_0']
@ -227,15 +243,14 @@ def execute_task(task, cmd_options):
zk.ensure_path(zk_task_path) zk.ensure_path(zk_task_path)
zk.create(zk_task_path + "/description", task.copier_task_config) zk.create(zk_task_path + "/description", task.copier_task_config)
# Run cluster-copier processes on each node # Run cluster-copier processes on each node
docker_api = docker.from_env().api docker_api = docker.from_env().api
copiers_exec_ids = [] copiers_exec_ids = []
cmd = ['/usr/bin/clickhouse', 'copier', cmd = ['/usr/bin/clickhouse', 'copier',
'--config', '/etc/clickhouse-server/config-copier.xml', '--config', '/etc/clickhouse-server/config-copier.xml',
'--task-path', zk_task_path, '--task-path', zk_task_path,
'--base-dir', '/var/log/clickhouse-server/copier'] '--base-dir', '/var/log/clickhouse-server/copier']
cmd += cmd_options cmd += cmd_options
copiers = random.sample(cluster.instances.keys(), 3) copiers = random.sample(cluster.instances.keys(), 3)
@ -243,7 +258,8 @@ def execute_task(task, cmd_options):
for instance_name in copiers: for instance_name in copiers:
instance = cluster.instances[instance_name] instance = cluster.instances[instance_name]
container = instance.get_docker_handle() container = instance.get_docker_handle()
instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"),
"/etc/clickhouse-server/config-copier.xml")
print "Copied copier config to {}".format(instance.name) print "Copied copier config to {}".format(instance.name)
exec_id = docker_api.exec_create(container.id, cmd, stderr=True) exec_id = docker_api.exec_create(container.id, cmd, stderr=True)
output = docker_api.exec_start(exec_id).decode('utf8') output = docker_api.exec_start(exec_id).decode('utf8')
@ -277,7 +293,6 @@ def execute_task(task, cmd_options):
True True
] ]
) )
def test_copy_simple(started_cluster, use_sample_offset): def test_copy_simple(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(Task1(started_cluster), ['--experimental-use-sample-offset', '1']) execute_task(Task1(started_cluster), ['--experimental-use-sample-offset', '1'])
@ -292,7 +307,6 @@ def test_copy_simple(started_cluster, use_sample_offset):
True True
] ]
) )
def test_copy_with_recovering(started_cluster, use_sample_offset): def test_copy_with_recovering(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY),
@ -300,6 +314,7 @@ def test_copy_with_recovering(started_cluster, use_sample_offset):
else: else:
execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) execute_task(Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
@pytest.mark.parametrize( @pytest.mark.parametrize(
('use_sample_offset'), ('use_sample_offset'),
[ [
@ -307,7 +322,6 @@ def test_copy_with_recovering(started_cluster, use_sample_offset):
True True
] ]
) )
def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset): def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY),
@ -315,29 +329,36 @@ def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offs
else: else:
execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) execute_task(Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)])
@pytest.mark.timeout(600) @pytest.mark.timeout(600)
def test_copy_month_to_week_partition(started_cluster): def test_copy_month_to_week_partition(started_cluster):
execute_task(Task2(started_cluster), []) execute_task(Task2(started_cluster), [])
@pytest.mark.timeout(600) @pytest.mark.timeout(600)
def test_copy_month_to_week_partition_with_recovering(started_cluster): def test_copy_month_to_week_partition_with_recovering(started_cluster):
execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) execute_task(Task2(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
@pytest.mark.timeout(600) @pytest.mark.timeout(600)
def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster):
execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) execute_task(Task2(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)])
def test_block_size(started_cluster): def test_block_size(started_cluster):
execute_task(Task_test_block_size(started_cluster), []) execute_task(Task_test_block_size(started_cluster), [])
def test_no_index(started_cluster): def test_no_index(started_cluster):
execute_task(Task_no_index(started_cluster), []) execute_task(Task_no_index(started_cluster), [])
def test_no_arg(started_cluster): def test_no_arg(started_cluster):
execute_task(Task_no_arg(started_cluster), []) execute_task(Task_no_arg(started_cluster), [])
if __name__ == '__main__': if __name__ == '__main__':
with contextmanager(started_cluster)() as cluster: with contextmanager(started_cluster)() as cluster:
for name, instance in cluster.instances.items(): for name, instance in cluster.instances.items():
print name, instance.ip_address print name, instance.ip_address
raw_input("Cluster created, press any key to destroy...") raw_input("Cluster created, press any key to destroy...")

26
tests/integration/test_cluster_copier/trivial_test.py
View File

@ -1,13 +1,10 @@
import os import os
import os.path as p
import sys import sys
import time import time
import datetime
import pytest
from contextlib import contextmanager from contextlib import contextmanager
import docker
from kazoo.client import KazooClient
import docker
import pytest
CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR))
@ -18,13 +15,14 @@ COPYING_FAIL_PROBABILITY = 0.33
MOVING_FAIL_PROBABILITY = 0.1 MOVING_FAIL_PROBABILITY = 0.1
cluster = None cluster = None
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def started_cluster(): def started_cluster():
global cluster global cluster
try: try:
clusters_schema = { clusters_schema = {
"0" : {"0" : ["0"]}, "0": {"0": ["0"]},
"1" : {"0" : ["0"]} "1": {"0": ["0"]}
} }
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -50,12 +48,11 @@ class TaskTrivial:
def __init__(self, cluster, use_sample_offset): def __init__(self, cluster, use_sample_offset):
self.cluster = cluster self.cluster = cluster
if use_sample_offset: if use_sample_offset:
self.zk_task_path="/clickhouse-copier/task_trivial_use_sample_offset" self.zk_task_path = "/clickhouse-copier/task_trivial_use_sample_offset"
else: else:
self.zk_task_path="/clickhouse-copier/task_trivial" self.zk_task_path = "/clickhouse-copier/task_trivial"
self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read() self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read()
def start(self): def start(self):
source = cluster.instances['s0_0_0'] source = cluster.instances['s0_0_0']
destination = cluster.instances['s1_0_0'] destination = cluster.instances['s1_0_0']
@ -68,8 +65,8 @@ class TaskTrivial:
"ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') " "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial', '1') "
"PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16")
source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", settings={"insert_distributed_sync": 1}) source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002",
settings={"insert_distributed_sync": 1})
def check(self): def check(self):
source = cluster.instances['s0_0_0'] source = cluster.instances['s0_0_0']
@ -138,7 +135,6 @@ def execute_task(task, cmd_options):
True True
] ]
) )
def test_trivial_copy(started_cluster, use_sample_offset): def test_trivial_copy(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--experimental-use-sample-offset', '1']) execute_task(TaskTrivial(started_cluster, use_sample_offset), ['--experimental-use-sample-offset', '1'])
@ -146,6 +142,7 @@ def test_trivial_copy(started_cluster, use_sample_offset):
print("AAAAA") print("AAAAA")
execute_task(TaskTrivial(started_cluster, use_sample_offset), []) execute_task(TaskTrivial(started_cluster, use_sample_offset), [])
@pytest.mark.parametrize( @pytest.mark.parametrize(
('use_sample_offset'), ('use_sample_offset'),
[ [
@ -153,7 +150,6 @@ def test_trivial_copy(started_cluster, use_sample_offset):
True True
] ]
) )
def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset): def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY),
@ -161,6 +157,7 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
else: else:
execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) execute_task(TaskTrivial(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)])
@pytest.mark.parametrize( @pytest.mark.parametrize(
('use_sample_offset'), ('use_sample_offset'),
[ [
@ -168,7 +165,6 @@ def test_trivial_copy_with_copy_fault(started_cluster, use_sample_offset):
True True
] ]
) )
def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset): def test_trivial_copy_with_move_fault(started_cluster, use_sample_offset):
if use_sample_offset: if use_sample_offset:
execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), execute_task(TaskTrivial(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY),

7
tests/integration/test_concurrent_queries_for_user_restriction/test.py
View File

@ -1,8 +1,7 @@
import time import time
from multiprocessing.dummy import Pool
import pytest import pytest
from multiprocessing.dummy import Pool
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -10,6 +9,7 @@ cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml']) node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml'])
node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml']) node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -21,6 +21,7 @@ def started_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_exception_message(started_cluster): def test_exception_message(started_cluster):
assert node1.query("select number from nums order by number") == "0\n1\n" assert node1.query("select number from nums order by number") == "0\n1\n"
@ -30,7 +31,7 @@ def test_exception_message(started_cluster):
busy_pool = Pool(3) busy_pool = Pool(3)
busy_pool.map_async(node_busy, xrange(3)) busy_pool.map_async(node_busy, xrange(3))
time.sleep(1) # wait a little until polling starts time.sleep(1) # wait a little until polling starts
try: try:
assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n" assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n"
except Exception as ex: except Exception as ex:

35
tests/integration/test_concurrent_ttl_merges/test.py
View File

@ -1,12 +1,9 @@
import time import time
import pytest import pytest
import helpers.client as client
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True)
@ -24,14 +21,16 @@ def started_cluster():
def count_ttl_merges_in_queue(node, table): def count_ttl_merges_in_queue(node, table):
result = node.query("SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) result = node.query(
"SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
if not result: if not result:
return 0 return 0
return int(result.strip()) return int(result.strip())
def count_ttl_merges_in_background_pool(node, table): def count_ttl_merges_in_background_pool(node, table):
result = node.query("SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table)) result = node.query(
"SELECT count() FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{}'".format(table))
if not result: if not result:
return 0 return 0
return int(result.strip()) return int(result.strip())
@ -55,12 +54,14 @@ def count_running_mutations(node, table):
# but it revealed a bug when we assign different merges to the same part # but it revealed a bug when we assign different merges to the same part
# on the borders of partitions. # on the borders of partitions.
def test_no_ttl_merges_in_busy_pool(started_cluster): def test_no_ttl_merges_in_busy_pool(started_cluster):
node1.query("CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0") node1.query(
"CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0")
node1.query("SYSTEM STOP TTL MERGES") node1.query("SYSTEM STOP TTL MERGES")
for i in range(1, 7): for i in range(1, 7):
node1.query("INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i)) node1.query(
"INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {}, number FROM numbers(5)".format(i))
node1.query("ALTER TABLE test_ttl UPDATE data = data + 1 WHERE sleepEachRow(1) = 0") node1.query("ALTER TABLE test_ttl UPDATE data = data + 1 WHERE sleepEachRow(1) = 0")
@ -85,7 +86,8 @@ def test_no_ttl_merges_in_busy_pool(started_cluster):
def test_limited_ttl_merges_in_empty_pool(started_cluster): def test_limited_ttl_merges_in_empty_pool(started_cluster):
node1.query("CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query(
"CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
node1.query("SYSTEM STOP TTL MERGES") node1.query("SYSTEM STOP TTL MERGES")
@ -107,7 +109,8 @@ def test_limited_ttl_merges_in_empty_pool(started_cluster):
def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster):
node1.query("CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query(
"CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
node1.query("SYSTEM STOP TTL MERGES") node1.query("SYSTEM STOP TTL MERGES")
@ -134,14 +137,17 @@ def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster):
def test_limited_ttl_merges_two_replicas(started_cluster): def test_limited_ttl_merges_two_replicas(started_cluster):
# Actually this test quite fast and often we cannot catch any merges. # Actually this test quite fast and often we cannot catch any merges.
# To check for sure just add some sleeps in mergePartsToTemporaryPart # To check for sure just add some sleeps in mergePartsToTemporaryPart
node1.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") node1.query(
node2.query("CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
node2.query(
"CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0")
node1.query("SYSTEM STOP TTL MERGES") node1.query("SYSTEM STOP TTL MERGES")
node2.query("SYSTEM STOP TTL MERGES") node2.query("SYSTEM STOP TTL MERGES")
for i in range(100): for i in range(100):
node1.query("INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i)) node1.query(
"INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {}, number FROM numbers(10000)".format(i))
node2.query("SYSTEM SYNC REPLICA replicated_ttl_2", timeout=10) node2.query("SYSTEM SYNC REPLICA replicated_ttl_2", timeout=10)
assert node1.query("SELECT COUNT() FROM replicated_ttl_2") == "1000000\n" assert node1.query("SELECT COUNT() FROM replicated_ttl_2") == "1000000\n"
@ -155,7 +161,8 @@ def test_limited_ttl_merges_two_replicas(started_cluster):
while True: while True:
merges_with_ttl_count_node1.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl_2")) merges_with_ttl_count_node1.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl_2"))
merges_with_ttl_count_node2.add(count_ttl_merges_in_background_pool(node2, "replicated_ttl_2")) merges_with_ttl_count_node2.add(count_ttl_merges_in_background_pool(node2, "replicated_ttl_2"))
if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n": if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query(
"SELECT COUNT() FROM replicated_ttl_2") == "0\n":
break break
# Both replicas can assign merges with TTL. If one will perform better than # Both replicas can assign merges with TTL. If one will perform better than

4
tests/integration/test_config_corresponding_root/test.py
View File

@ -1,6 +1,6 @@
import os import os
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@ -9,6 +9,7 @@ cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', main_configs=["configs/config.d/bad.xml"]) node = cluster.add_instance('node', main_configs=["configs/config.d/bad.xml"])
caught_exception = "" caught_exception = ""
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
global caught_exception global caught_exception
@ -17,6 +18,7 @@ def start_cluster():
except Exception as e: except Exception as e:
caught_exception = str(e) caught_exception = str(e)
def test_work(start_cluster): def test_work(start_cluster):
print(caught_exception) print(caught_exception)
assert caught_exception.find("Root element doesn't have the corresponding root element as the config file.") != -1 assert caught_exception.find("Root element doesn't have the corresponding root element as the config file.") != -1

54
tests/integration/test_config_substitutions/test.py
View File

@ -1,21 +1,26 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333 node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333
node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], env_variables={"MAX_QUERY_SIZE": "55555"}) node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'],
env_variables={"MAX_QUERY_SIZE": "55555"})
node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True)
node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], main_configs=['configs/max_query_size.xml']) # include value 77777 node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'],
main_configs=['configs/max_query_size.xml']) # include value 77777
node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml'])
node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, main_configs=['configs/max_query_size.xml']) node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'],
env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"},
main_configs=['configs/max_query_size.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
def create_zk_roots(zk): def create_zk_roots(zk):
zk.create(path="/setting/max_query_size", value="77777", makepath=True) zk.create(path="/setting/max_query_size", value="77777", makepath=True)
cluster.add_zookeeper_startup_command(create_zk_roots) cluster.add_zookeeper_startup_command(create_zk_roots)
cluster.start() cluster.start()
@ -23,25 +28,36 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_config(start_cluster): def test_config(start_cluster):
assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n" assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n"
assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n" assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n"
assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n" assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n"
assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n" assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n"
def test_allow_databases(start_cluster): def test_allow_databases(start_cluster):
node5.query("CREATE DATABASE db1") node5.query("CREATE DATABASE db1")
node5.query("CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)") node5.query(
"CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)")
node5.query("INSERT INTO db1.test_table VALUES('2000-01-01', 'test_key', 1)") node5.query("INSERT INTO db1.test_table VALUES('2000-01-01', 'test_key', 1)")
assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n" assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n"
assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n" assert node5.query(
assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n" "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n"
assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n" assert node5.query(
assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n" "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n"
assert node5.query(
"SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n"
assert node5.query(
"SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n"
assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == "" assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == ""
assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", user="test_allow").strip() == "" assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'",
assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" user="test_allow").strip() == ""
assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'",
assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", user="test_allow").strip() == "" user="test_allow").strip() == ""
assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'",
user="test_allow").strip() == ""
assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'",
user="test_allow").strip() == ""

9
tests/integration/test_consistant_parts_after_move_partition/test.py
View File

@ -1,12 +1,8 @@
import os
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
CLICKHOUSE_DATABASE = 'test' CLICKHOUSE_DATABASE = 'test'
@ -46,13 +42,14 @@ def test_consistent_part_after_move_partition(start_cluster):
# insert into all replicas # insert into all replicas
for i in range(100): for i in range(100):
node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE, node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE,
value=i)) value=i))
query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE) query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE)
query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE) query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)
assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_source, node1.query(query_source))
assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest))
node1.query('ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)) node1.query(
'ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE))
assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_source, node1.query(query_source))
assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest))

17
tests/integration/test_consistent_parts_after_clone_replica/test.py
View File

@ -8,19 +8,20 @@ from helpers.test_tools import assert_eq_with_retry
def fill_nodes(nodes, shard): def fill_nodes(nodes, shard):
for node in nodes: for node in nodes:
node.query( node.query(
''' '''
CREATE DATABASE test; CREATE DATABASE test;
CREATE TABLE test_table(date Date, id UInt32) CREATE TABLE test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
ORDER BY id PARTITION BY toYYYYMM(date) ORDER BY id PARTITION BY toYYYYMM(date)
SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name)) '''.format(shard=shard, replica=node.name))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:
@ -56,5 +57,3 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster):
# the first replica will be cloned from the second # the first replica will be cloned from the second
pm.heal_all() pm.heal_all()
assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table")) assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table"))

3
tests/integration/test_cross_replication/test.py
View File

@ -2,18 +2,17 @@ import time
from contextlib import contextmanager from contextlib import contextmanager
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:

16
tests/integration/test_custom_settings/test.py
View File

@ -2,7 +2,8 @@ import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"], user_configs=["configs/users.d/custom_settings.xml"]) node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"],
user_configs=["configs/users.d/custom_settings.xml"])
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
@ -22,9 +23,10 @@ def test():
assert node.query("SELECT getSetting('custom_d')") == "some text\n" assert node.query("SELECT getSetting('custom_d')") == "some text\n"
assert "custom_a = -5, custom_b = 10000000000, custom_c = -4.325, custom_d = \\'some text\\'" \ assert "custom_a = -5, custom_b = 10000000000, custom_c = -4.325, custom_d = \\'some text\\'" \
in node.query("SHOW CREATE SETTINGS PROFILE default") in node.query("SHOW CREATE SETTINGS PROFILE default")
assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting") assert "no settings profile" in node.query_and_get_error(
"SHOW CREATE SETTINGS PROFILE profile_with_unknown_setting")
assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_illformed_setting") assert "no settings profile" in node.query_and_get_error("SHOW CREATE SETTINGS PROFILE profile_illformed_setting")
@ -33,9 +35,9 @@ def test_invalid_settings():
node.query("SYSTEM FLUSH LOGS") node.query("SYSTEM FLUSH LOGS")
assert node.query("SELECT COUNT() FROM system.text_log WHERE" assert node.query("SELECT COUNT() FROM system.text_log WHERE"
" message LIKE '%Could not parse profile `profile_illformed_setting`%'" " message LIKE '%Could not parse profile `profile_illformed_setting`%'"
" AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n" " AND message LIKE '%Couldn\\'t restore Field from dump%'") == "1\n"
assert node.query("SELECT COUNT() FROM system.text_log WHERE" assert node.query("SELECT COUNT() FROM system.text_log WHERE"
" message LIKE '%Could not parse profile `profile_with_unknown_setting`%'" " message LIKE '%Could not parse profile `profile_with_unknown_setting`%'"
" AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n" " AND message LIKE '%Setting x is neither a builtin setting nor started with the prefix \\'custom_\\'%'") == "1\n"

12
tests/integration/test_ddl_alter_query/test.py
View File

@ -10,7 +10,6 @@ node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -18,10 +17,14 @@ def started_cluster():
for i, node in enumerate([node1, node2]): for i, node in enumerate([node1, node2]):
node.query("CREATE DATABASE testdb") node.query("CREATE DATABASE testdb")
node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(i)) node.query(
'''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format(
i))
for i, node in enumerate([node3, node4]): for i, node in enumerate([node3, node4]):
node.query("CREATE DATABASE testdb") node.query("CREATE DATABASE testdb")
node.query('''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(i)) node.query(
'''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format(
i))
yield cluster yield cluster
finally: finally:
@ -34,7 +37,8 @@ def test_alter(started_cluster):
node2.query("SYSTEM SYNC REPLICA testdb.test_table") node2.query("SYSTEM SYNC REPLICA testdb.test_table")
node4.query("SYSTEM SYNC REPLICA testdb.test_table") node4.query("SYSTEM SYNC REPLICA testdb.test_table")
node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", settings={"replication_alter_partitions_sync": "2"}) node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val",
settings={"replication_alter_partitions_sync": "2"})
node1.query("SYSTEM SYNC REPLICA testdb.test_table") node1.query("SYSTEM SYNC REPLICA testdb.test_table")
node2.query("SYSTEM SYNC REPLICA testdb.test_table") node2.query("SYSTEM SYNC REPLICA testdb.test_table")

105
tests/integration/test_default_compression_codec/test.py
View File

@ -1,14 +1,17 @@
import string
import random import random
import pytest import string
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml'], with_zookeeper=True)
node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True) node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml'],
image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True,
with_installed_binary=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
@ -21,12 +24,14 @@ def start_cluster():
def get_compression_codec_byte(node, table_name, part_name): def get_compression_codec_byte(node, table_name, part_name):
cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name) cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format(
table_name, part_name)
return node.exec_in_container(["bash", "-c", cmd]).strip() return node.exec_in_container(["bash", "-c", cmd]).strip()
def get_second_multiple_codec_byte(node, table_name, part_name): def get_second_multiple_codec_byte(node, table_name, part_name):
cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format(table_name, part_name) cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format(
table_name, part_name)
return node.exec_in_container(["bash", "-c", cmd]).strip() return node.exec_in_container(["bash", "-c", cmd]).strip()
@ -74,16 +79,22 @@ def test_default_codec_single(start_cluster):
# Same codec for all # Same codec for all
assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table", "2_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_compression_codec_byte(node1, "compression_table", "2_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table", "3_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_compression_codec_byte(node1, "compression_table", "3_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n"
# just to be sure that replication works # just to be sure that replication works
node1.query("OPTIMIZE TABLE compression_table FINAL") node1.query("OPTIMIZE TABLE compression_table FINAL")
@ -101,16 +112,22 @@ def test_default_codec_single(start_cluster):
node2.query("SYSTEM FLUSH LOGS") node2.query("SYSTEM FLUSH LOGS")
assert get_compression_codec_byte(node1, "compression_table", "1_0_0_1") == CODECS_MAPPING['ZSTD'] assert get_compression_codec_byte(node1, "compression_table", "1_0_0_1") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] assert get_compression_codec_byte(node1, "compression_table", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n"
assert get_compression_codec_byte(node1, "compression_table", "3_0_0_1") == CODECS_MAPPING['LZ4'] assert get_compression_codec_byte(node1, "compression_table", "3_0_0_1") == CODECS_MAPPING['LZ4']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n"
assert node1.query("SELECT COUNT() FROM compression_table") == "3\n" assert node1.query("SELECT COUNT() FROM compression_table") == "3\n"
assert node2.query("SELECT COUNT() FROM compression_table") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table") == "3\n"
@ -137,18 +154,24 @@ def test_default_codec_multiple(start_cluster):
# Same codec for all # Same codec for all
assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['ZSTD'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n"
node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15) node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15)
@ -156,18 +179,24 @@ def test_default_codec_multiple(start_cluster):
assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['ZSTD'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['ZSTD']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n"
assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n"
assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['LZ4'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['LZ4']
assert node1.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" assert node1.query(
assert node2.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
assert node2.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n"
assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n"
assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n"
@ -187,15 +216,21 @@ def test_default_codec_version_update(start_cluster):
node3.restart_with_latest_version() node3.restart_with_latest_version()
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n" assert node3.query(
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n"
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n" assert node3.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n"
assert node3.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n"
node3.query("OPTIMIZE TABLE compression_table FINAL") node3.query("OPTIMIZE TABLE compression_table FINAL")
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n" assert node3.query(
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n" "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n"
assert node3.query("SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n" assert node3.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n"
assert node3.query(
"SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n"
assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple']
assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC']
assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple'] assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple']

30
tests/integration/test_default_database_on_cluster/test.py
View File

@ -1,12 +1,20 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) ch1 = cluster.add_instance('ch1',
ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) with_zookeeper=True)
ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], with_zookeeper=True) ch2 = cluster.add_instance('ch2',
main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
with_zookeeper=True)
ch3 = cluster.add_instance('ch3',
main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
with_zookeeper=True)
ch4 = cluster.add_instance('ch4',
main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"],
with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
@ -20,13 +28,17 @@ def started_cluster():
def test_default_database_on_cluster(started_cluster): def test_default_database_on_cluster(started_cluster):
ch1.query(database='test_default_database', sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;") ch1.query(database='test_default_database',
sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;")
for node in [ch1, ch2, ch3, ch4]: for node in [ch1, ch2, ch3, ch4]:
assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_local_table\n" assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_local_table\n"
ch1.query(database='test_default_database', sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');") ch1.query(database='test_default_database',
sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');")
for node in [ch1, ch2, ch3, ch4]: for node in [ch1, ch2, ch3, ch4]:
assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n" assert node.query(
assert node.query("SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n" "SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n"
assert node.query(
"SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n"

19
tests/integration/test_default_role/test.py
View File

@ -1,7 +1,6 @@
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
import re
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance') instance = cluster.add_instance('instance')
@ -32,41 +31,41 @@ def test_set_default_roles():
assert instance.query("SHOW CURRENT ROLES", user="john") == "" assert instance.query("SHOW CURRENT ROLES", user="john") == ""
instance.query("GRANT rx, ry TO john") instance.query("GRANT rx, ry TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
instance.query("SET DEFAULT ROLE NONE TO john") instance.query("SET DEFAULT ROLE NONE TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == "" assert instance.query("SHOW CURRENT ROLES", user="john") == ""
instance.query("SET DEFAULT ROLE rx TO john") instance.query("SET DEFAULT ROLE rx TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]])
instance.query("SET DEFAULT ROLE ry TO john") instance.query("SET DEFAULT ROLE ry TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
instance.query("SET DEFAULT ROLE ALL TO john") instance.query("SET DEFAULT ROLE ALL TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john") instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
def test_alter_user(): def test_alter_user():
assert instance.query("SHOW CURRENT ROLES", user="john") == "" assert instance.query("SHOW CURRENT ROLES", user="john") == ""
instance.query("GRANT rx, ry TO john") instance.query("GRANT rx, ry TO john")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
instance.query("ALTER USER john DEFAULT ROLE NONE") instance.query("ALTER USER john DEFAULT ROLE NONE")
assert instance.query("SHOW CURRENT ROLES", user="john") == "" assert instance.query("SHOW CURRENT ROLES", user="john") == ""
instance.query("ALTER USER john DEFAULT ROLE rx") instance.query("ALTER USER john DEFAULT ROLE rx")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]])
instance.query("ALTER USER john DEFAULT ROLE ALL") instance.query("ALTER USER john DEFAULT ROLE ALL")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]])
instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx") instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx")
assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]])
def test_wrong_set_default_role(): def test_wrong_set_default_role():

15
tests/integration/test_delayed_replica_failover/test.py
View File

@ -1,14 +1,14 @@
import pytest import os
import sys
import time import time
import os, sys
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import helpers
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager from helpers.network import PartitionManager
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
# Cluster with 2 shards of 2 replicas each. node_1_1 is the instance with Distributed table. # Cluster with 2 shards of 2 replicas each. node_1_1 is the instance with Distributed table.
@ -19,6 +19,7 @@ node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True)
node_2_1 = cluster.add_instance('node_2_1', with_zookeeper=True) node_2_1 = cluster.add_instance('node_2_1', with_zookeeper=True)
node_2_2 = cluster.add_instance('node_2_2', with_zookeeper=True) node_2_2 = cluster.add_instance('node_2_2', with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -30,7 +31,7 @@ def started_cluster():
node.query(''' node.query('''
CREATE TABLE replicated (d Date, x UInt32) ENGINE = CREATE TABLE replicated (d Date, x UInt32) ENGINE =
ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)''' ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)'''
.format(shard=shard, instance=node.name)) .format(shard=shard, instance=node.name))
node_1_1.query( node_1_1.query(
"CREATE TABLE distributed (d Date, x UInt32) ENGINE = " "CREATE TABLE distributed (d Date, x UInt32) ENGINE = "
@ -51,7 +52,7 @@ def test(started_cluster):
node_1_2.query("INSERT INTO replicated VALUES ('2017-05-08', 1)") node_1_2.query("INSERT INTO replicated VALUES ('2017-05-08', 1)")
node_2_2.query("INSERT INTO replicated VALUES ('2017-05-08', 2)") node_2_2.query("INSERT INTO replicated VALUES ('2017-05-08', 2)")
time.sleep(1) # accrue replica delay time.sleep(1) # accrue replica delay
assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == '0' assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == '0'
assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == '1' assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == '1'
@ -78,7 +79,7 @@ SELECT sum(x) FROM distributed WITH TOTALS SETTINGS
pm.drop_instance_zk_connections(node_1_2) pm.drop_instance_zk_connections(node_1_2)
pm.drop_instance_zk_connections(node_2_2) pm.drop_instance_zk_connections(node_2_2)
time.sleep(4) # allow pings to zookeeper to timeout (must be greater than ZK session timeout). time.sleep(4) # allow pings to zookeeper to timeout (must be greater than ZK session timeout).
# At this point all replicas are stale, but the query must still go to second replicas which are the least stale ones. # At this point all replicas are stale, but the query must still go to second replicas which are the least stale ones.
assert instance_with_dist_table.query(''' assert instance_with_dist_table.query('''

22
tests/integration/test_dictionaries_all_layouts_and_sources/test.py
View File

@ -1,11 +1,12 @@
import pytest import math
import os import os
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceCassandra
from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceRedis, SourceCassandra from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, \
import math SourceExecutableHashed
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries')
@ -103,8 +104,6 @@ VALUES = {
] ]
} }
LAYOUTS = [ LAYOUTS = [
Layout("flat"), Layout("flat"),
Layout("hashed"), Layout("hashed"),
@ -135,6 +134,7 @@ DICTIONARIES = []
cluster = None cluster = None
node = None node = None
def get_dict(source, layout, fields, suffix_name=''): def get_dict(source, layout, fields, suffix_name=''):
global dict_configs_path global dict_configs_path
@ -173,7 +173,8 @@ def setup_module(module):
for fname in os.listdir(dict_configs_path): for fname in os.listdir(dict_configs_path):
dictionaries.append(os.path.join(dict_configs_path, fname)) dictionaries.append(os.path.join(dict_configs_path, fname))
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True, with_mongo=True, with_redis=True, with_cassandra=True) node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True,
with_mongo=True, with_redis=True, with_cassandra=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -195,7 +196,7 @@ def get_dictionaries(fold, total_folds, all_dicts):
chunk_len = int(math.ceil(len(all_dicts) / float(total_folds))) chunk_len = int(math.ceil(len(all_dicts) / float(total_folds)))
if chunk_len * fold >= len(all_dicts): if chunk_len * fold >= len(all_dicts):
return [] return []
return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] return all_dicts[fold * chunk_len: (fold + 1) * chunk_len]
def remove_mysql_dicts(): def remove_mysql_dicts():
@ -225,8 +226,8 @@ def remove_mysql_dicts():
TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed
""" """
#global DICTIONARIES # global DICTIONARIES
#DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")] # DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")]
@pytest.mark.parametrize("fold", list(range(10))) @pytest.mark.parametrize("fold", list(range(10)))
@ -281,7 +282,6 @@ def test_simple_dictionaries(started_cluster, fold):
@pytest.mark.parametrize("fold", list(range(10))) @pytest.mark.parametrize("fold", list(range(10)))
def test_complex_dictionaries(started_cluster, fold): def test_complex_dictionaries(started_cluster, fold):
if node.is_built_with_thread_sanitizer(): if node.is_built_with_thread_sanitizer():
remove_mysql_dicts() remove_mysql_dicts()

26
tests/integration/test_dictionaries_complex_key_cache_string/test.py
View File

@ -1,30 +1,40 @@
import pytest
import os import os
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def cluster(request): def cluster(request):
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
try: try:
if request.param == "memory": if request.param == "memory":
node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/complex_key_cache_string.xml']) node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/complex_key_cache_string.xml'])
if request.param == "ssd": if request.param == "ssd":
node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/ssd_complex_key_cache_string.xml']) node = cluster.add_instance('node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/ssd_complex_key_cache_string.xml'])
cluster.start() cluster.start()
node.query("create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id") node.query(
"create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id")
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
@pytest.mark.parametrize("cluster", ["memory", "ssd"], indirect=True) @pytest.mark.parametrize("cluster", ["memory", "ssd"], indirect=True)
def test_memory_consumption(cluster): def test_memory_consumption(cluster):
node = cluster.instances['node'] node = cluster.instances['node']
node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8)) node.query(
node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16)) "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8))
node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32)) node.query(
node.query("insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64)) "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16))
node.query(
"insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32))
node.query(
"insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64))
# Fill dictionary # Fill dictionary
node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)") node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)")

59
tests/integration/test_dictionaries_ddl/test.py
View File

@ -1,17 +1,24 @@
import pytest
import os import os
from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException
import pymysql
import warnings import warnings
import time import time
import pymysql
import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'],
node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'], user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'], dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml', 'configs/dictionaries/conflict_name_dictionary.xml'], user_configs=['configs/user_admin.xml']) node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'],
main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'],
user_configs=['configs/user_admin.xml', 'configs/user_default.xml'])
node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'],
dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml',
'configs/dictionaries/conflict_name_dictionary.xml'],
user_configs=['configs/user_admin.xml'])
node4 = cluster.add_instance('node4', user_configs=['configs/user_admin.xml', 'configs/config_password.xml']) node4 = cluster.add_instance('node4', user_configs=['configs/user_admin.xml', 'configs/config_password.xml'])
@ -22,6 +29,7 @@ def create_mysql_conn(user, password, hostname, port):
host=hostname, host=hostname,
port=port) port=port)
def execute_mysql_query(connection, query): def execute_mysql_query(connection, query):
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore") warnings.simplefilter("ignore")
@ -36,15 +44,18 @@ def started_cluster():
cluster.start() cluster.start()
for clickhouse in [node1, node2, node3, node4]: for clickhouse in [node1, node2, node3, node4]:
clickhouse.query("CREATE DATABASE test", user="admin") clickhouse.query("CREATE DATABASE test", user="admin")
clickhouse.query("CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", user="admin") clickhouse.query(
clickhouse.query("INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", user="admin") "CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id",
user="admin")
clickhouse.query(
"INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)",
user="admin")
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
@pytest.mark.parametrize("clickhouse,name,layout", [ @pytest.mark.parametrize("clickhouse,name,layout", [
(node1, 'complex_node1_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())'), (node1, 'complex_node1_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())'),
(node1, 'complex_node1_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))'), (node1, 'complex_node1_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))'),
@ -54,7 +65,9 @@ def started_cluster():
def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308) mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308)
execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse") execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse")
execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format(name)) execute_mysql_query(mysql_conn,
"CREATE TABLE clickhouse.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format(
name))
values = [] values = []
for i in range(1000): for i in range(1000):
values.append('(' + ','.join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ')') values.append('(' + ','.join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ')')
@ -81,12 +94,16 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
""".format(name, name, layout)) """.format(name, name, layout))
for i in range(172, 200): for i in range(172, 200):
assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 5 + '\n' assert clickhouse.query(
stroka = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip() "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i,
i * i)) == str(
i) * 5 + '\n'
stroka = clickhouse.query(
"SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i,
i * i)).strip()
value = float(stroka) value = float(stroka)
assert int(value) == int(i * 3.14) assert int(value) == int(i * 3.14)
for i in range(1000): for i in range(1000):
values.append('(' + ','.join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ')') values.append('(' + ','.join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ')')
execute_mysql_query(mysql_conn, "REPLACE INTO clickhouse.{} VALUES ".format(name) + ','.join(values)) execute_mysql_query(mysql_conn, "REPLACE INTO clickhouse.{} VALUES ".format(name) + ','.join(values))
@ -94,8 +111,13 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
clickhouse.query("SYSTEM RELOAD DICTIONARY 'default.{}'".format(name)) clickhouse.query("SYSTEM RELOAD DICTIONARY 'default.{}'".format(name))
for i in range(172, 200): for i in range(172, 200):
assert clickhouse.query("SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)) == str(i) * 3 + '\n' assert clickhouse.query(
string = clickhouse.query("SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, i * i)).strip() "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i,
i * i)) == str(
i) * 3 + '\n'
string = clickhouse.query(
"SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i,
i * i)).strip()
value = float(string) value = float(string)
assert int(value) == int(i * 2.718) assert int(value) == int(i * 2.718)
@ -183,6 +205,7 @@ def test_conflicting_name(started_cluster):
# old version still works # old version still works
node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n' node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n'
def test_http_dictionary_restrictions(started_cluster): def test_http_dictionary_restrictions(started_cluster):
try: try:
node3.query(""" node3.query("""
@ -199,6 +222,7 @@ def test_http_dictionary_restrictions(started_cluster):
except QueryRuntimeException as ex: except QueryRuntimeException as ex:
assert 'is not allowed in config.xml' in str(ex) assert 'is not allowed in config.xml' in str(ex)
def test_file_dictionary_restrictions(started_cluster): def test_file_dictionary_restrictions(started_cluster):
try: try:
node3.query(""" node3.query("""
@ -219,7 +243,8 @@ def test_file_dictionary_restrictions(started_cluster):
def test_dictionary_with_where(started_cluster): def test_dictionary_with_where(started_cluster):
mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308) mysql_conn = create_mysql_conn("root", "clickhouse", "localhost", 3308)
execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse") execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS clickhouse")
execute_mysql_query(mysql_conn, "CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))") execute_mysql_query(mysql_conn,
"CREATE TABLE clickhouse.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))")
execute_mysql_query(mysql_conn, "INSERT INTO clickhouse.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')") execute_mysql_query(mysql_conn, "INSERT INTO clickhouse.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')")
node1.query(""" node1.query("""

18
tests/integration/test_dictionaries_dependency/test.py
View File

@ -18,9 +18,9 @@ def start_cluster():
node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log")
node.query("INSERT INTO test.source VALUES (5,6)") node.query("INSERT INTO test.source VALUES (5,6)")
node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x "\ node.query("CREATE DICTIONARY test.dict(x UInt64, y UInt64) PRIMARY KEY x " \
"SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) "\ "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \
"LAYOUT(FLAT()) LIFETIME(0)") "LAYOUT(FLAT()) LIFETIME(0)")
yield cluster yield cluster
finally: finally:
@ -48,8 +48,8 @@ def cleanup_after_test():
def test_dependency_via_implicit_table(node): def test_dependency_via_implicit_table(node):
d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"]
for d_name in d_names: for d_name in d_names:
node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
"SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) "\ "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) " \
"LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) "LAYOUT(FLAT()) LIFETIME(0)".format(d_name))
def check(): def check():
@ -72,8 +72,8 @@ def test_dependency_via_explicit_table(node):
tbl_database, tbl_shortname = tbl_name.split('.') tbl_database, tbl_shortname = tbl_name.split('.')
d_name = d_names[i] d_name = d_names[i]
node.query("CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format(tbl_name)) node.query("CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format(tbl_name))
node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
"SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) "\ "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) " \
"LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database)) "LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database))
def check(): def check():
@ -93,8 +93,8 @@ def test_dependency_via_dictionary_database(node):
d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"]
for d_name in d_names: for d_name in d_names:
node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x "\ node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \
"SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) "\ "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test.dict' DB 'dict_db')) " \
"LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) "LAYOUT(FLAT()) LIFETIME(0)".format(d_name))
def check(): def check():

10
tests/integration/test_dictionaries_dependency_xml/test.py
View File

@ -1,13 +1,13 @@
import pytest import pytest
import os
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', 'configs/dictionaries/dep_z.xml'] DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml',
'configs/dictionaries/dep_z.xml']
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES,) instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES, )
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -60,14 +60,14 @@ def test_get_data(started_cluster):
query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)") query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)")
# Wait for dictionaries to be reloaded. # Wait for dictionaries to be reloaded.
assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time=2, retry_count=10)
assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n"
assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n"
assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n"
# dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed.
query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)") query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)")
assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time=2, retry_count=10)
assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n"
assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n"
assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n"

25
tests/integration/test_dictionaries_mysql/test.py
View File

@ -1,17 +1,13 @@
import pytest
import os
import time
## sudo -H pip install PyMySQL ## sudo -H pip install PyMySQL
import pymysql.cursors import pymysql.cursors
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml', 'configs/remote_servers.xml'] CONFIG_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml',
'configs/remote_servers.xml']
CONFIG_FILES += ['configs/enable_dictionaries.xml'] CONFIG_FILES += ['configs/enable_dictionaries.xml']
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql = True) instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql=True)
create_table_mysql_template = """ create_table_mysql_template = """
CREATE TABLE IF NOT EXISTS `test`.`{}` ( CREATE TABLE IF NOT EXISTS `test`.`{}` (
@ -25,10 +21,11 @@ create_clickhouse_dictionary_table_template = """
CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` UInt64, `value` String) ENGINE = Dictionary({}) CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` (`id` UInt64, `value` String) ENGINE = Dictionary({})
""" """
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
#time.sleep(30) # time.sleep(30)
cluster.start() cluster.start()
# Create a MySQL database # Create a MySQL database
@ -66,10 +63,12 @@ def test_load_mysql_dictionaries(started_cluster):
# Check number of row # Check number of row
assert query("SELECT count() FROM `test`.`dict_table_{}`".format('test' + str(n % 5))).rstrip() == '10000' assert query("SELECT count() FROM `test`.`dict_table_{}`".format('test' + str(n % 5))).rstrip() == '10000'
def create_mysql_db(mysql_connection, name): def create_mysql_db(mysql_connection, name):
with mysql_connection.cursor() as cursor: with mysql_connection.cursor() as cursor:
cursor.execute("CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET 'utf8'".format(name)) cursor.execute("CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET 'utf8'".format(name))
def prepare_mysql_table(table_name, index): def prepare_mysql_table(table_name, index):
mysql_connection = get_mysql_conn() mysql_connection = get_mysql_conn()
@ -78,17 +77,21 @@ def prepare_mysql_table(table_name, index):
# Insert rows using CH # Insert rows using CH
query = instance.query query = instance.query
query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name + str(index), table_name + str(index))) query(
"INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(
table_name + str(index), table_name + str(index)))
assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000' assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000'
mysql_connection.close() mysql_connection.close()
#Create CH Dictionary tables based on MySQL tables # Create CH Dictionary tables based on MySQL tables
query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index))) query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index)))
def get_mysql_conn(): def get_mysql_conn():
conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.10', port=3308) conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.10', port=3308)
return conn return conn
def create_mysql_table(conn, table_name): def create_mysql_table(conn, table_name):
with conn.cursor() as cursor: with conn.cursor() as cursor:
cursor.execute(create_table_mysql_template.format(table_name)) cursor.execute(create_table_mysql_template.format(table_name))

7
tests/integration/test_dictionaries_null_value/test.py
View File

@ -1,13 +1,11 @@
import pytest import pytest
import os
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV, assert_eq_with_retry
ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
DICTIONARY_FILES = ['configs/dictionaries/cache.xml'] DICTIONARY_FILES = ['configs/dictionaries/cache.xml']
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -42,4 +40,5 @@ def test_null_value(started_cluster):
# Check, that empty null_value interprets as default value # Check, that empty null_value interprets as default value
assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n" assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n"
assert query("select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n" assert query(
"select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n"

16
tests/integration/test_dictionaries_redis/test.py
View File

@ -1,7 +1,6 @@
import os import os
import pytest
import redis
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
from helpers.external_sources import SourceRedis from helpers.external_sources import SourceRedis
@ -22,10 +21,10 @@ KEY_FIELDS = {
} }
KEY_VALUES = { KEY_VALUES = {
"simple" : [ "simple": [
[1], [2] [1], [2]
], ],
"complex" : [ "complex": [
[1, 'world'], [2, 'qwerty2'] [1, 'world'], [2, 'qwerty2']
] ]
} }
@ -76,6 +75,7 @@ LAYOUTS = [
DICTIONARIES = [] DICTIONARIES = []
def get_dict(source, layout, fields, suffix_name=''): def get_dict(source, layout, fields, suffix_name=''):
global dict_configs_path global dict_configs_path
@ -99,8 +99,10 @@ def setup_module(module):
for i, field in enumerate(FIELDS): for i, field in enumerate(FIELDS):
DICTIONARIES.append([]) DICTIONARIES.append([])
sources = [] sources = []
sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2, storage_type="simple")) sources.append(SourceRedis("RedisSimple", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2,
sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1, storage_type="hash_map")) storage_type="simple"))
sources.append(SourceRedis("RedisHash", "localhost", "6380", "redis1", "6379", "", "clickhouse", i * 2 + 1,
storage_type="hash_map"))
for source in sources: for source in sources:
for layout in LAYOUTS: for layout in LAYOUTS:
if not source.compatible_with_layout(layout): if not source.compatible_with_layout(layout):
@ -118,6 +120,7 @@ def setup_module(module):
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_redis=True) node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_redis=True)
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def started_cluster(): def started_cluster():
try: try:
@ -134,6 +137,7 @@ def started_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
@pytest.mark.parametrize("id", range(len(FIELDS))) @pytest.mark.parametrize("id", range(len(FIELDS)))
def test_redis_dictionaries(started_cluster, id): def test_redis_dictionaries(started_cluster, id):
print 'id:', id print 'id:', id

4
tests/integration/test_dictionaries_select_all/generate_dictionaries.py
View File

@ -1,6 +1,5 @@
import os
import glob
import difflib import difflib
import os
files = ['key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv'] files = ['key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv']
@ -12,7 +11,6 @@ types = [
'Date', 'DateTime' 'Date', 'DateTime'
] ]
implicit_defaults = [ implicit_defaults = [
'1', '1', '1', '', '1', '1', '1', '',
'-1', '-1', '-1', '-1', '-1', '-1', '-1', '-1',

8
tests/integration/test_dictionaries_select_all/test.py
View File

@ -1,7 +1,9 @@
import pytest
import os import os
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV, assert_eq_with_retry from helpers.test_tools import TSV
from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@ -20,7 +22,7 @@ def setup_module(module):
dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure) dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure)
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=dictionary_files+['configs/enable_dictionaries.xml']) instance = cluster.add_instance('instance', main_configs=dictionary_files + ['configs/enable_dictionaries.xml'])
test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv')) test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv'))

45
tests/integration/test_dictionaries_update_and_reload/test.py
View File

@ -1,16 +1,18 @@
import pytest
import os import os
import time import time
from helpers.cluster import ClickHouseCluster
import pytest
from helpers.client import QueryTimeoutExceedException from helpers.client import QueryTimeoutExceedException
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml'] DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml',
'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml']
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) instance = cluster.add_instance('instance', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -30,24 +32,31 @@ def get_status(dictionary_name):
def get_last_exception(dictionary_name): def get_last_exception(dictionary_name):
return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
"\n").replace("\\'", "'")
def get_loading_start_time(dictionary_name): def get_loading_start_time(dictionary_name):
s = instance.query("SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") s = instance.query(
"SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
"\n")
if s == "1970-01-01 00:00:00": if s == "1970-01-01 00:00:00":
return None return None
return time.strptime(s, "%Y-%m-%d %H:%M:%S") return time.strptime(s, "%Y-%m-%d %H:%M:%S")
def get_last_successful_update_time(dictionary_name): def get_last_successful_update_time(dictionary_name):
s = instance.query("SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") s = instance.query(
"SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip(
"\n")
if s == "1970-01-01 00:00:00": if s == "1970-01-01 00:00:00":
return None return None
return time.strptime(s, "%Y-%m-%d %H:%M:%S") return time.strptime(s, "%Y-%m-%d %H:%M:%S")
def get_loading_duration(dictionary_name): def get_loading_duration(dictionary_name):
return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) return float(
instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'"))
def replace_in_file_in_container(file_name, what, replace_with): def replace_in_file_in_container(file_name, what, replace_with):
@ -63,14 +72,14 @@ def test_reload_while_loading(started_cluster):
# It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout. # It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout.
with pytest.raises(QueryTimeoutExceedException): with pytest.raises(QueryTimeoutExceedException):
query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 0.5) query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=0.5)
# The dictionary is now loading. # The dictionary is now loading.
assert get_status('slow') == "LOADING" assert get_status('slow') == "LOADING"
start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
assert duration > 0 assert duration > 0
time.sleep(0.5) # Still loading. time.sleep(0.5) # Still loading.
assert get_status('slow') == "LOADING" assert get_status('slow') == "LOADING"
prev_start_time, prev_duration = start_time, duration prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
@ -79,14 +88,14 @@ def test_reload_while_loading(started_cluster):
# SYSTEM RELOAD DICTIONARY should restart loading. # SYSTEM RELOAD DICTIONARY should restart loading.
with pytest.raises(QueryTimeoutExceedException): with pytest.raises(QueryTimeoutExceedException):
query("SYSTEM RELOAD DICTIONARY 'slow'", timeout = 0.5) query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=0.5)
assert get_status('slow') == "LOADING" assert get_status('slow') == "LOADING"
prev_start_time, prev_duration = start_time, duration prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
assert start_time > prev_start_time assert start_time > prev_start_time
assert duration < prev_duration assert duration < prev_duration
time.sleep(0.5) # Still loading. time.sleep(0.5) # Still loading.
assert get_status('slow') == "LOADING" assert get_status('slow') == "LOADING"
prev_start_time, prev_duration = start_time, duration prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow')
@ -95,7 +104,7 @@ def test_reload_while_loading(started_cluster):
# Changing the configuration file should restart loading again. # Changing the configuration file should restart loading again.
replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0') replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0')
time.sleep(5) # Configuration files are reloaded once in 5 seconds. time.sleep(5) # Configuration files are reloaded once in 5 seconds.
# This time loading should finish quickly. # This time loading should finish quickly.
assert get_status('slow') == "LOADED" assert get_status('slow') == "LOADED"
@ -129,7 +138,7 @@ def test_reload_after_loading(started_cluster):
assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n" assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n"
# SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries. # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries.
time.sleep(1) # see the comment above time.sleep(1) # see the comment above
replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '81', '82') replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '81', '82')
replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '101', '102') replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '101', '102')
query("SYSTEM RELOAD DICTIONARIES") query("SYSTEM RELOAD DICTIONARIES")
@ -138,7 +147,7 @@ def test_reload_after_loading(started_cluster):
# Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds. # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds.
# Wait slightly more, to be sure it did reload. # Wait slightly more, to be sure it did reload.
time.sleep(1) # see the comment above time.sleep(1) # see the comment above
replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '82', '83') replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '82', '83')
replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '102', '103') replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '102', '103')
time.sleep(7) time.sleep(7)
@ -163,7 +172,8 @@ def test_reload_after_fail_by_system_reload(started_cluster):
assert get_status("no_file") == "FAILED" assert get_status("no_file") == "FAILED"
# Creating the file source makes the dictionary able to load. # Creating the file source makes the dictionary able to load.
instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file.txt") instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"),
"/etc/clickhouse-server/config.d/no_file.txt")
query("SYSTEM RELOAD DICTIONARY 'no_file'") query("SYSTEM RELOAD DICTIONARY 'no_file'")
query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n"
assert get_status("no_file") == "LOADED" assert get_status("no_file") == "LOADED"
@ -192,7 +202,8 @@ def test_reload_after_fail_by_timer(started_cluster):
assert get_status("no_file_2") == "FAILED" assert get_status("no_file_2") == "FAILED"
# Creating the file source makes the dictionary able to load. # Creating the file source makes the dictionary able to load.
instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file_2.txt") instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"),
"/etc/clickhouse-server/config.d/no_file_2.txt")
time.sleep(6); time.sleep(6);
query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n"
assert get_status("no_file_2") == "LOADED" assert get_status("no_file_2") == "LOADED"

34
tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py
View File

@ -1,9 +1,8 @@
from __future__ import print_function from __future__ import print_function
import pytest
import time
import os
from contextlib import contextmanager
import time
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.cluster import ClickHouseKiller from helpers.cluster import ClickHouseKiller
from helpers.network import PartitionManager from helpers.network import PartitionManager
@ -14,6 +13,7 @@ dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/cache_ints_dictionary.xml']) 'configs/dictionaries/cache_ints_dictionary.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -32,6 +32,7 @@ def started_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
# @pytest.mark.skip(reason="debugging") # @pytest.mark.skip(reason="debugging")
def test_default_reading(started_cluster): def test_default_reading(started_cluster):
assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"
@ -39,14 +40,22 @@ def test_default_reading(started_cluster):
# Key 0 is not in dictionary, so default value will be returned # Key 0 is not in dictionary, so default value will be returned
def test_helper(): def test_helper():
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i8', toUInt64(13), toInt8(42));").rstrip() assert '42' == main_node.query(
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(13), toInt16(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i8', toUInt64(13), toInt8(42));").rstrip()
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(13), toInt32(42));").rstrip() assert '42' == main_node.query(
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(13), toInt64(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i16', toUInt64(13), toInt16(42));").rstrip()
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u8', toUInt64(13), toUInt8(42));").rstrip() assert '42' == main_node.query(
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(13), toUInt16(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i32', toUInt64(13), toInt32(42));").rstrip()
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(13), toUInt32(42));").rstrip() assert '42' == main_node.query(
assert '42' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(13), toUInt64(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i64', toUInt64(13), toInt64(42));").rstrip()
assert '42' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u8', toUInt64(13), toUInt8(42));").rstrip()
assert '42' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u16', toUInt64(13), toUInt16(42));").rstrip()
assert '42' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u32', toUInt64(13), toUInt32(42));").rstrip()
assert '42' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u64', toUInt64(13), toUInt64(42));").rstrip()
test_helper() test_helper()
@ -61,4 +70,3 @@ def test_default_reading(started_cluster):
time.sleep(3) time.sleep(3)
test_helper() test_helper()

12
tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py
View File

@ -1,10 +1,11 @@
from __future__ import print_function from __future__ import print_function
import pytest
import os import os
import random import random
import string import string
import time import time
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
@ -12,13 +13,16 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml','configs/dictionaries/cache_strings_default_settings.xml']) main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/cache_ints_dictionary.xml',
'configs/dictionaries/cache_strings_default_settings.xml'])
def get_random_string(string_length=8): def get_random_string(string_length=8):
alphabet = string.ascii_letters + string.digits alphabet = string.ascii_letters + string.digits
return ''.join((random.choice(alphabet) for _ in range(string_length))) return ''.join((random.choice(alphabet) for _ in range(string_length)))
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -31,13 +35,15 @@ def started_cluster():
ENGINE = Memory; ENGINE = Memory;
""") """)
values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) values_to_insert = ", ".join(
["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)])
dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert)) dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert))
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
# @pytest.mark.skip(reason="debugging") # @pytest.mark.skip(reason="debugging")
def test_return_real_values(started_cluster): def test_return_real_values(started_cluster):
assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"

12
tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py
View File

@ -1,18 +1,18 @@
from __future__ import print_function from __future__ import print_function
import pytest
import time
import os
from contextlib import contextmanager
import time
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.cluster import ClickHouseKiller from helpers.cluster import ClickHouseKiller
from helpers.network import PartitionManager from helpers.network import PartitionManager
from helpers.network import PartitionManagerDisabler
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml', 'configs/dictionaries/cache_ints_dictionary.xml']) main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/cache_ints_dictionary.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():

36
tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py
View File

@ -1,9 +1,8 @@
from __future__ import print_function from __future__ import print_function
import pytest
import time
import os
from contextlib import contextmanager
import time
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.cluster import ClickHouseKiller from helpers.cluster import ClickHouseKiller
from helpers.network import PartitionManager from helpers.network import PartitionManager
@ -11,7 +10,9 @@ from helpers.network import PartitionManager
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml','configs/dictionaries/cache_ints_dictionary.xml']) main_node = cluster.add_instance('main_node', main_configs=['configs/enable_dictionaries.xml',
'configs/dictionaries/cache_ints_dictionary.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
@ -31,19 +32,28 @@ def started_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
# @pytest.mark.skip(reason="debugging") # @pytest.mark.skip(reason="debugging")
def test_simple_dict_get_or_default(started_cluster): def test_simple_dict_get_or_default(started_cluster):
assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive"
def test_helper(): def test_helper():
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i8', toUInt64(5), toInt8(42));").rstrip() assert '5' == main_node.query(
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i16', toUInt64(5), toInt16(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i8', toUInt64(5), toInt8(42));").rstrip()
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i32', toUInt64(5), toInt32(42));").rstrip() assert '5' == main_node.query(
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'i64', toUInt64(5), toInt64(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i16', toUInt64(5), toInt16(42));").rstrip()
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u8', toUInt64(5), toUInt8(42));").rstrip() assert '5' == main_node.query(
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u16', toUInt64(5), toUInt16(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i32', toUInt64(5), toInt32(42));").rstrip()
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u32', toUInt64(5), toUInt32(42));").rstrip() assert '5' == main_node.query(
assert '5' == main_node.query("select dictGetOrDefault('anime_dict', 'u64', toUInt64(5), toUInt64(42));").rstrip() "select dictGetOrDefault('anime_dict', 'i64', toUInt64(5), toInt64(42));").rstrip()
assert '5' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u8', toUInt64(5), toUInt8(42));").rstrip()
assert '5' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u16', toUInt64(5), toUInt16(42));").rstrip()
assert '5' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u32', toUInt64(5), toUInt32(42));").rstrip()
assert '5' == main_node.query(
"select dictGetOrDefault('anime_dict', 'u64', toUInt64(5), toUInt64(42));").rstrip()
test_helper() test_helper()

7
tests/integration/test_dictionary_custom_settings/http_server.py
View File

@ -1,9 +1,9 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import argparse import argparse
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer import csv
import socket import socket
import ssl import ssl
import csv from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
# Decorator used to see if authentication works for external dictionary who use a HTTP source. # Decorator used to see if authentication works for external dictionary who use a HTTP source.
@ -15,6 +15,7 @@ def check_auth(fn):
req.send_response(401) req.send_response(401)
else: else:
fn(req) fn(req)
return wrapper return wrapper
@ -37,7 +38,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family):
self.send_header('Content-type', 'text/csv') self.send_header('Content-type', 'text/csv')
self.end_headers() self.end_headers()
def __send_data(self, only_ids = None): def __send_data(self, only_ids=None):
with open(data_path, 'r') as fl: with open(data_path, 'r') as fl:
reader = csv.reader(fl, delimiter='\t') reader = csv.reader(fl, delimiter='\t')
for row in reader: for row in reader:

5
tests/integration/test_dictionary_custom_settings/test.py
View File

@ -1,6 +1,6 @@
import os import os
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml'] ENABLE_DICT_CONFIG = ['configs/enable_dictionaries.xml']
@ -13,7 +13,8 @@ DICTIONARY_FILES = [
] ]
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG+DICTIONARY_FILES) instance = cluster.add_instance('node', main_configs=ENABLE_DICT_CONFIG + DICTIONARY_FILES)
def prepare(): def prepare():
node = instance node = instance

30
tests/integration/test_dictionary_ddl_on_cluster/test.py
View File

@ -1,19 +1,24 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) with_zookeeper=True)
ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], with_zookeeper=True) with_zookeeper=True)
ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
with_zookeeper=True)
ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"],
with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
cluster.start() cluster.start()
ch1.query("CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key") ch1.query(
"CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key")
yield cluster yield cluster
finally: finally:
@ -27,7 +32,6 @@ def test_dictionary_ddl_on_cluster(started_cluster):
for num, node in enumerate([ch1, ch2, ch3, ch4]): for num, node in enumerate([ch1, ch2, ch3, ch4]):
node.query("insert into sometbl values ({}, '{}')".format(num, node.name)) node.query("insert into sometbl values ({}, '{}')".format(num, node.name))
ch1.query( ch1.query(
""" """
CREATE DICTIONARY somedict ON CLUSTER 'cluster' ( CREATE DICTIONARY somedict ON CLUSTER 'cluster' (
@ -42,7 +46,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
for num, node in enumerate([ch1, ch2, ch3, ch4]): for num, node in enumerate([ch1, ch2, ch3, ch4]):
assert node.query("SELECT count() from sometbl") == "1\n" assert node.query("SELECT count() from sometbl") == "1\n"
assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' assert node.query(
"SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'") ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'")
@ -54,8 +59,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
for num, node in enumerate([ch1, ch2, ch3, ch4]): for num, node in enumerate([ch1, ch2, ch3, ch4]):
assert node.query("SELECT count() from sometbl") == "1\n" assert node.query("SELECT count() from sometbl") == "1\n"
assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' assert node.query(
"SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n'
for num, node in enumerate([ch1, ch2, ch3, ch4]): for num, node in enumerate([ch1, ch2, ch3, ch4]):
node.query("ALTER TABLE sometbl UPDATE value = 'new_key' WHERE 1") node.query("ALTER TABLE sometbl UPDATE value = 'new_key' WHERE 1")
@ -63,7 +68,8 @@ def test_dictionary_ddl_on_cluster(started_cluster):
ch1.query("SYSTEM RELOAD DICTIONARY ON CLUSTER 'cluster' `default.somedict`") ch1.query("SYSTEM RELOAD DICTIONARY ON CLUSTER 'cluster' `default.somedict`")
for num, node in enumerate([ch1, ch2, ch3, ch4]): for num, node in enumerate([ch1, ch2, ch3, ch4]):
assert node.query("SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n' assert node.query(
"SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n'
ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'") ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'")

27
tests/integration/test_disk_access_storage/test.py
View File

@ -39,15 +39,20 @@ def test_create():
def check(): def check():
assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" assert instance.query(
assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" assert instance.query(
"SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
assert instance.query(
"SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n"
assert instance.query("SHOW GRANTS FOR u1") == "" assert instance.query("SHOW GRANTS FOR u1") == ""
assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n"
assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n"
assert instance.query("SHOW GRANTS FOR rx") == "" assert instance.query("SHOW GRANTS FOR rx") == ""
assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" assert instance.query(
assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n"
assert instance.query(
"SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
check() check()
instance.restart_clickhouse() # Check persistency instance.restart_clickhouse() # Check persistency
@ -69,15 +74,18 @@ def test_alter():
def check(): def check():
assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" assert instance.query(
"SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n" assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n"
assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n"
assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n"
assert instance.query("SHOW CREATE ROLE ry") == "CREATE ROLE ry\n" assert instance.query("SHOW CREATE ROLE ry") == "CREATE ROLE ry\n"
assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n" assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n"
assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n" assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n"
assert instance.query("SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" assert instance.query(
assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n"
assert instance.query(
"SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n"
check() check()
instance.restart_clickhouse() # Check persistency instance.restart_clickhouse() # Check persistency
@ -98,7 +106,8 @@ def test_drop():
assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n"
assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n" assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n"
assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2") assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2")
assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error(
"SHOW CREATE ROW POLICY p ON mydb.mytable")
assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q") assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q")
check() check()

14
tests/integration/test_disk_types/test.py
View File

@ -1,15 +1,14 @@
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
disk_types = { disk_types = {
"default" : "local", "default": "local",
"disk_s3" : "s3", "disk_s3": "s3",
"disk_memory" : "memory", "disk_memory": "memory",
} }
@pytest.fixture(scope="module")
@pytest.fixture(scope="module")
def cluster(): def cluster():
try: try:
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -19,19 +18,20 @@ def cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_different_types(cluster): def test_different_types(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
responce = node.query("SELECT * FROM system.disks") responce = node.query("SELECT * FROM system.disks")
disks = responce.split("\n") disks = responce.split("\n")
for disk in disks: for disk in disks:
if disk == '': # skip empty line (after split at last position) if disk == '': # skip empty line (after split at last position)
continue continue
fields = disk.split("\t") fields = disk.split("\t")
assert len(fields) >= 6 assert len(fields) >= 6
assert disk_types.get(fields[0], "UNKNOWN") == fields[5] assert disk_types.get(fields[0], "UNKNOWN") == fields[5]
def test_select_by_type(cluster): def test_select_by_type(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
for name, disk_type in disk_types.items(): for name, disk_type in disk_types.items():
assert node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") == name + "\n" assert node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") == name + "\n"

15
tests/integration/test_distributed_backward_compatability/test.py
View File

@ -1,16 +1,14 @@
import pytest import pytest
import time
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server',
tag='19.17.8.54', stay_alive=True, with_installed_binary=True)
node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -22,14 +20,17 @@ def started_cluster():
node_old.query("INSERT INTO local_table VALUES (1, 'node1')") node_old.query("INSERT INTO local_table VALUES (1, 'node1')")
node_new.query("INSERT INTO local_table VALUES (2, 'node2')") node_new.query("INSERT INTO local_table VALUES (2, 'node2')")
node_old.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") node_old.query(
node_new.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
node_new.query(
"CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)")
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_distributed_in_tuple(started_cluster): def test_distributed_in_tuple(started_cluster):
query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))" query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))"
query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed"

30
tests/integration/test_distributed_ddl/cluster.py
View File

@ -17,18 +17,21 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
def prepare(self, replace_hostnames_with_ips=True): def prepare(self, replace_hostnames_with_ips=True):
try: try:
main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml","ddl.xml"] main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml",
"ddl.xml"]
main_configs = [os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files] main_configs = [os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files]
user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in ["restricted_user.xml", "query_log.xml"]] user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in
["restricted_user.xml", "query_log.xml"]]
if self.test_config_dir == "configs_secure": if self.test_config_dir == "configs_secure":
main_configs += [os.path.join(self.test_config_dir, f) for f in ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]] main_configs += [os.path.join(self.test_config_dir, f) for f in
["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]]
for i in xrange(4): for i in xrange(4):
self.add_instance( self.add_instance(
'ch{}'.format(i+1), 'ch{}'.format(i + 1),
main_configs=main_configs, main_configs=main_configs,
user_configs=user_configs, user_configs=user_configs,
macros={"layer": 0, "shard": i/2 + 1, "replica": i%2 + 1}, macros={"layer": 0, "shard": i / 2 + 1, "replica": i % 2 + 1},
with_zookeeper=True) with_zookeeper=True)
self.start() self.start()
@ -40,8 +43,12 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
# Select sacrifice instance to test CONNECTION_LOSS and server fail on it # Select sacrifice instance to test CONNECTION_LOSS and server fail on it
sacrifice = self.instances['ch4'] sacrifice = self.instances['ch4']
self.pm_random_drops = PartitionManager() self.pm_random_drops = PartitionManager()
self.pm_random_drops._add_rule({'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) self.pm_random_drops._add_rule(
self.pm_random_drops._add_rule({'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, 'action': 'REJECT --reject-with tcp-reset'}) {'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181,
'action': 'REJECT --reject-with tcp-reset'})
self.pm_random_drops._add_rule(
{'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181,
'action': 'REJECT --reject-with tcp-reset'})
# Initialize databases and service tables # Initialize databases and service tables
instance = self.instances['ch1'] instance = self.instances['ch1']
@ -67,7 +74,7 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
num_hosts = len(self.instances) num_hosts = len(self.instances)
M = TSV.toMat(tsv_content) M = TSV.toMat(tsv_content)
hosts = [(l[0], l[1]) for l in M] # (host, port) hosts = [(l[0], l[1]) for l in M] # (host, port)
codes = [l[2] for l in M] codes = [l[2] for l in M]
messages = [l[3] for l in M] messages = [l[3] for l in M]
@ -88,14 +95,17 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster):
for inst_name in instances_to_replace: for inst_name in instances_to_replace:
inst = self.instances[inst_name] inst = self.instances[inst_name]
self.instances[inst_name].exec_in_container(['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], environment={"NEW_CONFIG": clusters_config}, privileged=True) self.instances[inst_name].exec_in_container(
['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'],
environment={"NEW_CONFIG": clusters_config}, privileged=True)
# print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"])
@staticmethod @staticmethod
def ddl_check_there_are_no_dublicates(instance): def ddl_check_there_are_no_dublicates(instance):
query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)" query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)"
rows = instance.query(query) rows = instance.query(query)
assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name, instance.ip_address, query) assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}, query {}".format(instance.name,
instance.ip_address, query)
@staticmethod @staticmethod
def insert_reliable(instance, query_insert): def insert_reliable(instance, query_insert):

123
tests/integration/test_distributed_ddl/test.py
View File

@ -41,7 +41,8 @@ def test_default_database(test_cluster):
test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV")
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV")
test_cluster.ddl_check_query(instance, "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") test_cluster.ddl_check_query(instance,
"CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null")
contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h") contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h")
assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n") assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n")
@ -52,13 +53,18 @@ def test_default_database(test_cluster):
def test_create_view(test_cluster): def test_create_view(test_cluster):
instance = test_cluster.instances['ch3'] instance = test_cluster.instances['ch3']
test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") test_cluster.ddl_check_query(instance,
test_cluster.ddl_check_query(instance, "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV")
test_cluster.ddl_check_query(instance,
"CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV")
test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV")
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") test_cluster.ddl_check_query(instance,
"DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV")
test_cluster.ddl_check_query(instance, "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") test_cluster.ddl_check_query(instance,
test_cluster.ddl_check_query(instance, "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory")
test_cluster.ddl_check_query(instance,
"RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV")
test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'")
@ -69,7 +75,8 @@ def test_on_server_fail(test_cluster):
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'")
kill_instance.get_docker_handle().stop() kill_instance.get_docker_handle().stop()
request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", timeout=30) request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null",
timeout=30)
kill_instance.get_docker_handle().start() kill_instance.get_docker_handle().start()
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'")
@ -78,7 +85,8 @@ def test_on_server_fail(test_cluster):
test_cluster.check_all_hosts_successfully_executed(request.get_answer()) test_cluster.check_all_hosts_successfully_executed(request.get_answer())
# And check query artefacts # And check query artefacts
contents = instance.query("SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") contents = instance.query(
"SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h")
assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n")
test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'")
@ -98,11 +106,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout):
def test_on_connection_loss(test_cluster): def test_on_connection_loss(test_cluster):
_test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config) _test_on_connection_losses(test_cluster, 1.5) # connection loss will occur only (3 sec ZK timeout in config)
def test_on_session_expired(test_cluster): def test_on_session_expired(test_cluster):
_test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config) _test_on_connection_losses(test_cluster, 4) # session should be expired (3 sec ZK timeout in config)
def test_simple_alters(test_cluster): def test_simple_alters(test_cluster):
@ -127,28 +135,31 @@ ENGINE = Distributed('{cluster}', default, merge, i)
for i in xrange(0, 4, 2): for i in xrange(0, 4, 2):
k = (i / 2) * 2 k = (i / 2) * 2
test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k+1)) test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1))
assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)]))
assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(
''.join(['{}\n'.format(x) for x in xrange(4)]))
time.sleep(5) time.sleep(5)
test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64") test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64")
time.sleep(5) time.sleep(5)
test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV") test_cluster.ddl_check_query(instance,
"ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV")
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
for i in xrange(0, 4, 2): for i in xrange(0, 4, 2):
k = (i / 2) * 2 + 4 k = (i / 2) * 2 + 4
test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k+1)) test_cluster.instances['ch{}'.format(i + 1)].query(
"INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k + 1))
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)]))
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)]))
test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002") test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002")
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'") test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'")
test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'") test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'")
@ -160,9 +171,11 @@ def test_macro(test_cluster):
test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory")
for i in xrange(4): for i in xrange(4):
test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO tab VALUES ({})".format(i)) test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
"INSERT INTO tab VALUES ({})".format(i))
test_cluster.ddl_check_query(instance, "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") test_cluster.ddl_check_query(instance,
"CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)")
assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n')
assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n')
@ -197,22 +210,27 @@ def test_allowed_databases(test_cluster):
instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster") instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster")
instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster") instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster")
instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory",
settings={"user": "restricted_user"})
with pytest.raises(Exception): with pytest.raises(Exception):
instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory",
settings={"user": "restricted_user"})
with pytest.raises(Exception): with pytest.raises(Exception):
instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", settings={"user" : "restricted_user"}) instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory",
settings={"user": "restricted_user"})
with pytest.raises(Exception): with pytest.raises(Exception):
instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user" : "restricted_user"}) instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user": "restricted_user"})
instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user": "restricted_user"})
instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user" : "restricted_user"})
def test_kill_query(test_cluster): def test_kill_query(test_cluster):
instance = test_cluster.instances['ch3'] instance = test_cluster.instances['ch3']
test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV")
def test_detach_query(test_cluster): def test_detach_query(test_cluster):
instance = test_cluster.instances['ch3'] instance = test_cluster.instances['ch3']
@ -226,21 +244,25 @@ def test_optimize_query(test_cluster):
instance = test_cluster.instances['ch3'] instance = test_cluster.instances['ch3']
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV")
test_cluster.ddl_check_query(instance, "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") test_cluster.ddl_check_query(instance,
"CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)")
test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV")
def test_create_as_select(test_cluster): def test_create_as_select(test_cluster):
instance = test_cluster.instances['ch2'] instance = test_cluster.instances['ch2']
test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") test_cluster.ddl_check_query(instance,
"CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)")
assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n") assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n")
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster")
def test_create_reserved(test_cluster): def test_create_reserved(test_cluster):
instance = test_cluster.instances['ch2'] instance = test_cluster.instances['ch2']
test_cluster.ddl_check_query(instance, "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") test_cluster.ddl_check_query(instance,
test_cluster.ddl_check_query(instance, "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)")
test_cluster.ddl_check_query(instance,
"CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)")
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster")
test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster")
@ -248,11 +270,12 @@ def test_create_reserved(test_cluster):
def test_rename(test_cluster): def test_rename(test_cluster):
instance = test_cluster.instances['ch1'] instance = test_cluster.instances['ch1']
rules = test_cluster.pm_random_drops.pop_rules() rules = test_cluster.pm_random_drops.pop_rules()
test_cluster.ddl_check_query(instance, "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") test_cluster.ddl_check_query(instance,
test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)")
test_cluster.ddl_check_query(instance,
"CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;") test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;")
for i in range(10): for i in range(10):
instance.query("insert into rename (id) values ({})".format(i)) instance.query("insert into rename (id) values ({})".format(i))
@ -261,10 +284,13 @@ def test_rename(test_cluster):
# so ddl query will always fail on some replicas even if query was actually executed by leader # so ddl query will always fail on some replicas even if query was actually executed by leader
# Also such inconsistency in cluster configuration may lead to query duplication if leader suddenly changed # Also such inconsistency in cluster configuration may lead to query duplication if leader suddenly changed
# because path of lock in zk contains shard name, which is list of host names of replicas # because path of lock in zk contains shard name, which is list of host names of replicas
instance.query("ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", ignore_error=True) instance.query(
"ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))",
ignore_error=True)
time.sleep(1) time.sleep(1)
test_cluster.ddl_check_query(instance, "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") test_cluster.ddl_check_query(instance,
"CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)")
instance.query("system stop distributed sends rename") instance.query("system stop distributed sends rename")
@ -283,36 +309,43 @@ def test_rename(test_cluster):
# system stop distributed sends does not affect inserts into local shard, # system stop distributed sends does not affect inserts into local shard,
# so some ids in range (10, 20) will be inserted into rename_shard # so some ids in range (10, 20) will be inserted into rename_shard
assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360" assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360"
#assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290" # assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290"
assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115" assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115"
#assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45" # assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45"
assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245" assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245"
test_cluster.pm_random_drops.push_rules(rules) test_cluster.pm_random_drops.push_rules(rules)
def test_socket_timeout(test_cluster): def test_socket_timeout(test_cluster):
instance = test_cluster.instances['ch1'] instance = test_cluster.instances['ch1']
# queries should not fail with "Timeout exceeded while reading from socket" in case of EINTR caused by query profiler # queries should not fail with "Timeout exceeded while reading from socket" in case of EINTR caused by query profiler
for i in range(0, 100): for i in range(0, 100):
instance.query("select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host") instance.query("select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host")
def test_replicated_without_arguments(test_cluster): def test_replicated_without_arguments(test_cluster):
rules = test_cluster.pm_random_drops.pop_rules() rules = test_cluster.pm_random_drops.pop_rules()
instance = test_cluster.instances['ch1'] instance = test_cluster.instances['ch1']
test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic", test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic",
settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", test_cluster.ddl_check_query(instance,
"CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster")
test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", test_cluster.ddl_check_query(instance,
"CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster")
test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", test_cluster.ddl_check_query(instance,
"CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n",
settings={'show_table_uuid_in_table_create_query_if_not_nil': 1}) settings={'show_table_uuid_in_table_create_query_if_not_nil': 1})
test_cluster.ddl_check_query(instance, "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") test_cluster.ddl_check_query(instance,
"EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster")
test_cluster.pm_random_drops.push_rules(rules) test_cluster.pm_random_drops.push_rules(rules)
if __name__ == '__main__': if __name__ == '__main__':
with contextmanager(test_cluster)() as ctx_cluster: with contextmanager(test_cluster)() as ctx_cluster:
for name, instance in ctx_cluster.instances.items(): for name, instance in ctx_cluster.instances.items():
print name, instance.ip_address print name, instance.ip_address
raw_input("Cluster created, press any key to destroy...") raw_input("Cluster created, press any key to destroy...")

24
tests/integration/test_distributed_ddl/test_replicated_alter.py
View File

@ -1,6 +1,7 @@
import os import os
import sys import sys
import time import time
import pytest import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -60,29 +61,34 @@ ENGINE = Distributed(cluster, default, merge_for_alter, i)
for i in xrange(4): for i in xrange(4):
k = (i / 2) * 2 k = (i / 2) * 2
test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k+1)) test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
"INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k + 1))
test_cluster.sync_replicas("merge_for_alter") test_cluster.sync_replicas("merge_for_alter")
assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(''.join(['{}\n'.format(x) for x in xrange(4)])) assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV(
''.join(['{}\n'.format(x) for x in xrange(4)]))
test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64")
test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)") test_cluster.ddl_check_query(instance,
"ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)")
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)]))
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
for i in xrange(4): for i in xrange(4):
k = (i / 2) * 2 + 4 k = (i / 2) * 2 + 4
test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k+1)) test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)],
"INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k + 1))
test_cluster.sync_replicas("merge_for_alter") test_cluster.sync_replicas("merge_for_alter")
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(8)])) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(8)]))
test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002")
assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(''.join(['{}\t{}\n'.format(x,x) for x in xrange(4)])) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV(
''.join(['{}\t{}\n'.format(x, x) for x in xrange(4)]))
test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster")

9
tests/integration/test_distributed_ddl_on_cross_replication/test.py
View File

@ -1,4 +1,3 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
@ -12,6 +11,7 @@ node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True,
macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2}) macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2})
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -49,7 +49,8 @@ def started_cluster():
2017-06-17 31 2 2017-06-17 31 2
''' '''
node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert, settings={"insert_distributed_sync" : 1}) node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert,
settings={"insert_distributed_sync": 1})
yield cluster yield cluster
finally: finally:
@ -64,7 +65,8 @@ def test_alter_ddl(started_cluster):
WHERE part_key='2017-06-16'") WHERE part_key='2017-06-16'")
node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5)
assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3') assert_eq_with_retry(node1,
"SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3')
node1.query("ALTER TABLE replica_1.replicated_local \ node1.query("ALTER TABLE replica_1.replicated_local \
ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;") ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;")
@ -75,4 +77,3 @@ def test_alter_ddl(started_cluster):
node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5)
assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0') assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0')

70
tests/integration/test_distributed_ddl_password/test.py
View File

@ -1,17 +1,21 @@
import time
import pytest import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry from helpers.test_tools import assert_eq_with_retry
from helpers.client import QueryRuntimeException
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"],
node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"],
node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"],
node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"], user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"],
user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"],
user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"],
user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -21,27 +25,29 @@ def start_cluster():
for node, shard in [(node1, 1), (node2, 1), (node3, 2), (node4, 2), (node5, 3), (node6, 3)]: for node, shard in [(node1, 1), (node2, 1), (node3, 2), (node4, 2), (node5, 3), (node6, 3)]:
node.query( node.query(
''' '''
CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}')
PARTITION BY date PARTITION BY date
ORDER BY id ORDER BY id
'''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"}) '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"})
yield cluster yield cluster
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_truncate(start_cluster): def test_truncate(start_cluster):
node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"}) node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)",
settings={"password": "clickhouse"})
assert node1.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" assert node1.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
node2.query("system sync replica test_table", settings={"password": "clickhouse"}) node2.query("system sync replica test_table", settings={"password": "clickhouse"})
assert node2.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" assert node2.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)",
node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", settings={"password": "clickhouse"}) settings={"password": "clickhouse"})
assert node3.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" assert node3.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n"
node4.query("system sync replica test_table", settings={"password": "clickhouse"}) node4.query("system sync replica test_table", settings={"password": "clickhouse"})
@ -55,11 +61,15 @@ def test_truncate(start_cluster):
node2.query("drop table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"}) node2.query("drop table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"})
for node in [node1, node2, node3, node4]: for node in [node1, node2, node3, node4]:
assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0",
settings={"password": "clickhouse"})
def test_alter(start_cluster): def test_alter(start_cluster):
node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", settings={"password": "clickhouse"}) node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)",
node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", settings={"password": "clickhouse"}) settings={"password": "clickhouse"})
node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)",
settings={"password": "clickhouse"})
node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"})
node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"})
@ -75,24 +85,30 @@ def test_alter(start_cluster):
assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'",
settings={"password": "clickhouse"})
assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
with pytest.raises(QueryRuntimeException): with pytest.raises(QueryRuntimeException):
node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'",
settings={"password": "clickhouse"})
node5.query("ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) node5.query("ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"})
assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"})
node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", settings={"password": "clickhouse"}) node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String",
settings={"password": "clickhouse"})
assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1",
assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", settings={"password": "clickhouse"}) settings={"password": "clickhouse"})
assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1",
settings={"password": "clickhouse"})
node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", settings={"password": "clickhouse"}) node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'",
settings={"password": "clickhouse"})
assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"})
assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"})

25
tests/integration/test_distributed_format/test.py
View File

@ -1,12 +1,6 @@
import time
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from multiprocessing.dummy import Pool
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml']) node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml'])
@ -27,10 +21,13 @@ def started_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
@cluster_param @cluster_param
def test_single_file(started_cluster, cluster): def test_single_file(started_cluster, cluster):
node.query("create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) node.query(
node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) "create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')",
settings={"use_compact_format_in_distributed_parts_names": "1"})
query = "select * from file('/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin', 'Distributed')" query = "select * from file('/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin', 'Distributed')"
out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
@ -48,9 +45,12 @@ def test_single_file(started_cluster, cluster):
@cluster_param @cluster_param
def test_two_files(started_cluster, cluster): def test_two_files(started_cluster, cluster):
node.query("create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) node.query(
node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", settings={"use_compact_format_in_distributed_parts_names": "1"}) "create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", settings={"use_compact_format_in_distributed_parts_names": "1"}) node.query("insert into test.distr_2 values (0, '_'), (1, 'a')",
settings={"use_compact_format_in_distributed_parts_names": "1"})
node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')",
settings={"use_compact_format_in_distributed_parts_names": "1"})
query = "select * from file('/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x" query = "select * from file('/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x"
out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
@ -68,7 +68,8 @@ def test_two_files(started_cluster, cluster):
@cluster_param @cluster_param
def test_single_file_old(started_cluster, cluster): def test_single_file_old(started_cluster, cluster):
node.query("create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) node.query(
"create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster))
node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')") node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')")
query = "select * from file('/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin', 'Distributed')" query = "select * from file('/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin', 'Distributed')"

15
tests/integration/test_distributed_load_balancing/test.py
View File

@ -3,8 +3,8 @@
# pylint: disable=line-too-long # pylint: disable=line-too-long
import uuid import uuid
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -14,7 +14,8 @@ n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml'])
n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml']) n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml'])
nodes = len(cluster.instances) nodes = len(cluster.instances)
queries = nodes*5 queries = nodes * 5
def bootstrap(): def bootstrap():
for n in cluster.instances.values(): for n in cluster.instances.values():
@ -58,9 +59,11 @@ def bootstrap():
data) data)
""".format()) """.format())
def make_uuid(): def make_uuid():
return uuid.uuid4().hex return uuid.uuid4().hex
@pytest.fixture(scope='module', autouse=True) @pytest.fixture(scope='module', autouse=True)
def start_cluster(): def start_cluster():
try: try:
@ -70,6 +73,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def get_node(query_node, table='dist', *args, **kwargs): def get_node(query_node, table='dist', *args, **kwargs):
query_id = make_uuid() query_id = make_uuid()
@ -106,6 +110,7 @@ def get_node(query_node, table='dist', *args, **kwargs):
""".format(query_id=query_id)) """.format(query_id=query_id))
return rows.strip() return rows.strip()
# TODO: right now random distribution looks bad, but works # TODO: right now random distribution looks bad, but works
def test_load_balancing_default(): def test_load_balancing_default():
unique_nodes = set() unique_nodes = set()
@ -113,6 +118,7 @@ def test_load_balancing_default():
unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'})) unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'}))
assert len(unique_nodes) == nodes, unique_nodes assert len(unique_nodes) == nodes, unique_nodes
def test_load_balancing_nearest_hostname(): def test_load_balancing_nearest_hostname():
unique_nodes = set() unique_nodes = set()
for _ in range(0, queries): for _ in range(0, queries):
@ -120,6 +126,7 @@ def test_load_balancing_nearest_hostname():
assert len(unique_nodes) == 1, unique_nodes assert len(unique_nodes) == 1, unique_nodes
assert unique_nodes == set(['n1']) assert unique_nodes == set(['n1'])
def test_load_balancing_in_order(): def test_load_balancing_in_order():
unique_nodes = set() unique_nodes = set()
for _ in range(0, queries): for _ in range(0, queries):
@ -127,6 +134,7 @@ def test_load_balancing_in_order():
assert len(unique_nodes) == 1, unique_nodes assert len(unique_nodes) == 1, unique_nodes
assert unique_nodes == set(['n1']) assert unique_nodes == set(['n1'])
def test_load_balancing_first_or_random(): def test_load_balancing_first_or_random():
unique_nodes = set() unique_nodes = set()
for _ in range(0, queries): for _ in range(0, queries):
@ -134,6 +142,7 @@ def test_load_balancing_first_or_random():
assert len(unique_nodes) == 1, unique_nodes assert len(unique_nodes) == 1, unique_nodes
assert unique_nodes == set(['n1']) assert unique_nodes == set(['n1'])
def test_load_balancing_round_robin(): def test_load_balancing_round_robin():
unique_nodes = set() unique_nodes = set()
for _ in range(0, nodes): for _ in range(0, nodes):
@ -141,6 +150,7 @@ def test_load_balancing_round_robin():
assert len(unique_nodes) == nodes, unique_nodes assert len(unique_nodes) == nodes, unique_nodes
assert unique_nodes == set(['n1', 'n2', 'n3']) assert unique_nodes == set(['n1', 'n2', 'n3'])
@pytest.mark.parametrize('dist_table', [ @pytest.mark.parametrize('dist_table', [
('dist_priority'), ('dist_priority'),
('dist_priority_negative'), ('dist_priority_negative'),
@ -153,6 +163,7 @@ def test_load_balancing_priority_round_robin(dist_table):
# n2 has bigger priority in config # n2 has bigger priority in config
assert unique_nodes == set(['n1', 'n3']) assert unique_nodes == set(['n1', 'n3'])
def test_distributed_replica_max_ignored_errors(): def test_distributed_replica_max_ignored_errors():
settings = { settings = {
'load_balancing': 'in_order', 'load_balancing': 'in_order',

22
tests/integration/test_distributed_over_distributed/test.py
View File

@ -3,16 +3,8 @@
from __future__ import print_function from __future__ import print_function
import itertools
import timeit
import logging
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -44,6 +36,7 @@ ENGINE = Distributed('test_cluster', default, distributed_table);
INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})"
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def started_cluster(): def started_cluster():
try: try:
@ -59,11 +52,12 @@ def started_cluster():
@pytest.mark.parametrize("node", NODES.values()) @pytest.mark.parametrize("node", NODES.values())
@pytest.mark.parametrize("source", ["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"]) @pytest.mark.parametrize("source",
["distributed_over_distributed_table", "cluster('test_cluster', default, distributed_table)"])
class TestDistributedOverDistributedSuite: class TestDistributedOverDistributedSuite:
def test_select_with_order_by_node(self, started_cluster, node, source): def test_select_with_order_by_node(self, started_cluster, node, source):
assert node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) \ assert node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) \
== """node1 0 0 == """node1 0 0
node1 0 0 node1 0 0
node1 1 1 node1 1 1
node1 1 1 node1 1 1
@ -75,7 +69,7 @@ node2 1 11
def test_select_with_order_by_key(self, started_cluster, node, source): def test_select_with_order_by_key(self, started_cluster, node, source):
assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \ assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \
== """node1 0 0 == """node1 0 0
node1 0 0 node1 0 0
node2 0 10 node2 0 10
node2 0 10 node2 0 10
@ -87,12 +81,12 @@ node2 1 11
def test_select_with_group_by_node(self, started_cluster, node, source): def test_select_with_group_by_node(self, started_cluster, node, source):
assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \ assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \
== "node1 2\nnode2 42\n" == "node1 2\nnode2 42\n"
def test_select_with_group_by_key(self, started_cluster, node, source): def test_select_with_group_by_key(self, started_cluster, node, source):
assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \ assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \
== "0 20\n1 24\n" == "0 20\n1 24\n"
def test_select_sum(self, started_cluster, node, source): def test_select_sum(self, started_cluster, node, source):
assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \ assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \
== "44\n" == "44\n"

25
tests/integration/test_distributed_over_live_view/test.py
View File

@ -2,16 +2,10 @@ from __future__ import print_function
import sys import sys
import time import time
import itertools
import timeit
import logging
import pytest import pytest
from helpers.uclient import client, prompt, end_of_block
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager from helpers.uclient import client, prompt, end_of_block
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -46,6 +40,7 @@ ENGINE = Distributed(test_cluster, default, base_table, rand());
INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})"
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def started_cluster(): def started_cluster():
try: try:
@ -77,7 +72,8 @@ class TestLiveViewOverDistributedSuite:
client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.send("DROP TABLE IF EXISTS distributed_over_lv")
client1.expect(prompt) client1.expect(prompt)
client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.send(
"CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
client1.expect(prompt) client1.expect(prompt)
client1.send(select_query) client1.send(select_query)
@ -115,7 +111,8 @@ class TestLiveViewOverDistributedSuite:
client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.send("DROP TABLE IF EXISTS distributed_over_lv")
client1.expect(prompt) client1.expect(prompt)
client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.send(
"CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
client1.expect(prompt) client1.expect(prompt)
client1.send(select_query) client1.send(select_query)
@ -153,7 +150,8 @@ class TestLiveViewOverDistributedSuite:
client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.send("DROP TABLE IF EXISTS distributed_over_lv")
client1.expect(prompt) client1.expect(prompt)
client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.send(
"CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
client1.expect(prompt) client1.expect(prompt)
client1.send(select_query) client1.send(select_query)
@ -192,7 +190,8 @@ class TestLiveViewOverDistributedSuite:
client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.send("DROP TABLE IF EXISTS distributed_over_lv")
client1.expect(prompt) client1.expect(prompt)
client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.send(
"CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
client1.expect(prompt) client1.expect(prompt)
client1.send(select_query) client1.send(select_query)
@ -230,7 +229,8 @@ class TestLiveViewOverDistributedSuite:
client1.send("DROP TABLE IF EXISTS distributed_over_lv") client1.send("DROP TABLE IF EXISTS distributed_over_lv")
client1.expect(prompt) client1.expect(prompt)
client1.send("CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") client1.send(
"CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)")
client1.expect(prompt) client1.expect(prompt)
client1.send("SELECT sum(value) FROM distributed_over_lv") client1.send("SELECT sum(value) FROM distributed_over_lv")
@ -254,4 +254,3 @@ class TestLiveViewOverDistributedSuite:
client1.send("SELECT sum(value) FROM distributed_over_lv") client1.send("SELECT sum(value) FROM distributed_over_lv")
client1.expect(r"31" + end_of_block) client1.expect(r"31" + end_of_block)
client1.expect(prompt) client1.expect(prompt)

7
tests/integration/test_distributed_respect_user_timeouts/test.py
View File

@ -1,13 +1,12 @@
import itertools import itertools
import timeit
import os.path import os.path
import pytest import timeit
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager from helpers.network import PartitionManager
from helpers.test_tools import TSV from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
NODES = {'node' + str(i): None for i in (1, 2)} NODES = {'node' + str(i): None for i in (1, 2)}
@ -62,6 +61,7 @@ TIMEOUT_DIFF_UPPER_BOUND = {
}, },
} }
def _check_exception(exception, expected_tries=3): def _check_exception(exception, expected_tries=3):
lines = exception.split('\n') lines = exception.split('\n')
@ -88,7 +88,6 @@ def _check_exception(exception, expected_tries=3):
@pytest.fixture(scope="module", params=["configs", "configs_secure"]) @pytest.fixture(scope="module", params=["configs", "configs_secure"])
def started_cluster(request): def started_cluster(request):
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
cluster.__with_ssl_config = request.param == "configs_secure" cluster.__with_ssl_config = request.param == "configs_secure"
main_configs = [] main_configs = []

10
tests/integration/test_distributed_storage_configuration/test.py
View File

@ -9,8 +9,9 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', node = cluster.add_instance('node',
main_configs=["configs/config.d/storage_configuration.xml"], main_configs=["configs/config.d/storage_configuration.xml"],
tmpfs=['/disk1:size=100M', '/disk2:size=100M']) tmpfs=['/disk1:size=100M', '/disk2:size=100M'])
@pytest.fixture(scope='module') @pytest.fixture(scope='module')
def start_cluster(): def start_cluster():
@ -21,14 +22,17 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def _files_in_dist_mon(node, root, table): def _files_in_dist_mon(node, root, table):
return int(node.exec_in_container([ return int(node.exec_in_container([
'bash', 'bash',
'-c', '-c',
# `-maxdepth 1` to avoid /tmp/ subdirectory # `-maxdepth 1` to avoid /tmp/ subdirectory
'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format(root=root, table=table) 'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format(
root=root, table=table)
]).split('\n')[0]) ]).split('\n')[0])
def test_insert(start_cluster): def test_insert(start_cluster):
node.query('CREATE TABLE test.foo (key Int) Engine=Memory()') node.query('CREATE TABLE test.foo (key Int) Engine=Memory()')
node.query(""" node.query("""

6
tests/integration/test_distributed_system_query/test.py
View File

@ -1,5 +1,3 @@
from contextlib import contextmanager
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
@ -18,7 +16,8 @@ def started_cluster():
for node in (node1, node2): for node in (node1, node2):
node.query('''CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;''') node.query('''CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;''')
node1.query('''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''') node1.query(
'''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''')
yield cluster yield cluster
@ -38,4 +37,3 @@ def test_start_and_stop_replica_send(started_cluster):
node1.query("SYSTEM START DISTRIBUTED SENDS distributed_table;") node1.query("SYSTEM START DISTRIBUTED SENDS distributed_table;")
node1.query("SYSTEM FLUSH DISTRIBUTED distributed_table;") node1.query("SYSTEM FLUSH DISTRIBUTED distributed_table;")
assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '2' assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '2'

111
tests/integration/test_drop_replica/test.py
View File

@ -1,53 +1,52 @@
import time import time
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.cluster import ClickHouseKiller
from helpers.test_tools import assert_eq_with_retry
from helpers.network import PartitionManager from helpers.network import PartitionManager
def fill_nodes(nodes, shard): def fill_nodes(nodes, shard):
for node in nodes: for node in nodes:
node.query( node.query(
''' '''
CREATE DATABASE test; CREATE DATABASE test;
CREATE TABLE test.test_table(date Date, id UInt32) CREATE TABLE test.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name)) '''.format(shard=shard, replica=node.name))
node.query( node.query(
''' '''
CREATE DATABASE test1; CREATE DATABASE test1;
CREATE TABLE test1.test_table(date Date, id UInt32) CREATE TABLE test1.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name)) '''.format(shard=shard, replica=node.name))
node.query( node.query(
''' '''
CREATE DATABASE test2; CREATE DATABASE test2;
CREATE TABLE test2.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name))
CREATE TABLE test2.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name))
node.query( node.query(
''' '''
CREATE DATABASE test3; CREATE DATABASE test3;
CREATE TABLE test3.test_table(date Date, id UInt32) CREATE TABLE test3.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name)) '''.format(shard=shard, replica=node.name))
node.query( node.query(
''' '''
CREATE DATABASE test4; CREATE DATABASE test4;
CREATE TABLE test4.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name))
CREATE TABLE test4.test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0;
'''.format(shard=shard, replica=node.name))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
@ -71,6 +70,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_drop_replica(start_cluster): def test_drop_replica(start_cluster):
for i in range(100): for i in range(100):
node_1_1.query("INSERT INTO test.test_table VALUES (1, {})".format(i)) node_1_1.query("INSERT INTO test.test_table VALUES (1, {})".format(i))
@ -81,17 +81,25 @@ def test_drop_replica(start_cluster):
zk = cluster.get_kazoo_client('zoo1') zk = cluster.get_kazoo_client('zoo1')
assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'")
assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") assert "can't drop local replica" in node_1_1.query_and_get_error(
assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test")
assert "can't drop local replica" in node_1_1.query_and_get_error(
"SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'")
assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test")
assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
assert "it's active" in \ assert "it's active" in \
node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) node_1_3.query_and_get_error(
"SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
shard=1))
assert "There is a local table" in \ assert "There is a local table" in \
node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) node_1_2.query_and_get_error(
"SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
shard=1))
assert "There is a local table" in \ assert "There is a local table" in \
node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(shard=1)) node_1_1.query_and_get_error(
"SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format(
shard=1))
assert "does not look like a table path" in \ assert "does not look like a table path" in \
node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'") node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'")
@ -100,31 +108,48 @@ def test_drop_replica(start_cluster):
pm.drop_instance_zk_connections(node_1_1) pm.drop_instance_zk_connections(node_1_1)
time.sleep(10) time.sleep(10)
assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") assert "doesn't exist" in node_1_3.query_and_get_error(
"SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1")
node_1_3.query("SYSTEM DROP REPLICA 'node_1_1'") node_1_3.query("SYSTEM DROP REPLICA 'node_1_1'")
exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 != None) assert (exists_replica_1_1 != None)
## If you want to drop a inactive/stale replicate table that does not have a local replica, you can following syntax(ZKPATH): ## If you want to drop a inactive/stale replicate table that does not have a local replica, you can following syntax(ZKPATH):
node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format(shard=1)) node_1_3.query(
exists_replica_1_1 = zk.exists("/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format(
shard=1))
exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 == None) assert (exists_replica_1_1 == None)
node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table")
exists_replica_1_1 = zk.exists("/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 == None) assert (exists_replica_1_1 == None)
node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1")
exists_replica_1_1 = zk.exists("/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 == None) assert (exists_replica_1_1 == None)
node_1_3.query("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format(shard=1)) node_1_3.query(
exists_replica_1_1 = zk.exists("/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format(
shard=1))
exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 == None) assert (exists_replica_1_1 == None)
node_1_2.query("SYSTEM DROP REPLICA 'node_1_1'") node_1_2.query("SYSTEM DROP REPLICA 'node_1_1'")
exists_replica_1_1 = zk.exists("/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, replica='node_1_1')) exists_replica_1_1 = zk.exists(
"/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1,
replica='node_1_1'))
assert (exists_replica_1_1 == None) assert (exists_replica_1_1 == None)

1
tests/integration/test_enabling_access_management/test.py
View File

@ -4,6 +4,7 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', user_configs=["configs/users.d/extra_users.xml"]) instance = cluster.add_instance('instance', user_configs=["configs/users.d/extra_users.xml"])
@pytest.fixture(scope="module", autouse=True) @pytest.fixture(scope="module", autouse=True)
def started_cluster(): def started_cluster():
try: try:

26
tests/integration/test_extreme_deduplication/test.py
View File

@ -1,21 +1,22 @@
import time import time
from contextlib import contextmanager
import pytest import pytest
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
from helpers.client import CommandRequest from helpers.client import CommandRequest
from helpers.client import QueryTimeoutExceedException from helpers.client import QueryTimeoutExceedException
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) node1 = cluster.add_instance('node1',
node2 = cluster.add_instance('node2', main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"],
with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1})
node2 = cluster.add_instance('node2',
main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"],
with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2})
nodes = [node1, node2] nodes = [node1, node2]
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def started_cluster(): def started_cluster():
try: try:
@ -36,15 +37,18 @@ def test_deduplication_window_in_seconds(started_cluster):
node.query("INSERT INTO simple VALUES (0, 0)") node.query("INSERT INTO simple VALUES (0, 0)")
time.sleep(1) time.sleep(1)
node.query("INSERT INTO simple VALUES (0, 0)") # deduplication works here node.query("INSERT INTO simple VALUES (0, 0)") # deduplication works here
node.query("INSERT INTO simple VALUES (0, 1)") node.query("INSERT INTO simple VALUES (0, 1)")
assert TSV(node.query("SELECT count() FROM simple")) == TSV("2\n") assert TSV(node.query("SELECT count() FROM simple")) == TSV("2\n")
# wait clean thread # wait clean thread
time.sleep(2) time.sleep(2)
assert TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[0][0] == "1" assert \
node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[
0][
0] == "1"
node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted
assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3" assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3"
node1.query("""DROP TABLE simple ON CLUSTER test_cluster""") node1.query("""DROP TABLE simple ON CLUSTER test_cluster""")

8
tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py
View File

@ -1,15 +1,13 @@
from __future__ import print_function from __future__ import print_function
from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException
import helpers
import pytest
import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
try: try:

3
tests/integration/test_filesystem_layout/test.py
View File

@ -25,4 +25,5 @@ def test_file_path_escaping(started_cluster):
node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''') node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''')
node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])
node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) node.exec_in_container(
["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])

9
tests/integration/test_force_deduplication/test.py
View File

@ -2,14 +2,14 @@
# pylint: disable=redefined-outer-name # pylint: disable=redefined-outer-name
import pytest import pytest
from helpers.cluster import ClickHouseCluster
from helpers.client import QueryRuntimeException from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node', with_zookeeper=True) node = cluster.add_instance('node', with_zookeeper=True)
@pytest.fixture(scope='module') @pytest.fixture(scope='module')
def start_cluster(): def start_cluster():
try: try:
@ -19,6 +19,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def get_counts(): def get_counts():
src = int(node.query("SELECT count() FROM test")) src = int(node.query("SELECT count() FROM test"))
a = int(node.query("SELECT count() FROM test_mv_a")) a = int(node.query("SELECT count() FROM test_mv_a"))
@ -69,7 +70,7 @@ def test_basic(start_cluster):
) )
src, a, b, c = get_counts() src, a, b, c = get_counts()
assert src == 11 assert src == 11
assert a == old_a + 10 # first insert could be succesfull with disabled dedup assert a == old_a + 10 # first insert could be succesfull with disabled dedup
assert b == 11 assert b == 11
assert c == old_c + 10 assert c == old_c + 10
@ -94,5 +95,3 @@ def test_basic(start_cluster):
assert a == old_a + 20 assert a == old_a + 20
assert b == 21 assert b == 21
assert c == old_c + 20 assert c == old_c + 20

14
tests/integration/test_format_avro_confluent/test.py
View File

@ -1,20 +1,15 @@
import json
import logging
import io import io
import logging
import pytest
from helpers.cluster import ClickHouseCluster, ClickHouseInstance
import helpers.client
import avro.schema import avro.schema
from confluent.schemaregistry.client import CachedSchemaRegistryClient import pytest
from confluent.schemaregistry.serializers import MessageSerializer from confluent.schemaregistry.serializers import MessageSerializer
from helpers.cluster import ClickHouseCluster, ClickHouseInstance
logging.getLogger().setLevel(logging.INFO) logging.getLogger().setLevel(logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler()) logging.getLogger().addHandler(logging.StreamHandler())
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def cluster(): def cluster():
try: try:
@ -42,7 +37,6 @@ def run_query(instance, query, data=None, settings=None):
return result return result
def test_select(cluster): def test_select(cluster):
# type: (ClickHouseCluster) -> None # type: (ClickHouseCluster) -> None

3
tests/integration/test_format_schema_on_server/test.py
View File

@ -36,5 +36,6 @@ def test_protobuf_format_input(started_cluster):
def test_protobuf_format_output(started_cluster): def test_protobuf_format_output(started_cluster):
create_simple_table() create_simple_table()
instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')"); instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')");
assert instance.http_query("SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \ assert instance.http_query(
"SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \
"\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def" "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def"

4
tests/integration/test_freeze_table/test.py
View File

@ -39,7 +39,7 @@ def test_freeze_table(started_cluster):
''')) '''))
assert 11 == len(freeze_result) assert 11 == len(freeze_result)
path_col_ix = freeze_result[0].index('part_backup_path') path_col_ix = freeze_result[0].index('part_backup_path')
for row in freeze_result[1:]: # skip header for row in freeze_result[1:]: # skip header
part_backup_path = row[path_col_ix] part_backup_path = row[path_col_ix]
node.exec_in_container( node.exec_in_container(
["bash", "-c", "test -d {}".format(part_backup_path)] ["bash", "-c", "test -d {}".format(part_backup_path)]
@ -55,7 +55,7 @@ def test_freeze_table(started_cluster):
''')) '''))
assert 2 == len(freeze_result) assert 2 == len(freeze_result)
path_col_ix = freeze_result[0].index('part_backup_path') path_col_ix = freeze_result[0].index('part_backup_path')
for row in freeze_result[1:]: # skip header for row in freeze_result[1:]: # skip header
part_backup_path = row[path_col_ix] part_backup_path = row[path_col_ix]
assert 'test_01417_single_part' in part_backup_path assert 'test_01417_single_part' in part_backup_path
node.exec_in_container( node.exec_in_container(

38
tests/integration/test_globs_in_filepath/test.py
View File

@ -4,7 +4,8 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node = cluster.add_instance('node') node = cluster.add_instance('node')
path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_cluster(): def start_cluster():
@ -19,11 +20,13 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_strange_filenames(start_cluster): def test_strange_filenames(start_cluster):
# 2 rows data # 2 rows data
some_data = "\t111.222\nData\t333.444" some_data = "\t111.222\nData\t333.444"
node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)],
privileged=True, user='root')
files = ["p.o.i.n.t.s", files = ["p.o.i.n.t.s",
"b}{ra{ces", "b}{ra{ces",
@ -31,7 +34,10 @@ def test_strange_filenames(start_cluster):
# filename inside testing data for debug simplicity # filename inside testing data for debug simplicity
for filename in files: for filename in files:
node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data,
path_to_userfiles_from_defaut_config,
filename)], privileged=True,
user='root')
test_requests = [("p.o.??n.t.s", "2"), test_requests = [("p.o.??n.t.s", "2"),
("p.o.*t.s", "2"), ("p.o.*t.s", "2"),
@ -47,6 +53,7 @@ def test_strange_filenames(start_cluster):
select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64') select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64')
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
def test_linear_structure(start_cluster): def test_linear_structure(start_cluster):
# 2 rows data # 2 rows data
some_data = "\t123.456\nData\t789.012" some_data = "\t123.456\nData\t789.012"
@ -58,7 +65,9 @@ def test_linear_structure(start_cluster):
# filename inside testing data for debug simplicity # filename inside testing data for debug simplicity
for filename in files: for filename in files:
node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') node.exec_in_container(['bash', '-c',
'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
filename)], privileged=True, user='root')
test_requests = [("file{0..9}", "10"), test_requests = [("file{0..9}", "10"),
("file?", "10"), ("file?", "10"),
@ -81,6 +90,7 @@ def test_linear_structure(start_cluster):
select count(*) from file('{}{}', 'TSV', 'text String, number Float64') select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
def test_deep_structure(start_cluster): def test_deep_structure(start_cluster):
# 2 rows data # 2 rows data
some_data = "\t135.791\nData\t246.802" some_data = "\t135.791\nData\t246.802"
@ -92,7 +102,8 @@ def test_deep_structure(start_cluster):
"we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"] "we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"]
for dir in dirs: for dir in dirs:
node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)],
privileged=True, user='root')
# all directories appeared in files must be listed in dirs # all directories appeared in files must be listed in dirs
files = [] files = []
@ -102,13 +113,15 @@ def test_deep_structure(start_cluster):
files.append("directory1/big_dir/file" + str(i) + str(j) + str(k)) files.append("directory1/big_dir/file" + str(i) + str(j) + str(k))
for dir in dirs: for dir in dirs:
files.append(dir+"file") files.append(dir + "file")
# filename inside testing data for debug simplicity # filename inside testing data for debug simplicity
for filename in files: for filename in files:
node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, filename)], privileged=True, user='root') node.exec_in_container(['bash', '-c',
'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config,
filename)], privileged=True, user='root')
test_requests = [ ("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"), test_requests = [("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"),
("?", "0"), ("?", "0"),
("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"), ("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"),
("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")] ("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")]
@ -121,13 +134,16 @@ def test_deep_structure(start_cluster):
select count(*) from file('{}{}', 'TSV', 'text String, number Float64') select count(*) from file('{}{}', 'TSV', 'text String, number Float64')
'''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value)
def test_table_function_and_virtual_columns(start_cluster): def test_table_function_and_virtual_columns(start_cluster):
node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)]) node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)])
node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)]) node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)])
node.query("insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") node.query(
assert node.query("select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' "insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)")
assert node.query(
"select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000'
node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)") node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)")
assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()== '1' assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() == '1'
assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()
assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip()
assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip()

77
tests/integration/test_grant_and_revoke/test.py
View File

@ -1,7 +1,6 @@
import pytest import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
import re
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance') instance = cluster.add_instance('instance')
@ -111,7 +110,8 @@ def test_grant_all_on_table():
instance.query("CREATE USER A, B") instance.query("CREATE USER A, B")
instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION") instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION")
instance.query("GRANT ALL ON test.table TO B", user='A') instance.query("GRANT ALL ON test.table TO B", user='A')
assert instance.query("SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" assert instance.query(
"SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
instance.query("REVOKE ALL ON test.table FROM B", user='A') instance.query("REVOKE ALL ON test.table FROM B", user='A')
assert instance.query("SHOW GRANTS FOR B") == "" assert instance.query("SHOW GRANTS FOR B") == ""
@ -120,36 +120,42 @@ def test_implicit_show_grants():
instance.query("CREATE USER A") instance.query("CREATE USER A")
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "0\n"
instance.query("GRANT SELECT(x) ON test.table TO A") instance.query("GRANT SELECT(x) ON test.table TO A")
assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT(x) ON test.table TO A\n" assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT(x) ON test.table TO A\n"
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "1\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "1\n"
instance.query("GRANT SELECT ON test.table TO A") instance.query("GRANT SELECT ON test.table TO A")
assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.table TO A\n" assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.table TO A\n"
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "2\n"
instance.query("GRANT SELECT ON test.* TO A") instance.query("GRANT SELECT ON test.* TO A")
assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.* TO A\n" assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.* TO A\n"
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "2\n"
instance.query("GRANT SELECT ON *.* TO A") instance.query("GRANT SELECT ON *.* TO A")
assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON *.* TO A\n" assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON *.* TO A\n"
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "2\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "2\n"
instance.query("REVOKE ALL ON *.* FROM A") instance.query("REVOKE ALL ON *.* FROM A")
assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n"
assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n"
assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", user="A") == "0\n" assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'",
user="A") == "0\n"
def test_implicit_create_view_grant(): def test_implicit_create_view_grant():
@ -184,45 +190,52 @@ def test_introspection():
instance.query('GRANT SELECT ON test.table TO A') instance.query('GRANT SELECT ON test.table TO A')
instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION')
assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) assert instance.query("SHOW USERS") == TSV(["A", "B", "default"])
assert instance.query("SHOW CREATE USERS A") == TSV([ "CREATE USER A" ]) assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"])
assert instance.query("SHOW CREATE USERS B") == TSV([ "CREATE USER B" ]) assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"])
assert instance.query("SHOW CREATE USERS A,B") == TSV([ "CREATE USER A", "CREATE USER B" ]) assert instance.query("SHOW CREATE USERS A,B") == TSV(["CREATE USER A", "CREATE USER B"])
assert instance.query("SHOW CREATE USERS") == TSV([ "CREATE USER A", "CREATE USER B", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" ]) assert instance.query("SHOW CREATE USERS") == TSV(["CREATE USER A", "CREATE USER B",
"CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"])
assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"])
assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
assert instance.query("SHOW GRANTS FOR A,B") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS FOR A,B") == TSV(
assert instance.query("SHOW GRANTS FOR B,A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
assert instance.query("SHOW GRANTS FOR ALL") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT ALL ON *.* TO default WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS FOR B,A") == TSV(
["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
assert instance.query("SHOW GRANTS FOR ALL") == TSV(
["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION",
"GRANT ALL ON *.* TO default WITH GRANT OPTION"])
assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"])
assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION" ]) assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"])
expected_access1 = "CREATE USER A\n"\ expected_access1 = "CREATE USER A\n" \
"CREATE USER B\n"\ "CREATE USER B\n" \
"CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"
expected_access2 = "GRANT SELECT ON test.table TO A\n"\ expected_access2 = "GRANT SELECT ON test.table TO A\n" \
"GRANT CREATE ON *.* TO B WITH GRANT OPTION\n"\ "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" \
"GRANT ALL ON *.* TO default WITH GRANT OPTION\n" "GRANT ALL ON *.* TO default WITH GRANT OPTION\n"
assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access1 in instance.query("SHOW ACCESS")
assert expected_access2 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS")
assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ assert instance.query(
TSV([[ "A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], "SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") == \
[ "B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) TSV([["A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"],
["B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"]])
assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") ==\ assert instance.query(
TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], "SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") == \
[ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ]]) TSV([["A", "\N", "SELECT", "test", "table", "\N", 0, 0],
["B", "\N", "CREATE", "\N", "\N", "\N", 0, 1]])
def test_current_database(): def test_current_database():
instance.query("CREATE USER A") instance.query("CREATE USER A")
instance.query("GRANT SELECT ON table TO A", database="test") instance.query("GRANT SELECT ON table TO A", database="test")
assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"])
assert instance.query("SHOW GRANTS FOR A", database="test") == TSV([ "GRANT SELECT ON test.table TO A" ]) assert instance.query("SHOW GRANTS FOR A", database="test") == TSV(["GRANT SELECT ON test.table TO A"])
assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n"
assert instance.query("SELECT * FROM table", user='A', database='test') == "1\t5\n2\t10\n" assert instance.query("SELECT * FROM table", user='A', database='test') == "1\t5\n2\t10\n"

27
tests/integration/test_graphite_merge_tree/test.py
View File

@ -1,12 +1,11 @@
import datetime
import os.path as p import os.path as p
import time import time
import datetime
import pytest
import pytest
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', instance = cluster.add_instance('instance',
main_configs=['configs/graphite_rollup.xml']) main_configs=['configs/graphite_rollup.xml'])
@ -317,20 +316,20 @@ def test_combined_rules(graphite_table):
expected_unmerged = '' expected_unmerged = ''
for i in range(384): for i in range(384):
to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format( to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format(
v=1, t=1487970000+(i*300) v=1, t=1487970000 + (i * 300)
) )
to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format( to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format(
v=i, t=1487970000+(i*300) v=i, t=1487970000 + (i * 300)
) )
expected_unmerged += ("five_min.count\t{v1}\t{t}\n" expected_unmerged += ("five_min.count\t{v1}\t{t}\n"
"five_min.max\t{v2}\t{t}\n").format( "five_min.max\t{v2}\t{t}\n").format(
v1=1, v2=i, v1=1, v2=i,
t=1487970000+(i*300) t=1487970000 + (i * 300)
) )
q(to_insert) q(to_insert)
assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' assert TSV(q('SELECT metric, value, timestamp FROM test.graphite'
' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged)
q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL') q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL')
expected_merged = ''' expected_merged = '''
@ -370,16 +369,16 @@ CREATE TABLE test.graphite
expected_unmerged = '' expected_unmerged = ''
for i in range(100): for i in range(100):
to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format( to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format(
v=1, t=1487970000+(i*60) v=1, t=1487970000 + (i * 60)
) )
to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format( to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format(
v=i, t=1487970000+(i*60) v=i, t=1487970000 + (i * 60)
) )
expected_unmerged += ("top_level.count\t{v1}\t{t}\n" expected_unmerged += ("top_level.count\t{v1}\t{t}\n"
"top_level.max\t{v2}\t{t}\n").format( "top_level.max\t{v2}\t{t}\n").format(
v1=1, v2=i, v1=1, v2=i,
t=1487970000+(i*60) t=1487970000 + (i * 60)
) )
q(to_insert) q(to_insert)
assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' assert TSV(q('SELECT metric, value, timestamp FROM test.graphite'

53
tests/integration/test_host_ip_change/test.py
View File

@ -1,15 +1,12 @@
import time
import pytest import pytest
import subprocess
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
from helpers.client import QueryRuntimeException from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
def _fill_nodes(nodes, table_name): def _fill_nodes(nodes, table_name):
for node in nodes: for node in nodes:
node.query( node.query(
@ -21,9 +18,12 @@ def _fill_nodes(nodes, table_name):
'''.format(table_name, node.name) '''.format(table_name, node.name)
) )
node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True, ipv6_address='2001:3984:3989::1:1111')
node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True,
ipv6_address='2001:3984:3989::1:1111')
node2 = cluster.add_instance('node2', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], node2 = cluster.add_instance('node2', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'],
with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112') with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112')
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def cluster_without_dns_cache_update(): def cluster_without_dns_cache_update():
@ -41,6 +41,7 @@ def cluster_without_dns_cache_update():
cluster.shutdown() cluster.shutdown()
pass pass
# node1 is a source, node2 downloads data # node1 is a source, node2 downloads data
# node2 has long dns_cache_update_period, so dns cache update wouldn't work # node2 has long dns_cache_update_period, so dns cache update wouldn't work
def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update):
@ -73,9 +74,11 @@ def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update):
node3 = cluster.add_instance('node3', main_configs=['configs/listen_host.xml'], node3 = cluster.add_instance('node3', main_configs=['configs/listen_host.xml'],
with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113') with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113')
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml', 'configs/dns_update_short.xml'], node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml',
with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114') 'configs/dns_update_short.xml'],
with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114')
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def cluster_with_dns_cache_update(): def cluster_with_dns_cache_update():
@ -93,6 +96,7 @@ def cluster_with_dns_cache_update():
cluster.shutdown() cluster.shutdown()
pass pass
# node3 is a source, node4 downloads data # node3 is a source, node4 downloads data
# node4 has short dns_cache_update_period, so testing update of dns cache # node4 has short dns_cache_update_period, so testing update of dns cache
def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
@ -107,7 +111,6 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
# Put some data to source node3 # Put some data to source node3
node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)") node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)")
# Check that data is placed on node3 # Check that data is placed on node3
assert node3.query("SELECT count(*) from test_table_update") == "6\n" assert node3.query("SELECT count(*) from test_table_update") == "6\n"
@ -126,9 +129,12 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update):
assert node3.query("SELECT count(*) from test_table_update") == "7\n" assert node3.query("SELECT count(*) from test_table_update") == "7\n"
assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "7") assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "7")
def set_hosts(node, hosts): def set_hosts(node, hosts):
new_content = '\\n'.join(['127.0.0.1 localhost', '::1 localhost'] + hosts) new_content = '\\n'.join(['127.0.0.1 localhost', '::1 localhost'] + hosts)
node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True,
user='root')
def test_dns_cache_update(cluster_with_dns_cache_update): def test_dns_cache_update(cluster_with_dns_cache_update):
set_hosts(node4, ['127.255.255.255 lost_host']) set_hosts(node4, ['127.255.255.255 lost_host'])
@ -136,7 +142,8 @@ def test_dns_cache_update(cluster_with_dns_cache_update):
with pytest.raises(QueryRuntimeException): with pytest.raises(QueryRuntimeException):
node4.query("SELECT * FROM remote('lost_host', 'system', 'one')") node4.query("SELECT * FROM remote('lost_host', 'system', 'one')")
node4.query("CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") node4.query(
"CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')")
with pytest.raises(QueryRuntimeException): with pytest.raises(QueryRuntimeException):
node4.query("SELECT * FROM distributed_lost_host") node4.query("SELECT * FROM distributed_lost_host")
@ -146,21 +153,26 @@ def test_dns_cache_update(cluster_with_dns_cache_update):
assert_eq_with_retry(node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0") assert_eq_with_retry(node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0")
assert_eq_with_retry(node4, "SELECT * FROM distributed_lost_host", "0") assert_eq_with_retry(node4, "SELECT * FROM distributed_lost_host", "0")
assert TSV(node4.query("SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV("lost_host\t127.0.0.1\n") assert TSV(node4.query(
"SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV(
"lost_host\t127.0.0.1\n")
assert TSV(node4.query("SELECT hostName()")) == TSV("node4") assert TSV(node4.query("SELECT hostName()")) == TSV("node4")
# Check SYSTEM DROP DNS CACHE on node5 and background cache update on node6 # Check SYSTEM DROP DNS CACHE on node5 and background cache update on node6
node5 = cluster.add_instance('node5', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], node5 = cluster.add_instance('node5', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'],
user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1115') user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1115')
node6 = cluster.add_instance('node6', main_configs=['configs/listen_host.xml', 'configs/dns_update_short.xml'], node6 = cluster.add_instance('node6', main_configs=['configs/listen_host.xml', 'configs/dns_update_short.xml'],
user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1116') user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1116')
@pytest.mark.parametrize("node", [node5, node6]) @pytest.mark.parametrize("node", [node5, node6])
def test_user_access_ip_change(cluster_with_dns_cache_update, node): def test_user_access_ip_change(cluster_with_dns_cache_update, node):
node_name = node.name node_name = node.name
node_num = node.name[-1] node_num = node.name[-1]
# getaddrinfo(...) may hang for a log time without this options # getaddrinfo(...) may hang for a log time without this options
node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'],
privileged=True, user='root')
assert node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" assert node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n"
assert node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" assert node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n"
@ -180,8 +192,11 @@ def test_user_access_ip_change(cluster_with_dns_cache_update, node):
retry_count = 60 retry_count = 60
if node_name == 'node5': if node_name == 'node5':
# client is not allowed to connect, so execute it directly in container to send query from localhost # client is not allowed to connect, so execute it directly in container to send query from localhost
node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True, user='root') node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True,
user='root')
retry_count = 1 retry_count = 1
assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1) assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0",
assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", retry_count=retry_count, sleep_time=1) retry_count=retry_count, sleep_time=1)
assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0",
retry_count=retry_count, sleep_time=1)

6
tests/integration/test_http_and_readonly/test.py
View File

@ -16,5 +16,7 @@ def setup_nodes():
def test_http_get_is_readonly(): def test_http_get_is_readonly():
assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)") assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error(
assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error("CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0}) "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)")
assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error(
"CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0})

150
tests/integration/test_http_handlers_config/test.py
View File

@ -1,6 +1,6 @@
import contextlib
import os import os
import urllib import urllib
import contextlib
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
@ -20,101 +20,145 @@ class SimpleCluster:
def test_dynamic_query_handler(): def test_dynamic_query_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster:
test_query = urllib.quote_plus('SELECT * FROM system.settings WHERE name = \'max_threads\'') test_query = urllib.quote_plus('SELECT * FROM system.settings WHERE name = \'max_threads\'')
assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST',
headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET',
headers={'XXX': 'bad'}).status_code
assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET',
headers={'XXX': 'xxx'}).status_code
assert 200 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query, assert 200 == cluster.instance.http_request(
method='GET', headers={'XXX': 'xxx'}).status_code 'test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query,
method='GET', headers={'XXX': 'xxx'}).status_code
def test_predefined_query_handler(): def test_predefined_query_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster:
assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST',
headers={'XXX': 'xxx'}).status_code
assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET',
headers={'XXX': 'xxx'}).status_code
assert 'max_threads\t1\n' == cluster.instance.http_request('test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET', headers={'XXX': 'xxx'}).content assert 'max_threads\t1\n' == cluster.instance.http_request(
'test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET',
headers={'XXX': 'xxx'}).content
assert 'max_threads\t1\nmax_alter_threads\t1\n' == cluster.instance.http_request( assert 'max_threads\t1\nmax_alter_threads\t1\n' == cluster.instance.http_request(
'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1', headers={'XXX': 'max_alter_threads'}).content 'query_param_with_url/max_threads?max_threads=1&max_alter_threads=1',
headers={'XXX': 'max_alter_threads'}).content
def test_fixed_static_handler(): def test_fixed_static_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST',
headers={'XXX': 'xxx'}).status_code
assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] headers={'XXX': 'xxx'}).status_code
assert 'Test get static handler and fix content' == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content assert 'text/html; charset=UTF-8' == \
cluster.instance.http_request('test_get_fixed_static_handler', method='GET',
headers={'XXX': 'xxx'}).headers['Content-Type']
assert 'Test get static handler and fix content' == cluster.instance.http_request(
'test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content
def test_config_static_handler(): def test_config_static_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST',
headers={'XXX': 'xxx'}).status_code
# check default status code # check default status code
assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET',
assert 'text/plain; charset=UTF-8' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] headers={'XXX': 'xxx'}).status_code
assert 'Test get static handler and config content' == cluster.instance.http_request('test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content assert 'text/plain; charset=UTF-8' == \
cluster.instance.http_request('test_get_config_static_handler', method='GET',
headers={'XXX': 'xxx'}).headers['Content-Type']
assert 'Test get static handler and config content' == cluster.instance.http_request(
'test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content
def test_absolute_path_static_handler(): def test_absolute_path_static_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
cluster.instance.exec_in_container( cluster.instance.exec_in_container(
['bash', '-c', 'echo "<html><body>Absolute Path File</body></html>" > /var/lib/clickhouse/user_files/absolute_path_file.html'], ['bash', '-c',
'echo "<html><body>Absolute Path File</body></html>" > /var/lib/clickhouse/user_files/absolute_path_file.html'],
privileged=True, user='root') privileged=True, user='root')
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST',
headers={'XXX': 'xxx'}).status_code
# check default status code # check default status code
assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] headers={'XXX': 'xxx'}).status_code
assert '<html><body>Absolute Path File</body></html>\n' == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content assert 'text/html; charset=UTF-8' == \
cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET',
headers={'XXX': 'xxx'}).headers['Content-Type']
assert '<html><body>Absolute Path File</body></html>\n' == cluster.instance.http_request(
'test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
def test_relative_path_static_handler(): def test_relative_path_static_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster:
cluster.instance.exec_in_container( cluster.instance.exec_in_container(
['bash', '-c', 'echo "<html><body>Relative Path File</body></html>" > /var/lib/clickhouse/user_files/relative_path_file.html'], ['bash', '-c',
'echo "<html><body>Relative Path File</body></html>" > /var/lib/clickhouse/user_files/relative_path_file.html'],
privileged=True, user='root') privileged=True, user='root')
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST',
headers={'XXX': 'xxx'}).status_code
# check default status code # check default status code
assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type'] headers={'XXX': 'xxx'}).status_code
assert '<html><body>Relative Path File</body></html>\n' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content assert 'text/html; charset=UTF-8' == \
cluster.instance.http_request('test_get_relative_path_static_handler', method='GET',
headers={'XXX': 'xxx'}).headers['Content-Type']
assert '<html><body>Relative Path File</body></html>\n' == cluster.instance.http_request(
'test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
def test_defaults_http_handlers(): def test_defaults_http_handlers():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster:
assert 200 == cluster.instance.http_request('', method='GET').status_code assert 200 == cluster.instance.http_request('', method='GET').status_code
assert 'Default server response' == cluster.instance.http_request('', method='GET').content assert 'Default server response' == cluster.instance.http_request('', method='GET').content
@ -130,24 +174,34 @@ def test_defaults_http_handlers():
assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code
assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content
def test_prometheus_handler(): def test_prometheus_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: with contextlib.closing(
SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_prometheus', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_prometheus', method='POST',
headers={'XXX': 'xxx'}).status_code
assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code
assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).content assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET',
headers={'XXX': 'xxx'}).content
def test_replicas_status_handler(): def test_replicas_status_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", "test_replicas_status_handler")) as cluster: with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler",
"test_replicas_status_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'bad'}).status_code assert 404 == cluster.instance.http_request('test_replicas_status', method='GET',
headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code assert 404 == cluster.instance.http_request('test_replicas_status', method='POST',
headers={'XXX': 'xxx'}).status_code
assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code assert 200 == cluster.instance.http_request('test_replicas_status', method='GET',
assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content headers={'XXX': 'xxx'}).status_code
assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET',
headers={'XXX': 'xxx'}).content

54
tests/integration/test_https_replication/test.py
View File

@ -1,30 +1,36 @@
import time
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
from helpers.network import PartitionManager
from multiprocessing.dummy import Pool
import random import random
import time
from multiprocessing.dummy import Pool
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry
""" """
Both ssl_conf.xml and no_ssl_conf.xml have the same port Both ssl_conf.xml and no_ssl_conf.xml have the same port
""" """
def _fill_nodes(nodes, shard): def _fill_nodes(nodes, shard):
for node in nodes: for node in nodes:
node.query( node.query(
''' '''
CREATE DATABASE test; CREATE DATABASE test;
CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192);
'''.format(shard=shard, replica=node.name))
CREATE TABLE test_table(date Date, id UInt32, dummy UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192);
'''.format(shard=shard, replica=node.name))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) node1 = cluster.add_instance('node1',
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
"configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
node2 = cluster.add_instance('node2',
main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
"configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def both_https_cluster(): def both_https_cluster():
@ -38,6 +44,7 @@ def both_https_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_both_https(both_https_cluster): def test_both_https(both_https_cluster):
node1.query("insert into test_table values ('2017-06-16', 111, 0)") node1.query("insert into test_table values ('2017-06-16', 111, 0)")
@ -77,9 +84,11 @@ def test_replication_after_partition(both_https_cluster):
assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100') assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100')
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
with_zookeeper=True)
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
with_zookeeper=True)
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True)
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def both_http_cluster(): def both_http_cluster():
@ -93,6 +102,7 @@ def both_http_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_both_http(both_http_cluster): def test_both_http(both_http_cluster):
node3.query("insert into test_table values ('2017-06-16', 111, 0)") node3.query("insert into test_table values ('2017-06-16', 111, 0)")
@ -104,8 +114,13 @@ def test_both_http(both_http_cluster):
assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111\n222') assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111\n222')
assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111\n222') assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111\n222')
node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) node5 = cluster.add_instance('node5',
main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt",
"configs/server.key", "configs/dhparam.pem"], with_zookeeper=True)
node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'],
with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def mixed_protocol_cluster(): def mixed_protocol_cluster():
@ -119,6 +134,7 @@ def mixed_protocol_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def test_mixed_protocol(mixed_protocol_cluster): def test_mixed_protocol(mixed_protocol_cluster):
node5.query("insert into test_table values ('2017-06-16', 111, 0)") node5.query("insert into test_table values ('2017-06-16', 111, 0)")

1
tests/integration/test_inherit_multiple_profiles/test.py
View File

@ -4,7 +4,6 @@ from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV from helpers.test_tools import TSV
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', instance = cluster.add_instance('instance',
user_configs=['configs/combined_profile.xml']) user_configs=['configs/combined_profile.xml'])

7
tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py
View File

@ -10,6 +10,7 @@ cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=['configs/conf.xml']) instance = cluster.add_instance('instance', main_configs=['configs/conf.xml'])
@pytest.fixture(scope='module', autouse=True) @pytest.fixture(scope='module', autouse=True)
def start_cluster(): def start_cluster():
try: try:
@ -18,6 +19,7 @@ def start_cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
# max_memory_usage_for_user cannot be used, since the memory for user accounted # max_memory_usage_for_user cannot be used, since the memory for user accounted
# correctly, only total is not # correctly, only total is not
def test_memory_tracking_total(): def test_memory_tracking_total():
@ -25,9 +27,10 @@ def test_memory_tracking_total():
CREATE TABLE null (row String) ENGINE=Null; CREATE TABLE null (row String) ENGINE=Null;
''') ''')
instance.exec_in_container(['bash', '-c', instance.exec_in_container(['bash', '-c',
'clickhouse client -q "SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), \' \') from numbers(10000)" > data.json']) 'clickhouse client -q "SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), \' \') from numbers(10000)" > data.json'])
for it in range(0, 20): for it in range(0, 20):
# the problem can be triggered only via HTTP, # the problem can be triggered only via HTTP,
# since clickhouse-client parses the data by itself. # since clickhouse-client parses the data by itself.
assert instance.exec_in_container(['curl', '--silent', '--show-error', '--data-binary', '@data.json', assert instance.exec_in_container(['curl', '--silent', '--show-error', '--data-binary', '@data.json',
'http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV']) == '', 'Failed on {} iteration'.format(it) 'http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV']) == '', 'Failed on {} iteration'.format(
it)

Some files were not shown because too many files have changed in this diff Show More