thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Yatsishin Ilya 2021-03-05 16:39:51 +03:00
parent 298824c926
commit 534500ef84
4 changed files with 73 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docker/test/integration/runner/compose/docker_compose_mysql.yml
View File

@ -5,6 +5,20 @@ services:
restart: always
environment:
MYSQL_ROOT_PASSWORD: clickhouse
ports:
- ${MYSQL_EXTERNAL_PORT}:${MYSQL_INTERNAL_PORT}
command: --server_id=100 --log-bin='mysql-bin-1.log' --default-time-zone='+3:00' --gtid-mode="ON" --enforce-gtid-consistency
MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST}
DATADIR: /mysql/
expose:
- ${MYSQL_PORT}
command: --server_id=100
--log-bin='mysql-bin-1.log'
--default-time-zone='+3:00'
--gtid-mode="ON"
--enforce-gtid-consistency
--log-error-verbosity=3
--log-error=/mysql/error.log
--general-log=ON
--general-log-file=/mysql/general.log
volumes:
- type: ${MYSQL_LOGS_FS:-tmpfs}
source: ${MYSQL_LOGS:-}
target: /mysql/

63
tests/integration/helpers/cluster.py
View File

@ -3,6 +3,7 @@ import errno
import http.client
import logging
import os
import stat
import os.path as p
import pprint
import pwd
@ -111,7 +112,6 @@ def check_kafka_is_available(kafka_id, kafka_port):
p.communicate()
return p.returncode == 0
class ClickHouseCluster:
"""ClickHouse cluster with several instances and (possibly) ZooKeeper.
@ -242,11 +242,16 @@ class ClickHouseCluster:
# available when with_mysql == True
self.mysql_host = "mysql57"
self.mysql_port = get_open_port()
self.mysql_port = 3306
self.mysql_ip = None
self.mysql_dir = p.abspath(p.join(self.instances_dir, "mysql"))
self.mysql_logs_dir = os.path.join(self.mysql_dir, "logs")
# available when with_mysql8 == True
self.mysql8_host = "mysql80"
self.mysql8_port = get_open_port()
self.mysql8_dir = p.abspath(p.join(self.instances_dir, "mysql8"))
self.mysql8_logs_dir = os.path.join(self.mysql8_dir, "logs")
self.zookeeper_use_tmpfs = True
@ -263,8 +268,10 @@ class ClickHouseCluster:
def setup_mysql_cmd(self, instance, env_variables, docker_compose_yml_dir):
self.with_mysql = True
env_variables['MYSQL_HOST'] = self.mysql_host
env_variables['MYSQL_EXTERNAL_PORT'] = str(self.mysql_port)
env_variables['MYSQL_INTERNAL_PORT'] = "3306"
env_variables['MYSQL_PORT'] = str(self.mysql_port)
env_variables['MYSQL_ROOT_HOST'] = '%'
env_variables['MYSQL_LOGS'] = self.mysql_logs_dir
env_variables['MYSQL_LOGS_FS'] = "bind"
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')])
self.base_mysql_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name,
'--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]
@ -276,6 +283,8 @@ class ClickHouseCluster:
env_variables['MYSQL8_HOST'] = self.mysql8_host
env_variables['MYSQL8_EXTERNAL_PORT'] = str(self.mysql8_port)
env_variables['MYSQL8_INTERNAL_PORT'] = "3306"
env_variables['MYSQL8_LOGS'] = self.mysql8_logs_dir
env_variables['MYSQL8_LOGS_FS'] = "bind"
self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_8_0.yml')])
self.base_mysql8_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name,
'--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_8_0.yml')]
@ -627,12 +636,13 @@ class ClickHouseCluster:
["bash", "-c", "echo {} | base64 --decode > {}".format(encodedStr, dest_path)],
user='root')
def wait_mysql_to_start(self, timeout=60):
def wait_mysql_to_start(self, timeout=30):
self.mysql_ip = self.get_instance_ip('mysql57')
start = time.time()
errors = []
while time.time() - start < timeout:
try:
conn = pymysql.connect(user='root', password='clickhouse', host='127.0.0.1', port=self.mysql_port)
conn = pymysql.connect(user='root', password='clickhouse', host=self.mysql_ip, port=self.mysql_port)
conn.close()
logging.debug("Mysql Started")
return
@ -812,6 +822,7 @@ class ClickHouseCluster:
if self.is_up:
return
# Just in case kill unstopped containers from previous launch
try:
logging.debug("Trying to kill unstopped containers...")
@ -822,6 +833,15 @@ class ClickHouseCluster:
except:
pass
# # Just in case remove unused networks
# try:
# logging.debug("Trying to prune unused networks...")
# subprocess_call(['docker', 'network', 'prune', '-f'])
# logging.debug("Networks pruned")
# except:
# pass
try:
if destroy_dirs and p.exists(self.instances_dir):
logging.debug(("Removing instances dir %s", self.instances_dir))
@ -831,7 +851,7 @@ class ClickHouseCluster:
logging.debug(('Setup directory for instance: {} destroy_dirs: {}'.format(instance.name, destroy_dirs)))
instance.create_dir(destroy_dir=destroy_dirs)
self.docker_client = docker.from_env(version=self.docker_api_version)
self.docker_client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180)
common_opts = ['up', '-d', '--force-recreate']
@ -856,11 +876,14 @@ class ClickHouseCluster:
if self.with_mysql and self.base_mysql_cmd:
logging.debug('Setup MySQL')
os.makedirs(self.mysql_logs_dir)
os.chmod(self.mysql_logs_dir, stat.S_IRWXO)
subprocess_check_call(self.base_mysql_cmd + common_opts)
self.wait_mysql_to_start(120)
self.wait_mysql_to_start(60)
if self.with_mysql8 and self.base_mysql8_cmd:
logging.debug('Setup MySQL 8')
os.makedirs(self.mysql8_logs_dir)
subprocess_check_call(self.base_mysql8_cmd + common_opts)
self.wait_mysql8_to_start(120)
@ -935,7 +958,7 @@ class ClickHouseCluster:
subprocess_check_call(clickhouse_start_cmd)
logging.debug("ClickHouse instance created")
start_deadline = time.time() + 20.0 # seconds
start_deadline = time.time() + 30.0 # seconds
for instance in self.instances.values():
instance.docker_client = self.docker_client
instance.ip_address = self.get_instance_ip(instance.name)
@ -970,13 +993,19 @@ class ClickHouseCluster:
break
for instance in list(self.instances.values()):
if instance.contains_in_log(SANITIZER_SIGN):
sanitizer_assert_instance = instance.grep_in_log(SANITIZER_SIGN)
logging.ERROR(f"Sanitizer in instance {instance.name} fatal log {fatal_log}")
try:
if not instance.is_up:
continue
if instance.contains_in_log(SANITIZER_SIGN):
sanitizer_assert_instance = instance.grep_in_log(SANITIZER_SIGN)
logging.ERROR(f"Sanitizer in instance {instance.name} log {sanitizer_assert_instance}")
if instance.contains_in_log("Fatal"):
fatal_log = instance.grep_in_log("Fatal")
logging.ERROR(f"Crash in instance {instance.name} fatal log {fatal_log}")
if instance.contains_in_log("Fatal"):
fatal_log = instance.grep_in_log("Fatal")
name = instance.name
logging.ERROR(f"Crash in instance {name} fatal log {fatal_log}")
except Exception as e:
logging.error(f"Failed to check fails in logs: {e}")
if kill:
try:
@ -1177,6 +1206,7 @@ class ClickHouseInstance:
self.ipv6_address = ipv6_address
self.with_installed_binary = with_installed_binary
self.env_file = os.path.join(os.path.dirname(self.docker_compose_path), DEFAULT_ENV_NAME)
self.is_up = False
def is_built_with_thread_sanitizer(self):
@ -1342,7 +1372,7 @@ class ClickHouseInstance:
return None
return None
def restart_with_latest_version(self, stop_start_wait_sec=10, callback_onstop=None, signal=15):
def restart_with_latest_version(self, stop_start_wait_sec=15, callback_onstop=None, signal=15):
if not self.stay_alive:
raise Exception("Cannot restart not stay alive container")
self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root')
@ -1410,6 +1440,7 @@ class ClickHouseInstance:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(time_left)
sock.connect((self.ip_address, 9000))
self.is_up = True
return
except socket.timeout:
continue

4
tests/integration/helpers/external_sources.py
View File

@ -9,6 +9,7 @@ import cassandra.cluster
import pymongo
import pymysql.cursors
import redis
import logging
from tzlocal import get_localzone
@ -59,6 +60,7 @@ class SourceMySQL(ExternalSource):
}
def create_mysql_conn(self):
logging.debug(f"pymysql connect {self.user}, {self.password}, {self.internal_hostname}, {self.internal_port}")
self.connection = pymysql.connect(
user=self.user,
password=self.password,
@ -98,6 +100,8 @@ class SourceMySQL(ExternalSource):
)
def prepare(self, structure, table_name, cluster):
if self.internal_hostname is None:
self.internal_hostname = cluster.mysql_ip
self.create_mysql_conn()
self.execute_mysql_query("create database if not exists test default character set 'utf8'")
self.execute_mysql_query("drop table if exists test.{}".format(table_name))

10
tests/integration/test_mysql_protocol/test.py
View File

@ -36,7 +36,7 @@ def mysql_client():
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_client.yml')
run_and_check(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build'])
yield docker.from_env().containers.get(cluster.project_name + '_mysql1_1')
yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180).containers.get(cluster.project_name + '_mysql1_1')
@pytest.fixture(scope='module')
@ -63,7 +63,7 @@ def golang_container():
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_golang_client.yml')
run_and_check(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build'])
yield docker.from_env().containers.get(cluster.project_name + '_golang1_1')
yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180).containers.get(cluster.project_name + '_golang1_1')
@pytest.fixture(scope='module')
@ -71,7 +71,7 @@ def php_container():
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_php_client.yml')
run_and_check(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build'])
yield docker.from_env().containers.get(cluster.project_name + '_php1_1')
yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180).containers.get(cluster.project_name + '_php1_1')
@pytest.fixture(scope='module')
@ -79,7 +79,7 @@ def nodejs_container():
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_js_client.yml')
run_and_check(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build'])
yield docker.from_env().containers.get(cluster.project_name + '_mysqljs1_1')
yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180).containers.get(cluster.project_name + '_mysqljs1_1')
@pytest.fixture(scope='module')
@ -87,7 +87,7 @@ def java_container():
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_java_client.yml')
run_and_check(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build'])
yield docker.from_env().containers.get(cluster.project_name + '_java1_1')
yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=180).containers.get(cluster.project_name + '_java1_1')
def test_mysql_client(mysql_client, server_address):