ClickHouse/tests/integration/test_materialize_mysql_database/test.py

149 lines
7.2 KiB
Python
Raw Normal View History

import os
import subprocess
2020-07-13 14:43:05 +00:00
import time
import pymysql.cursors
import pytest
from helpers.cluster import ClickHouseCluster, get_docker_compose_path
2020-07-13 14:43:05 +00:00
import materialize_with_ddl
2020-07-13 14:43:05 +00:00
2020-08-27 11:42:53 +00:00
DOCKER_COMPOSE_PATH = get_docker_compose_path()
2020-07-13 14:43:05 +00:00
cluster = ClickHouseCluster(__file__)
clickhouse_node = cluster.add_instance('node1', user_configs=["configs/users.xml"], with_mysql=False)
2020-07-13 14:43:05 +00:00
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
2020-07-13 14:43:05 +00:00
class MySQLNodeInstance:
def __init__(self, user='root', password='clickhouse', hostname='127.0.0.1', port=3308):
self.user = user
self.port = port
self.hostname = hostname
self.password = password
self.mysql_connection = None # lazy init
def alloc_connection(self):
2020-07-13 14:43:05 +00:00
if self.mysql_connection is None:
self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname,
port=self.port, autocommit=True)
return self.mysql_connection
def query(self, execution_query):
with self.alloc_connection().cursor() as cursor:
2020-07-13 14:43:05 +00:00
cursor.execute(execution_query)
def close(self):
if self.mysql_connection is not None:
self.mysql_connection.close()
def wait_mysql_to_start(self, timeout=60):
start = time.time()
while time.time() - start < timeout:
try:
self.alloc_connection()
print "Mysql Started"
return
except Exception as ex:
print "Can't connect to MySQL " + str(ex)
time.sleep(0.5)
2020-07-13 14:43:05 +00:00
2020-07-17 10:05:46 +00:00
subprocess.check_call(['docker-compose', 'ps', '--services', 'all'])
raise Exception("Cannot wait MySQL container")
2020-07-15 05:26:37 +00:00
@pytest.fixture(scope="module")
def started_mysql_5_7():
mysql_node = MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', 3308)
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql.yml')
2020-07-15 05:26:37 +00:00
try:
subprocess.check_call(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d'])
mysql_node.wait_mysql_to_start(120)
yield mysql_node
finally:
mysql_node.close()
subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes',
'--remove-orphans'])
2020-07-15 05:26:37 +00:00
@pytest.fixture(scope="module")
def started_mysql_8_0():
mysql_node = MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', 33308)
2020-08-27 11:42:53 +00:00
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_8_0.yml')
try:
subprocess.check_call(
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d'])
mysql_node.wait_mysql_to_start(120)
yield mysql_node
finally:
mysql_node.close()
subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes',
'--remove-orphans'])
2020-07-20 17:44:38 +00:00
def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7):
materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql1")
2020-07-20 17:44:38 +00:00
def test_materialize_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0):
materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
2020-09-08 03:29:18 +00:00
materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_8_0, "mysql8_0")
2020-07-20 17:44:38 +00:00
2020-08-19 10:05:40 +00:00
def test_materialize_database_ddl_with_mysql_5_7(started_cluster, started_mysql_5_7):
2020-08-17 19:55:27 +00:00
try:
materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
"mysql1")
materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
"mysql1")
2020-08-17 19:55:27 +00:00
# mysql 5.7 cannot support alter rename column
# materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
"mysql1")
materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
"mysql1")
2020-08-17 19:55:27 +00:00
except:
print(clickhouse_node.query(
"select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw"))
2020-08-17 19:55:27 +00:00
raise
def test_materialize_database_ddl_with_mysql_8_0(started_cluster, started_mysql_8_0):
materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
"mysql8_0")
materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
"mysql8_0")
materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
"mysql8_0")
materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
"mysql8_0")
materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
"mysql8_0")
2020-07-20 17:44:38 +00:00
def test_materialize_database_ddl_with_empty_transaction_5_7(started_cluster, started_mysql_5_7):
2020-09-03 10:27:44 +00:00
materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_5_7, "mysql1")
def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0):
materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql8_0")