2020-07-17 08:24:21 +00:00
|
|
|
import os
|
|
|
|
import subprocess
|
2020-07-13 14:43:05 +00:00
|
|
|
import time
|
|
|
|
|
|
|
|
import pymysql.cursors
|
|
|
|
import pytest
|
2020-09-16 04:26:10 +00:00
|
|
|
from helpers.cluster import ClickHouseCluster, get_docker_compose_path
|
2020-07-13 14:43:05 +00:00
|
|
|
|
2020-10-02 16:54:07 +00:00
|
|
|
from . import materialize_with_ddl
|
2020-07-13 14:43:05 +00:00
|
|
|
|
2020-08-27 11:42:53 +00:00
|
|
|
DOCKER_COMPOSE_PATH = get_docker_compose_path()
|
2020-07-17 08:24:21 +00:00
|
|
|
|
2020-07-13 14:43:05 +00:00
|
|
|
cluster = ClickHouseCluster(__file__)
|
2020-08-13 18:20:47 +00:00
|
|
|
clickhouse_node = cluster.add_instance('node1', user_configs=["configs/users.xml"], with_mysql=False)
|
2020-07-17 08:24:21 +00:00
|
|
|
|
2020-07-13 14:43:05 +00:00
|
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
|
|
def started_cluster():
|
|
|
|
try:
|
|
|
|
cluster.start()
|
|
|
|
yield cluster
|
|
|
|
finally:
|
2020-11-10 09:08:12 +00:00
|
|
|
#cluster.shutdown()
|
|
|
|
pass
|
2020-07-13 14:43:05 +00:00
|
|
|
|
2020-07-17 08:24:21 +00:00
|
|
|
|
2020-07-13 14:43:05 +00:00
|
|
|
class MySQLNodeInstance:
|
2020-11-10 09:08:12 +00:00
|
|
|
def __init__(self, user='root', password='clickhouse', ip_address='127.0.0.1', port=3308):
|
2020-07-13 14:43:05 +00:00
|
|
|
self.user = user
|
|
|
|
self.port = port
|
2020-11-10 09:08:12 +00:00
|
|
|
self.ip_address = ip_address
|
2020-07-13 14:43:05 +00:00
|
|
|
self.password = password
|
|
|
|
self.mysql_connection = None # lazy init
|
|
|
|
|
2020-11-09 09:45:41 +00:00
|
|
|
|
2020-07-17 08:24:21 +00:00
|
|
|
def alloc_connection(self):
|
2020-07-13 14:43:05 +00:00
|
|
|
if self.mysql_connection is None:
|
2020-11-10 09:08:12 +00:00
|
|
|
self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.ip_address,
|
2020-09-16 04:26:10 +00:00
|
|
|
port=self.port, autocommit=True)
|
2020-07-17 08:24:21 +00:00
|
|
|
return self.mysql_connection
|
|
|
|
|
|
|
|
def query(self, execution_query):
|
|
|
|
with self.alloc_connection().cursor() as cursor:
|
2020-07-13 14:43:05 +00:00
|
|
|
cursor.execute(execution_query)
|
|
|
|
|
2020-11-11 15:37:44 +00:00
|
|
|
def create_min_priv_user(self, user, password):
|
|
|
|
self.query("CREATE USER '" + user + "'@'%' IDENTIFIED BY '" + password + "'")
|
|
|
|
self.grant_min_priv_for_user(user)
|
|
|
|
|
2020-11-12 01:49:12 +00:00
|
|
|
def grant_min_priv_for_user(self, user, db='priv_err_db'):
|
2020-11-11 15:37:44 +00:00
|
|
|
self.query("GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO '" + user + "'@'%'")
|
|
|
|
self.query("GRANT SELECT ON " + db + ".* TO '" + user + "'@'%'")
|
2020-10-30 01:31:45 +00:00
|
|
|
|
2020-10-29 08:09:28 +00:00
|
|
|
def result(self, execution_query):
|
|
|
|
with self.alloc_connection().cursor() as cursor:
|
2020-10-30 01:31:45 +00:00
|
|
|
result = cursor.execute(execution_query)
|
|
|
|
if result is not None:
|
|
|
|
print(cursor.fetchall())
|
2020-10-29 08:09:28 +00:00
|
|
|
|
2020-11-10 11:31:20 +00:00
|
|
|
def query_and_get_data(self, executio_query):
|
|
|
|
with self.alloc_connection().cursor() as cursor:
|
|
|
|
cursor.execute(executio_query)
|
|
|
|
return cursor.fetchall()
|
|
|
|
|
2020-07-13 14:43:05 +00:00
|
|
|
def close(self):
|
|
|
|
if self.mysql_connection is not None:
|
|
|
|
self.mysql_connection.close()
|
|
|
|
|
2020-07-17 08:24:21 +00:00
|
|
|
def wait_mysql_to_start(self, timeout=60):
|
|
|
|
start = time.time()
|
|
|
|
while time.time() - start < timeout:
|
|
|
|
try:
|
|
|
|
self.alloc_connection()
|
2020-10-02 16:54:07 +00:00
|
|
|
print("Mysql Started")
|
2020-11-11 15:37:44 +00:00
|
|
|
self.create_min_priv_user("test", "123")
|
2020-10-30 01:31:45 +00:00
|
|
|
print("min priv user created")
|
2020-07-17 08:24:21 +00:00
|
|
|
return
|
|
|
|
except Exception as ex:
|
2020-10-02 16:54:07 +00:00
|
|
|
print("Can't connect to MySQL " + str(ex))
|
2020-07-17 08:24:21 +00:00
|
|
|
time.sleep(0.5)
|
2020-07-13 14:43:05 +00:00
|
|
|
|
2020-07-17 10:05:46 +00:00
|
|
|
subprocess.check_call(['docker-compose', 'ps', '--services', 'all'])
|
2020-07-17 08:24:21 +00:00
|
|
|
raise Exception("Cannot wait MySQL container")
|
2020-07-15 05:26:37 +00:00
|
|
|
|
|
|
|
|
2020-07-17 08:24:21 +00:00
|
|
|
@pytest.fixture(scope="module")
|
|
|
|
def started_mysql_5_7():
|
2020-08-13 18:20:47 +00:00
|
|
|
mysql_node = MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', 3308)
|
|
|
|
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql.yml')
|
2020-07-15 05:26:37 +00:00
|
|
|
|
2020-07-17 08:24:21 +00:00
|
|
|
try:
|
2020-09-16 04:26:10 +00:00
|
|
|
subprocess.check_call(
|
|
|
|
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d'])
|
2020-07-17 08:24:21 +00:00
|
|
|
mysql_node.wait_mysql_to_start(120)
|
|
|
|
yield mysql_node
|
|
|
|
finally:
|
|
|
|
mysql_node.close()
|
2020-09-16 04:26:10 +00:00
|
|
|
subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes',
|
|
|
|
'--remove-orphans'])
|
2020-07-15 05:26:37 +00:00
|
|
|
|
|
|
|
|
2020-07-18 04:13:41 +00:00
|
|
|
@pytest.fixture(scope="module")
|
|
|
|
def started_mysql_8_0():
|
|
|
|
mysql_node = MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', 33308)
|
2020-08-27 11:42:53 +00:00
|
|
|
docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_8_0.yml')
|
2020-07-18 04:13:41 +00:00
|
|
|
|
|
|
|
try:
|
2020-09-16 04:26:10 +00:00
|
|
|
subprocess.check_call(
|
|
|
|
['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d'])
|
2020-07-18 04:13:41 +00:00
|
|
|
mysql_node.wait_mysql_to_start(120)
|
|
|
|
yield mysql_node
|
|
|
|
finally:
|
|
|
|
mysql_node.close()
|
2020-09-16 04:26:10 +00:00
|
|
|
subprocess.check_call(['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'down', '--volumes',
|
|
|
|
'--remove-orphans'])
|
2020-07-18 04:13:41 +00:00
|
|
|
|
|
|
|
|
2020-07-20 17:44:38 +00:00
|
|
|
def test_materialize_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7):
|
2020-08-13 18:20:47 +00:00
|
|
|
materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
2020-09-09 15:42:03 +00:00
|
|
|
materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql1")
|
2020-07-20 17:44:38 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_materialize_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.dml_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
2020-09-08 03:29:18 +00:00
|
|
|
materialize_with_ddl.materialize_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
|
2020-07-20 17:44:38 +00:00
|
|
|
|
2020-09-16 04:26:10 +00:00
|
|
|
|
2020-08-19 10:05:40 +00:00
|
|
|
def test_materialize_database_ddl_with_mysql_5_7(started_cluster, started_mysql_5_7):
|
2020-08-17 19:55:27 +00:00
|
|
|
try:
|
2020-09-02 09:48:07 +00:00
|
|
|
materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
2020-09-16 04:26:10 +00:00
|
|
|
materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
|
|
|
|
"mysql1")
|
|
|
|
materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
|
|
|
|
"mysql1")
|
2020-08-17 19:55:27 +00:00
|
|
|
# mysql 5.7 cannot support alter rename column
|
2020-09-02 09:48:07 +00:00
|
|
|
# materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
2020-09-16 04:26:10 +00:00
|
|
|
materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
|
|
|
|
"mysql1")
|
|
|
|
materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7,
|
|
|
|
"mysql1")
|
2020-08-17 19:55:27 +00:00
|
|
|
except:
|
2020-10-02 16:54:07 +00:00
|
|
|
print((clickhouse_node.query(
|
|
|
|
"select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw")))
|
2020-08-17 19:55:27 +00:00
|
|
|
raise
|
2020-07-18 04:13:41 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_materialize_database_ddl_with_mysql_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.drop_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
materialize_with_ddl.create_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
materialize_with_ddl.rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
2020-09-16 04:26:10 +00:00
|
|
|
materialize_with_ddl.alter_add_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
|
|
|
|
"mysql8_0")
|
|
|
|
materialize_with_ddl.alter_drop_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
|
|
|
|
"mysql8_0")
|
|
|
|
materialize_with_ddl.alter_rename_table_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
|
|
|
|
"mysql8_0")
|
|
|
|
materialize_with_ddl.alter_rename_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
|
|
|
|
"mysql8_0")
|
|
|
|
materialize_with_ddl.alter_modify_column_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0,
|
|
|
|
"mysql8_0")
|
|
|
|
|
2020-07-20 17:44:38 +00:00
|
|
|
|
2020-09-02 00:31:51 +00:00
|
|
|
def test_materialize_database_ddl_with_empty_transaction_5_7(started_cluster, started_mysql_5_7):
|
2020-09-03 10:27:44 +00:00
|
|
|
materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_5_7, "mysql1")
|
2020-09-02 00:31:51 +00:00
|
|
|
|
2020-09-16 04:26:10 +00:00
|
|
|
|
2020-09-02 00:31:51 +00:00
|
|
|
def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
2020-10-08 20:39:24 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_select_without_columns_5_7(started_cluster, started_mysql_5_7):
|
|
|
|
materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
|
2020-11-16 13:46:36 +00:00
|
|
|
|
2020-10-08 20:39:24 +00:00
|
|
|
def test_select_without_columns_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
2020-11-16 13:46:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_insert_with_modify_binlog_checksum_5_7(started_cluster, started_mysql_5_7):
|
|
|
|
materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
|
|
|
|
|
2020-11-19 05:01:23 +00:00
|
|
|
def test_insert_with_modify_binlog_checksum_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
|
2020-11-16 13:53:22 +00:00
|
|
|
|
2020-10-27 11:17:35 +00:00
|
|
|
def test_materialize_database_err_sync_user_privs_5_7(started_cluster, started_mysql_5_7):
|
2020-10-30 01:31:45 +00:00
|
|
|
try:
|
|
|
|
materialize_with_ddl.err_sync_user_privs_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
except:
|
|
|
|
print((clickhouse_node.query(
|
|
|
|
"select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw")))
|
|
|
|
raise
|
2020-10-27 11:17:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_materialize_database_err_sync_user_privs_8_0(started_cluster, started_mysql_8_0):
|
2020-10-30 01:31:45 +00:00
|
|
|
try:
|
|
|
|
materialize_with_ddl.err_sync_user_privs_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
except:
|
|
|
|
print((clickhouse_node.query(
|
|
|
|
"select '\n', thread_id, query_id, arrayStringConcat(arrayMap(x -> concat(demangle(addressToSymbol(x)), '\n ', addressToLine(x)), trace), '\n') AS sym from system.stack_trace format TSVRaw")))
|
|
|
|
raise
|
2020-10-30 01:31:45 +00:00
|
|
|
|
2020-11-09 09:45:41 +00:00
|
|
|
def test_network_partition_5_7(started_cluster, started_mysql_5_7):
|
|
|
|
materialize_with_ddl.network_partition_test(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
|
|
|
|
def test_network_partition_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.network_partition_test(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
|
|
|
|
2020-11-10 11:31:20 +00:00
|
|
|
def test_mysql_kill_sync_thread_restore_5_7(started_cluster, started_mysql_5_7):
|
|
|
|
materialize_with_ddl.mysql_kill_sync_thread_restore_test(clickhouse_node, started_mysql_5_7, "mysql1")
|
|
|
|
|
|
|
|
def test_mysql_kill_sync_thread_restore_8_0(started_cluster, started_mysql_8_0):
|
|
|
|
materialize_with_ddl.mysql_kill_sync_thread_restore_test(clickhouse_node, started_mysql_8_0, "mysql8_0")
|
2020-11-10 11:31:20 +00:00
|
|
|
<<<<<<< HEAD
|
2020-11-10 11:31:20 +00:00
|
|
|
|
2020-11-10 11:31:20 +00:00
|
|
|
=======
|
|
|
|
>>>>>>> add mysql kill sync id test
|