2019-10-10 16:10:46 +00:00
import pytest
import os
import time
## sudo -H pip install PyMySQL
import pymysql . cursors
from helpers . cluster import ClickHouseCluster
from helpers . test_tools import assert_eq_with_retry
SCRIPT_DIR = os . path . dirname ( os . path . realpath ( __file__ ) )
2020-02-27 09:34:06 +00:00
CONFIG_FILES = [ ' configs/dictionaries/mysql_dict1.xml ' , ' configs/dictionaries/mysql_dict2.xml ' , ' configs/remote_servers.xml ' ]
2019-10-10 16:10:46 +00:00
cluster = ClickHouseCluster ( __file__ , base_configs_dir = os . path . join ( SCRIPT_DIR , ' configs ' ) )
2020-02-27 09:34:06 +00:00
instance = cluster . add_instance ( ' instance ' , main_configs = CONFIG_FILES , with_mysql = True )
2019-10-10 16:10:46 +00:00
create_table_mysql_template = """
2020-02-27 09:34:06 +00:00
CREATE TABLE IF NOT EXISTS ` test ` . ` { } ` (
2019-10-10 16:10:46 +00:00
` id ` int ( 11 ) NOT NULL ,
` value ` varchar ( 50 ) NOT NULL ,
PRIMARY KEY ( ` id ` )
) ENGINE = InnoDB ;
"""
create_clickhouse_dictionary_table_template = """
2020-02-27 09:34:06 +00:00
CREATE TABLE IF NOT EXISTS ` test ` . ` dict_table_ { } ` ( ` id ` Int32 , ` value ` String ) ENGINE = Dictionary ( { } )
2019-10-10 16:10:46 +00:00
"""
@pytest.fixture ( scope = " module " )
def started_cluster ( ) :
try :
2020-02-27 09:34:06 +00:00
#time.sleep(30)
2019-10-10 16:10:46 +00:00
cluster . start ( )
2019-10-11 07:25:22 +00:00
2019-10-10 16:10:46 +00:00
# Create a MySQL database
2020-02-27 09:34:06 +00:00
mysql_connection = get_mysql_conn ( )
create_mysql_db ( mysql_connection , ' test ' )
mysql_connection . close ( )
2019-10-11 07:25:22 +00:00
# Create database in ClickHouse
instance . query ( " CREATE DATABASE IF NOT EXISTS test " )
# Create database in ClickChouse using MySQL protocol (will be used for data insertion)
2019-10-10 16:10:46 +00:00
instance . query ( " CREATE DATABASE clickhouse_mysql ENGINE = MySQL( ' mysql1:3306 ' , ' test ' , ' root ' , ' clickhouse ' ) " )
yield cluster
finally :
cluster . shutdown ( )
def test_load_mysql_dictionaries ( started_cluster ) :
# Load dictionaries
query = instance . query
query ( " SYSTEM RELOAD DICTIONARIES " )
for n in range ( 0 , 5 ) :
2019-10-11 07:25:22 +00:00
# Create MySQL tables, fill them and create CH dict tables
2020-02-27 09:34:06 +00:00
prepare_mysql_table ( ' test ' , str ( n ) )
2019-10-10 16:10:46 +00:00
# Check dictionaries are loaded and have correct number of elements
for n in range ( 0 , 100 ) :
2019-10-11 07:25:22 +00:00
# Force reload of dictionaries (each 10 iteration)
2019-10-10 16:10:46 +00:00
if ( n % 10 ) == 0 :
query ( " SYSTEM RELOAD DICTIONARIES " )
2019-10-11 07:25:22 +00:00
2020-02-27 09:34:06 +00:00
# Check number of row
assert query ( " SELECT count() FROM `test`.`dict_table_ {} ` " . format ( ' test ' + str ( n % 5 ) ) ) . rstrip ( ) == ' 10000 '
2019-10-10 16:10:46 +00:00
def create_mysql_db ( mysql_connection , name ) :
with mysql_connection . cursor ( ) as cursor :
2020-02-27 09:34:06 +00:00
cursor . execute ( " CREATE DATABASE IF NOT EXISTS {} DEFAULT CHARACTER SET ' utf8 ' " . format ( name ) )
2019-10-10 16:10:46 +00:00
2020-02-27 09:34:06 +00:00
def prepare_mysql_table ( table_name , index ) :
2019-10-10 16:10:46 +00:00
mysql_connection = get_mysql_conn ( )
# Create table
2020-02-27 09:34:06 +00:00
create_mysql_table ( mysql_connection , table_name + str ( index ) )
2019-10-10 16:10:46 +00:00
# Insert rows using CH
query = instance . query
2020-02-27 09:34:06 +00:00
query ( " INSERT INTO `clickhouse_mysql`. {} (id, value) select number, concat( ' {} value ' , toString(number)) from numbers(10000) " . format ( table_name + str ( index ) , table_name + str ( index ) ) )
assert query ( " SELECT count() FROM `clickhouse_mysql`. {} " . format ( table_name + str ( index ) ) ) . rstrip ( ) == ' 10000 '
2019-10-10 16:10:46 +00:00
mysql_connection . close ( )
2019-10-11 07:25:22 +00:00
#Create CH Dictionary tables based on MySQL tables
2020-02-27 09:34:06 +00:00
query ( create_clickhouse_dictionary_table_template . format ( table_name + str ( index ) , ' dict ' + str ( index ) ) )
2019-10-10 16:10:46 +00:00
def get_mysql_conn ( ) :
2020-02-27 09:34:06 +00:00
conn = pymysql . connect ( user = ' root ' , password = ' clickhouse ' , host = ' 127.0.0.10 ' , port = 3308 )
2019-10-10 16:10:46 +00:00
return conn
def create_mysql_table ( conn , table_name ) :
with conn . cursor ( ) as cursor :
cursor . execute ( create_table_mysql_template . format ( table_name ) )