mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge pull request #15110 from nikitamikhaylov/split-huge-dictionary-test
Splitting test_dictionaries_all_layouts_and_sources
This commit is contained in:
alesapin
GitHub
commit
9bf27fb42a
@ -486,8 +486,8 @@ class ClickHouseCluster:
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
connection.database_names()
|
||||
print "Connected to Mongo dbs:", connection.database_names()
|
||||
connection.list_database_names()
|
||||
print "Connected to Mongo dbs:", connection.list_database_names()
|
||||
return
|
||||
except Exception as ex:
|
||||
print "Can't connect to Mongo " + str(ex)
|
||||
|
||||
@ -333,16 +333,16 @@ class _SourceExecutableBase(ExternalSource):
|
||||
user='root')
|
||||
|
||||
|
||||
class SourceExecutableCache(_SourceExecutableBase):
|
||||
class SourceExecutableHashed(_SourceExecutableBase):
|
||||
|
||||
def _get_cmd(self, path):
|
||||
return "cat {}".format(path)
|
||||
|
||||
def compatible_with_layout(self, layout):
|
||||
return 'cache' not in layout.name
|
||||
return 'hashed' in layout.name
|
||||
|
||||
|
||||
class SourceExecutableHashed(_SourceExecutableBase):
|
||||
class SourceExecutableCache(_SourceExecutableBase):
|
||||
|
||||
def _get_cmd(self, path):
|
||||
return "cat - >/dev/null;cat {}".format(path)
|
||||
|
||||
@ -1,346 +0,0 @@
|
||||
import math
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceMongo, SourceMongoURI, SourceHTTP, SourceHTTPS, SourceCassandra
|
||||
from helpers.external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, \
|
||||
SourceExecutableHashed
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries')
|
||||
|
||||
FIELDS = {
|
||||
"simple": [
|
||||
Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999),
|
||||
Field("UInt8_", 'UInt8', default_value_for_get=55),
|
||||
Field("UInt16_", 'UInt16', default_value_for_get=66),
|
||||
Field("UInt32_", 'UInt32', default_value_for_get=77),
|
||||
Field("UInt64_", 'UInt64', default_value_for_get=88),
|
||||
Field("Int8_", 'Int8', default_value_for_get=-55),
|
||||
Field("Int16_", 'Int16', default_value_for_get=-66),
|
||||
Field("Int32_", 'Int32', default_value_for_get=-77),
|
||||
Field("Int64_", 'Int64', default_value_for_get=-88),
|
||||
Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'),
|
||||
Field("Date_", 'Date', default_value_for_get='2018-12-30'),
|
||||
Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'),
|
||||
Field("String_", 'String', default_value_for_get='hi'),
|
||||
Field("Float32_", 'Float32', default_value_for_get=555.11),
|
||||
Field("Float64_", 'Float64', default_value_for_get=777.11),
|
||||
Field("ParentKeyField", "UInt64", default_value_for_get=444, hierarchical=True)
|
||||
],
|
||||
"complex": [
|
||||
Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999),
|
||||
Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx'),
|
||||
Field("UInt8_", 'UInt8', default_value_for_get=55),
|
||||
Field("UInt16_", 'UInt16', default_value_for_get=66),
|
||||
Field("UInt32_", 'UInt32', default_value_for_get=77),
|
||||
Field("UInt64_", 'UInt64', default_value_for_get=88),
|
||||
Field("Int8_", 'Int8', default_value_for_get=-55),
|
||||
Field("Int16_", 'Int16', default_value_for_get=-66),
|
||||
Field("Int32_", 'Int32', default_value_for_get=-77),
|
||||
Field("Int64_", 'Int64', default_value_for_get=-88),
|
||||
Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'),
|
||||
Field("Date_", 'Date', default_value_for_get='2018-12-30'),
|
||||
Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'),
|
||||
Field("String_", 'String', default_value_for_get='hi'),
|
||||
Field("Float32_", 'Float32', default_value_for_get=555.11),
|
||||
Field("Float64_", 'Float64', default_value_for_get=777.11),
|
||||
],
|
||||
"ranged": [
|
||||
Field("KeyField1", 'UInt64', is_key=True),
|
||||
Field("KeyField2", 'Date', is_range_key=True),
|
||||
Field("StartDate", 'Date', range_hash_type='min'),
|
||||
Field("EndDate", 'Date', range_hash_type='max'),
|
||||
Field("UInt8_", 'UInt8', default_value_for_get=55),
|
||||
Field("UInt16_", 'UInt16', default_value_for_get=66),
|
||||
Field("UInt32_", 'UInt32', default_value_for_get=77),
|
||||
Field("UInt64_", 'UInt64', default_value_for_get=88),
|
||||
Field("Int8_", 'Int8', default_value_for_get=-55),
|
||||
Field("Int16_", 'Int16', default_value_for_get=-66),
|
||||
Field("Int32_", 'Int32', default_value_for_get=-77),
|
||||
Field("Int64_", 'Int64', default_value_for_get=-88),
|
||||
Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'),
|
||||
Field("Date_", 'Date', default_value_for_get='2018-12-30'),
|
||||
Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'),
|
||||
Field("String_", 'String', default_value_for_get='hi'),
|
||||
Field("Float32_", 'Float32', default_value_for_get=555.11),
|
||||
Field("Float64_", 'Float64', default_value_for_get=777.11),
|
||||
]
|
||||
}
|
||||
|
||||
VALUES = {
|
||||
"simple": [
|
||||
[1, 22, 333, 4444, 55555, -6, -77,
|
||||
-888, -999, '550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25', 'hello', 22.543, 3332154213.4, 0],
|
||||
[2, 3, 4, 5, 6, -7, -8,
|
||||
-9, -10, '550e8400-e29b-41d4-a716-446655440002',
|
||||
'1978-06-28', '1986-02-28 23:42:25', 'hello', 21.543, 3222154213.4, 1]
|
||||
],
|
||||
"complex": [
|
||||
[1, 'world', 22, 333, 4444, 55555, -6,
|
||||
-77, -888, -999, '550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25',
|
||||
'hello', 22.543, 3332154213.4],
|
||||
[2, 'qwerty2', 52, 2345, 6544, 9191991, -2,
|
||||
-717, -81818, -92929, '550e8400-e29b-41d4-a716-446655440007',
|
||||
'1975-09-28', '2000-02-28 23:33:24',
|
||||
'my', 255.543, 3332221.44]
|
||||
|
||||
],
|
||||
"ranged": [
|
||||
[1, '2019-02-10', '2019-02-01', '2019-02-28',
|
||||
22, 333, 4444, 55555, -6, -77, -888, -999,
|
||||
'550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25', 'hello',
|
||||
22.543, 3332154213.4],
|
||||
[2, '2019-04-10', '2019-04-01', '2019-04-28',
|
||||
11, 3223, 41444, 52515, -65, -747, -8388, -9099,
|
||||
'550e8400-e29b-41d4-a716-446655440004',
|
||||
'1973-06-29', '2002-02-28 23:23:25', '!!!!',
|
||||
32.543, 3332543.4]
|
||||
]
|
||||
}
|
||||
|
||||
LAYOUTS = [
|
||||
Layout("flat"),
|
||||
Layout("hashed"),
|
||||
Layout("cache"),
|
||||
Layout("complex_key_hashed"),
|
||||
Layout("complex_key_cache"),
|
||||
Layout("range_hashed"),
|
||||
Layout("direct"),
|
||||
Layout("complex_key_direct")
|
||||
]
|
||||
|
||||
SOURCES = [
|
||||
SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", ""),
|
||||
SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"),
|
||||
SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse"),
|
||||
SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse"),
|
||||
SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", ""),
|
||||
SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", ""),
|
||||
SourceFile("File", "localhost", "9000", "node", "9000", "", ""),
|
||||
SourceExecutableHashed("ExecutableHashed", "localhost", "9000", "node", "9000", "", ""),
|
||||
SourceExecutableCache("ExecutableCache", "localhost", "9000", "node", "9000", "", ""),
|
||||
SourceHTTP("SourceHTTP", "localhost", "9000", "clickhouse1", "9000", "", ""),
|
||||
SourceHTTPS("SourceHTTPS", "localhost", "9000", "clickhouse1", "9000", "", ""),
|
||||
]
|
||||
|
||||
DICTIONARIES = []
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
|
||||
|
||||
def get_dict(source, layout, fields, suffix_name=''):
|
||||
global dict_configs_path
|
||||
|
||||
structure = DictionaryStructure(layout, fields)
|
||||
dict_name = source.name + "_" + layout.name + '_' + suffix_name
|
||||
dict_path = os.path.join(dict_configs_path, dict_name + '.xml')
|
||||
dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields)
|
||||
dictionary.generate_config()
|
||||
return dictionary
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global DICTIONARIES
|
||||
global cluster
|
||||
global node
|
||||
global dict_configs_path
|
||||
|
||||
for f in os.listdir(dict_configs_path):
|
||||
os.remove(os.path.join(dict_configs_path, f))
|
||||
|
||||
for layout in LAYOUTS:
|
||||
for source in SOURCES:
|
||||
if source.compatible_with_layout(layout):
|
||||
DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type]))
|
||||
else:
|
||||
print "Source", source.name, "incompatible with layout", layout.name
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
cluster.add_instance('clickhouse1', main_configs=main_configs)
|
||||
|
||||
dictionaries = []
|
||||
for fname in os.listdir(dict_configs_path):
|
||||
dictionaries.append(os.path.join(dict_configs_path, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True,
|
||||
with_mongo=True, with_redis=True, with_cassandra=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
for dictionary in DICTIONARIES:
|
||||
print "Preparing", dictionary.name
|
||||
dictionary.prepare_source(cluster)
|
||||
print "Prepared"
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def get_dictionaries(fold, total_folds, all_dicts):
|
||||
chunk_len = int(math.ceil(len(all_dicts) / float(total_folds)))
|
||||
if chunk_len * fold >= len(all_dicts):
|
||||
return []
|
||||
return all_dicts[fold * chunk_len: (fold + 1) * chunk_len]
|
||||
|
||||
|
||||
def remove_mysql_dicts():
|
||||
"""
|
||||
We have false-positive race condition in our openSSL version.
|
||||
MySQL dictionary use OpenSSL, so to prevent known failure we
|
||||
disable tests for these dictionaries.
|
||||
|
||||
Read of size 8 at 0x7b3c00005dd0 by thread T61 (mutexes: write M1010349240585225536):
|
||||
#0 EVP_CIPHER_mode <null> (clickhouse+0x13b2223b)
|
||||
#1 do_ssl3_write <null> (clickhouse+0x13a137bc)
|
||||
#2 ssl3_write_bytes <null> (clickhouse+0x13a12387)
|
||||
#3 ssl3_write <null> (clickhouse+0x139db0e6)
|
||||
#4 ssl_write_internal <null> (clickhouse+0x139eddce)
|
||||
#5 SSL_write <null> (clickhouse+0x139edf20)
|
||||
#6 ma_tls_write <null> (clickhouse+0x139c7557)
|
||||
#7 ma_pvio_tls_write <null> (clickhouse+0x139a8f59)
|
||||
#8 ma_pvio_write <null> (clickhouse+0x139a8488)
|
||||
#9 ma_net_real_write <null> (clickhouse+0x139a4e2c)
|
||||
#10 ma_net_write_command <null> (clickhouse+0x139a546d)
|
||||
#11 mthd_my_send_cmd <null> (clickhouse+0x13992546)
|
||||
#12 mysql_close_slow_part <null> (clickhouse+0x13999afd)
|
||||
#13 mysql_close <null> (clickhouse+0x13999071)
|
||||
#14 mysqlxx::Connection::~Connection() <null> (clickhouse+0x1370f814)
|
||||
#15 mysqlxx::Pool::~Pool() <null> (clickhouse+0x13715a7b)
|
||||
|
||||
TODO remove this when open ssl will be fixed or thread sanitizer will be suppressed
|
||||
"""
|
||||
|
||||
# global DICTIONARIES
|
||||
# DICTIONARIES = [d for d in DICTIONARIES if not d.name.startswith("MySQL")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fold", list(range(10)))
|
||||
def test_simple_dictionaries(started_cluster, fold):
|
||||
if node.is_built_with_thread_sanitizer():
|
||||
remove_mysql_dicts()
|
||||
|
||||
fields = FIELDS["simple"]
|
||||
values = VALUES["simple"]
|
||||
data = [Row(fields, vals) for vals in values]
|
||||
|
||||
all_simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"]
|
||||
simple_dicts = get_dictionaries(fold, 10, all_simple_dicts)
|
||||
|
||||
print "Length of dicts:", len(simple_dicts)
|
||||
for dct in simple_dicts:
|
||||
dct.load_data(data)
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
|
||||
queries_with_answers = []
|
||||
for dct in simple_dicts:
|
||||
for row in data:
|
||||
for field in fields:
|
||||
if not field.is_key:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query in dct.get_select_has_queries(field, row):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query in dct.get_select_get_or_default_queries(field, row):
|
||||
queries_with_answers.append((query, field.default_value_for_get))
|
||||
for query in dct.get_hierarchical_queries(data[0]):
|
||||
queries_with_answers.append((query, [1]))
|
||||
|
||||
for query in dct.get_hierarchical_queries(data[1]):
|
||||
queries_with_answers.append((query, [2, 1]))
|
||||
|
||||
for query in dct.get_is_in_queries(data[0], data[1]):
|
||||
queries_with_answers.append((query, 0))
|
||||
|
||||
for query in dct.get_is_in_queries(data[1], data[0]):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
print query
|
||||
if isinstance(answer, list):
|
||||
answer = str(answer).replace(' ', '')
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fold", list(range(10)))
|
||||
def test_complex_dictionaries(started_cluster, fold):
|
||||
if node.is_built_with_thread_sanitizer():
|
||||
remove_mysql_dicts()
|
||||
|
||||
fields = FIELDS["complex"]
|
||||
values = VALUES["complex"]
|
||||
data = [Row(fields, vals) for vals in values]
|
||||
|
||||
all_complex_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "complex"]
|
||||
complex_dicts = get_dictionaries(fold, 10, all_complex_dicts)
|
||||
|
||||
for dct in complex_dicts:
|
||||
dct.load_data(data)
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
|
||||
queries_with_answers = []
|
||||
for dct in complex_dicts:
|
||||
for row in data:
|
||||
for field in fields:
|
||||
if not field.is_key:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query in dct.get_select_has_queries(field, row):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query in dct.get_select_get_or_default_queries(field, row):
|
||||
queries_with_answers.append((query, field.default_value_for_get))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
print query
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fold", list(range(10)))
|
||||
def test_ranged_dictionaries(started_cluster, fold):
|
||||
if node.is_built_with_thread_sanitizer():
|
||||
remove_mysql_dicts()
|
||||
|
||||
fields = FIELDS["ranged"]
|
||||
values = VALUES["ranged"]
|
||||
data = [Row(fields, vals) for vals in values]
|
||||
|
||||
all_ranged_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "ranged"]
|
||||
ranged_dicts = get_dictionaries(fold, 10, all_ranged_dicts)
|
||||
|
||||
for dct in ranged_dicts:
|
||||
dct.load_data(data)
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
|
||||
queries_with_answers = []
|
||||
for dct in ranged_dicts:
|
||||
for row in data:
|
||||
for field in fields:
|
||||
if not field.is_key and not field.is_range:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
print query
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
@ -0,0 +1,239 @@
|
||||
import os
|
||||
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
|
||||
KEY_FIELDS = {
|
||||
"simple": [
|
||||
Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999)
|
||||
],
|
||||
"complex": [
|
||||
Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999),
|
||||
Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx')
|
||||
],
|
||||
"ranged": [
|
||||
Field("KeyField1", 'UInt64', is_key=True),
|
||||
Field("KeyField2", 'Date', is_range_key=True)
|
||||
]
|
||||
}
|
||||
|
||||
START_FIELDS = {
|
||||
"simple": [],
|
||||
"complex": [],
|
||||
"ranged" : [
|
||||
Field("StartDate", 'Date', range_hash_type='min'),
|
||||
Field("EndDate", 'Date', range_hash_type='max')
|
||||
]
|
||||
}
|
||||
|
||||
MIDDLE_FIELDS = [
|
||||
Field("UInt8_", 'UInt8', default_value_for_get=55),
|
||||
Field("UInt16_", 'UInt16', default_value_for_get=66),
|
||||
Field("UInt32_", 'UInt32', default_value_for_get=77),
|
||||
Field("UInt64_", 'UInt64', default_value_for_get=88),
|
||||
Field("Int8_", 'Int8', default_value_for_get=-55),
|
||||
Field("Int16_", 'Int16', default_value_for_get=-66),
|
||||
Field("Int32_", 'Int32', default_value_for_get=-77),
|
||||
Field("Int64_", 'Int64', default_value_for_get=-88),
|
||||
Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'),
|
||||
Field("Date_", 'Date', default_value_for_get='2018-12-30'),
|
||||
Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'),
|
||||
Field("String_", 'String', default_value_for_get='hi'),
|
||||
Field("Float32_", 'Float32', default_value_for_get=555.11),
|
||||
Field("Float64_", 'Float64', default_value_for_get=777.11),
|
||||
]
|
||||
|
||||
END_FIELDS = {
|
||||
"simple" : [
|
||||
Field("ParentKeyField", "UInt64", default_value_for_get=444, hierarchical=True)
|
||||
],
|
||||
"complex" : [],
|
||||
"ranged" : []
|
||||
}
|
||||
|
||||
LAYOUTS_SIMPLE = ["flat", "hashed", "cache", "direct"]
|
||||
LAYOUTS_COMPLEX = ["complex_key_hashed", "complex_key_cache", "complex_key_direct"]
|
||||
LAYOUTS_RANGED = ["range_hashed"]
|
||||
|
||||
VALUES = {
|
||||
"simple": [
|
||||
[1, 22, 333, 4444, 55555, -6, -77,
|
||||
-888, -999, '550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25', 'hello', 22.543, 3332154213.4, 0],
|
||||
[2, 3, 4, 5, 6, -7, -8,
|
||||
-9, -10, '550e8400-e29b-41d4-a716-446655440002',
|
||||
'1978-06-28', '1986-02-28 23:42:25', 'hello', 21.543, 3222154213.4, 1]
|
||||
],
|
||||
"complex": [
|
||||
[1, 'world', 22, 333, 4444, 55555, -6,
|
||||
-77, -888, -999, '550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25',
|
||||
'hello', 22.543, 3332154213.4],
|
||||
[2, 'qwerty2', 52, 2345, 6544, 9191991, -2,
|
||||
-717, -81818, -92929, '550e8400-e29b-41d4-a716-446655440007',
|
||||
'1975-09-28', '2000-02-28 23:33:24',
|
||||
'my', 255.543, 3332221.44]
|
||||
],
|
||||
"ranged": [
|
||||
[1, '2019-02-10', '2019-02-01', '2019-02-28',
|
||||
22, 333, 4444, 55555, -6, -77, -888, -999,
|
||||
'550e8400-e29b-41d4-a716-446655440003',
|
||||
'1973-06-28', '1985-02-28 23:43:25', 'hello',
|
||||
22.543, 3332154213.4],
|
||||
[2, '2019-04-10', '2019-04-01', '2019-04-28',
|
||||
11, 3223, 41444, 52515, -65, -747, -8388, -9099,
|
||||
'550e8400-e29b-41d4-a716-446655440004',
|
||||
'1973-06-29', '2002-02-28 23:23:25', '!!!!',
|
||||
32.543, 3332543.4]
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
DICT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'configs/dictionaries')
|
||||
|
||||
def get_dict(source, layout, fields, suffix_name=''):
|
||||
global DICT_CONFIG_PATH
|
||||
structure = DictionaryStructure(layout, fields)
|
||||
dict_name = source.name + "_" + layout.name + '_' + suffix_name
|
||||
dict_path = os.path.join(DICT_CONFIG_PATH, dict_name + '.xml')
|
||||
dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields)
|
||||
dictionary.generate_config()
|
||||
return dictionary
|
||||
|
||||
class SimpleLayoutTester:
|
||||
def __init__(self):
|
||||
self.fields = KEY_FIELDS["simple"] + START_FIELDS["simple"] + MIDDLE_FIELDS + END_FIELDS["simple"]
|
||||
self.values = VALUES["simple"]
|
||||
self.data = [Row(self.fields, vals) for vals in self.values]
|
||||
self.layout_to_dictionary = dict()
|
||||
|
||||
def create_dictionaries(self, source_):
|
||||
for layout in LAYOUTS_SIMPLE:
|
||||
if source_.compatible_with_layout(Layout(layout)):
|
||||
self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields)
|
||||
|
||||
def prepare(self, cluster_):
|
||||
for _, dictionary in self.layout_to_dictionary.items():
|
||||
dictionary.prepare_source(cluster_)
|
||||
dictionary.load_data(self.data)
|
||||
|
||||
def execute(self, layout_name, node):
|
||||
if not self.layout_to_dictionary.has_key(layout_name):
|
||||
raise RuntimeError("Source doesn't support layout: {}".format(layout_name))
|
||||
|
||||
dct = self.layout_to_dictionary[layout_name]
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
queries_with_answers = []
|
||||
|
||||
for row in self.data:
|
||||
for field in self.fields:
|
||||
if not field.is_key:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query in dct.get_select_has_queries(field, row):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query in dct.get_select_get_or_default_queries(field, row):
|
||||
queries_with_answers.append((query, field.default_value_for_get))
|
||||
|
||||
for query in dct.get_hierarchical_queries(self.data[0]):
|
||||
queries_with_answers.append((query, [1]))
|
||||
|
||||
for query in dct.get_hierarchical_queries(self.data[1]):
|
||||
queries_with_answers.append((query, [2, 1]))
|
||||
|
||||
for query in dct.get_is_in_queries(self.data[0], self.data[1]):
|
||||
queries_with_answers.append((query, 0))
|
||||
|
||||
for query in dct.get_is_in_queries(self.data[1], self.data[0]):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
# print query
|
||||
if isinstance(answer, list):
|
||||
answer = str(answer).replace(' ', '')
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
|
||||
|
||||
class ComplexLayoutTester:
|
||||
def __init__(self):
|
||||
self.fields = KEY_FIELDS["complex"] + START_FIELDS["complex"] + MIDDLE_FIELDS + END_FIELDS["complex"]
|
||||
self.values = VALUES["complex"]
|
||||
self.data = [Row(self.fields, vals) for vals in self.values]
|
||||
self.layout_to_dictionary = dict()
|
||||
|
||||
def create_dictionaries(self, source_):
|
||||
for layout in LAYOUTS_COMPLEX:
|
||||
if source_.compatible_with_layout(Layout(layout)):
|
||||
self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields)
|
||||
|
||||
def prepare(self, cluster_):
|
||||
for _, dictionary in self.layout_to_dictionary.items():
|
||||
dictionary.prepare_source(cluster_)
|
||||
dictionary.load_data(self.data)
|
||||
|
||||
def execute(self, layout_name, node):
|
||||
if not self.layout_to_dictionary.has_key(layout_name):
|
||||
raise RuntimeError("Source doesn't support layout: {}".format(layout_name))
|
||||
|
||||
dct = self.layout_to_dictionary[layout_name]
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
queries_with_answers = []
|
||||
|
||||
for row in self.data:
|
||||
for field in self.fields:
|
||||
if not field.is_key:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query in dct.get_select_has_queries(field, row):
|
||||
queries_with_answers.append((query, 1))
|
||||
|
||||
for query in dct.get_select_get_or_default_queries(field, row):
|
||||
queries_with_answers.append((query, field.default_value_for_get))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
# print query
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
|
||||
|
||||
class RangedLayoutTester:
|
||||
def __init__(self):
|
||||
self.fields = KEY_FIELDS["ranged"] + START_FIELDS["ranged"] + MIDDLE_FIELDS + END_FIELDS["ranged"]
|
||||
self.values = VALUES["ranged"]
|
||||
self.data = [Row(self.fields, vals) for vals in self.values]
|
||||
self.layout_to_dictionary = dict()
|
||||
|
||||
def create_dictionaries(self, source_):
|
||||
for layout in LAYOUTS_RANGED:
|
||||
if source_.compatible_with_layout(Layout(layout)):
|
||||
self.layout_to_dictionary[layout] = get_dict(source_, Layout(layout), self.fields)
|
||||
|
||||
def prepare(self, cluster_):
|
||||
for _, dictionary in self.layout_to_dictionary.items():
|
||||
dictionary.prepare_source(cluster_)
|
||||
dictionary.load_data(self.data)
|
||||
|
||||
def execute(self, layout_name, node):
|
||||
|
||||
if not self.layout_to_dictionary.has_key(layout_name):
|
||||
raise RuntimeError("Source doesn't support layout: {}".format(layout_name))
|
||||
|
||||
dct = self.layout_to_dictionary[layout_name]
|
||||
|
||||
node.query("system reload dictionaries")
|
||||
|
||||
queries_with_answers = []
|
||||
for row in self.data:
|
||||
for field in self.fields:
|
||||
if not field.is_key and not field.is_range:
|
||||
for query in dct.get_select_get_queries(field, row):
|
||||
queries_with_answers.append((query, row.get_value_by_name(field.name)))
|
||||
|
||||
for query, answer in queries_with_answers:
|
||||
# print query
|
||||
assert node.query(query) == str(answer) + '\n'
|
||||
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceCassandra
|
||||
|
||||
SOURCE = SourceCassandra("Cassandra", "localhost", "9043", "cassandra1", "9042", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceClickHouse
|
||||
|
||||
SOURCE = SourceClickHouse("LocalClickHouse", "localhost", "9000", "node", "9000", "default", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,84 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceClickHouse
|
||||
|
||||
SOURCE = SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse1", "9000", "default", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
cluster.add_instance('clickhouse1', main_configs=main_configs)
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_SIMPLE).difference(set("cache"))) )
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_COMPLEX).difference(set("complex_key_cache"))))
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,78 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceExecutableCache
|
||||
|
||||
SOURCE = SourceExecutableCache("ExecutableCache", "localhost", "9000", "node", "9000", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", ['cache'])
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", ['complex_key_cache'])
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceExecutableHashed
|
||||
|
||||
SOURCE = SourceExecutableHashed("ExecutableHashed", "localhost", "9000", "node", "9000", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", ['hashed'])
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", ['complex_key_hashed'])
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceFile
|
||||
|
||||
SOURCE = SourceFile("File", "localhost", "9000", "node", "9000", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", set(LAYOUTS_SIMPLE).difference({'cache', 'direct'}) )
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", list(set(LAYOUTS_COMPLEX).difference({'complex_key_cache', 'complex_key_direct'})))
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,84 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceHTTP
|
||||
|
||||
SOURCE = SourceHTTP("SourceHTTP", "localhost", "9000", "clickhouse1", "9000", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
cluster.add_instance('clickhouse1', main_configs=main_configs)
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,84 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceHTTPS
|
||||
|
||||
SOURCE = SourceHTTPS("SourceHTTPS", "localhost", "9000", "clickhouse1", "9000", "", "")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
cluster.add_instance('clickhouse1', main_configs=main_configs)
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceMongo
|
||||
|
||||
SOURCE = SourceMongo("MongoDB", "localhost", "27018", "mongo1", "27017", "root", "clickhouse")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,75 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceMongoURI
|
||||
|
||||
SOURCE = SourceMongoURI("MongoDB_URI", "localhost", "27018", "mongo1", "27017", "root", "clickhouse")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
# See comment in SourceMongoURI
|
||||
@pytest.mark.parametrize("layout_name", ["flat"])
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
@ -0,0 +1,82 @@
|
||||
import os
|
||||
import math
|
||||
import pytest
|
||||
|
||||
from .common import *
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout
|
||||
from helpers.external_sources import SourceMySQL
|
||||
|
||||
SOURCE = SourceMySQL("MySQL", "localhost", "3308", "mysql1", "3306", "root", "clickhouse")
|
||||
|
||||
cluster = None
|
||||
node = None
|
||||
simple_tester = None
|
||||
complex_tester = None
|
||||
ranged_tester = None
|
||||
|
||||
|
||||
def setup_module(module):
|
||||
global cluster
|
||||
global node
|
||||
global simple_tester
|
||||
global complex_tester
|
||||
global ranged_tester
|
||||
|
||||
for f in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, f))
|
||||
|
||||
simple_tester = SimpleLayoutTester()
|
||||
simple_tester.create_dictionaries(SOURCE)
|
||||
|
||||
complex_tester = ComplexLayoutTester()
|
||||
complex_tester.create_dictionaries(SOURCE)
|
||||
|
||||
ranged_tester = RangedLayoutTester()
|
||||
ranged_tester.create_dictionaries(SOURCE)
|
||||
# Since that all .xml configs were created
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
dictionaries = []
|
||||
main_configs = []
|
||||
main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml'))
|
||||
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
dictionaries.append(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True)
|
||||
|
||||
|
||||
def teardown_module(module):
|
||||
global DICT_CONFIG_PATH
|
||||
for fname in os.listdir(DICT_CONFIG_PATH):
|
||||
os.remove(os.path.join(DICT_CONFIG_PATH, fname))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
simple_tester.prepare(cluster)
|
||||
complex_tester.prepare(cluster)
|
||||
ranged_tester.prepare(cluster)
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_SIMPLE)
|
||||
def test_simple(started_cluster, layout_name):
|
||||
simple_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_COMPLEX)
|
||||
def test_complex(started_cluster, layout_name):
|
||||
complex_tester.execute(layout_name, node)
|
||||
|
||||
@pytest.mark.parametrize("layout_name", LAYOUTS_RANGED)
|
||||
def test_ranged(started_cluster, layout_name):
|
||||
ranged_tester.execute(layout_name, node)
|
||||
Loading…
Reference in New Issue
Block a user