integration test added

This commit is contained in:
Nikita Mikhaylov 2019-12-27 23:31:40 +03:00
parent b194174ae1
commit 62f265576a
10 changed files with 230 additions and 22 deletions

View File

@ -709,6 +709,7 @@ void CacheDictionary::updateThreadFunction()
while (!finished) while (!finished)
{ {
UpdateUnitPtr unit_ptr; UpdateUnitPtr unit_ptr;
/// TODO: pop as many as we can + concatenate requested_ids
update_queue.pop(unit_ptr); update_queue.pop(unit_ptr);
try try
{ {

View File

@ -361,24 +361,30 @@ void CacheDictionary::prepareAnswer(
continue; continue;
} }
if (error_count) /// TODO: understand properly this code and remove
{ // if (error_count)
if (find_result.outdated) // {
{ // if (find_result.outdated)
/// We have expired data for that `id` so we can continue using it. // {
bool was_default = cell.isDefault(); // /// We have expired data for that `id` so we can continue using it.
cell.setExpiresAt(backoff_end_time); // bool was_default = cell.isDefault();
if (was_default) //
cell.setDefault(); // if (was_default)
if (was_default) // LOG_FATAL(log, "WAS DEFAULT");
on_id_not_found(id, cell_idx); //
else // cell.setExpiresAt(backoff_end_time);
on_cell_updated(id, cell_idx); // if (was_default)
continue; // cell.setDefault();
} //
/// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`. // if (was_default)
std::rethrow_exception(last_exception); // on_id_not_found(id, cell_idx);
} // else
// on_cell_updated(id, cell_idx);
// continue;
// }
// /// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`.
// std::rethrow_exception(last_exception);
// }
/// Check if cell had not been occupied before and increment element counter if it hadn't /// Check if cell had not been occupied before and increment element counter if it hadn't
if (cell.id == 0 && cell_idx != zero_cell_idx) if (cell.id == 0 && cell_idx != zero_cell_idx)

View File

@ -643,6 +643,13 @@ class ClickHouseInstance:
return urllib.urlopen(url, data).read() return urllib.urlopen(url, data).read()
def kill_clickhouse(self, stop_start_wait_sec=5):
pid = self.get_process_pid("clickhouse")
if not pid:
raise Exception("No clickhouse found")
self.exec_in_container(["bash", "-c", "kill -9 {}".format(pid)], user='root')
time.sleep(stop_start_wait_sec)
def restart_clickhouse(self, stop_start_wait_sec=5, kill=False): def restart_clickhouse(self, stop_start_wait_sec=5, kill=False):
if not self.stay_alive: if not self.stay_alive:
raise Exception("clickhouse can be restarted only with stay_alive=True instance") raise Exception("clickhouse can be restarted only with stay_alive=True instance")

View File

@ -90,7 +90,7 @@ class PartitionManager:
self.heal_all() self.heal_all()
class PartitionManagerDisbaler: class PartitionManagerDisabler:
def __init__(self, manager): def __init__(self, manager):
self.manager = manager self.manager = manager
self.rules = self.manager.pop_rules() self.rules = self.manager.pop_rules()

View File

@ -293,13 +293,16 @@ class DictionaryStructure(object):
class Dictionary(object): class Dictionary(object):
def __init__(self, name, structure, source, config_path, table_name, fields): def __init__(self, name, structure, source, config_path,
table_name, fields, min_lifetime=3, max_lifetime=5):
self.name = name self.name = name
self.structure = copy.deepcopy(structure) self.structure = copy.deepcopy(structure)
self.source = copy.deepcopy(source) self.source = copy.deepcopy(source)
self.config_path = config_path self.config_path = config_path
self.table_name = table_name self.table_name = table_name
self.fields = fields self.fields = fields
self.min_lifetime = min_lifetime
self.max_lifetime = max_lifetime
def generate_config(self): def generate_config(self):
with open(self.config_path, 'w') as result: with open(self.config_path, 'w') as result:
@ -307,8 +310,8 @@ class Dictionary(object):
<yandex> <yandex>
<dictionary> <dictionary>
<lifetime> <lifetime>
<min>3</min> <min>{min_lifetime}</min>
<max>5</max> <max>{max_lifetime}</max>
</lifetime> </lifetime>
<name>{name}</name> <name>{name}</name>
{structure} {structure}
@ -318,6 +321,8 @@ class Dictionary(object):
</dictionary> </dictionary>
</yandex> </yandex>
'''.format( '''.format(
min_lifetime=self.min_lifetime,
max_lifetime=self.max_lifetime,
name=self.name, name=self.name,
structure=self.structure.get_structure_str(), structure=self.structure.get_structure_str(),
source=self.source.get_source_str(self.table_name), source=self.source.get_source_str(self.table_name),

View File

@ -0,0 +1,30 @@
<?xml version="1.0"?>
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>10</count>
</logger>
<tcp_port>9000</tcp_port>
<listen_host>127.0.0.1</listen_host>
<openSSL>
<client>
<cacheSessions>true</cacheSessions>
<verificationMode>none</verificationMode>
<invalidCertificateHandler>
<name>AcceptCertificateHandler</name>
</invalidCertificateHandler>
</client>
</openSSL>
<max_concurrent_queries>500</max_concurrent_queries>
<mark_cache_size>5368709120</mark_cache_size>
<path>./clickhouse/</path>
<users_config>users.xml</users_config>
<dictionaries_config>/etc/clickhouse-server/config.d/*.xml</dictionaries_config>
</yandex>

View File

@ -0,0 +1,72 @@
<yandex>
<dictionary>
<name>anime_dict</name>
<source>
<clickhouse>
<host>dictionary_node</host>
<port>9000</port>
<user>default</user>
<password></password>
<db>test</db>
<table>ints</table>
</clickhouse>
</source>
<lifetime>
<max>2</max>
<min>1</min>
</lifetime>
<layout>
<cache>
<size_in_cells>10000</size_in_cells>
<max_update_queue_size>10000</max_update_queue_size>
<allow_read_expired_keys>1</allow_read_expired_keys>
<update_queue_push_timeout_milliseconds>10</update_queue_push_timeout_milliseconds>
</cache>
</layout>
<structure>
<id>
<name>key</name>
</id>
<attribute>
<name>i8</name>
<type>Int8</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>i16</name>
<type>Int16</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>i32</name>
<type>Int32</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>i64</name>
<type>Int64</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>u8</name>
<type>UInt8</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>u16</name>
<type>UInt16</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>u32</name>
<type>UInt32</type>
<null_value>0</null_value>
</attribute>
<attribute>
<name>u64</name>
<type>UInt64</type>
<null_value>0</null_value>
</attribute>
</structure>
</dictionary>
</yandex>

View File

@ -0,0 +1,23 @@
<?xml version="1.0"?>
<yandex>
<profiles>
<default>
</default>
</profiles>
<users>
<default>
<password></password>
<networks incl="networks" replace="replace">
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
</default>
</users>
<quotas>
<default>
</default>
</quotas>
</yandex>

View File

@ -0,0 +1,64 @@
from __future__ import print_function
import pytest
import time
import os
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.network import PartitionManagerDisabler
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs'))
dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True)
main_node = cluster.add_instance('main_node', main_configs=['configs/dictionaries/cache_ints_dictionary.xml'])
pm = PartitionManager()
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
dictionary_node.query("create database if not exists test;")
dictionary_node.query("drop table if exists test.ints;")
dictionary_node.query("create table test.ints (key UInt64, i8 Int8, i16 Int16, i32 Int32, "
"i64 Int64, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) "
"Engine = Memory;")
dictionary_node.query("insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);")
print(dictionary_node.query("describe test.ints"))
print(dictionary_node.query("select * from test.ints"))
yield cluster
finally:
cluster.shutdown()
def test_simple_dict_get(started_cluster):
assert '7' == main_node.query("select dictGet('anime_dict', 'i8', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i16', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i32', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i64', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u8', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u16', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u32', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u64', toUInt64(7));").rstrip(), "Wrong answer."
with PartitionManager() as pm:
pm.partition_instances(main_node, dictionary_node, port=9000)
# Kill the instance for sure
dictionary_node.kill_clickhouse()
assert None == dictionary_node.get_process_pid("clickhouse")
pm.heal_all()
pm.partition_instances(main_node, dictionary_node, port=9000)
# print(dictionary_node.query("select * from test.ints"))
assert '7' == main_node.query("select dictGet('anime_dict', 'i8', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i16', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i32', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'i64', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u8', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u16', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u32', toUInt64(7));").rstrip(), "Wrong answer."
assert '7' == main_node.query("select dictGet('anime_dict', 'u64', toUInt64(7));").rstrip(), "Wrong answer."