diff --git a/dbms/src/Functions/array/arrayEnumerateRanked.h b/dbms/src/Functions/array/arrayEnumerateRanked.h index ab46af2266d..a1019ba83bf 100644 --- a/dbms/src/Functions/array/arrayEnumerateRanked.h +++ b/dbms/src/Functions/array/arrayEnumerateRanked.h @@ -336,7 +336,9 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( /// Skipping offsets if no data in this array if (prev_off == off) { - want_clear = true; + + if (depth_to_look > 2) + want_clear = true; if (depth_to_look >= 2) { diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/.gitignore b/dbms/tests/integration/test_dictionaries/configs/dictionaries/.gitignore deleted file mode 100644 index 8f1b0e23a85..00000000000 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!.gitignore -!source.tsv -!dictionary_preset* \ No newline at end of file diff --git a/dbms/tests/integration/test_dictionaries/test.py b/dbms/tests/integration/test_dictionaries/test.py deleted file mode 100644 index 95f82f65c0d..00000000000 --- a/dbms/tests/integration/test_dictionaries/test.py +++ /dev/null @@ -1,411 +0,0 @@ -import pytest -import os -import time - -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV, assert_eq_with_retry -from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -cluster = None -instance = None -test_table = None - - -def get_status(dictionary_name): - return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") - - -def get_last_exception(dictionary_name): - return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") - - -def get_loading_start_time(dictionary_name): - s = instance.query("SELECT loading_start_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") - if s == "0000-00-00 00:00:00": - return None - return time.strptime(s, "%Y-%m-%d %H:%M:%S") - - -def get_loading_duration(dictionary_name): - return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) - - -def replace_in_file_in_container(file_name, what, replace_with): - instance.exec_in_container('sed -i "s/' + what + '/' + replace_with + '/g" ' + file_name) - - -def setup_module(module): - global cluster - global instance - global test_table - - structure = generate_structure() - dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure) - - cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) - instance = cluster.add_instance('instance', main_configs=dictionary_files) - test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv')) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - instance.query("CREATE DATABASE IF NOT EXISTS dict ENGINE=Dictionary") - test_table.create_clickhouse_source(instance) - for line in TSV(instance.query('select name from system.dictionaries')).lines: - print line, - - # Create table `test.small_dict_source` - instance.query(''' - drop table if exists test.small_dict_source; - create table test.small_dict_source (id UInt64, a String, b Int32, c Float64) engine=Log; - insert into test.small_dict_source values (0, 'water', 10, 1), (1, 'air', 40, 0.01), (2, 'earth', 100, 1.7); - ''') - - yield cluster - - finally: - cluster.shutdown() - - -@pytest.fixture(params=[ - # name, keys, use_parent - ('clickhouse_hashed', ('id',), True), - ('clickhouse_flat', ('id',), True), - ('clickhouse_complex_integers_key_hashed', ('key0', 'key1'), False), - ('clickhouse_complex_mixed_key_hashed', ('key0_str', 'key1'), False), - ('clickhouse_range_hashed', ('id', 'StartDate', 'EndDate'), False), -], - ids=['clickhouse_hashed', 'clickhouse_flat', - 'clickhouse_complex_integers_key_hashed', - 'clickhouse_complex_mixed_key_hashed', - 'clickhouse_range_hashed'] -) -def dictionary_structure(started_cluster, request): - return request.param - - -def test_select_all(dictionary_structure): - name, keys, use_parent = dictionary_structure - query = instance.query - - structure = test_table.get_structure_for_keys(keys, use_parent) - query(''' - DROP TABLE IF EXISTS test.{0} - '''.format(name)) - - create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) - TSV(query(create_query)) - - result = TSV(query('select * from test.{0}'.format(name))) - - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) - print test_table.process_diff(diff) - assert not diff - - -@pytest.fixture(params=[ - # name, keys, use_parent - ('clickhouse_cache', ('id',), True), - ('clickhouse_complex_integers_key_cache', ('key0', 'key1'), False), - ('clickhouse_complex_mixed_key_cache', ('key0_str', 'key1'), False) -], - ids=['clickhouse_cache', 'clickhouse_complex_integers_key_cache', 'clickhouse_complex_mixed_key_cache'] -) -def cached_dictionary_structure(started_cluster, request): - return request.param - - -def test_select_all_from_cached(cached_dictionary_structure): - name, keys, use_parent = cached_dictionary_structure - query = instance.query - - structure = test_table.get_structure_for_keys(keys, use_parent) - query(''' - DROP TABLE IF EXISTS test.{0} - '''.format(name)) - - create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) - TSV(query(create_query)) - - for i in range(4): - result = TSV(query('select * from test.{0}'.format(name))) - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=False) - print test_table.process_diff(diff) - assert not diff - - key = [] - for key_name in keys: - if key_name.endswith('str'): - key.append("'" + str(i) + "'") - else: - key.append(str(i)) - if len(key) == 1: - key = 'toUInt64(' + str(i) + ')' - else: - key = str('(' + ','.join(key) + ')') - query("select dictGetUInt8('{0}', 'UInt8_', {1})".format(name, key)) - - result = TSV(query('select * from test.{0}'.format(name))) - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) - print test_table.process_diff(diff) - assert not diff - - -def test_null_value(started_cluster): - query = instance.query - - assert TSV(query("select dictGetUInt8('clickhouse_cache', 'UInt8_', toUInt64(12121212))")) == TSV("1") - assert TSV(query("select dictGetString('clickhouse_cache', 'String_', toUInt64(12121212))")) == TSV("implicit-default") - assert TSV(query("select dictGetDate('clickhouse_cache', 'Date_', toUInt64(12121212))")) == TSV("2015-11-25") - - # Check, that empty null_value interprets as default value - assert TSV(query("select dictGetUInt64('clickhouse_cache', 'UInt64_', toUInt64(12121212))")) == TSV("0") - assert TSV(query("select dictGetDateTime('clickhouse_cache', 'DateTime_', toUInt64(12121212))")) == TSV("0000-00-00 00:00:00") - - -def test_dictionary_dependency(started_cluster): - query = instance.query - - # dictionaries_lazy_load == false, so these dictionary are not loaded. - assert get_status('dep_x') == 'NOT_LOADED' - assert get_status('dep_y') == 'NOT_LOADED' - assert get_status('dep_z') == 'NOT_LOADED' - - # Dictionary 'dep_x' depends on 'dep_z', which depends on 'dep_y'. - # So they all should be loaded at once. - assert query("SELECT dictGetString('dep_x', 'a', toUInt64(1))") == "air\n" - assert get_status('dep_x') == 'LOADED' - assert get_status('dep_y') == 'LOADED' - assert get_status('dep_z') == 'LOADED' - - # Other dictionaries should work too. - assert query("SELECT dictGetString('dep_y', 'a', toUInt64(1))") == "air\n" - assert query("SELECT dictGetString('dep_z', 'a', toUInt64(1))") == "air\n" - - assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" - assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "YY\n" - assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" - - # Update the source table. - query("insert into test.small_dict_source values (3, 'fire', 30, 8)") - - # Wait for dictionaries to be reloaded. - assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) - assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" - assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" - assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" - - # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. - query("insert into test.small_dict_source values (4, 'ether', 404, 0.001)") - assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) - assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" - assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" - assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" - assert query("SELECT dictGetString('dep_x', 'a', toUInt64(4))") == "ether\n" - assert query("SELECT dictGetString('dep_y', 'a', toUInt64(4))") == "ether\n" - assert query("SELECT dictGetString('dep_z', 'a', toUInt64(4))") == "ether\n" - - -def test_reload_while_loading(started_cluster): - query = instance.query - - # dictionaries_lazy_load == false, so this dictionary is not loaded. - assert get_status('longload') == "NOT_LOADED" - assert get_loading_duration('longload') == 0 - - # It's not possible to get a value from the dictionary within 1.0 second, so the following query fails by timeout. - assert query("SELECT dictGetInt32('longload', 'a', toUInt64(5))", timeout = 1, ignore_error = True) == "" - - # The dictionary is now loading. - assert get_status('longload') == "LOADING" - start_time, duration = get_loading_start_time('longload'), get_loading_duration('longload') - assert duration > 0 - - time.sleep(0.5) # Still loading. - assert get_status('longload') == "LOADING" - prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('longload'), get_loading_duration('longload') - assert start_time == prev_start_time - assert duration >= prev_duration - - # SYSTEM RELOAD DICTIONARY should restart loading. - query("SYSTEM RELOAD DICTIONARY 'longload'") - assert get_status('longload') == "LOADING" - prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('longload'), get_loading_duration('longload') - assert start_time > prev_start_time - assert duration < prev_duration - - time.sleep(0.5) # Still loading. - assert get_status('longload') == "LOADING" - prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('longload'), get_loading_duration('longload') - assert start_time == prev_start_time - assert duration >= prev_duration - - # SYSTEM RELOAD DICTIONARIES should restart loading again. - query("SYSTEM RELOAD DICTIONARIES") - assert get_status('longload') == "LOADING" - prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('longload'), get_loading_duration('longload') - assert start_time > prev_start_time - assert duration < prev_duration - - # Changing the configuration file should restart loading one more time. - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_longload.xml', 'sleep 100', 'sleep 0') - time.sleep(5) # Configuration files are reloaded once in 5 seconds. - - # This time loading should finish quickly. - assert get_status('longload') == "LOADED" - assert query("SELECT dictGetInt32('longload', 'a', toUInt64(5))") == "6\n" - - -def test_reload_after_loading(started_cluster): - query = instance.query - - assert query("SELECT dictGetInt32('cmd', 'a', toUInt64(7))") == "8\n" - assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "10\n" - - # Change the dictionaries' data. - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_cmd.xml', '8', '81') - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_file.txt', '10', '101') - - # SYSTEM RELOAD 'name' reloads only the specified dictionary. - query("SYSTEM RELOAD DICTIONARY 'cmd'") - assert query("SELECT dictGetInt32('cmd', 'a', toUInt64(7))") == "81\n" - assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "10\n" - - query("SYSTEM RELOAD DICTIONARY 'file'") - assert query("SELECT dictGetInt32('cmd', 'a', toUInt64(7))") == "81\n" - assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n" - - # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries. - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_cmd.xml', '81', '82') - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_file.txt', '101', '102') - query("SYSTEM RELOAD DICTIONARIES") - assert query("SELECT dictGetInt32('cmd', 'a', toUInt64(7))") == "82\n" - assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "102\n" - - # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds. - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_cmd.xml', '82', '83') - replace_in_file_in_container('/etc/clickhouse-server/config.d/dictionary_preset_file.txt', '102', '103') - time.sleep(5) - assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "103\n" - assert query("SELECT dictGetInt32('cmd', 'a', toUInt64(7))") == "83\n" - - -def test_reload_after_fail_by_system_reload(started_cluster): - query = instance.query - - # dictionaries_lazy_load == false, so this dictionary is not loaded. - assert get_status("no_file") == "NOT_LOADED" - - # We expect an error because the file source doesn't exist. - expected_error = "No such file" - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") - assert get_status("no_file") == "FAILED" - - # SYSTEM RELOAD should not change anything now, the status is still FAILED. - query("SYSTEM RELOAD DICTIONARY 'no_file'") - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") - assert get_status("no_file") == "FAILED" - - # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/dictionary_preset_file.txt"), "/etc/clickhouse-server/config.d/dictionary_preset_no_file.txt") - query("SYSTEM RELOAD DICTIONARY 'no_file'") - query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" - assert get_status("no_file") == "LOADED" - - # Removing the file source should not spoil the loaded dictionary. - instance.exec_in_container("rm /etc/clickhouse-server/config.d/dictionary_preset_no_file.txt") - query("SYSTEM RELOAD DICTIONARY 'no_file'") - query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" - assert get_status("no_file") == "LOADED" - - -def test_reload_after_fail_by_timer(started_cluster): - query = instance.query - - # dictionaries_lazy_load == false, so this dictionary is not loaded. - assert get_status("no_file_2") == "NOT_LOADED" - - # We expect an error because the file source doesn't exist. - expected_error = "No such file" - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") - assert get_status("no_file_2") == "FAILED" - - # Passed time should not change anything now, the status is still FAILED. - time.sleep(6); - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") - assert get_status("no_file_2") == "FAILED" - - # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/dictionary_preset_file.txt"), "/etc/clickhouse-server/config.d/dictionary_preset_no_file_2.txt") - time.sleep(6); - query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" - assert get_status("no_file_2") == "LOADED" - - # Removing the file source should not spoil the loaded dictionary. - instance.exec_in_container("rm /etc/clickhouse-server/config.d/dictionary_preset_no_file_2.txt") - time.sleep(6); - query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" - assert get_status("no_file_2") == "LOADED" - - -def test_reload_after_fail_in_cache_dictionary(started_cluster): - query = instance.query - query_and_get_error = instance.query_and_get_error - - # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond. - expected_error = "Table test.xypairs doesn't exist" - assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))") - assert get_status("cache_xypairs") == "LOADED" - assert expected_error in get_last_exception("cache_xypairs") - - # Create table `test.xypairs`. - query(''' - drop table if exists test.xypairs; - create table test.xypairs (x UInt64, y UInt64) engine=Log; - insert into test.xypairs values (1, 56), (3, 78); - ''') - - # Cache dictionary now works. - assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True) - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" - assert get_last_exception("cache_xypairs") == "" - - # Drop table `test.xypairs`. - query('drop table if exists test.xypairs') - - # Values are cached so we can get them. - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" - assert get_last_exception("cache_xypairs") == "" - - # But we can't get a value from the source table which isn't cached. - assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") - assert expected_error in get_last_exception("cache_xypairs") - - # Passed time should not spoil the cache. - time.sleep(5); - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" - assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") - assert expected_error in get_last_exception("cache_xypairs") - - # Create table `test.xypairs` again with changed values. - query(''' - drop table if exists test.xypairs; - create table test.xypairs (x UInt64, y UInt64) engine=Log; - insert into test.xypairs values (1, 57), (3, 79); - ''') - - # The cache dictionary returns new values now. - assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57") - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" - query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79" - assert get_last_exception("cache_xypairs") == "" diff --git a/dbms/tests/integration/test_cached_dictionary_string/__init__.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/__init__.py similarity index 100% rename from dbms/tests/integration/test_cached_dictionary_string/__init__.py rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/__init__.py diff --git a/dbms/tests/integration/test_external_dictionaries/configs/config.xml b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/config.xml similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/configs/config.xml rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/config.xml diff --git a/dbms/tests/integration/test_external_dictionaries/configs/dictionaries/.gitkeep b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/dictionaries/.gitkeep similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/configs/dictionaries/.gitkeep rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/dictionaries/.gitkeep diff --git a/dbms/tests/integration/test_cached_dictionary_string/configs/users.xml b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/users.xml similarity index 100% rename from dbms/tests/integration/test_cached_dictionary_string/configs/users.xml rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/configs/users.xml diff --git a/dbms/tests/integration/test_external_dictionaries/dictionary.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/dictionary.py rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/dictionary.py diff --git a/dbms/tests/integration/test_external_dictionaries/external_sources.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/external_sources.py rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/external_sources.py diff --git a/dbms/tests/integration/test_external_dictionaries/fake_cert.pem b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/fake_cert.pem similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/fake_cert.pem rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/fake_cert.pem diff --git a/dbms/tests/integration/test_external_dictionaries/http_server.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/http_server.py similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/http_server.py rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/http_server.py diff --git a/dbms/tests/integration/test_external_dictionaries/test.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/test.py rename to dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py diff --git a/dbms/tests/integration/test_dictionaries/__init__.py b/dbms/tests/integration/test_dictionaries_complex_key_cache_string/__init__.py similarity index 100% rename from dbms/tests/integration/test_dictionaries/__init__.py rename to dbms/tests/integration/test_dictionaries_complex_key_cache_string/__init__.py diff --git a/dbms/tests/integration/test_cached_dictionary_string/configs/config.xml b/dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/config.xml similarity index 100% rename from dbms/tests/integration/test_cached_dictionary_string/configs/config.xml rename to dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/config.xml diff --git a/dbms/tests/integration/test_cached_dictionary_string/configs/dictionaries/complex_key_cache_string.xml b/dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/dictionaries/complex_key_cache_string.xml similarity index 100% rename from dbms/tests/integration/test_cached_dictionary_string/configs/dictionaries/complex_key_cache_string.xml rename to dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/dictionaries/complex_key_cache_string.xml diff --git a/dbms/tests/integration/test_dictionaries/configs/users.xml b/dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/users.xml similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/users.xml rename to dbms/tests/integration/test_dictionaries_complex_key_cache_string/configs/users.xml diff --git a/dbms/tests/integration/test_cached_dictionary_string/test.py b/dbms/tests/integration/test_dictionaries_complex_key_cache_string/test.py similarity index 100% rename from dbms/tests/integration/test_cached_dictionary_string/test.py rename to dbms/tests/integration/test_dictionaries_complex_key_cache_string/test.py diff --git a/dbms/tests/integration/test_external_dictionaries/__init__.py b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/__init__.py similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/__init__.py rename to dbms/tests/integration/test_dictionaries_depend_on_dictionaries/__init__.py diff --git a/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/config.xml b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/config.xml new file mode 100644 index 00000000000..b60daf72dcf --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/config.xml @@ -0,0 +1,30 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_x.xml similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml rename to dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_x.xml diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_y.xml similarity index 95% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml rename to dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_y.xml index 8806c724111..227d87ca92a 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml +++ b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_y.xml @@ -8,7 +8,7 @@ default test - small_dict_source
+ elements
5 diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_z.xml similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml rename to dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/dictionaries/dep_z.xml diff --git a/dbms/tests/integration/test_external_dictionaries/configs/users.xml b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/users.xml similarity index 100% rename from dbms/tests/integration/test_external_dictionaries/configs/users.xml rename to dbms/tests/integration/test_dictionaries_depend_on_dictionaries/configs/users.xml diff --git a/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/test.py b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/test.py new file mode 100644 index 00000000000..c0ce0af0313 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_depend_on_dictionaries/test.py @@ -0,0 +1,76 @@ +import pytest +import os +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', 'configs/dictionaries/dep_z.xml'] + +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) +instance = cluster.add_instance('instance', main_configs=DICTIONARY_FILES) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + instance.query(''' + CREATE DATABASE IF NOT EXISTS dict ENGINE=Dictionary; + CREATE DATABASE IF NOT EXISTS test; + DROP TABLE IF EXISTS test.elements; + CREATE TABLE test.elements (id UInt64, a String, b Int32, c Float64) ENGINE=Log; + INSERT INTO test.elements VALUES (0, 'water', 10, 1), (1, 'air', 40, 0.01), (2, 'earth', 100, 1.7); + ''') + + yield cluster + + finally: + cluster.shutdown() + + +def get_status(dictionary_name): + return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + + +def test_get_data(started_cluster): + query = instance.query + + # dictionaries_lazy_load == false, so these dictionary are not loaded. + assert get_status('dep_x') == 'NOT_LOADED' + assert get_status('dep_y') == 'NOT_LOADED' + assert get_status('dep_z') == 'NOT_LOADED' + + # Dictionary 'dep_x' depends on 'dep_z', which depends on 'dep_y'. + # So they all should be loaded at once. + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(1))") == "air\n" + assert get_status('dep_x') == 'LOADED' + assert get_status('dep_y') == 'LOADED' + assert get_status('dep_z') == 'LOADED' + + # Other dictionaries should work too. + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(1))") == "air\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(1))") == "air\n" + + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "YY\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # Update the source table. + query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)") + + # Wait for dictionaries to be reloaded. + assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. + query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)") + assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(4))") == "ether\n" diff --git a/dbms/tests/integration/test_dictionaries_null_value/__init__.py b/dbms/tests/integration/test_dictionaries_null_value/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_dictionaries/configs/config.xml b/dbms/tests/integration/test_dictionaries_null_value/configs/config.xml similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/config.xml rename to dbms/tests/integration/test_dictionaries_null_value/configs/config.xml diff --git a/dbms/tests/integration/test_dictionaries_null_value/configs/dictionaries/cache.xml b/dbms/tests/integration/test_dictionaries_null_value/configs/dictionaries/cache.xml new file mode 100644 index 00000000000..9a1ae0732db --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_null_value/configs/dictionaries/cache.xml @@ -0,0 +1,113 @@ + + + cache + + + + localhost + 9000 + default + + test + source
+ + + + 0 + + + 128 + + + + + id + + + + UInt8_ + UInt8 + 1 + + + + UInt16_ + UInt16 + 1 + + + + UInt32_ + UInt32 + 1 + + + + UInt64_ + UInt64 + + + + + Int8_ + Int8 + -1 + + + + Int16_ + Int16 + -1 + + + + Int32_ + Int32 + -1 + + + + Int64_ + Int64 + -1 + + + + Float32_ + Float32 + 2.71828 + + + + Float64_ + Float64 + 2.71828 + + + + String_ + String + implicit-default + + + + Date_ + Date + 2015-11-25 + + + + DateTime_ + DateTime + + + + + Parent + UInt64 + true + 0 + + + + diff --git a/dbms/tests/integration/test_dictionaries_null_value/configs/users.xml b/dbms/tests/integration/test_dictionaries_null_value/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_null_value/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_dictionaries_null_value/test.py b/dbms/tests/integration/test_dictionaries_null_value/test.py new file mode 100644 index 00000000000..e31f397c246 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_null_value/test.py @@ -0,0 +1,45 @@ +import pytest +import os +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +DICTIONARY_FILES = ['configs/dictionaries/cache.xml'] + +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) +instance = cluster.add_instance('instance', main_configs=DICTIONARY_FILES) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + instance.query(''' + CREATE DATABASE IF NOT EXISTS test; + DROP TABLE IF EXISTS test.source; + CREATE TABLE test.source (id UInt64, key0 UInt8, key0_str String, key1 UInt8, + StartDate Date, EndDate Date, + UInt8_ UInt8, UInt16_ UInt16, UInt32_ UInt32, UInt64_ UInt64, + Int8_ Int8, Int16_ Int16, Int32_ Int32, Int64_ Int64, + Float32_ Float32, Float64_ Float64, + String_ String, + Date_ Date, DateTime_ DateTime, Parent UInt64) ENGINE=Log; + ''') + + yield cluster + + finally: + cluster.shutdown() + + +def test_null_value(started_cluster): + query = instance.query + + assert query("select dictGetUInt8('cache', 'UInt8_', toUInt64(12121212))") == "1\n" + assert query("select dictGetString('cache', 'String_', toUInt64(12121212))") == "implicit-default\n" + assert query("select dictGetDate('cache', 'Date_', toUInt64(12121212))") == "2015-11-25\n" + + # Check, that empty null_value interprets as default value + assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n" + assert query("select dictGetDateTime('cache', 'DateTime_', toUInt64(12121212))") == "0000-00-00 00:00:00\n" diff --git a/dbms/tests/integration/test_dictionaries_select_all/__init__.py b/dbms/tests/integration/test_dictionaries_select_all/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_dictionaries_select_all/configs/config.xml b/dbms/tests/integration/test_dictionaries_select_all/configs/config.xml new file mode 100644 index 00000000000..1e4c14585a9 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_select_all/configs/config.xml @@ -0,0 +1,30 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_dictionaries_select_all/configs/dictionaries/.gitignore b/dbms/tests/integration/test_dictionaries_select_all/configs/dictionaries/.gitignore new file mode 100644 index 00000000000..cc461064a39 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_select_all/configs/dictionaries/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!source.tsv \ No newline at end of file diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/source.tsv b/dbms/tests/integration/test_dictionaries_select_all/configs/dictionaries/source.tsv similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/source.tsv rename to dbms/tests/integration/test_dictionaries_select_all/configs/dictionaries/source.tsv diff --git a/dbms/tests/integration/test_dictionaries_select_all/configs/users.xml b/dbms/tests/integration/test_dictionaries_select_all/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_select_all/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_dictionaries/generate_dictionaries.py b/dbms/tests/integration/test_dictionaries_select_all/generate_dictionaries.py similarity index 96% rename from dbms/tests/integration/test_dictionaries/generate_dictionaries.py rename to dbms/tests/integration/test_dictionaries_select_all/generate_dictionaries.py index c644bd8f644..30a5648fdbe 100644 --- a/dbms/tests/integration/test_dictionaries/generate_dictionaries.py +++ b/dbms/tests/integration/test_dictionaries_select_all/generate_dictionaries.py @@ -12,13 +12,6 @@ types = [ 'Date', 'DateTime' ] -explicit_defaults = [ - '42', '42', '42', '42', - '-42', '-42', '-42', '-42', - '1.5', '1.6', - "'explicit-default'", - "'2015-01-01'", "'2015-01-01 00:00:00'" -] implicit_defaults = [ '1', '1', '1', '', @@ -182,9 +175,6 @@ def generate_dictionaries(path, structure): file_names = [] - # Add ready dictionaries. - file_names.extend(glob.glob(os.path.join(path, "*dictionary_preset*"))) - # Generate dictionaries. for (name, key_idx, has_parent), (source, layout) in zip(structure, sources_and_layouts): filename = os.path.join(path, 'dictionary_%s.xml' % name) diff --git a/dbms/tests/integration/test_dictionaries_select_all/test.py b/dbms/tests/integration/test_dictionaries_select_all/test.py new file mode 100644 index 00000000000..8bad8a9b214 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_select_all/test.py @@ -0,0 +1,122 @@ +import pytest +import os +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV, assert_eq_with_retry +from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = None +instance = None +test_table = None + + +def setup_module(module): + global cluster + global instance + global test_table + + structure = generate_structure() + dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure) + + cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) + instance = cluster.add_instance('instance', main_configs=dictionary_files) + test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv')) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + test_table.create_clickhouse_source(instance) + for line in TSV(instance.query('select name from system.dictionaries')).lines: + print line, + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(params=[ + # name, keys, use_parent + ('clickhouse_hashed', ('id',), True), + ('clickhouse_flat', ('id',), True), + ('clickhouse_complex_integers_key_hashed', ('key0', 'key1'), False), + ('clickhouse_complex_mixed_key_hashed', ('key0_str', 'key1'), False), + ('clickhouse_range_hashed', ('id', 'StartDate', 'EndDate'), False), +], + ids=['clickhouse_hashed', 'clickhouse_flat', + 'clickhouse_complex_integers_key_hashed', + 'clickhouse_complex_mixed_key_hashed', + 'clickhouse_range_hashed'] +) +def dictionary_structure(started_cluster, request): + return request.param + + +def test_select_all(dictionary_structure): + name, keys, use_parent = dictionary_structure + query = instance.query + + structure = test_table.get_structure_for_keys(keys, use_parent) + query(''' + DROP TABLE IF EXISTS test.{0} + '''.format(name)) + + create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) + TSV(query(create_query)) + + result = TSV(query('select * from test.{0}'.format(name))) + + diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) + print test_table.process_diff(diff) + assert not diff + + +@pytest.fixture(params=[ + # name, keys, use_parent + ('clickhouse_cache', ('id',), True), + ('clickhouse_complex_integers_key_cache', ('key0', 'key1'), False), + ('clickhouse_complex_mixed_key_cache', ('key0_str', 'key1'), False) +], + ids=['clickhouse_cache', 'clickhouse_complex_integers_key_cache', 'clickhouse_complex_mixed_key_cache'] +) +def cached_dictionary_structure(started_cluster, request): + return request.param + + +def test_select_all_from_cached(cached_dictionary_structure): + name, keys, use_parent = cached_dictionary_structure + query = instance.query + + structure = test_table.get_structure_for_keys(keys, use_parent) + query(''' + DROP TABLE IF EXISTS test.{0} + '''.format(name)) + + create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) + TSV(query(create_query)) + + for i in range(4): + result = TSV(query('select * from test.{0}'.format(name))) + diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=False) + print test_table.process_diff(diff) + assert not diff + + key = [] + for key_name in keys: + if key_name.endswith('str'): + key.append("'" + str(i) + "'") + else: + key.append(str(i)) + if len(key) == 1: + key = 'toUInt64(' + str(i) + ')' + else: + key = str('(' + ','.join(key) + ')') + query("select dictGetUInt8('{0}', 'UInt8_', {1})".format(name, key)) + + result = TSV(query('select * from test.{0}'.format(name))) + diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) + print test_table.process_diff(diff) + assert not diff diff --git a/dbms/tests/integration/test_dictionaries_update_and_reload/__init__.py b/dbms/tests/integration/test_dictionaries_update_and_reload/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_dictionaries_update_and_reload/configs/config.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/config.xml new file mode 100644 index 00000000000..b60daf72dcf --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/config.xml @@ -0,0 +1,30 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/cache_xypairs.xml similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cache_xypairs.xml rename to dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/cache_xypairs.xml diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cmd.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/executable.xml similarity index 93% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cmd.xml rename to dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/executable.xml index 9f1e259e2d7..519a2915a59 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_cmd.xml +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/executable.xml @@ -1,7 +1,7 @@ - cmd + executable echo '7\t8'; diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_file.txt b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.txt similarity index 100% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_file.txt rename to dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.txt diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_file.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml similarity index 82% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_file.xml rename to dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml index 0e6db1f1637..2a937b5444d 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_file.xml +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/file.xml @@ -4,7 +4,7 @@ file - /etc/clickhouse-server/config.d/dictionary_preset_file.txt + /etc/clickhouse-server/config.d/file.txt TabSeparated @@ -21,7 +21,7 @@ no_file - /etc/clickhouse-server/config.d/dictionary_preset_no_file.txt + /etc/clickhouse-server/config.d/no_file.txt TabSeparated @@ -38,7 +38,7 @@ no_file_2 - /etc/clickhouse-server/config.d/dictionary_preset_no_file_2.txt + /etc/clickhouse-server/config.d/no_file_2.txt TabSeparated diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_longload.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/slow.xml similarity index 94% rename from dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_longload.xml rename to dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/slow.xml index f5d4cdec583..c6814c5fe9c 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_longload.xml +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/dictionaries/slow.xml @@ -1,7 +1,7 @@ - longload + slow sleep 100 && echo '5\t6'; diff --git a/dbms/tests/integration/test_dictionaries_update_and_reload/configs/users.xml b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_dictionaries_update_and_reload/test.py b/dbms/tests/integration/test_dictionaries_update_and_reload/test.py new file mode 100644 index 00000000000..b972dc6c918 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_update_and_reload/test.py @@ -0,0 +1,246 @@ +import pytest +import os +import time +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml'] + +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) +instance = cluster.add_instance('instance', main_configs=DICTIONARY_FILES) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + instance.query("CREATE DATABASE IF NOT EXISTS test") + + yield cluster + + finally: + cluster.shutdown() + + +def get_status(dictionary_name): + return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + + +def get_last_exception(dictionary_name): + return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n").replace("\\'", "'") + + +def get_loading_start_time(dictionary_name): + s = instance.query("SELECT loading_start_time FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + if s == "0000-00-00 00:00:00": + return None + return time.strptime(s, "%Y-%m-%d %H:%M:%S") + + +def get_loading_duration(dictionary_name): + return float(instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) + + +def replace_in_file_in_container(file_name, what, replace_with): + instance.exec_in_container('sed -i "s/' + what + '/' + replace_with + '/g" ' + file_name) + + +def test_reload_while_loading(started_cluster): + query = instance.query + + # dictionaries_lazy_load == false, so this dictionary is not loaded. + assert get_status('slow') == "NOT_LOADED" + assert get_loading_duration('slow') == 0 + + # It's not possible to get a value from the dictionary within 1.0 second, so the following query fails by timeout. + assert query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout = 1, ignore_error = True) == "" + + # The dictionary is now loading. + assert get_status('slow') == "LOADING" + start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert duration > 0 + + time.sleep(0.5) # Still loading. + assert get_status('slow') == "LOADING" + prev_start_time, prev_duration = start_time, duration + start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert start_time == prev_start_time + assert duration >= prev_duration + + # SYSTEM RELOAD DICTIONARY should restart loading. + query("SYSTEM RELOAD DICTIONARY 'slow'") + assert get_status('slow') == "LOADING" + prev_start_time, prev_duration = start_time, duration + start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert start_time > prev_start_time + assert duration < prev_duration + + time.sleep(0.5) # Still loading. + assert get_status('slow') == "LOADING" + prev_start_time, prev_duration = start_time, duration + start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert start_time == prev_start_time + assert duration >= prev_duration + + # SYSTEM RELOAD DICTIONARIES should restart loading again. + query("SYSTEM RELOAD DICTIONARIES") + assert get_status('slow') == "LOADING" + prev_start_time, prev_duration = start_time, duration + start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert start_time > prev_start_time + assert duration < prev_duration + + # Changing the configuration file should restart loading one more time. + replace_in_file_in_container('/etc/clickhouse-server/config.d/slow.xml', 'sleep 100', 'sleep 0') + time.sleep(5) # Configuration files are reloaded once in 5 seconds. + + # This time loading should finish quickly. + assert get_status('slow') == "LOADED" + assert query("SELECT dictGetInt32('slow', 'a', toUInt64(5))") == "6\n" + + +def test_reload_after_loading(started_cluster): + query = instance.query + + assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "8\n" + assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "10\n" + + # Change the dictionaries' data. + replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '8', '81') + replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '10', '101') + + # SYSTEM RELOAD 'name' reloads only the specified dictionary. + query("SYSTEM RELOAD DICTIONARY 'executable'") + assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "81\n" + assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "10\n" + + query("SYSTEM RELOAD DICTIONARY 'file'") + assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "81\n" + assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "101\n" + + # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries. + replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '81', '82') + replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '101', '102') + query("SYSTEM RELOAD DICTIONARIES") + assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "82\n" + assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "102\n" + + # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds. + replace_in_file_in_container('/etc/clickhouse-server/config.d/executable.xml', '82', '83') + replace_in_file_in_container('/etc/clickhouse-server/config.d/file.txt', '102', '103') + time.sleep(5) + assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "103\n" + assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "83\n" + + +def test_reload_after_fail_by_system_reload(started_cluster): + query = instance.query + + # dictionaries_lazy_load == false, so this dictionary is not loaded. + assert get_status("no_file") == "NOT_LOADED" + + # We expect an error because the file source doesn't exist. + expected_error = "No such file" + assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + assert get_status("no_file") == "FAILED" + + # SYSTEM RELOAD should not change anything now, the status is still FAILED. + query("SYSTEM RELOAD DICTIONARY 'no_file'") + assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + assert get_status("no_file") == "FAILED" + + # Creating the file source makes the dictionary able to load. + instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file.txt") + query("SYSTEM RELOAD DICTIONARY 'no_file'") + query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" + assert get_status("no_file") == "LOADED" + + # Removing the file source should not spoil the loaded dictionary. + instance.exec_in_container("rm /etc/clickhouse-server/config.d/no_file.txt") + query("SYSTEM RELOAD DICTIONARY 'no_file'") + query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" + assert get_status("no_file") == "LOADED" + + +def test_reload_after_fail_by_timer(started_cluster): + query = instance.query + + # dictionaries_lazy_load == false, so this dictionary is not loaded. + assert get_status("no_file_2") == "NOT_LOADED" + + # We expect an error because the file source doesn't exist. + expected_error = "No such file" + assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") + assert get_status("no_file_2") == "FAILED" + + # Passed time should not change anything now, the status is still FAILED. + time.sleep(6); + assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") + assert get_status("no_file_2") == "FAILED" + + # Creating the file source makes the dictionary able to load. + instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), "/etc/clickhouse-server/config.d/no_file_2.txt") + time.sleep(6); + query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" + assert get_status("no_file_2") == "LOADED" + + # Removing the file source should not spoil the loaded dictionary. + instance.exec_in_container("rm /etc/clickhouse-server/config.d/no_file_2.txt") + time.sleep(6); + query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" + assert get_status("no_file_2") == "LOADED" + + +def test_reload_after_fail_in_cache_dictionary(started_cluster): + query = instance.query + query_and_get_error = instance.query_and_get_error + + # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond. + expected_error = "Table test.xypairs doesn't exist" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))") + assert get_status("cache_xypairs") == "LOADED" + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs`. + query(''' + DROP TABLE IF EXISTS test.xypairs; + CREATE TABLE test.xypairs (x UInt64, y UInt64) ENGINE=Log; + INSERT INTO test.xypairs VALUES (1, 56), (3, 78); + ''') + + # Cache dictionary now works. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True) + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # Drop table `test.xypairs`. + query('DROP TABLE test.xypairs') + + # Values are cached so we can get them. + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert get_last_exception("cache_xypairs") == "" + + # But we can't get a value from the source table which isn't cached. + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Passed time should not spoil the cache. + time.sleep(5); + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in get_last_exception("cache_xypairs") + + # Create table `test.xypairs` again with changed values. + query(''' + CREATE TABLE test.xypairs (x UInt64, y UInt64) ENGINE=Log; + INSERT INTO test.xypairs VALUES (1, 57), (3, 79); + ''') + + # The cache dictionary returns new values now. + assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57") + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" + query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79" + assert get_last_exception("cache_xypairs") == "" diff --git a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference index f97d393cc32..595dcdf3803 100644 --- a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference +++ b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference @@ -278,3 +278,9 @@ a1,a2 12 [1,2] 1 2019-06-06 1 4 2 1 5 1 [1,2] [1001,1002] [1,1] 1 2019-06-06 1 4 2 1 5 0 [1,2] [1002,1003] [1,1] 1 2019-06-06 1 4 2 1 6 0 [3] [2001] [1] +-- empty +[[1],[],[2]] +[[1],[],[2]] +[[1],[],[2],[],[3],[],[4],[],[5],[],[6],[],[7],[],[8],[],[9]] +[[],[1],[],[2],[],[3],[],[4],[],[5],[],[6],[],[7],[],[8]] +[[1],[2],[],[3]] diff --git a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql index 5f4b12e1988..9cf82a368d6 100644 --- a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql +++ b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql @@ -305,3 +305,11 @@ ARRAY JOIN Test.PuidVal AS PuidValArr; DROP TABLE arr_tests_visits; + + +select '-- empty'; +SELECT arrayEnumerateUniqRanked([['a'], [], ['a']]); +SELECT arrayEnumerateUniqRanked([[1], [], [1]]); +SELECT arrayEnumerateUniqRanked([[1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1]]); +SELECT arrayEnumerateUniqRanked([[], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1]]); +SELECT arrayEnumerateUniqRanked([[1], [1], [], [1]]); diff --git a/dbms/tests/queries/0_stateless/00933_ttl_with_default.sql b/dbms/tests/queries/0_stateless/00933_ttl_with_default.sql index e6c0a6e700c..d3f3b62126c 100644 --- a/dbms/tests/queries/0_stateless/00933_ttl_with_default.sql +++ b/dbms/tests/queries/0_stateless/00933_ttl_with_default.sql @@ -5,6 +5,7 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4); +select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a from ttl_00933_2 order by a; @@ -15,6 +16,7 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1, 100); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2, 200); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3, 300); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4, 400); +select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a, b from ttl_00933_2 order by a; @@ -25,6 +27,7 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1, 5); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2, 10); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3, 15); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4, 20); +select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a, b from ttl_00933_2 order by a;