diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 3f1676679dd..3577553be34 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -87,14 +87,16 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, silent=False, settings={}): exception = None - for _ in range(retry_count): + for cnt in range(retry_count): try: - instance.query(query, timeout=30, settings=settings) + res = instance.query(query, timeout=30, settings=settings) + if not silent: + logging.debug(f"Result of {query} on {cnt} try is {res}") break except Exception as ex: exception = ex if not silent: - logging.exception(f"Failed to execute query '{query}' on instance '{instance.name}' will retry") + logging.exception(f"Failed to execute query '{query}' on {cnt} try on instance '{instance.name}' will retry") time.sleep(sleep_time) else: raise exception diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 3d28295d40e..7c7e4b4e76e 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -155,7 +155,7 @@ class Task2: assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV( "85\n") assert TSV(self.cluster.instances['s1_0_0'].query( - "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n") + "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("test_copy_month_to_week_partition_with_recovering\t85\n") assert TSV(self.cluster.instances['s1_0_0'].query( "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n") diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 4d111db59ab..d114954d739 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -1,7 +1,8 @@ import random import string - +import logging import pytest +import time from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) @@ -130,6 +131,9 @@ def test_default_codec_single(start_cluster): assert node1.query("SELECT COUNT() FROM compression_table") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table") == "3\n" + node1.query("DROP TABLE compression_table SYNC") + node2.query("DROP TABLE compression_table SYNC") + def test_default_codec_multiple(start_cluster): for i, node in enumerate([node1, node2]): @@ -199,6 +203,9 @@ def test_default_codec_multiple(start_cluster): assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" + node1.query("DROP TABLE compression_table_multiple SYNC") + node2.query("DROP TABLE compression_table_multiple SYNC") + def test_default_codec_version_update(start_cluster): node3.query(""" @@ -212,8 +219,10 @@ def test_default_codec_version_update(start_cluster): node3.query("INSERT INTO compression_table VALUES (2, '{}')".format(get_random_string(2048))) node3.query("INSERT INTO compression_table VALUES (3, '{}')".format(get_random_string(22048))) + old_version = node3.query("SELECT version()") node3.restart_with_latest_version() - + new_version = node3.query("SELECT version()") + logging.debug(f"Updated from {old_version} to {new_version}") assert node3.query( "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n" assert node3.query( @@ -230,6 +239,16 @@ def test_default_codec_version_update(start_cluster): assert node3.query( "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n" + node3.query("DROP TABLE compression_table SYNC") + + def callback(n): + n.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/metadata/system /var/lib/clickhouse/data/system '], user='root') + node3.restart_with_original_version(callback_onstop=callback) + + cur_version = node3.query("SELECT version()") + logging.debug(f"End with {cur_version}") + + def test_default_codec_for_compact_parts(start_cluster): node4.query(""" CREATE TABLE compact_parts_table ( @@ -254,3 +273,4 @@ def test_default_codec_for_compact_parts(start_cluster): node4.query("ATTACH TABLE compact_parts_table") assert node4.query("SELECT COUNT() FROM compact_parts_table") == "1\n" + node4.query("DROP TABLE compact_parts_table SYNC") diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/.gitignore b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/.gitignore new file mode 100644 index 00000000000..12657916b22 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/.gitignore @@ -0,0 +1,2 @@ +dictionaries/* +!.gitignore diff --git a/tests/integration/test_dictionaries_update_field/test.py b/tests/integration/test_dictionaries_update_field/test.py index c52c836b4f7..2e46403c63b 100644 --- a/tests/integration/test_dictionaries_update_field/test.py +++ b/tests/integration/test_dictionaries_update_field/test.py @@ -4,7 +4,6 @@ import time import pytest from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller -from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py index 9ab790c0fd6..cfd5f4d5607 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py @@ -51,7 +51,7 @@ def test_default_reading(started_cluster): test_helper() with PartitionManager() as pm, ClickHouseKiller(dictionary_node): - assert None == dictionary_node.get_process_pid("clickhouse"), "CLickHouse must be alive" + assert None == dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" # Remove connection between main_node and dictionary for sure pm.heal_all() diff --git a/tests/integration/test_match_process_uid_against_data_owner/test.py b/tests/integration/test_match_process_uid_against_data_owner/test.py index 6ad67630713..00149d7d2da 100644 --- a/tests/integration/test_match_process_uid_against_data_owner/test.py +++ b/tests/integration/test_match_process_uid_against_data_owner/test.py @@ -1,7 +1,6 @@ import os import pwd import re -import logging import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_START_COMMAND diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index fa1e9cf49fa..ea80f01e18e 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -37,6 +37,7 @@ def started_cluster(): cluster.shutdown() def test_create_replicated_table(started_cluster): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") assert "Explicit zookeeper_path and replica_name are specified" in \ main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);") @@ -57,9 +58,11 @@ def test_create_replicated_table(started_cluster): assert_create_query([main_node, dummy_node], "testdb.replicated_table", expected) # assert without replacing uuid assert main_node.query("show create testdb.replicated_table") == dummy_node.query("show create testdb.replicated_table") + main_node.query("DROP DATABASE testdb SYNC") @pytest.mark.parametrize("engine", ['MergeTree', 'ReplicatedMergeTree']) def test_simple_alter_table(started_cluster, engine): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") # test_simple_alter_table name = "testdb.alter_test_{}".format(engine) main_node.query("CREATE TABLE {} " @@ -100,12 +103,11 @@ def test_simple_alter_table(started_cluster, engine): "SETTINGS index_granularity = 8192".format(name, full_engine) assert_create_query([main_node, dummy_node, competing_node], name, expected) - + main_node.query("DROP DATABASE testdb SYNC") def get_table_uuid(database, name): return main_node.query(f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'").strip() - @pytest.fixture(scope="module", name="attachable_part") def fixture_attachable_part(started_cluster): main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") @@ -115,10 +117,11 @@ def fixture_attachable_part(started_cluster): table_uuid = get_table_uuid("testdb_attach_atomic", "test") return os.path.join(main_node.path, f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0") - - @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_attach(started_cluster, attachable_part, engine): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + name = "alter_attach_test_{}".format(engine) main_node.query(f"CREATE TABLE testdb.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") table_uuid = get_table_uuid("testdb", name) @@ -134,10 +137,14 @@ def test_alter_attach(started_cluster, attachable_part, engine): assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "123\n" else: assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "" - + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_part(started_cluster, engine): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + table = f"alter_drop_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") @@ -151,10 +158,14 @@ def test_alter_drop_part(started_cluster, engine): assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "" else: assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "456\n" - + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_detach_part(started_cluster, engine): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + table = f"alter_detach_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") @@ -169,10 +180,14 @@ def test_alter_detach_part(started_cluster, engine): assert dummy_node.query(detached_parts_query) == f"{part_name}\n" else: assert dummy_node.query(detached_parts_query) == "" - + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_detached_part(started_cluster, engine): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + table = f"alter_drop_detached_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") @@ -186,8 +201,14 @@ def test_alter_drop_detached_part(started_cluster, engine): assert main_node.query(detached_parts_query) == "" assert dummy_node.query(detached_parts_query) == f"{part_name}\n" + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") + def test_alter_fetch(started_cluster): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query("CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") main_node.query("CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") main_node.query("INSERT INTO testdb.fetch_source VALUES (123)") @@ -197,8 +218,13 @@ def test_alter_fetch(started_cluster): assert main_node.query(detached_parts_query) == "all_0_0_0\n" assert dummy_node.query(detached_parts_query) == "" + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") def test_alters_from_different_replicas(started_cluster): + main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + # test_alters_from_different_replicas competing_node.query("CREATE DATABASE IF NOT EXISTS testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');") @@ -294,6 +320,8 @@ def test_alters_from_different_replicas(started_cluster): "9\t2021-02-11\t1241149650\n" assert_eq_with_retry(dummy_node, "SELECT CounterID, StartDate, UserID FROM testdb.dist ORDER BY CounterID", expected) + main_node.query("DROP DATABASE testdb SYNC") + dummy_node.query("DROP DATABASE testdb SYNC") def test_recover_staled_replica(started_cluster): main_node.query("CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');") @@ -377,10 +405,10 @@ def test_recover_staled_replica(started_cluster): dummy_node.query("DROP TABLE recover.tmp") assert_eq_with_retry(main_node, "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", "0\n") + main_node.query("DROP DATABASE recover SYNC") + dummy_node.query("DROP DATABASE recover SYNC") def test_startup_without_zk(started_cluster): - main_node.query("DROP DATABASE IF EXISTS testdb SYNC") - main_node.query("DROP DATABASE IF EXISTS recover SYNC") with PartitionManager() as pm: pm.drop_instance_zk_connections(main_node) err = main_node.query_and_get_error("CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');") @@ -403,7 +431,7 @@ def test_startup_without_zk(started_cluster): main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" - + main_node.query("DROP DATABASE startup") def test_server_uuid(started_cluster): uuid1 = main_node.query("select serverUUID()") diff --git a/tests/integration/test_restore_replica/test.py b/tests/integration/test_restore_replica/test.py index 4197c064243..4013b5b474c 100644 --- a/tests/integration/test_restore_replica/test.py +++ b/tests/integration/test_restore_replica/test.py @@ -4,7 +4,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller from helpers.test_tools import assert_eq_with_retry -from helpers.network import PartitionManager def fill_nodes(nodes): for node in nodes: