From 24d67082a91797c1b950e592f18c93a81032b548 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 14 Dec 2020 22:57:31 -0800 Subject: [PATCH] few more changes and simplification to integration test --- .../System/StorageSystemDDLWorkerQueue.cpp | 4 +- src/Storages/System/attachSystemTables.cpp | 2 +- .../configs/config.d/clusters.xml | 93 ------------------- .../configs/config.d/ddl.xml | 8 -- .../configs/config.d/macro.xml | 5 - .../configs/config.d/query_log.xml | 14 --- .../config.d/zookeeper_session_timeout.xml | 7 -- .../configs/remote_servers.xml | 28 ++++++ .../configs/users.d/query_log.xml | 8 -- .../configs/users.d/restricted_user.xml | 16 ---- .../test_system_ddl_worker_queue/test.py | 85 +++++++---------- 11 files changed, 65 insertions(+), 205 deletions(-) delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/config.d/clusters.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/config.d/ddl.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/config.d/macro.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/config.d/query_log.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/config.d/zookeeper_session_timeout.xml create mode 100644 tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/users.d/query_log.xml delete mode 100644 tests/integration/test_system_ddl_worker_queue/configs/users.d/restricted_user.xml diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp index f2167c3378b..e12634892b4 100644 --- a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -90,7 +90,7 @@ NamesAndTypesList StorageSystemDDLWorkerQueue::getNamesAndTypes() {"port", std::make_shared()}, {"status", std::make_shared(getStatusEnumsAndValues())}, {"cluster", std::make_shared()}, - {"values", std::make_shared()}, + {"value", std::make_shared()}, {"query_start_time", std::make_shared()}, {"query_finish_time", std::make_shared()}, {"query_duration_ms", std::make_shared()}, @@ -275,7 +275,7 @@ void StorageSystemDDLWorkerQueue::fillData(MutableColumns & res_columns, const C Coordination::GetResponse res; if (!futures.empty()) res = futures[query_id].get(); - res_columns[i++]->insert(res.data); // values + res_columns[i++]->insert(res.data); // value auto query_start_time = res.stat.mtime; res_columns[i++]->insert(UInt64(query_start_time / 1000)); // query_start_time diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index e6fc00ec544..673cf671548 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -136,7 +136,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) attach(system_database, "mutations"); attach(system_database, "replicas"); attach(system_database, "replication_queue"); - attach(system_database, "ddl_worker_queue"); + attach(system_database, "distributed_ddl_queue"); attach(system_database, "distribution_queue"); attach(system_database, "dictionaries"); attach(system_database, "models"); diff --git a/tests/integration/test_system_ddl_worker_queue/configs/config.d/clusters.xml b/tests/integration/test_system_ddl_worker_queue/configs/config.d/clusters.xml deleted file mode 100644 index 9a8dde66756..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/config.d/clusters.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - - - - true - - ch1 - 9000 - - - ch2 - 9000 - - - - true - - ch3 - 9000 - - - ch4 - 9000 - - - - - - - - true - - ch1 - 9000 - default - - - ch2 - 9000 - test2 - - - - true - - ch3 - 9000 - default - - - ch4 - 9000 - test2 - - - - - - - - false - - ch1 - 9000 - - - - false - - ch2 - 9000 - - - - false - - ch3 - 9000 - - - - false - - ch4 - 9000 - - - - - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/config.d/ddl.xml b/tests/integration/test_system_ddl_worker_queue/configs/config.d/ddl.xml deleted file mode 100644 index 1f11ae24dc0..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/config.d/ddl.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - /clickhouse/task_queue/ddl - 10 - 3600 - 5 - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/config.d/macro.xml b/tests/integration/test_system_ddl_worker_queue/configs/config.d/macro.xml deleted file mode 100644 index b7bc11da782..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/config.d/macro.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - cluster_no_replicas - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/config.d/query_log.xml b/tests/integration/test_system_ddl_worker_queue/configs/config.d/query_log.xml deleted file mode 100644 index 7f57bb1eb85..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/config.d/query_log.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - system - query_log
- - - 1000 - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/config.d/zookeeper_session_timeout.xml b/tests/integration/test_system_ddl_worker_queue/configs/config.d/zookeeper_session_timeout.xml deleted file mode 100644 index 6af9cc8cc99..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/config.d/zookeeper_session_timeout.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 3000 - - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml b/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml new file mode 100644 index 00000000000..4c3de4b3905 --- /dev/null +++ b/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml @@ -0,0 +1,28 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/tests/integration/test_system_ddl_worker_queue/configs/users.d/query_log.xml b/tests/integration/test_system_ddl_worker_queue/configs/users.d/query_log.xml deleted file mode 100644 index 0f389295bec..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/users.d/query_log.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - 1 - - - diff --git a/tests/integration/test_system_ddl_worker_queue/configs/users.d/restricted_user.xml b/tests/integration/test_system_ddl_worker_queue/configs/users.d/restricted_user.xml deleted file mode 100644 index 5b6084eea7b..00000000000 --- a/tests/integration/test_system_ddl_worker_queue/configs/users.d/restricted_user.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - default - default - - ::/0 - - - - db1 - - - - diff --git a/tests/integration/test_system_ddl_worker_queue/test.py b/tests/integration/test_system_ddl_worker_queue/test.py index 69ce6859cbb..c5037fc400e 100644 --- a/tests/integration/test_system_ddl_worker_queue/test.py +++ b/tests/integration/test_system_ddl_worker_queue/test.py @@ -1,65 +1,48 @@ -import os -import sys -import time -from contextlib import contextmanager - import pytest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helpers.cluster import ClickHouseCluster -# just import cluster setup from test_didstributed_ddl to avoid code duplication -from test_distributed_ddl.cluster import ClickHouseClusterWithDDLHelpers -from helpers.test_tools import assert_eq_with_retry +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + +nodes = [node1, node2, node3, node4] -@pytest.fixture(scope="module", params=["configs"]) -def test_cluster(request): - cluster = ClickHouseClusterWithDDLHelpers(__file__, request.param) - +@pytest.fixture(scope="module") +def started_cluster(): try: - cluster.prepare() + cluster.start() + for i, node in enumerate([node1, node2]): + node.query("CREATE DATABASE testdb") + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format( + i)) + for i, node in enumerate([node3, node4]): + node.query("CREATE DATABASE testdb") + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format( + i)) yield cluster - instance = cluster.instances['ch1'] - cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") - - # Check query log to ensure that DDL queries are not executed twice - time.sleep(1.5) - for instance in list(cluster.instances.values()): - cluster.ddl_check_there_are_no_dublicates(instance) - - cluster.pm_random_drops.heal_all() - finally: cluster.shutdown() -def test_ddl_worker_queue_table_entries(test_cluster): - instance = test_cluster.instances['ch1'] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, """ -CREATE TABLE IF NOT EXISTS merge ON CLUSTER '{cluster}' (p Date, i Int32) -ENGINE = MergeTree(p, p, 1) -""") - for i in range(0, 4, 2): - k = (i / 2) * 2 - test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1)) - time.sleep(2) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002") - time.sleep(2) - # query the ddl_worker_queue table to ensure that the columns are populated as expected - assert_eq_with_retry(instance, - "SELECT If((SELECT count(*) FROM system.ddl_worker_queue WHERE cluster='cluster' AND status='finished') > 0, 'ok', 'fail')", - "ok\n"); - assert_eq_with_retry(instance, - "SELECT If((SELECT count(*) FROM system.ddl_worker_queue WHERE cluster='cluster' AND status='active') >= 0, 'ok', 'fail')", - "ok\n") - test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'") +def test_distributed_ddl_queue(started_cluster): + node1.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node3.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node2.query("SYSTEM SYNC REPLICA testdb.test_table") + node4.query("SYSTEM SYNC REPLICA testdb.test_table") - -if __name__ == '__main__': - with contextmanager(test_cluster)() as ctx_cluster: - for name, instance in list(ctx_cluster.instances.items()): - print(name, instance.ip_address) - input("Cluster created, press any key to destroy...") + node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", + settings={"replication_alter_partitions_sync": "2"}) + for node in nodes: + node.query("SYSTEM SYNC REPLICA testdb.test_table") + assert node.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" + assert node.query( + "SELECT If((SELECT count(*) FROM system.distributed_ddl_queue WHERE cluster='test_cluster' AND entry='query-0000000000') > 0, 'ok', 'fail')") == "ok\n"