from os import path as p import pytest from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.keeper_utils import ( get_active_zk_connections, replace_zookeeper_config, reset_zookeeper_config, ) from helpers.test_tools import assert_eq_with_retry from helpers.utility import random_string default_zk_config = p.join(p.dirname(p.realpath(__file__)), "configs/zookeeper.xml") cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") node = cluster.add_instance("node", with_zookeeper=True) @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() yield cluster finally: cluster.shutdown() def test_reload_zookeeper(start_cluster): # random is used for flaky tests, where ZK is not fast enough to clear the node node.query( f""" CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/shard1/{random_string(7)}/test_table', '1') PARTITION BY toYYYYMM(date) ORDER BY id """ ) node.query( "INSERT INTO test_table(date, id) select today(), number FROM numbers(1000)" ) ## remove zoo2, zoo3 from configs new_config = """ zoo1 2181 2000 """ replace_zookeeper_config(node, new_config) ## config reloads, but can still work assert_eq_with_retry( node, "SELECT COUNT() FROM test_table", "1000", retry_count=120, sleep_time=0.5 ) ## stop all zookeepers, table will be readonly cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) node.query("SELECT COUNT() FROM test_table") with pytest.raises(QueryRuntimeException): node.query( "SELECT COUNT() FROM test_table", settings={"select_sequential_consistency": 1}, ) ## start zoo2, zoo3, table will be readonly too, because it only connect to zoo1 cluster.start_zookeeper_nodes(["zoo2", "zoo3"]) cluster.wait_zookeeper_nodes_to_start(["zoo2", "zoo3"]) node.query("SELECT COUNT() FROM test_table") with pytest.raises(QueryRuntimeException): node.query( "SELECT COUNT() FROM test_table", settings={"select_sequential_consistency": 1}, ) ## set config to zoo2, server will be normal new_config = """ zoo2 2181 2000 """ replace_zookeeper_config(node, new_config) active_zk_connections = get_active_zk_connections(node) assert ( len(active_zk_connections) == 1 ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) assert_eq_with_retry( node, "SELECT COUNT() FROM test_table", "1000", retry_count=120, sleep_time=0.5 ) active_zk_connections = get_active_zk_connections(node) assert ( len(active_zk_connections) == 1 ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) # Reset cluster state cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) cluster.wait_zookeeper_nodes_to_start(["zoo1", "zoo2", "zoo3"]) reset_zookeeper_config(node, default_zk_config) node.query("DROP TABLE test_table")