ClickHouse/tests/integration/test_ddl_worker_non_leader/test.py

import pytest
import time
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.client import QueryRuntimeException

cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True)

@pytest.fixture(scope="module")
def started_cluster():
    try:
        cluster.start()

        yield cluster
    finally:
        cluster.shutdown()


def test_non_leader_replica(started_cluster):

    node1.query_with_retry('''CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String)
    ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '1') ORDER BY tuple()''')

    node2.query_with_retry('''CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String)
    ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '2') ORDER BY tuple() SETTINGS replicated_can_become_leader = 0''')

    node1.query("INSERT INTO sometable SELECT number, toString(number) FROM numbers(100)")
    node2.query_with_retry("SYSTEM SYNC REPLICA sometable", timeout=10)

    assert node1.query("SELECT COUNT() FROM sometable") == "100\n"
    assert node2.query("SELECT COUNT() FROM sometable") == "100\n"


    with PartitionManager() as pm:
        pm.drop_instance_zk_connections(node1)

        # this query should be executed by leader, but leader partitioned from zookeeper
        with pytest.raises(Exception):
            node2.query("ALTER TABLE sometable ON CLUSTER 'test_cluster' MODIFY COLUMN value UInt64 SETTINGS distributed_ddl_task_timeout=5")

    for _ in range(100):
        if 'UInt64' in node1.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'"):
            break
        time.sleep(0.1)

    for _ in range(100):
        if 'UInt64' in node2.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'"):
            break
        time.sleep(0.1)

    assert 'UInt64' in node1.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'")
    assert 'UInt64' in node2.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'")

    # Checking that DDLWorker doesn't hung and still able to execute DDL queries
    node1.query("CREATE TABLE new_table_with_ddl ON CLUSTER 'test_cluster' (key UInt32) ENGINE=MergeTree() ORDER BY tuple()", settings={"distributed_ddl_task_timeout": "10"})
    assert node1.query("EXISTS new_table_with_ddl") == "1\n"
    assert node2.query("EXISTS new_table_with_ddl") == "1\n"