ClickHouse/tests/integration/test_keeper_three_nodes_two_alive/test.py
2021-12-02 13:41:04 +03:00

106 lines
3.5 KiB
Python

#!/usr/bin/env python3
import pytest
from helpers.cluster import ClickHouseCluster
import random
import string
import os
import time
from multiprocessing.dummy import Pool
from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry
from kazoo.client import KazooClient, KazooState
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/keeper_conf.xml'], stay_alive=True)
node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/keeper_conf.xml'], stay_alive=True)
node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml', 'configs/keeper_conf.xml'], stay_alive=True)
def get_fake_zk(nodename, timeout=30.0):
_fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout)
_fake_zk_instance.start()
return _fake_zk_instance
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def start(node):
node.start_clickhouse()
def delete_with_retry(node_name, path):
for _ in range(30):
try:
get_fake_zk(node_name).delete(path)
return
except:
time.sleep(0.5)
raise Exception(f"Cannot delete {path} from node {node_name}")
def test_start_offline(started_cluster):
p = Pool(3)
try:
node1_zk = get_fake_zk("node1")
node1_zk.create("/test_alive", b"aaaa")
node1.stop_clickhouse()
node2.stop_clickhouse()
node3.stop_clickhouse()
time.sleep(3)
p.map(start, [node2, node3])
assert node2.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start")
assert node3.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start")
node2_zk = get_fake_zk("node2")
node2_zk.create("/c", b"data")
finally:
p.map(start, [node1, node2, node3])
delete_with_retry("node1", "/test_alive")
def test_start_non_existing(started_cluster):
p = Pool(3)
try:
node1.stop_clickhouse()
node2.stop_clickhouse()
node3.stop_clickhouse()
node1.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper1.xml', 'node3', 'non_existing_node')
node2.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper2.xml', 'node3', 'non_existing_node')
time.sleep(3)
p.map(start, [node2, node1])
assert node1.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start")
assert node2.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start")
node2_zk = get_fake_zk("node2")
node2_zk.create("/test_non_exising", b"data")
finally:
node1.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper1.xml', 'non_existing_node', 'node3')
node2.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper2.xml', 'non_existing_node', 'node3')
p.map(start, [node1, node2, node3])
delete_with_retry("node2", "/test_non_exising")
def test_restart_third_node(started_cluster):
node1_zk = get_fake_zk("node1")
node1_zk.create("/test_restart", b"aaaa")
node3.restart_clickhouse()
assert node3.contains_in_log("Connected to ZooKeeper (or Keeper) before internal Keeper start")
node1_zk.delete("/test_restart")