thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

better test

Browse Source
This commit is contained in:
zhangxiao871 2021-11-01 18:26:50 +08:00
parent 4e53eb0e4e
commit 49e03025dd
5 changed files with 26 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_log.xml
View File

@ -1,7 +0,0 @@
<clickhouse>
<zookeeper_log>
<database>system</database>
<table>zookeeper_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</zookeeper_log>
</clickhouse>

34
tests/integration/test_zookeeper_config_load_balancing/test_first_or_random.py
View File

@ -6,11 +6,11 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_first_or_random.xml')
node1 = cluster.add_instance('node1', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml"])
node2 = cluster.add_instance('node2', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml"])
node3 = cluster.add_instance('node3', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_first_or_random.xml"])
@pytest.fixture(scope="module", autouse=True)
@ -23,31 +23,9 @@ def started_cluster():
finally:
cluster.shutdown()
def wait_zookeeper_node_to_start(started_cluster, zk_nodes, timeout=60):
start = time.time()
while time.time() - start < timeout:
try:
for instance in zk_nodes:
conn = started_cluster.get_kazoo_client(instance)
conn.get_children('/')
print("All instances of ZooKeeper started")
return
except Exception as ex:
print(("Can't connect to ZooKeeper " + str(ex)))
time.sleep(0.5)
def test_first_or_random(started_cluster):
wait_zookeeper_node_to_start(started_cluster, ["zoo1", "zoo2", "zoo3"])
time.sleep(10)
zoo1_ip = started_cluster.get_instance_ip("zoo1")
for i, node in enumerate([node1, node3]):
node.query('DROP TABLE IF EXISTS simple SYNC')
node.query('''
CREATE TABLE simple (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192);
'''.format(replica=node.name))
time.sleep(5)
assert '::ffff:' + str(zoo1_ip) + '\n' == node1.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo1_ip) + '\n' == node2.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo1_ip) + '\n' == node3.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '1' == str(node1.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node2.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node3.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()

35
tests/integration/test_zookeeper_config_load_balancing/test_in_order.py
View File

@ -6,11 +6,11 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_in_order.xml')
node1 = cluster.add_instance('node1', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml"])
node2 = cluster.add_instance('node2', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml"])
node3 = cluster.add_instance('node3', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_in_order.xml"])
@pytest.fixture(scope="module", autouse=True)
@ -23,31 +23,8 @@ def started_cluster():
finally:
cluster.shutdown()
def wait_zookeeper_node_to_start(started_cluster, zk_nodes, timeout=60):
start = time.time()
while time.time() - start < timeout:
try:
for instance in zk_nodes:
conn = started_cluster.get_kazoo_client(instance)
conn.get_children('/')
print("All instances of ZooKeeper started")
return
except Exception as ex:
print(("Can't connect to ZooKeeper " + str(ex)))
time.sleep(0.5)
def test_in_order(started_cluster):
wait_zookeeper_node_to_start(started_cluster, ["zoo1", "zoo2", "zoo3"])
time.sleep(10)
zoo1_ip = started_cluster.get_instance_ip("zoo1")
for i, node in enumerate([node1, node3]):
node.query('DROP TABLE IF EXISTS simple SYNC')
node.query('''
CREATE TABLE simple (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192);
'''.format(replica=node.name))
time.sleep(5)
assert '::ffff:' + str(zoo1_ip) + '\n' == node1.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo1_ip) + '\n' == node2.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo1_ip) + '\n' == node3.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '1' == str(node1.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node2.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node3.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()

38
tests/integration/test_zookeeper_config_load_balancing/test_nearest_hostname.py
View File

@ -6,11 +6,11 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_nearest_hostname.xml')
node1 = cluster.add_instance('nod1', with_zookeeper=True,
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml"])
node2 = cluster.add_instance('nod2', with_zookeeper=True,
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml"])
node3 = cluster.add_instance('nod3', with_zookeeper=True,
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers_nearest_hostname.xml", "configs/zookeeper_config_nearest_hostname.xml"])
@pytest.fixture(scope="module", autouse=True)
@ -23,34 +23,8 @@ def started_cluster():
finally:
cluster.shutdown()
def wait_zookeeper_node_to_start(started_cluster, zk_nodes, timeout=60):
start = time.time()
while time.time() - start < timeout:
try:
for instance in zk_nodes:
conn = started_cluster.get_kazoo_client(instance)
conn.get_children('/')
print("All instances of ZooKeeper started")
return
except Exception as ex:
print(("Can't connect to ZooKeeper " + str(ex)))
time.sleep(0.5)
def test_nearest_hostname(started_cluster):
wait_zookeeper_node_to_start(started_cluster, ["zoo1", "zoo2", "zoo3"])
time.sleep(10)
zoo1_ip = started_cluster.get_instance_ip("zoo1")
zoo2_ip = started_cluster.get_instance_ip("zoo2")
zoo3_ip = started_cluster.get_instance_ip("zoo3")
for i, node in enumerate([node1, node3]):
node.query('DROP TABLE IF EXISTS simple SYNC')
node.query('''
CREATE TABLE simple (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192);
'''.format(replica=node.name))
time.sleep(5)
assert '::ffff:' + str(zoo1_ip) + '\n' == node1.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo2_ip) + '\n' == node2.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo3_ip) + '\n' == node3.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '1' == str(node1.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo1_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node2.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo2_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node3.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo3_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()

41
tests/integration/test_zookeeper_config_load_balancing/test_round_robin.py
View File

@ -6,11 +6,11 @@ from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_round_robin.xml')
node1 = cluster.add_instance('node1', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml"])
node2 = cluster.add_instance('node2', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml"])
node3 = cluster.add_instance('node3', with_zookeeper=True,
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml", "configs/zookeeper_log.xml"])
main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_round_robin.xml"])
@pytest.fixture(scope="module", autouse=True)
@ -23,39 +23,14 @@ def started_cluster():
finally:
cluster.shutdown()
def wait_zookeeper_node_to_start(started_cluster, zk_nodes, timeout=60):
start = time.time()
while time.time() - start < timeout:
try:
for instance in zk_nodes:
conn = started_cluster.get_kazoo_client(instance)
conn.get_children('/')
print("All instances of ZooKeeper started")
return
except Exception as ex:
print(("Can't connect to ZooKeeper " + str(ex)))
time.sleep(0.5)
def test_round_robin(started_cluster):
wait_zookeeper_node_to_start(started_cluster, ["zoo1", "zoo2", "zoo3"])
started_cluster.stop_zookeeper_nodes(["zoo1"])
time.sleep(10)
zoo2_ip = started_cluster.get_instance_ip("zoo2")
for i, node in enumerate([node1, node3]):
node.query('DROP TABLE IF EXISTS simple SYNC')
node.query('''
CREATE TABLE simple (date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192);
'''.format(replica=node.name))
time.sleep(1)
time.sleep(5)
print("zoo2", zoo2_ip)
assert '::ffff:' + str(zoo2_ip) + '\n' == node1.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo2_ip) + '\n' == node2.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '::ffff:' + str(zoo2_ip) + '\n' == node3.query('SELECT IPv6NumToString(address) FROM system.zookeeper_log ORDER BY event_time DESC LIMIT 1')
assert '1' == str(node1.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo2_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node2.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo2_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
assert '1' == str(node3.exec_in_container(['bash', '-c', "lsof -a -i4 -i6 -itcp -w | grep 'roottestzookeeperconfigloadbalancing_zoo2_1.roottestzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l"], privileged=True, user='root')).strip()
## start zoo2, zoo3, table will be readonly too, because it only connect to zoo1
started_cluster.start_zookeeper_nodes(["zoo1"])
wait_zookeeper_node_to_start(started_cluster, ["zoo1"])