Better tests

This commit is contained in:
alesapin 2021-05-15 15:33:01 +03:00
parent 340cd20274
commit 33e9f1bcf1
4 changed files with 31 additions and 67 deletions

View File

@ -80,3 +80,16 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti
time.sleep(sleep_time)
else:
raise AssertionError("'{}' not found in logs".format(substring))
def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, settings={}):
exception = None
for _ in range(retry_count):
try:
instance.query(query, timeout=30, settings=settings)
break
except Exception as ex:
exception = ex
print("Failed to execute query '", query, "' on instance", instance.name, "will retry")
time.sleep(sleep_time)
else:
raise exception

View File

@ -7,7 +7,7 @@ import pytest
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import TSV
from helpers.test_tools import assert_eq_with_retry
from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry
cluster = ClickHouseCluster(__file__)
@ -408,8 +408,9 @@ def test_in_memory_wal_rotate(start_cluster):
def test_in_memory_deduplication(start_cluster):
for i in range(3):
node9.query("INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')")
node10.query("INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')")
# table can be in readonly node
exec_query_with_retry(node9, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')")
exec_query_with_retry(node10, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')")
node9.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20)
node10.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20)
@ -430,10 +431,10 @@ def test_in_memory_alters(start_cluster):
node9.restart_clickhouse(kill=True)
expected = "1\tab\t0\n2\tcd\t0\n"
assert node9.query("SELECT id, s, col1 FROM alters_table") == expected
assert node9.query("SELECT id, s, col1 FROM alters_table ORDER BY id") == expected
check_parts_type(1)
node9.query("INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)")
# After hard restart table can be in readonly mode
exec_query_with_retry(node9, "INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)")
node9.query("ALTER TABLE alters_table MODIFY COLUMN col1 String")
node9.query("ALTER TABLE alters_table DROP COLUMN s")
node9.restart_clickhouse(kill=True)
@ -442,8 +443,10 @@ def test_in_memory_alters(start_cluster):
with pytest.raises(Exception):
node9.query("SELECT id, s, col1 FROM alters_table")
expected = expected = "1\t0_foo\n2\t0_foo\n3\t100_foo\n"
assert node9.query("SELECT id, col1 || '_foo' FROM alters_table")
# Values of col1 was not materialized as integers, so they have
# default string values after alter
expected = "1\t_foo\n2\t_foo\n3\t100_foo\n"
assert node9.query("SELECT id, col1 || '_foo' FROM alters_table ORDER BY id") == expected
def test_polymorphic_parts_index(start_cluster):

View File

@ -3,7 +3,7 @@ import time
import helpers.client as client
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import TSV
from helpers.test_tools import TSV, exec_query_with_retry
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True)
@ -393,31 +393,11 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run):
time.sleep(5) # Wait for TTL
# after restart table can be in readonly mode
exception = None
for _ in range(40):
try:
node_right.query("OPTIMIZE TABLE test_ttl_delete FINAL")
break
except Exception as ex:
print("Cannot optimaze table on node", node_right.name, "exception", ex)
time.sleep(0.5)
exception = ex
else:
raise ex
exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL")
node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL")
node_right.query("OPTIMIZE TABLE test_ttl_where FINAL")
for _ in range(40):
try:
node_left.query("SYSTEM SYNC REPLICA test_ttl_delete", timeout=20)
break
except Exception as ex:
print("Cannot sync replica table on node", node_left.name, "exception", ex)
time.sleep(0.5)
exception = ex
else:
raise ex
exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete")
node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20)
node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20)

View File

@ -2,7 +2,7 @@ import pytest
import time
from helpers.cluster import ClickHouseCluster
from helpers.test_tools import assert_eq_with_retry
from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry
cluster = ClickHouseCluster(__file__)
@ -38,29 +38,9 @@ def test_mutate_and_upgrade(start_cluster):
node1.restart_with_latest_version(signal=9)
node2.restart_with_latest_version(signal=9)
exception = None
# After hard restart table can be in readonly mode
for _ in range(40):
try:
node2.query("INSERT INTO mt VALUES ('2020-02-13', 3);")
break
except Exception as ex:
print("Cannot insert into node2 with error {}", ex)
time.sleep(0.5)
exception = ex
else:
raise exception
for _ in range(40):
try:
node1.query("SYSTEM SYNC REPLICA mt", timeout=5)
break
except Exception as ex:
print("Cannot sync node1 with error {}", ex)
time.sleep(0.5)
exception = ex
else:
raise exception
exec_query_with_retry(node2, "INSERT INTO mt VALUES ('2020-02-13', 3)")
exec_query_with_retry(node1, "SYSTEM SYNC REPLICA mt")
assert node1.query("SELECT COUNT() FROM mt") == "2\n"
assert node2.query("SELECT COUNT() FROM mt") == "2\n"
@ -99,19 +79,7 @@ def test_upgrade_while_mutation(start_cluster):
node3.restart_with_latest_version(signal=9)
# After hard restart table can be in readonly mode
exception = None
for _ in range(40):
try:
node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"})
break
except Exception as ex:
print("Cannot alter node3 with error {}", ex)
time.sleep(0.5)
exception = ex
else:
raise exception
exec_query_with_retry(node3, "ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"})
# will delete nothing, but previous async mutation will finish with this query
assert_eq_with_retry(node3, "SELECT COUNT() from mt1", "50000\n")