mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-18 21:51:57 +00:00
b75963d370
This PR formats all the `*.py` files found under the `tests/integration` folder. It also reorders the imports and cleans up a bunch of unused imports. The formatting also takes care of other things like wrapping lines and fixing spaces and indents such that the tests look more readable.
409 lines
19 KiB
Python
409 lines
19 KiB
Python
import time
|
|
|
|
import pytest
|
|
from helpers.client import QueryRuntimeException, QueryTimeoutExceedException
|
|
from helpers.cluster import ClickHouseCluster
|
|
from helpers.test_tools import assert_eq_with_retry
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True)
|
|
node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True)
|
|
|
|
node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18',
|
|
with_installed_binary=True)
|
|
node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True)
|
|
|
|
node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15',
|
|
with_installed_binary=True)
|
|
node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True)
|
|
|
|
node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
|
|
with_installed_binary=True)
|
|
node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
|
|
with_installed_binary=True)
|
|
|
|
node9 = cluster.add_instance('node9', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
|
|
'configs/merge_tree_settings.xml'], with_zookeeper=True,
|
|
image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
|
|
with_installed_binary=True)
|
|
node10 = cluster.add_instance('node10', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml',
|
|
'configs/merge_tree_settings.xml'], with_zookeeper=True,
|
|
image='yandex/clickhouse-server', tag='19.6.3.18', stay_alive=True,
|
|
with_installed_binary=True)
|
|
|
|
node11 = cluster.add_instance('node11', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
|
|
with_installed_binary=True)
|
|
node12 = cluster.add_instance('node12', main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'],
|
|
with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.15', stay_alive=True,
|
|
with_installed_binary=True)
|
|
|
|
|
|
def prepare_single_pair_with_setting(first_node, second_node, group):
|
|
for node in (first_node, second_node):
|
|
node.query("CREATE DATABASE IF NOT EXISTS test")
|
|
|
|
# Two tables with adaptive granularity
|
|
first_node.query(
|
|
'''
|
|
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 10485760
|
|
'''.format(g=group))
|
|
|
|
second_node.query(
|
|
'''
|
|
CREATE TABLE table_by_default(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_by_default', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 10485760
|
|
'''.format(g=group))
|
|
|
|
# Two tables with fixed granularity
|
|
first_node.query(
|
|
'''
|
|
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 0
|
|
'''.format(g=group))
|
|
|
|
second_node.query(
|
|
'''
|
|
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 0
|
|
'''.format(g=group))
|
|
|
|
# Two tables with different granularity
|
|
with pytest.raises(QueryRuntimeException):
|
|
first_node.query(
|
|
'''
|
|
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 10485760
|
|
'''.format(g=group))
|
|
|
|
second_node.query(
|
|
'''
|
|
CREATE TABLE table_with_different_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_different_granularity', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 0
|
|
'''.format(g=group))
|
|
|
|
# Two tables with different granularity, but enabled mixed parts
|
|
first_node.query(
|
|
'''
|
|
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 10485760, enable_mixed_granularity_parts=1
|
|
'''.format(g=group))
|
|
|
|
second_node.query(
|
|
'''
|
|
CREATE TABLE table_with_mixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_mixed_granularity', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 0, enable_mixed_granularity_parts=1
|
|
'''.format(g=group))
|
|
|
|
|
|
def prepare_single_pair_without_setting(first_node, second_node, group):
|
|
for node in (first_node, second_node):
|
|
node.query("CREATE DATABASE IF NOT EXISTS test")
|
|
|
|
# Two tables with fixed granularity
|
|
first_node.query(
|
|
'''
|
|
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
'''.format(g=group))
|
|
|
|
second_node.query(
|
|
'''
|
|
CREATE TABLE table_with_fixed_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{g}/table_with_fixed_granularity', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes = 0
|
|
'''.format(g=group))
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_static_cluster():
|
|
try:
|
|
cluster.start()
|
|
|
|
prepare_single_pair_with_setting(node1, node2, "shard1")
|
|
prepare_single_pair_with_setting(node3, node4, "shard2")
|
|
prepare_single_pair_without_setting(node5, node6, "shard3")
|
|
yield cluster
|
|
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
('first_node', 'second_node', 'table'),
|
|
[
|
|
(node1, node2, 'table_by_default'),
|
|
(node1, node2, 'table_with_fixed_granularity'),
|
|
(node3, node4, 'table_by_default'),
|
|
(node3, node4, 'table_with_fixed_granularity'),
|
|
(node5, node6, 'table_with_fixed_granularity'),
|
|
]
|
|
)
|
|
def test_different_versions_cluster(start_static_cluster, first_node, second_node, table):
|
|
counter = 1
|
|
for n1, n2 in ((first_node, second_node), (second_node, first_node)):
|
|
n1.query("INSERT INTO {tbl} VALUES (toDate('2018-10-01'), {c1}, 333), (toDate('2018-10-02'), {c2}, 444)".format(
|
|
tbl=table, c1=counter * 2, c2=counter * 2 + 1))
|
|
n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
|
|
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
|
n1.query("DETACH TABLE {tbl}".format(tbl=table))
|
|
n2.query("DETACH TABLE {tbl}".format(tbl=table))
|
|
n1.query("ATTACH TABLE {tbl}".format(tbl=table))
|
|
n2.query("ATTACH TABLE {tbl}".format(tbl=table))
|
|
assert_eq_with_retry(n1, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
|
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
|
n1.query("OPTIMIZE TABLE {tbl} FINAL".format(tbl=table))
|
|
n2.query("SYSTEM SYNC REPLICA {tbl}".format(tbl=table))
|
|
assert_eq_with_retry(n1, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
|
assert_eq_with_retry(n2, "SELECT count() from {tbl}".format(tbl=table), str(counter * 2))
|
|
counter += 1
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def start_dynamic_cluster():
|
|
try:
|
|
cluster.start()
|
|
node7.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node7.query(
|
|
'''
|
|
CREATE TABLE table_with_adaptive_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/7/table_with_adaptive_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
SETTINGS index_granularity_bytes=10485760
|
|
''')
|
|
|
|
node8.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/8/table_with_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node9.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/9/table_with_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node10.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/10/table_with_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node11.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node12.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
yield cluster
|
|
|
|
finally:
|
|
cluster.shutdown()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
('n', 'tables'),
|
|
[
|
|
(node7, ['table_with_default_granularity', 'table_with_adaptive_default_granularity']),
|
|
(node8, ['table_with_default_granularity']),
|
|
]
|
|
)
|
|
def test_version_single_node_update(start_dynamic_cluster, n, tables):
|
|
for table in tables:
|
|
n.query(
|
|
"INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)".format(tbl=table))
|
|
n.restart_with_latest_version()
|
|
for table in tables:
|
|
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '2\n'
|
|
n.query(
|
|
"INSERT INTO {tbl} VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)".format(tbl=table))
|
|
assert n.query("SELECT count() from {tbl}".format(tbl=table)) == '4\n'
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
('node',),
|
|
[
|
|
(node9,),
|
|
(node10,)
|
|
]
|
|
)
|
|
def test_mixed_granularity_single_node(start_dynamic_cluster, node):
|
|
node.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
|
node.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-09-01'), 1, 333), (toDate('2018-09-02'), 2, 444)")
|
|
|
|
def callback(n):
|
|
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
|
|
"<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
|
|
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
|
|
"<yandex><merge_tree><enable_mixed_granularity_parts>1</enable_mixed_granularity_parts></merge_tree></yandex>")
|
|
|
|
node.restart_with_latest_version(callback_onstop=callback)
|
|
node.query("SYSTEM RELOAD CONFIG")
|
|
assert_eq_with_retry(node,
|
|
"SELECT value FROM system.merge_tree_settings WHERE name='enable_mixed_granularity_parts'",
|
|
'1')
|
|
assert node.query("SELECT count() from table_with_default_granularity") == '4\n'
|
|
node.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
|
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
|
node.query("OPTIMIZE TABLE table_with_default_granularity PARTITION 201810 FINAL")
|
|
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
|
path_to_merged_part = node.query(
|
|
"SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition DESC LIMIT 1").strip()
|
|
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk2' | grep '.*'".format(
|
|
p=path_to_merged_part)]) # check that we have adaptive files
|
|
|
|
path_to_old_part = node.query(
|
|
"SELECT path FROM system.parts WHERE table = 'table_with_default_granularity' AND active=1 ORDER BY partition ASC LIMIT 1").strip()
|
|
|
|
node.exec_in_container(["bash", "-c", "find {p} -name '*.mrk' | grep '.*'".format(
|
|
p=path_to_old_part)]) # check that we have non adaptive files
|
|
|
|
node.query("ALTER TABLE table_with_default_granularity UPDATE dummy = dummy + 1 WHERE 1")
|
|
# still works
|
|
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
|
|
|
node.query("ALTER TABLE table_with_default_granularity MODIFY COLUMN dummy String")
|
|
node.query("ALTER TABLE table_with_default_granularity ADD COLUMN dummy2 Float64")
|
|
|
|
# still works
|
|
assert node.query("SELECT count() from table_with_default_granularity") == '6\n'
|
|
|
|
|
|
@pytest.mark.skip(reason="flaky")
|
|
def test_version_update_two_nodes(start_dynamic_cluster):
|
|
node11.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
|
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
|
|
assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '2\n'
|
|
|
|
def callback(n):
|
|
n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml",
|
|
"<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
|
|
n.replace_config("/etc/clickhouse-server/config.d/merge_tree_settings.xml",
|
|
"<yandex><merge_tree><enable_mixed_granularity_parts>0</enable_mixed_granularity_parts></merge_tree></yandex>")
|
|
|
|
node12.restart_with_latest_version(callback_onstop=callback)
|
|
|
|
node12.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
|
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=20)
|
|
assert node11.query("SELECT COUNT() FROM table_with_default_granularity") == '4\n'
|
|
|
|
node12.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '2')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node11.query(
|
|
'''
|
|
CREATE TABLE table_with_default_granularity_new(date Date, id UInt32, dummy UInt32)
|
|
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard11/table_with_default_granularity_new', '1')
|
|
PARTITION BY toYYYYMM(date)
|
|
ORDER BY id
|
|
''')
|
|
|
|
node12.query(
|
|
"INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 1, 333), (toDate('2018-10-02'), 2, 444)")
|
|
with pytest.raises(QueryTimeoutExceedException):
|
|
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=20)
|
|
node12.query(
|
|
"INSERT INTO table_with_default_granularity_new VALUES (toDate('2018-10-01'), 3, 333), (toDate('2018-10-02'), 4, 444)")
|
|
|
|
node11.restart_with_latest_version(callback_onstop=callback) # just to be sure
|
|
|
|
for i in range(3):
|
|
try:
|
|
node11.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=120)
|
|
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity_new", timeout=120)
|
|
break
|
|
except Exception as ex:
|
|
print("Exception during replica sync", ex)
|
|
node11.query("SYSTEM RESTART REPLICA table_with_default_granularity_new")
|
|
node12.query("SYSTEM RESTART REPLICA table_with_default_granularity_new")
|
|
time.sleep(2 * i)
|
|
|
|
assert node11.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
|
|
assert node12.query("SELECT COUNT() FROM table_with_default_granularity_new") == "4\n"
|
|
|
|
node11.query(
|
|
"INSERT INTO table_with_default_granularity VALUES (toDate('2018-10-01'), 5, 333), (toDate('2018-10-02'), 6, 444)")
|
|
for i in range(3):
|
|
try:
|
|
node12.query("SYSTEM SYNC REPLICA table_with_default_granularity", timeout=120)
|
|
break
|
|
except Exception as ex:
|
|
print("Exception during replica sync", ex)
|
|
node11.query("SYSTEM RESTART REPLICA table_with_default_granularity")
|
|
node12.query("SYSTEM RESTART REPLICA table_with_default_granularity")
|
|
time.sleep(2 * i)
|
|
|
|
assert node12.query("SELECT COUNT() FROM table_with_default_granularity") == '6\n'
|