ClickHouse/tests/integration/test_parts_delete_zookeeper/test.py

119 lines
3.6 KiB
Python
Raw Normal View History

import time
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance(
"node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True
)
@pytest.fixture(scope="module")
def start_cluster():
try:
cluster.start()
node1.query(
"""
CREATE DATABASE test;
CREATE TABLE test_table(date Date, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1')
ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS old_parts_lifetime=4, cleanup_delay_period=1, cleanup_thread_preferred_points_per_iteration=0;
"""
)
yield cluster
except Exception as ex:
2020-10-02 16:54:07 +00:00
print(ex)
finally:
cluster.shutdown()
# Test that outdated parts are not removed when they cannot be removed from zookeeper
def test_merge_doesnt_work_without_zookeeper(start_cluster):
node1.query(
"INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
)
node1.query(
"INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)"
)
assert (
node1.query("SELECT count(*) from system.parts where table = 'test_table'")
== "2\n"
)
node1.query("OPTIMIZE TABLE test_table FINAL")
assert (
node1.query("SELECT count(*) from system.parts where table = 'test_table'")
== "3\n"
)
assert_eq_with_retry(
node1,
"SELECT count(*) from system.parts where table = 'test_table' and active = 1",
"1",
)
node1.query("TRUNCATE TABLE test_table")
2023-11-04 14:56:15 +00:00
total_parts = node1.query(
"SELECT count(*) from system.parts where table = 'test_table'"
)
2023-11-04 14:22:18 +00:00
assert total_parts == "0\n" or total_parts == "1\n"
assert (
2023-11-04 14:56:15 +00:00
node1.query(
"SELECT count(*) from system.parts where table = 'test_table' and active = 1"
)
== "0\n"
)
2023-11-04 21:48:29 +00:00
node1.query("DETACH TABLE test_table SYNC")
node1.query("ATTACH TABLE test_table")
node1.query(
"INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)"
)
node1.query(
"INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)"
)
assert (
node1.query(
"SELECT count(*) from system.parts where table = 'test_table' and active"
)
== "2\n"
)
2022-06-09 14:53:52 +00:00
# Parts may be moved to Deleting state and then back in Outdated state.
# But system.parts returns only Active and Outdated parts if _state column is not queried.
with PartitionManager() as pm:
node1.query("OPTIMIZE TABLE test_table FINAL")
pm.drop_instance_zk_connections(node1)
2021-12-27 09:24:19 +00:00
# unfortunately we can be too fast and delete node before partition with ZK
if (
2022-06-09 14:53:52 +00:00
node1.query(
"SELECT count(*) from system.parts where table = 'test_table' and _state!='dummy'"
)
== "1\n"
):
2021-12-27 09:24:19 +00:00
print("We were too fast and deleted parts before partition with ZK")
else:
time.sleep(10) # > old_parts_lifetime
assert (
node1.query(
2022-06-09 14:53:52 +00:00
"SELECT count(*) from system.parts where table = 'test_table' and _state!='dummy'"
)
== "3\n"
)
assert_eq_with_retry(
node1,
"SELECT count(*) from system.parts where table = 'test_table' and active = 1",
"1",
)