ClickHouse/tests/integration/test_s3_plain_rewritable/test.py

151 lines
4.1 KiB
Python
Raw Normal View History

2024-03-28 03:48:57 +00:00
import pytest
import random
import string
2024-05-15 02:18:18 +00:00
import threading
2024-03-28 03:48:57 +00:00
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
2024-05-15 02:18:18 +00:00
NUM_WORKERS = 5
MAX_ROWS = 1000
def gen_insert_values(size):
return ",".join(
2024-03-28 03:48:57 +00:00
f"({i},'{''.join(random.choices(string.ascii_lowercase, k=5))}')"
2024-05-15 02:18:18 +00:00
for i in range(size)
)
insert_values = ",".join(
f"({i},'{''.join(random.choices(string.ascii_lowercase, k=5))}')" for i in range(10)
)
2024-03-28 03:48:57 +00:00
@pytest.fixture(scope="module", autouse=True)
def start_cluster():
for i in range(NUM_WORKERS):
cluster.add_instance(
f"node{i + 1}",
main_configs=["configs/storage_conf.xml"],
with_minio=True,
env_variables={"ENDPOINT_SUBPATH": f"node{i + 1}"},
stay_alive=True,
# Override ENDPOINT_SUBPATH.
instance_env_variables=i > 0,
)
2024-03-28 03:48:57 +00:00
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
@pytest.mark.parametrize(
"storage_policy",
[
pytest.param("s3_plain_rewritable"),
pytest.param("cache_s3_plain_rewritable"),
],
)
def test(storage_policy):
2024-05-15 02:18:18 +00:00
def create_insert(node, insert_values):
2024-03-28 03:48:57 +00:00
node.query(
"""
2024-05-15 02:18:18 +00:00
CREATE TABLE test (
2024-03-28 03:48:57 +00:00
id Int64,
data String
) ENGINE=MergeTree()
ORDER BY id
SETTINGS storage_policy='{}'
""".format(
storage_policy
)
2024-03-28 03:48:57 +00:00
)
2024-05-15 02:18:18 +00:00
node.query("INSERT INTO test VALUES {}".format(insert_values))
insert_values_arr = [
gen_insert_values(random.randint(1, MAX_ROWS)) for _ in range(0, NUM_WORKERS)
]
threads = []
assert len(cluster.instances) == NUM_WORKERS
2024-05-15 02:18:18 +00:00
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
t = threading.Thread(target=create_insert, args=(node, insert_values_arr[i]))
2024-05-15 02:18:18 +00:00
threads.append(t)
t.start()
for t in threads:
t.join()
2024-03-28 03:48:57 +00:00
2024-05-15 02:18:18 +00:00
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
2024-03-28 03:48:57 +00:00
assert (
node.query("SELECT * FROM test ORDER BY id FORMAT Values")
2024-05-15 02:18:18 +00:00
== insert_values_arr[i]
2024-03-28 03:48:57 +00:00
)
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
node.query("ALTER TABLE test MODIFY SETTING old_parts_lifetime = 59")
assert (
node.query(
"SELECT engine_full from system.tables WHERE database = currentDatabase() AND name = 'test'"
).find("old_parts_lifetime = 59")
!= -1
)
node.query("ALTER TABLE test RESET SETTING old_parts_lifetime")
assert (
node.query(
"SELECT engine_full from system.tables WHERE database = currentDatabase() AND name = 'test'"
).find("old_parts_lifetime")
== -1
)
node.query("ALTER TABLE test MODIFY COMMENT 'new description'")
assert (
node.query(
"SELECT comment from system.tables WHERE database = currentDatabase() AND name = 'test'"
).find("new description")
!= -1
)
2024-05-15 02:18:18 +00:00
insert_values_arr = []
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
2024-05-15 02:18:18 +00:00
insert_values_arr.append(
node.query("SELECT * FROM test ORDER BY id FORMAT Values")
2024-03-28 03:48:57 +00:00
)
2024-05-15 02:18:18 +00:00
def restart(node):
node.restart_clickhouse()
threads = []
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
t = threading.Thread(target=restart, args=(node,))
2024-05-15 02:18:18 +00:00
threads.append(t)
t.start()
for t in threads:
t.join()
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
2024-03-28 03:48:57 +00:00
assert (
node.query("SELECT * FROM test ORDER BY id FORMAT Values")
2024-05-15 02:18:18 +00:00
== insert_values_arr[i]
2024-03-28 03:48:57 +00:00
)
2024-05-15 02:18:18 +00:00
for i in range(NUM_WORKERS):
node = cluster.instances[f"node{i + 1}"]
node.query("DROP TABLE IF EXISTS test SYNC")
2024-03-28 03:48:57 +00:00
it = cluster.minio_client.list_objects(
cluster.minio_bucket, "data/", recursive=True
)
assert len(list(it)) == 0