2022-04-08 07:52:16 +00:00
|
|
|
from helpers.cluster import ClickHouseCluster
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
|
|
def started_cluster():
|
|
|
|
global cluster
|
|
|
|
try:
|
|
|
|
cluster = ClickHouseCluster(__file__)
|
|
|
|
cluster.add_instance(
|
2024-06-14 11:06:56 +00:00
|
|
|
"disks_app_test",
|
2024-06-14 13:54:12 +00:00
|
|
|
main_configs=["config.xml"],
|
2024-06-14 11:06:56 +00:00
|
|
|
with_minio=True,
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
cluster.start()
|
2024-09-10 15:14:53 +00:00
|
|
|
|
|
|
|
# local disk requires its `path` directory to exist.
|
|
|
|
# the two paths below belong to `test1` and `test2` disks
|
|
|
|
node = cluster.instances["disks_app_test"]
|
|
|
|
for path in ["path1", "path2"]:
|
|
|
|
node.exec_in_container(
|
|
|
|
[
|
|
|
|
"bash",
|
|
|
|
"-c",
|
|
|
|
f"mkdir -p /var/lib/clickhouse/{path}",
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
yield cluster
|
|
|
|
|
|
|
|
finally:
|
|
|
|
cluster.shutdown()
|
|
|
|
|
|
|
|
|
|
|
|
def init_data(source):
|
|
|
|
source.query("DROP TABLE IF EXISTS test_table")
|
|
|
|
|
|
|
|
source.query(
|
|
|
|
"CREATE TABLE test_table(word String, value UInt64) "
|
|
|
|
"ENGINE=MergeTree() "
|
|
|
|
"ORDER BY word SETTINGS storage_policy = 'test1'"
|
|
|
|
)
|
|
|
|
|
|
|
|
source.query("INSERT INTO test_table(*) VALUES ('test1', 2)")
|
|
|
|
|
2022-12-08 17:27:23 +00:00
|
|
|
|
2022-12-08 17:20:54 +00:00
|
|
|
def init_data_s3(source):
|
|
|
|
source.query("DROP TABLE IF EXISTS test_table_s3")
|
|
|
|
|
|
|
|
source.query(
|
|
|
|
"CREATE TABLE test_table_s3(word String, value UInt64) "
|
|
|
|
"ENGINE=MergeTree() "
|
|
|
|
"ORDER BY word SETTINGS storage_policy = 'test3'"
|
|
|
|
)
|
|
|
|
|
|
|
|
source.query("INSERT INTO test_table_s3(*) VALUES ('test1', 2)")
|
2022-04-08 07:52:16 +00:00
|
|
|
|
2022-12-08 17:27:23 +00:00
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
def test_disks_app_func_ld(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
out = source.exec_in_container(
|
2024-05-30 14:07:25 +00:00
|
|
|
["/usr/bin/clickhouse", "disks", "--save-logs", "--query", "list-disks"]
|
2022-06-21 14:40:22 +00:00
|
|
|
)
|
2022-04-08 07:52:16 +00:00
|
|
|
|
2024-05-31 13:17:28 +00:00
|
|
|
disks = list(
|
|
|
|
sorted(
|
|
|
|
map(
|
|
|
|
lambda x: x.split(":")[0], filter(lambda x: len(x) > 1, out.split("\n"))
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
2022-04-08 07:52:16 +00:00
|
|
|
|
2024-05-30 14:07:25 +00:00
|
|
|
assert disks[:4] == ["default", "local", "test1", "test2"]
|
2022-04-08 07:52:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_disks_app_func_ls(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
init_data(source)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test1",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
|
|
|
|
|
|
|
assert files[0] == "store"
|
|
|
|
|
2022-06-23 21:57:28 +00:00
|
|
|
out = source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-23 21:57:28 +00:00
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"list . --recursive",
|
2022-06-23 21:57:28 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
assert ".:\nstore\n" in out
|
|
|
|
assert "\n./store:\n" in out
|
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
|
|
|
|
def test_disks_app_func_cp(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
init_data(source)
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"bash",
|
|
|
|
"-c",
|
|
|
|
"echo 'tester' |"
|
|
|
|
+ " ".join(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-21 14:40:22 +00:00
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"'write path1'",
|
2022-06-21 14:40:22 +00:00
|
|
|
]
|
|
|
|
),
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
2024-07-01 12:46:17 +00:00
|
|
|
"copy --recursive --disk-from test1 --disk-to test2 . .",
|
2022-04-08 07:52:16 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test2",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
assert "path1" in out
|
2022-09-29 11:21:41 +00:00
|
|
|
|
2022-09-29 11:03:03 +00:00
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-09-29 11:03:03 +00:00
|
|
|
"--disk",
|
|
|
|
"test2",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"remove path1",
|
2022-09-29 11:03:03 +00:00
|
|
|
]
|
|
|
|
)
|
2022-10-01 11:55:58 +00:00
|
|
|
|
2022-10-01 11:49:50 +00:00
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-10-01 11:49:50 +00:00
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"remove path1",
|
2022-10-01 11:49:50 +00:00
|
|
|
]
|
|
|
|
)
|
2022-09-29 11:03:03 +00:00
|
|
|
|
2022-10-01 15:16:25 +00:00
|
|
|
# alesapin: Why we need list one more time?
|
|
|
|
# kssenii: it is an assertion that the file is indeed deleted
|
2022-09-29 11:03:03 +00:00
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test2",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-09-29 11:03:03 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
assert "path1" not in out
|
2022-10-01 11:55:58 +00:00
|
|
|
|
2022-10-01 11:49:50 +00:00
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test1",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-10-01 11:49:50 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
assert "path1" not in out
|
2022-04-08 07:52:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_disks_app_func_ln(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
init_data(source)
|
|
|
|
|
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"link data/default/test_table data/default/z_tester",
|
2022-04-08 07:52:16 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-30 14:07:25 +00:00
|
|
|
["/usr/bin/clickhouse", "disks", "--save-logs", "--query", "list data/default/"]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
|
|
|
|
|
|
|
assert "z_tester" in files
|
|
|
|
|
|
|
|
|
|
|
|
def test_disks_app_func_rm(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
init_data(source)
|
|
|
|
|
|
|
|
source.exec_in_container(
|
|
|
|
[
|
2022-06-21 14:40:22 +00:00
|
|
|
"bash",
|
|
|
|
"-c",
|
|
|
|
"echo 'tester' |"
|
|
|
|
+ " ".join(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-21 14:40:22 +00:00
|
|
|
"--disk",
|
|
|
|
"test2",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"'write path3'",
|
2022-06-21 14:40:22 +00:00
|
|
|
]
|
|
|
|
),
|
2022-04-08 07:52:16 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test2",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-06-21 14:40:22 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
assert "path3" in out
|
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
source.exec_in_container(
|
2022-06-21 14:40:22 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-21 14:40:22 +00:00
|
|
|
"--disk",
|
|
|
|
"test2",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"remove path3",
|
2022-06-21 14:40:22 +00:00
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test2",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
assert "path3" not in out
|
2022-04-08 07:52:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_disks_app_func_mv(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
init_data(source)
|
|
|
|
|
2022-06-21 14:40:22 +00:00
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test1",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-06-21 14:40:22 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
|
|
|
assert "old_store" not in files
|
|
|
|
assert "store" in files
|
|
|
|
|
2022-04-08 07:52:16 +00:00
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"move store old_store",
|
2022-04-08 07:52:16 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test1",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
2022-06-21 14:40:22 +00:00
|
|
|
assert "old_store" in files
|
|
|
|
assert "store" not in files
|
2022-04-08 07:52:16 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_disks_app_func_read_write(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
|
|
|
|
source.exec_in_container(
|
|
|
|
[
|
|
|
|
"bash",
|
|
|
|
"-c",
|
|
|
|
"echo 'tester' |"
|
|
|
|
+ " ".join(
|
2022-06-21 14:40:22 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-21 14:40:22 +00:00
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"'write 5.txt'",
|
2022-06-21 14:40:22 +00:00
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
),
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2022-06-21 14:40:22 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
2022-10-20 23:14:19 +00:00
|
|
|
"--save-logs",
|
2022-06-21 14:40:22 +00:00
|
|
|
"--disk",
|
|
|
|
"test1",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
|
|
|
"read 5.txt",
|
2022-06-21 14:40:22 +00:00
|
|
|
]
|
2022-04-08 07:52:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
|
|
|
|
|
|
|
assert files[0] == "tester"
|
2022-12-08 17:20:54 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_remote_disk_list(started_cluster):
|
|
|
|
source = cluster.instances["disks_app_test"]
|
|
|
|
init_data_s3(source)
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
2024-05-31 13:17:28 +00:00
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test3",
|
|
|
|
"--query",
|
|
|
|
"list .",
|
|
|
|
]
|
2022-12-08 17:20:54 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
files = out.split("\n")
|
|
|
|
|
|
|
|
assert files[0] == "store"
|
|
|
|
|
|
|
|
out = source.exec_in_container(
|
|
|
|
[
|
|
|
|
"/usr/bin/clickhouse",
|
|
|
|
"disks",
|
|
|
|
"--save-logs",
|
|
|
|
"--disk",
|
|
|
|
"test3",
|
2024-05-30 14:07:25 +00:00
|
|
|
"--query",
|
2024-05-31 13:17:28 +00:00
|
|
|
"list . --recursive",
|
2022-12-08 17:20:54 +00:00
|
|
|
]
|
|
|
|
)
|
|
|
|
|
|
|
|
assert ".:\nstore\n" in out
|
|
|
|
assert "\n./store:\n" in out
|