From 1033c148598c1e6a4775acff4c658cccbd1d59e1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 24 Aug 2022 15:01:21 +0200 Subject: [PATCH 1/6] Add test --- .../configs/config.d/storage_conf.xml | 13 +++ .../test_replicated_merge_tree_s3/test.py | 92 +++++++++++++------ 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml index 92706ff445d..00aa03b1a92 100644 --- a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -7,6 +7,12 @@ minio minio123 + + cache + s3 + ./s3_cache/ + 1000000 + @@ -16,6 +22,13 @@ + + +
+ s3_cache +
+ + diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index cc85a4eab02..2296d28866b 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -5,6 +5,8 @@ import string import pytest from helpers.cluster import ClickHouseCluster +TABLE_NAME = "s3_test" + @pytest.fixture(scope="module") def cluster(): @@ -58,8 +60,13 @@ def generate_values(date_str, count, sign=1): def create_table(cluster, additional_settings=None): - create_table_statement = """ - CREATE TABLE s3_test ON CLUSTER cluster( + settings = { + "storage_policy": "s3", + } + settings.update(additional_settings) + + create_table_statement = f""" + CREATE TABLE {TABLE_NAME} ON CLUSTER cluster( dt Date, id Int64, data String, @@ -67,20 +74,42 @@ def create_table(cluster, additional_settings=None): ) ENGINE=ReplicatedMergeTree() PARTITION BY dt ORDER BY (dt, id) - SETTINGS storage_policy='s3' + SETTINGS {",".join((k+"="+repr(v) for k, v in settings.items()))} """ - if additional_settings: - create_table_statement += "," - create_table_statement += additional_settings list(cluster.instances.values())[0].query(create_table_statement) +def insert(cluster, node_idxs, verify=True): + all_values = "" + for node_idx in node_idxs: + node = cluster.instances["node" + str(node_idx)] + values = generate_values("2020-01-0" + str(node_idx), 4096) + node.query( + f"INSERT INTO {TABLE_NAME} VALUES {values}", + settings={"insert_quorum": 3}, + ) + if node_idx != 1: + all_values += "," + all_values += values + + if verify: + for node_idx in node_idxs: + node = cluster.instances["node" + str(node_idx)] + assert ( + node.query( + f"SELECT * FROM {TABLE_NAME} order by dt, id FORMAT Values", + settings={"select_sequential_consistency": 1}, + ) + == all_values + ) + + @pytest.fixture(autouse=True) def drop_table(cluster): yield for node in list(cluster.instances.values()): - node.query("DROP TABLE IF EXISTS s3_test") + node.query(f"DROP TABLE IF EXISTS {TABLE_NAME}") minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing @@ -95,32 +124,37 @@ def drop_table(cluster): def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): create_table( cluster, - additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part), + additional_settings=f"min_rows_for_wide_part={min_rows_for_wide_part}", ) - all_values = "" - for node_idx in range(1, 4): - node = cluster.instances["node" + str(node_idx)] - values = generate_values("2020-01-0" + str(node_idx), 4096) - node.query( - "INSERT INTO s3_test VALUES {}".format(values), - settings={"insert_quorum": 3}, - ) - if node_idx != 1: - all_values += "," - all_values += values - - for node_idx in range(1, 4): - node = cluster.instances["node" + str(node_idx)] - assert ( - node.query( - "SELECT * FROM s3_test order by dt, id FORMAT Values", - settings={"select_sequential_consistency": 1}, - ) - == all_values - ) + insert(node_idxs=[1, 2, 3, 4], verify=True) minio = cluster.minio_client assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * ( FILES_OVERHEAD + files_per_part * 3 ) + + +def test_drop_cache_on_cluster(cluster): + create_table( + cluster, + additional_settings={"storage_policy":"s3_cache"}, + ) + + insert(cluster, node_idxs=[1, 2, 3], verify=True) + + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + node3 = cluster.instances["node2"] + + node1.query(f"select * from clusterAllReplicas(cluster, default, {TABLE_NAME}) format Null") + + assert int(node1.query("select count() from system.filesystem_cache")) > 0 + assert int(node2.query("select count() from system.filesystem_cache")) > 0 + assert int(node3.query("select count() from system.filesystem_cache")) > 0 + + node1.query("system drop filesystem cache on cluster cluster") + + assert int(node1.query("select count() from system.filesystem_cache")) == 0 + assert int(node2.query("select count() from system.filesystem_cache")) == 0 + assert int(node3.query("select count() from system.filesystem_cache")) == 0 From 6ac35dabcf34d3f80373f27bf419034ff5ef5dcc Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 24 Aug 2022 15:03:02 +0200 Subject: [PATCH 2/6] Update test.py --- tests/integration/test_replicated_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 2296d28866b..e20aa948486 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -145,7 +145,7 @@ def test_drop_cache_on_cluster(cluster): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node3 = cluster.instances["node2"] + node3 = cluster.instances["node3"] node1.query(f"select * from clusterAllReplicas(cluster, default, {TABLE_NAME}) format Null") From bfc3180c9e2585bb100ded7b2e1b34f7ec32d7e6 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 25 Aug 2022 00:20:50 +0200 Subject: [PATCH 3/6] Update test.py --- tests/integration/test_replicated_merge_tree_s3/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index e20aa948486..ce2bbbfc1ac 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -138,7 +138,7 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par def test_drop_cache_on_cluster(cluster): create_table( cluster, - additional_settings={"storage_policy":"s3_cache"}, + additional_settings={"storage_policy": "s3_cache"}, ) insert(cluster, node_idxs=[1, 2, 3], verify=True) @@ -147,7 +147,9 @@ def test_drop_cache_on_cluster(cluster): node2 = cluster.instances["node2"] node3 = cluster.instances["node3"] - node1.query(f"select * from clusterAllReplicas(cluster, default, {TABLE_NAME}) format Null") + node1.query( + f"select * from clusterAllReplicas(cluster, default, {TABLE_NAME}) format Null" + ) assert int(node1.query("select count() from system.filesystem_cache")) > 0 assert int(node2.query("select count() from system.filesystem_cache")) > 0 From 73bb5a6b9ef84e04f1a6dc2e8554adab7928d9f8 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 25 Aug 2022 12:10:46 +0200 Subject: [PATCH 4/6] Update test.py --- tests/integration/test_replicated_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index ce2bbbfc1ac..46e54f01463 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -124,7 +124,7 @@ def drop_table(cluster): def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): create_table( cluster, - additional_settings=f"min_rows_for_wide_part={min_rows_for_wide_part}", + additional_settings={"min_rows_for_wide_part": min_rows_for_wide_part}, ) insert(node_idxs=[1, 2, 3, 4], verify=True) From e46d34c7220050a1fbb5abe4155a5c1e72c18a30 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 25 Aug 2022 18:37:25 +0200 Subject: [PATCH 5/6] Update test.py --- tests/integration/test_replicated_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index 46e54f01463..e10950aec45 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -127,7 +127,7 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par additional_settings={"min_rows_for_wide_part": min_rows_for_wide_part}, ) - insert(node_idxs=[1, 2, 3, 4], verify=True) + insert(node_idxs=[1, 2, 3], verify=True) minio = cluster.minio_client assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * ( From e2014c9872eb5672aa779174a1d8ff3299b35570 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:01:31 +0200 Subject: [PATCH 6/6] Update test.py --- tests/integration/test_replicated_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index e10950aec45..37027d07969 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -127,7 +127,7 @@ def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_par additional_settings={"min_rows_for_wide_part": min_rows_for_wide_part}, ) - insert(node_idxs=[1, 2, 3], verify=True) + insert(cluster, node_idxs=[1, 2, 3], verify=True) minio = cluster.minio_client assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * (