From 524edd8460431b447c4bcd2f932bed489f7a83f6 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 21 Nov 2023 00:08:20 +0100 Subject: [PATCH 1/4] turn test off --- tests/integration/test_merge_tree_s3/test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 3b2f1c0f6a6..0856b1681f8 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -851,6 +851,8 @@ def test_merge_canceled_by_s3_errors_when_move(cluster, broken_s3, node_name): def test_s3_engine_heavy_write_check_mem( cluster, broken_s3, node_name, in_flight_memory ): + pytest.skip("Disabled, will be fixed after https://github.com/ClickHouse/ClickHouse/issues/51152") + in_flight = in_flight_memory[0] memory = in_flight_memory[1] @@ -870,12 +872,16 @@ def test_s3_engine_heavy_write_check_mem( ) broken_s3.setup_fake_multpartuploads() - broken_s3.setup_slow_answers(10 * 1024 * 1024, timeout=15, count=10) + slow_responces = 10 + slow_timeout = 15 + broken_s3.setup_slow_answers(10 * 1024 * 1024, timeout=slow_timeout, count=slow_responces) query_id = f"INSERT_INTO_S3_ENGINE_QUERY_ID_{in_flight}" node.query( "INSERT INTO s3_test SELECT number, toString(number) FROM numbers(50000000)" - f" SETTINGS max_memory_usage={2*memory}" + f" SETTINGS " + f" max_memory_usage={2*memory}" + # f", max_threads=1" # ParallelFormattingOutputFormat consumption depends on it f", s3_max_inflight_parts_for_one_file={in_flight}", query_id=query_id, ) @@ -892,7 +898,8 @@ def test_s3_engine_heavy_write_check_mem( assert int(memory_usage) < 1.2 * memory assert int(memory_usage) > 0.8 * memory - assert int(wait_inflight) > in_flight * 1000 * 1000 + # The more in_flight value is the less time CH waits. + assert int(wait_inflight) / 1000 / 1000 > slow_responces * slow_timeout / in_flight check_no_objects_after_drop(cluster, node_name=node_name) From 2e1c142d6d968642636ef3ba3360f28ec4630dc3 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 21 Nov 2023 14:03:49 +0100 Subject: [PATCH 2/4] black style --- tests/integration/test_merge_tree_s3/test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 0856b1681f8..a3752323a53 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -851,7 +851,9 @@ def test_merge_canceled_by_s3_errors_when_move(cluster, broken_s3, node_name): def test_s3_engine_heavy_write_check_mem( cluster, broken_s3, node_name, in_flight_memory ): - pytest.skip("Disabled, will be fixed after https://github.com/ClickHouse/ClickHouse/issues/51152") + pytest.skip( + "Disabled, will be fixed after https://github.com/ClickHouse/ClickHouse/issues/51152" + ) in_flight = in_flight_memory[0] memory = in_flight_memory[1] @@ -874,7 +876,9 @@ def test_s3_engine_heavy_write_check_mem( broken_s3.setup_fake_multpartuploads() slow_responces = 10 slow_timeout = 15 - broken_s3.setup_slow_answers(10 * 1024 * 1024, timeout=slow_timeout, count=slow_responces) + broken_s3.setup_slow_answers( + 10 * 1024 * 1024, timeout=slow_timeout, count=slow_responces + ) query_id = f"INSERT_INTO_S3_ENGINE_QUERY_ID_{in_flight}" node.query( From 63bd45fa5f2c6b57d99c1d13d29e54c8e66bd9a5 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:42:37 +0100 Subject: [PATCH 3/4] Update tests/integration/test_merge_tree_s3/test.py --- tests/integration/test_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index a3752323a53..6a9aa1738d6 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -885,7 +885,7 @@ def test_s3_engine_heavy_write_check_mem( "INSERT INTO s3_test SELECT number, toString(number) FROM numbers(50000000)" f" SETTINGS " f" max_memory_usage={2*memory}" - # f", max_threads=1" # ParallelFormattingOutputFormat consumption depends on it + f", max_threads=1" # ParallelFormattingOutputFormat consumption depends on it f", s3_max_inflight_parts_for_one_file={in_flight}", query_id=query_id, ) From adf4ade00aa18540bfea7f0076d71501aa924b2f Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:38:15 +0100 Subject: [PATCH 4/4] Update tests/integration/test_merge_tree_s3/test.py --- tests/integration/test_merge_tree_s3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 6a9aa1738d6..1a34a2c3566 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -885,7 +885,7 @@ def test_s3_engine_heavy_write_check_mem( "INSERT INTO s3_test SELECT number, toString(number) FROM numbers(50000000)" f" SETTINGS " f" max_memory_usage={2*memory}" - f", max_threads=1" # ParallelFormattingOutputFormat consumption depends on it + f", max_threads=1" # ParallelFormattingOutputFormat consumption depends on it f", s3_max_inflight_parts_for_one_file={in_flight}", query_id=query_id, )