From d1bf8d7bfe6b52cbcb0d79080d1b0cf66608debc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 2 Sep 2024 11:06:24 +0000 Subject: [PATCH 01/16] Test 03231_pr_duplicate_announcement --- .../03231_pr_duplicate_announcement.reference | 2 ++ .../03231_pr_duplicate_announcement.sql | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/03231_pr_duplicate_announcement.reference create mode 100644 tests/queries/0_stateless/03231_pr_duplicate_announcement.sql diff --git a/tests/queries/0_stateless/03231_pr_duplicate_announcement.reference b/tests/queries/0_stateless/03231_pr_duplicate_announcement.reference new file mode 100644 index 00000000000..b32da0d591a --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_duplicate_announcement.reference @@ -0,0 +1,2 @@ +0 Value_0 +1 Value_1 diff --git a/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql b/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql new file mode 100644 index 00000000000..198f9622897 --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql @@ -0,0 +1,20 @@ +SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=ReplicatedMergeTree('/clickhouse/test/{database}/test_table', 'r1') ORDER BY tuple(); + +INSERT INTO test_table VALUES (0, 'Value_0'), (1, 'Value_1'), (2, 'Value_2'); + +DROP TABLE IF EXISTS test_table_for_in; +CREATE TABLE test_table_for_in +( + id UInt64 +) ENGINE=ReplicatedMergeTree('/clickhouse/test/{database}/test_table_for_in', 'r1') ORDER BY tuple(); + +INSERT INTO test_table_for_in VALUES (0), (1); + +SELECT id, value FROM test_table WHERE id IN (SELECT id FROM test_table_for_in UNION DISTINCT SELECT id FROM test_table_for_in); From 4b4976c9b00de476f71094898a5784357291c202 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 2 Sep 2024 12:37:53 +0000 Subject: [PATCH 02/16] Fix --- src/Planner/Planner.cpp | 3 ++- .../0_stateless/03231_pr_duplicate_announcement.sql | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 7b5101c5c7d..69a652a74a0 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1287,7 +1287,8 @@ void Planner::buildPlanForUnionNode() for (const auto & query_node : union_queries_nodes) { - Planner query_planner(query_node, select_query_options); + Planner query_planner(query_node, select_query_options, planner_context->getGlobalPlannerContext()); + query_planner.buildQueryPlanIfNeeded(); for (const auto & row_policy : query_planner.getUsedRowPolicies()) used_row_policies.insert(row_policy); diff --git a/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql b/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql index 198f9622897..cd1fc277fe1 100644 --- a/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql +++ b/tests/queries/0_stateless/03231_pr_duplicate_announcement.sql @@ -1,6 +1,4 @@ -SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; - -DROP TABLE IF EXISTS test_table; +DROP TABLE IF EXISTS test_table SYNC; CREATE TABLE test_table ( id UInt64, @@ -9,7 +7,7 @@ CREATE TABLE test_table INSERT INTO test_table VALUES (0, 'Value_0'), (1, 'Value_1'), (2, 'Value_2'); -DROP TABLE IF EXISTS test_table_for_in; +DROP TABLE IF EXISTS test_table_for_in SYNC; CREATE TABLE test_table_for_in ( id UInt64 @@ -17,4 +15,9 @@ CREATE TABLE test_table_for_in INSERT INTO test_table_for_in VALUES (0), (1); +SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + SELECT id, value FROM test_table WHERE id IN (SELECT id FROM test_table_for_in UNION DISTINCT SELECT id FROM test_table_for_in); + +DROP TABLE test_table SYNC; +DROP TABLE test_table_for_in SYNC; From ab6843e65e489fb25d6f9442dc07c6a261364326 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 2 Sep 2024 13:11:45 +0000 Subject: [PATCH 03/16] One more test --- ...3231_pr_duplicate_announcement_2.reference | 1 + .../03231_pr_duplicate_announcement_2.sql | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/queries/0_stateless/03231_pr_duplicate_announcement_2.reference create mode 100644 tests/queries/0_stateless/03231_pr_duplicate_announcement_2.sql diff --git a/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.reference b/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.reference new file mode 100644 index 00000000000..38f7ecf84a4 --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.reference @@ -0,0 +1 @@ +CAT 2 diff --git a/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.sql b/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.sql new file mode 100644 index 00000000000..eec1443ad2a --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_duplicate_announcement_2.sql @@ -0,0 +1,45 @@ +DROP TABLE IF EXISTS ANIMAL SYNC; + +CREATE TABLE ANIMAL ( ANIMAL Nullable(String) ) ENGINE = ReplicatedMergeTree('/clickhouse/test/{database}/animal', 'r1') ORDER BY tuple(); + +INSERT INTO ANIMAL (ANIMAL) VALUES ('CAT'), ('FISH'), ('DOG'), ('HORSE'), ('BIRD'); + +SET joined_subquery_requires_alias = 0; +SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT * +FROM +( + SELECT + x.b AS x, + countDistinct(x.c) AS ANIMAL + FROM + ( + SELECT + a.ANIMAL AS a, + 'CAT' AS b, + c.ANIMAL AS c, + d.ANIMAL AS d + FROM ANIMAL AS a + INNER JOIN ANIMAL AS b ON a.ANIMAL = b.ANIMAL + LEFT JOIN ANIMAL AS c ON b.ANIMAL = c.ANIMAL + RIGHT JOIN + ( + SELECT * + FROM ANIMAL + UNION ALL + SELECT * + FROM ANIMAL + UNION ALL + SELECT * + FROM ANIMAL + ) AS d ON a.ANIMAL = d.ANIMAL + WHERE (d.ANIMAL != 'CAT') AND (c.ANIMAL != 'DOG') AND (b.ANIMAL != 'FISH') + ) AS x + WHERE x.b >= 'CAT' + GROUP BY x.b + HAVING ANIMAL >= 0 +) AS ANIMAL +WHERE ANIMAL.ANIMAL >= 0; + +DROP TABLE ANIMAL SYNC; From 214f8a010257f4c2678bd517ce1f27a51439e8a1 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Tue, 3 Sep 2024 11:44:25 +0200 Subject: [PATCH 04/16] CI: Rerun check: do not check if manual rerun --- tests/ci/ci.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index d201b6602f5..3321d641c27 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -16,7 +16,7 @@ import upload_result_helper from build_check import get_release_or_pr from ci_config import CI from ci_metadata import CiMetadata -from ci_utils import GH, Utils +from ci_utils import GH, Utils, Envs from clickhouse_helper import ( CiLogsCredentials, ClickHouseHelper, @@ -333,11 +333,10 @@ def _pre_action(s3, job_name, batch, indata, pr_info): CI.JobNames.BUILD_CHECK, ): # we might want to rerun build report job rerun_helper = RerunHelper(commit, _get_ext_check_name(job_name)) - if ( - rerun_helper.is_already_finished_by_status() - and not Utils.is_job_triggered_manually() - ): - print("WARNING: Rerunning job with GH status ") + if rerun_helper.is_already_finished_by_status(): + print( + f"WARNING: Rerunning job with GH status, rerun triggered by {Envs.GITHUB_ACTOR}" + ) status = rerun_helper.get_finished_status() assert status print("::group::Commit Status") From 31a37c7736e0771396cce3e8aad748593dace349 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Sep 2024 13:23:18 +0200 Subject: [PATCH 05/16] Fix flaky 02915_move_partition_inactive_replica --- .../02915_move_partition_inactive_replica.sql | 16 +++++++++++----- ...er_move_partition_inactive_replica.reference} | 0 ..._another_move_partition_inactive_replica.sql} | 7 +++++++ 3 files changed, 18 insertions(+), 5 deletions(-) rename tests/queries/0_stateless/{02916_move_partition_inactive_replica.reference => 02916_another_move_partition_inactive_replica.reference} (100%) rename tests/queries/0_stateless/{02916_move_partition_inactive_replica.sql => 02916_another_move_partition_inactive_replica.sql} (87%) diff --git a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql index 3b30a2b6c2c..b679a0c24e8 100644 --- a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql @@ -10,11 +10,11 @@ drop table if exists shard_1.from_1; drop table if exists shard_0.to; drop table if exists shard_1.to; -create table shard_0.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; -create table shard_1.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_0.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; +create table shard_1.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; -create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; -create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; +create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; insert into shard_0.from_0 select number from numbers(10); insert into shard_0.from_0 select number + 10 from numbers(10); @@ -40,10 +40,17 @@ OPTIMIZE TABLE shard_0.from_0; OPTIMIZE TABLE shard_1.from_0; OPTIMIZE TABLE shard_0.from_1; OPTIMIZE TABLE shard_1.from_1; + OPTIMIZE TABLE shard_0.to; +-- If moved parts are not merged by OPTIMIZE or background merge restart +-- can log Warning about metadata version on disk. It's normal situation +-- and test shouldn't rarely fail because of it. +set send_logs_level = 'Error'; + system restart replica shard_0.to; +-- Doesn't lead to test flakyness, because we don't check anything after it select sleep(2); attach table shard_1.to; @@ -54,4 +61,3 @@ drop table if exists shard_0.from_1; drop table if exists shard_1.from_1; drop table if exists shard_0.to; drop table if exists shard_1.to; - diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.reference b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference similarity index 100% rename from tests/queries/0_stateless/02916_move_partition_inactive_replica.reference rename to tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql similarity index 87% rename from tests/queries/0_stateless/02916_move_partition_inactive_replica.sql rename to tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql index b0699539ac1..1b23d9f2815 100644 --- a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql @@ -35,8 +35,15 @@ OPTIMIZE TABLE shard_0.to; OPTIMIZE TABLE shard_0.to; select name, active from system.parts where database='shard_0' and table='to' and active order by name; +-- If moved parts are not merged by OPTIMIZE or background merge restart +-- can log Warning about metadata version on disk. It's normal situation +-- and test shouldn't rarely fail because of it. +set send_logs_level = 'Error'; + system restart replica shard_0.to; +-- Doesn't lead to test flakyness, because we don't check content in table +-- which doesn't depend on any background operation select sleep(3); attach table shard_1.to; From ebdd3af16ca9547f45646ff5ec2e9d24d3d5939f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Sep 2024 13:24:02 +0200 Subject: [PATCH 06/16] Fix flaky 02915_move_partition_inactive_replica --- .../0_stateless/02915_move_partition_inactive_replica.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql index b679a0c24e8..4ecbe05547c 100644 --- a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql @@ -10,11 +10,11 @@ drop table if exists shard_1.from_1; drop table if exists shard_0.to; drop table if exists shard_1.to; -create table shard_0.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; -create table shard_1.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; +create table shard_0.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_1.from_0 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_0_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; -create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; -create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, ratio_of_defaults_for_sparse_serialization=0, vertical_merge_algorithm_min_rows_to_activate=612278, vertical_merge_algorithm_min_columns_to_activate=1, allow_vertical_merges_from_compact_to_wide_parts=1; +create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; insert into shard_0.from_0 select number from numbers(10); insert into shard_0.from_0 select number + 10 from numbers(10); From 2ae4cf375b9a58565832ef86cd4fc0dda726041d Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 3 Sep 2024 15:19:42 +0200 Subject: [PATCH 07/16] Fix jepsen for aarch64 --- tests/ci/jepsen_check.py | 5 ++++- tests/jepsen.clickhouse/project.clj | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 772467d4245..998a97273ae 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -241,7 +241,10 @@ def main(): additional_data = [] try: test_result = _parse_jepsen_output(jepsen_log_path) - if any(r.status == "FAIL" for r in test_result): + if len(test_result) == 0: + status = FAILURE + description = "No test results found" + elif any(r.status == "FAIL" for r in test_result): status = FAILURE description = "Found invalid analysis (ノಥ益ಥ)ノ ┻━┻" diff --git a/tests/jepsen.clickhouse/project.clj b/tests/jepsen.clickhouse/project.clj index bb41be1ba10..79ebc2a9064 100644 --- a/tests/jepsen.clickhouse/project.clj +++ b/tests/jepsen.clickhouse/project.clj @@ -7,10 +7,13 @@ :main jepsen.clickhouse.main :plugins [[lein-cljfmt "0.7.0"]] :dependencies [[org.clojure/clojure "1.10.1"] - [jepsen "0.2.7"] + [jepsen "0.2.7":exclusions [net.java.dev.jna/jna + net.java.dev.jna/jna-platform]] [zookeeper-clj "0.9.4"] [org.clojure/java.jdbc "0.7.12"] [com.hierynomus/sshj "0.34.0"] + [net.java.dev.jna/jna "5.14.0"] + [net.java.dev.jna/jna-platform "5.14.0"] [com.clickhouse/clickhouse-jdbc "0.3.2-patch11"] [org.apache.zookeeper/zookeeper "3.6.1" :exclusions [org.slf4j/slf4j-log4j12]]] :repl-options {:init-ns jepsen.clickhouse-keeper.main} From 7b4a38453ae8db65687e8353835ca8a59d659661 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Sep 2024 13:49:54 +0000 Subject: [PATCH 08/16] Test 03231_pr_reverse_in_order --- .../03231_pr_reverse_in_order.reference | 150 ++++++++++++++++++ .../0_stateless/03231_pr_reverse_in_order.sql | 31 ++++ 2 files changed, 181 insertions(+) create mode 100644 tests/queries/0_stateless/03231_pr_reverse_in_order.reference create mode 100644 tests/queries/0_stateless/03231_pr_reverse_in_order.sql diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.reference b/tests/queries/0_stateless/03231_pr_reverse_in_order.reference new file mode 100644 index 00000000000..07af27dc224 --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.reference @@ -0,0 +1,150 @@ +2024-06-11 02:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-11 00:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 22:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 20:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 18:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 16:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 14:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 12:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 10:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 08:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 06:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 04:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 02:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 02:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 00:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-10 00:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 22:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 22:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 20:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 20:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 18:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 18:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 16:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 16:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 14:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 14:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 12:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 12:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 10:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 10:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 08:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 08:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 06:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 06:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 04:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 04:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 02:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 02:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 02:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 00:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 00:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-09 00:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 22:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 22:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 22:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 20:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 20:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 20:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 18:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 18:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 18:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 16:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 16:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 16:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 14:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 14:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 14:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 12:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 12:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 12:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 10:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 10:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 10:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 08:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 08:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 08:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 06:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 06:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 06:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 04:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 04:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 04:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 02:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 02:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 02:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 00:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 00:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-08 00:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 22:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 22:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 22:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 20:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 20:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 20:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 18:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 18:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 18:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 16:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 16:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 16:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 14:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 14:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 14:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 12:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 12:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 12:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 10:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 10:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 10:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 08:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 08:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 08:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 06:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 06:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 06:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 04:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 04:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 04:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 02:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 02:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 02:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 00:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 00:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-07 00:00:01 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 22:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 22:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 20:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 20:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 18:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 18:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 16:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 16:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 14:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 14:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 12:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 12:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 10:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 10:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 08:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 08:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 06:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 06:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 04:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 04:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 02:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 02:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 00:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-06 00:00:02 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 22:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 20:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 18:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 16:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 14:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 12:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 10:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 08:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 06:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 04:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 02:00:03 asan 02221_parallel_replicas_bug FAIL fail +2024-06-05 00:00:03 asan 02221_parallel_replicas_bug FAIL fail diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql new file mode 100644 index 00000000000..59e8dbd5578 --- /dev/null +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS checks SYNC; + +CREATE TABLE checks +( + `check_name` LowCardinality(String), + `check_status` LowCardinality(String), + `check_start_time` DateTime, + `test_name` LowCardinality(String), + `test_status` LowCardinality(String), +) +ENGINE = ReplicatedMergeTree('/clickhouse/{database}/checks', '{replica}') +ORDER BY check_start_time; + +SYSTEM STOP MERGES checks; + +INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime('2024-06-07 00:00:01') + INTERVAL number HOUR, '02221_parallel_replicas_bug', 'FAIL' from numbers(100); +INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime('2024-06-06 00:00:02') + INTERVAL number HOUR, '02221_parallel_replicas_bug', 'FAIL' from numbers(100); +INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime('2024-06-05 00:00:03') + INTERVAL number HOUR, '02221_parallel_replicas_bug', 'FAIL' from numbers(100); + +SET allow_experimental_parallel_reading_from_replicas = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; + +SELECT check_start_time, check_name, test_name, test_status, check_status +FROM checks +WHERE 1 + AND test_status != 'SKIPPED' + AND test_status != 'OK' + AND check_status != 'success' + AND test_name ilike '%parallel_replicas%' +ORDER BY check_start_time desc, check_name, test_name; + + From 6f1511c9a205ef0efef33bf5cefd6d0e5fcd1a9a Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 3 Sep 2024 14:17:41 +0000 Subject: [PATCH 09/16] Collect sanitizer report from client to `client_log` --- tests/clickhouse-test | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index ecc5acbb363..c409b4e7632 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1637,6 +1637,19 @@ class TestCase: if args.client_log: log_opt = " --client_logs_file=" + args.client_log + " " client_options += log_opt + + for env_name in [ + "TSAN_OPTIONS", + "ASAN_OPTIONS", + "MSAN_OPTIONS", + "UBSAN_OPTIONS", + ]: + current_options = os.environ.get(env_name, None) + if current_options is None: + os.environ[env_name] = f"log_path={args.client_log}" + elif "log_path=" not in current_options: + os.environ[env_name] += f":log_path={args.client_log}" + os.environ["CLICKHOUSE_CLIENT_OPT"] = ( os.environ["CLICKHOUSE_CLIENT_OPT"] if "CLICKHOUSE_CLIENT_OPT" in os.environ @@ -2134,9 +2147,9 @@ class TestSuite: ) ) self.all_tags: Dict[str, Set[str]] = all_tags_and_random_settings_limits[0] - self.all_random_settings_limits: Dict[str, Dict[str, (int, int)]] = ( - all_tags_and_random_settings_limits[1] - ) + self.all_random_settings_limits: Dict[ + str, Dict[str, (int, int)] + ] = all_tags_and_random_settings_limits[1] self.sequential_tests = [] self.parallel_tests = [] for test_name in self.all_tests: @@ -3108,13 +3121,15 @@ def main(args): print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) if args.client_log: - if os.path.exists(args.client_log): - with open(args.client_log, "rb") as stream: + for log_file in [args.client_log, *glob.glob(f"{args.client_log}.*")]: + if not os.path.exists(log_file): + continue + with open(log_file, "rb") as stream: content = stream.read().decode() if len(content): - print("Has fatal logs from client:\n") + print(f"Has fatal logs from client in '{log_file}':\n") print(content) - os.remove(args.client_log) + os.remove(log_file) if len(restarted_tests) > 0: print("\nSome tests were restarted:\n") From 4a45d53dd966db63b4719e8d3c8a59e8d265e364 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Sep 2024 16:39:33 +0200 Subject: [PATCH 10/16] Update 02915_move_partition_inactive_replica.sql --- .../0_stateless/02915_move_partition_inactive_replica.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql index 4ecbe05547c..81eabce4fc4 100644 --- a/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02915_move_partition_inactive_replica.sql @@ -46,7 +46,7 @@ OPTIMIZE TABLE shard_0.to; -- If moved parts are not merged by OPTIMIZE or background merge restart -- can log Warning about metadata version on disk. It's normal situation -- and test shouldn't rarely fail because of it. -set send_logs_level = 'Error'; +set send_logs_level = 'error'; system restart replica shard_0.to; From 8ccabe9659b8dd455c967b5beaceaf2c3f4b1526 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 3 Sep 2024 16:39:44 +0200 Subject: [PATCH 11/16] Update 02916_another_move_partition_inactive_replica.sql --- .../02916_another_move_partition_inactive_replica.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql index 1b23d9f2815..fa5b6357811 100644 --- a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql @@ -38,7 +38,7 @@ select name, active from system.parts where database='shard_0' and table='to' an -- If moved parts are not merged by OPTIMIZE or background merge restart -- can log Warning about metadata version on disk. It's normal situation -- and test shouldn't rarely fail because of it. -set send_logs_level = 'Error'; +set send_logs_level = 'error'; system restart replica shard_0.to; From fa773f177bcd22b849aa43a15b4a411cd09b3317 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Sep 2024 16:08:13 +0000 Subject: [PATCH 12/16] Polish test --- tests/queries/0_stateless/03231_pr_reverse_in_order.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql index 59e8dbd5578..80f251e0f1f 100644 --- a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql @@ -28,4 +28,4 @@ WHERE 1 AND test_name ilike '%parallel_replicas%' ORDER BY check_start_time desc, check_name, test_name; - +DROP TABLE checks SYNC; From 8967e6f9b88930f077aaf6450e7cb94d449f7620 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Sep 2024 16:11:18 +0000 Subject: [PATCH 13/16] Automatic style fix --- tests/clickhouse-test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c409b4e7632..6cb7d026417 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2147,9 +2147,9 @@ class TestSuite: ) ) self.all_tags: Dict[str, Set[str]] = all_tags_and_random_settings_limits[0] - self.all_random_settings_limits: Dict[ - str, Dict[str, (int, int)] - ] = all_tags_and_random_settings_limits[1] + self.all_random_settings_limits: Dict[str, Dict[str, (int, int)]] = ( + all_tags_and_random_settings_limits[1] + ) self.sequential_tests = [] self.parallel_tests = [] for test_name in self.all_tests: From 5684db7edcb8de0528f11cd6d1cfb241c49e6554 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Sep 2024 17:24:36 +0000 Subject: [PATCH 14/16] Improve the test --- .../03231_pr_reverse_in_order.reference | 1 + .../0_stateless/03231_pr_reverse_in_order.sql | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.reference b/tests/queries/0_stateless/03231_pr_reverse_in_order.reference index 07af27dc224..a1601578f34 100644 --- a/tests/queries/0_stateless/03231_pr_reverse_in_order.reference +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.reference @@ -1,3 +1,4 @@ +ReadType: InReverseOrder 2024-06-11 02:00:01 asan 02221_parallel_replicas_bug FAIL fail 2024-06-11 00:00:01 asan 02221_parallel_replicas_bug FAIL fail 2024-06-10 22:00:01 asan 02221_parallel_replicas_bug FAIL fail diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql index 80f251e0f1f..5927e4f7920 100644 --- a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql @@ -17,7 +17,24 @@ INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime( INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime('2024-06-06 00:00:02') + INTERVAL number HOUR, '02221_parallel_replicas_bug', 'FAIL' from numbers(100); INSERT INTO checks SELECT 'asan', if(number % 2, 'success', 'fail'), toDateTime('2024-06-05 00:00:03') + INTERVAL number HOUR, '02221_parallel_replicas_bug', 'FAIL' from numbers(100); -SET allow_experimental_parallel_reading_from_replicas = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SELECT trimBoth(explain) +FROM +( + EXPLAIN actions=1 SELECT + check_start_time, + check_name, + test_name, + test_status, + check_status + FROM checks + WHERE 1 AND (test_status != 'SKIPPED') AND (test_status != 'OK') AND (check_status != 'success') AND (test_name ILIKE '%parallel_replicas%') + ORDER BY + check_start_time DESC, + check_name ASC, + test_name ASC + SETTINGS query_plan_read_in_order = 1, max_parallel_replicas = 1 +) +WHERE explain LIKE '%InReverseOrder%'; SELECT check_start_time, check_name, test_name, test_status, check_status FROM checks @@ -26,6 +43,7 @@ WHERE 1 AND test_status != 'OK' AND check_status != 'success' AND test_name ilike '%parallel_replicas%' -ORDER BY check_start_time desc, check_name, test_name; +ORDER BY check_start_time desc, check_name, test_name +SETTINGS query_plan_read_in_order = 1, allow_experimental_parallel_reading_from_replicas = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; DROP TABLE checks SYNC; From 6debeafe2e54b9075b9d091b9e003e42112549de Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Sep 2024 19:43:27 +0000 Subject: [PATCH 15/16] Fix --- tests/queries/0_stateless/03231_pr_reverse_in_order.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql index 5927e4f7920..f5a4db44831 100644 --- a/tests/queries/0_stateless/03231_pr_reverse_in_order.sql +++ b/tests/queries/0_stateless/03231_pr_reverse_in_order.sql @@ -32,7 +32,7 @@ FROM check_start_time DESC, check_name ASC, test_name ASC - SETTINGS query_plan_read_in_order = 1, max_parallel_replicas = 1 + SETTINGS query_plan_read_in_order = 1, optimize_read_in_order = 1, max_parallel_replicas = 1 ) WHERE explain LIKE '%InReverseOrder%'; @@ -44,6 +44,6 @@ WHERE 1 AND check_status != 'success' AND test_name ilike '%parallel_replicas%' ORDER BY check_start_time desc, check_name, test_name -SETTINGS query_plan_read_in_order = 1, allow_experimental_parallel_reading_from_replicas = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS query_plan_read_in_order = 1, optimize_read_in_order = 1, allow_experimental_parallel_reading_from_replicas = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; DROP TABLE checks SYNC; From 7caacb7c0c81201c6de1d14925e529b323f007ea Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Tue, 3 Sep 2024 20:57:36 +0000 Subject: [PATCH 16/16] Fix progress bar when reading from Memory tables --- src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp | 5 ++++- src/Storages/StorageMemory.cpp | 3 +++ src/Storages/StorageMemory.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp b/src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp index 6dc0c021a14..538e35ea554 100644 --- a/src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp +++ b/src/Processors/QueryPlan/ReadFromMemoryStorageStep.cpp @@ -173,7 +173,10 @@ Pipe ReadFromMemoryStorageStep::makePipe() for (size_t stream = 0; stream < num_streams; ++stream) { - pipes.emplace_back(std::make_shared(columns_to_read, storage_snapshot, current_data, parallel_execution_index)); + auto source = std::make_shared(columns_to_read, storage_snapshot, current_data, parallel_execution_index); + if (stream == 0) + source->addTotalRowsApprox(snapshot_data.rows_approx); + pipes.emplace_back(std::move(source)); } return Pipe::unitePipes(std::move(pipes)); } diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 42bac783618..be7ec5afae7 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -161,6 +161,9 @@ StorageSnapshotPtr StorageMemory::getStorageSnapshot(const StorageMetadataPtr & { auto snapshot_data = std::make_unique(); snapshot_data->blocks = data.get(); + /// Not guaranteed to match `blocks`, but that's ok. It would probably be better to move + /// rows and bytes counters into the MultiVersion-ed struct, then everything would be consistent. + snapshot_data->rows_approx = total_size_rows.load(std::memory_order_relaxed); if (!hasDynamicSubcolumns(metadata_snapshot->getColumns())) return std::make_shared(*this, metadata_snapshot, ColumnsDescription{}, std::move(snapshot_data)); diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 5d269cf814d..57fccb98e06 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -42,6 +42,7 @@ public: struct SnapshotData : public StorageSnapshot::Data { std::shared_ptr blocks; + size_t rows_approx = 0; }; StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override;