From 5709dc585f35a62251f91fe7a069670ba68f17f3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 4 Sep 2024 15:09:21 +0000 Subject: [PATCH 1/6] Fix: Not-ready Set with parallel replicas --- src/Planner/Planner.cpp | 1 + .../QueryPlan/Optimizations/Optimizations.h | 2 +- .../QueryPlanOptimizationSettings.h | 2 ++ .../QueryPlan/Optimizations/optimizeTree.cpp | 2 +- src/Processors/QueryPlan/QueryPlan.cpp | 3 ++- .../03232_pr_not_ready_set.reference | 0 .../0_stateless/03232_pr_not_ready_set.sql | 18 ++++++++++++++++++ 7 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/03232_pr_not_ready_set.reference create mode 100644 tests/queries/0_stateless/03232_pr_not_ready_set.sql diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 69a652a74a0..7ed16f17087 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -198,6 +198,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr & auto & result_query_plan = planner.getQueryPlan(); auto optimization_settings = QueryPlanOptimizationSettings::fromContext(query_context); + optimization_settings.build_sets = false; // no need to build sets to collect filters result_query_plan.optimize(optimization_settings); FiltersForTableExpressionMap res; diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index c48bdf1552a..43f07ced696 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -16,7 +16,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & settings, Query void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_settings, QueryPlan::Node & root, QueryPlan::Nodes & nodes); /// Third pass is used to apply filters such as key conditions and skip indexes to the storages that support them. /// After that it add CreateSetsStep for the subqueries that has not be used in the filters. -void optimizeTreeThirdPass(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes); +void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes); /// Optimization (first pass) is a function applied to QueryPlan::Node. /// It can read and update subtree of specified node. diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 539ff2eafbb..a2b22495800 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -75,6 +75,8 @@ struct QueryPlanOptimizationSettings String force_projection_name; bool optimize_use_implicit_projections = false; + bool build_sets = true; + static QueryPlanOptimizationSettings fromSettings(const Settings & from); static QueryPlanOptimizationSettings fromContext(ContextPtr from); }; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 25895788e2e..f8504d84d12 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -216,7 +216,7 @@ void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_s optimization_settings.force_projection_name); } -void optimizeTreeThirdPass(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes) +void addStepsToBuildSets(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::Nodes & nodes) { Stack stack; stack.push_back({.node = &root}); diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index b78f7a29cde..9a39df26241 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -504,7 +504,8 @@ void QueryPlan::optimize(const QueryPlanOptimizationSettings & optimization_sett QueryPlanOptimizations::optimizeTreeFirstPass(optimization_settings, *root, nodes); QueryPlanOptimizations::optimizeTreeSecondPass(optimization_settings, *root, nodes); - QueryPlanOptimizations::optimizeTreeThirdPass(*this, *root, nodes); + if (optimization_settings.build_sets) + QueryPlanOptimizations::addStepsToBuildSets(*this, *root, nodes); updateDataStreams(*root); } diff --git a/tests/queries/0_stateless/03232_pr_not_ready_set.reference b/tests/queries/0_stateless/03232_pr_not_ready_set.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03232_pr_not_ready_set.sql b/tests/queries/0_stateless/03232_pr_not_ready_set.sql new file mode 100644 index 00000000000..1a724085903 --- /dev/null +++ b/tests/queries/0_stateless/03232_pr_not_ready_set.sql @@ -0,0 +1,18 @@ +SELECT + is_initial_query, + count() AS c, + replaceRegexpAll(query, '_data_(\\d+)_(\\d+)', '_data_') AS query +FROM system.query_log +WHERE (event_date >= yesterday()) AND (type = 'QueryFinish') AND (ignore(54, 0, ignore('QueryFinish', 11, toLowCardinality(toLowCardinality(11)), 11, 11, 11), 'QueryFinish', materialize(11), toUInt128(11)) IN ( + SELECT query_id + FROM system.query_log + WHERE (current_database = currentDatabase()) AND (event_date >= yesterday()) AND (type = 'QueryFinish') AND (query LIKE '-- Parallel inner query alone%') +)) +GROUP BY + is_initial_query, + query +ORDER BY + is_initial_query ASC, + c ASC, + query ASC +SETTINGS allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree=1, parallel_replicas_min_number_of_rows_per_replica=10; From 2ace0c55eb7e51919aa26422e73c1c6d16722430 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 6 Sep 2024 08:35:39 +0000 Subject: [PATCH 2/6] Increase timeout for intergration test tsan job --- tests/ci/ci_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index cd499aee581..30b2ce5f361 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -410,7 +410,9 @@ class CI: num_batches=6, ), JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_TSAN], num_batches=6 + required_builds=[BuildNames.PACKAGE_TSAN], + num_batches=6, + timeout = 9000 # the job timed out with default value (7200) ), JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], From b3b8abb4133f8bc80aec84b7647e5cafdc8a7bc3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2024 09:13:37 +0000 Subject: [PATCH 3/6] Automatic style fix --- tests/ci/ci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 30b2ce5f361..436fc5b6bb2 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -412,7 +412,7 @@ class CI: JobNames.INTEGRATION_TEST_TSAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN], num_batches=6, - timeout = 9000 # the job timed out with default value (7200) + timeout=9000, # the job timed out with default value (7200) ), JobNames.INTEGRATION_TEST_ARM: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_AARCH64], From 66bd32bd189a55fd7007d4c1fefad4a725fb23cd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:02:29 +0000 Subject: [PATCH 4/6] Bump curl to v8.8.0 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index de7b3e89218..fd567d4f068 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit de7b3e89218467159a7af72d58cea8425946e97d +Subproject commit fd567d4f06857f4fc8e2f64ea727b1318f76ad33 From 1b6fa8ef53a209b1bdffe988180f49a5373f14f4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:03:08 +0000 Subject: [PATCH 5/6] Bump curl to v8.9.0 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index fd567d4f068..5040f7e94cd 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit fd567d4f06857f4fc8e2f64ea727b1318f76ad33 +Subproject commit 5040f7e94cd01decbe7ba8fdacbf489182d503dc From b3f4f3459bf1907eb8c6b92f639206c2df9d2175 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Sep 2024 12:03:53 +0000 Subject: [PATCH 6/6] Bump curl to v8.9.1 --- contrib/curl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/curl b/contrib/curl index 5040f7e94cd..83bedbd730d 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit 5040f7e94cd01decbe7ba8fdacbf489182d503dc +Subproject commit 83bedbd730d62b83744cc26fa0433d3f6e2e4cd6