From 101217470e5a6414881a5d766764d0bb38c09932 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 1 Aug 2020 02:21:33 +0300 Subject: [PATCH 1/6] Use "Not using primary index on part" over "Not using index on part" (add "primary") --- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index acd4602f3a6..dc32e225697 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1343,7 +1343,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( /// If index is not used. if (key_condition.alwaysUnknownOrTrue()) { - LOG_TRACE(log, "Not using index on part {}", part->name); + LOG_TRACE(log, "Not using primary index on part {}", part->name); if (has_final_mark) res.push_back(MarkRange(0, marks_count - 1)); From e37c42c56cba01de045b30035d02871a21bcee0f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 1 Aug 2020 02:55:08 +0300 Subject: [PATCH 2/6] Fix logging in MergeTreeDataSelectExecutor for multiple threads (attach to thread group) --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index dc32e225697..7ac8ca6143b 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1,4 +1,5 @@ #include /// For calculations related to sampling coefficients. +#include #include #include @@ -613,7 +614,16 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( ThreadPool pool(num_threads); for (size_t part_index = 0; part_index < parts.size(); ++part_index) - pool.scheduleOrThrowOnError([&, part_index] { process_part(part_index); }); + pool.scheduleOrThrowOnError([&, part_index, thread_group = CurrentThread::getGroup()] { + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + if (thread_group) + CurrentThread::attachTo(thread_group); + + process_part(part_index); + }); pool.wait(); } From 43af55b9a109279fea3147a95f9fef26824fdca0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 1 Aug 2020 11:44:11 +0300 Subject: [PATCH 3/6] Fix 01323_too_many_threads_bug (use arrayUniq() over length()) --- tests/queries/0_stateless/01323_too_many_threads_bug.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index 62758e17b97..8c5e3793c5c 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -9,11 +9,11 @@ set log_queries = 1; select x from table_01323_many_parts limit 10 format Null; system flush logs; -select length(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; select x from table_01323_many_parts order by x limit 10 format Null; system flush logs; -select length(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; drop table if exists table_01323_many_parts; From 1fe329ac1874d902981e90d1731b06b5729442e5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 2 Aug 2020 17:37:24 +0300 Subject: [PATCH 4/6] Use ILIKE over lower() LIKE in 01323_too_many_threads_bug --- tests/queries/0_stateless/01323_too_many_threads_bug.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index 8c5e3793c5c..1c6a7f13dea 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -9,11 +9,11 @@ set log_queries = 1; select x from table_01323_many_parts limit 10 format Null; system flush logs; -select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; select x from table_01323_many_parts order by x limit 10 format Null; system flush logs; -select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and lower(query) like '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; drop table if exists table_01323_many_parts; From f0aeba432d779ae624de84773a7a4ec12145341c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 2 Aug 2020 17:38:24 +0300 Subject: [PATCH 5/6] Exclude threads used for query_log processing 01323_too_many_threads_bug --- tests/queries/0_stateless/01323_too_many_threads_bug.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index 1c6a7f13dea..80fc2283239 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -9,11 +9,11 @@ set log_queries = 1; select x from table_01323_many_parts limit 10 format Null; system flush logs; -select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts%' and query not like '%system.query_log%' and type = 'QueryFinish' order by query_start_time desc limit 1; select x from table_01323_many_parts order by x limit 10 format Null; system flush logs; -select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts order by x%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts order by x%' and query not like '%system.query_log%' and type = 'QueryFinish' order by query_start_time desc limit 1; drop table if exists table_01323_many_parts; From daa73c59413281886f3bb6acc891813a8817ad17 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 2 Aug 2020 18:41:33 +0300 Subject: [PATCH 6/6] Add more threads for the 01323_too_many_threads_bug Since now PK stage is handled in multiple threads there should be more (and counting number of threads from the log is not accurate, since not all threads logging something). --- tests/queries/0_stateless/01323_too_many_threads_bug.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index 80fc2283239..1c70831c293 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -14,6 +14,6 @@ select arrayUniq(thread_ids) <= 4 from system.query_log where event_date >= toda select x from table_01323_many_parts order by x limit 10 format Null; system flush logs; -select arrayUniq(thread_ids) <= 20 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts order by x%' and query not like '%system.query_log%' and type = 'QueryFinish' order by query_start_time desc limit 1; +select arrayUniq(thread_ids) <= 36 from system.query_log where event_date >= today() - 1 and query ilike '%select x from table_01323_many_parts order by x%' and query not like '%system.query_log%' and type = 'QueryFinish' order by query_start_time desc limit 1; drop table if exists table_01323_many_parts;