From 9482cf24696dda8f2c02cc4c0511425f1ac17b94 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 23 Dec 2019 19:49:41 +0300 Subject: [PATCH 1/6] Enable processors by default. --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 5a8af895610..4060a8b6527 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -367,7 +367,7 @@ struct Settings : public SettingsCollection M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.", 0) \ \ - M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \ + M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \ \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ From a82079ccc27cebbbade7cf4d3b586b8999521937 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 23 Dec 2019 19:55:11 +0300 Subject: [PATCH 2/6] Set affinity for PipelineExecutor threads. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 9013b83486a..315e20247df 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace DB { @@ -452,6 +454,14 @@ void PipelineExecutor::execute(size_t num_threads) void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) { + /// Specify CPU core for thread if can. + /// It may reduce the number of context swithches. + cpu_set_t cpu_set; + CPU_ZERO(&cpu_set); + CPU_SET(thread_num, &cpu_set); + if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set) == -1) + LOG_TRACE(log, "Cannot set affinity for thread " << num_threads); + UInt64 total_time_ns = 0; UInt64 execution_time_ns = 0; UInt64 processing_time_ns = 0; From e53b68d00f4282eb503a53897b68e8799cfa37d1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 24 Dec 2019 16:55:28 +0300 Subject: [PATCH 3/6] Fix apple build. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 315e20247df..98ace0a7edb 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -454,6 +454,7 @@ void PipelineExecutor::execute(size_t num_threads) void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) { +#if !defined(__APPLE__) && !defined(__FreeBSD__) /// Specify CPU core for thread if can. /// It may reduce the number of context swithches. cpu_set_t cpu_set; @@ -462,6 +463,8 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set) == -1) LOG_TRACE(log, "Cannot set affinity for thread " << num_threads); +#endif + UInt64 total_time_ns = 0; UInt64 execution_time_ns = 0; UInt64 processing_time_ns = 0; From 2148f09220fd17ce4b86c61dad9c9d00aa39f381 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 24 Dec 2019 20:00:43 +0300 Subject: [PATCH 4/6] Disable affinity for apple and FreeBSD. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 98ace0a7edb..6addec11975 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -11,7 +11,9 @@ #include #include +#if !defined(__APPLE__) && !defined(__FreeBSD__) #include +#endif namespace DB { From 5b2847fb90c2690b7c48877c36beaf255e8eab93 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 25 Dec 2019 21:46:37 +0300 Subject: [PATCH 5/6] Fix mysql_protocol integration tests. --- dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index f913087da9b..d40dea42248 100644 --- a/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -50,6 +50,9 @@ void MySQLOutputFormat::initialize() void MySQLOutputFormat::consume(Chunk chunk) { + + initialize(); + for (size_t i = 0; i < chunk.getNumRows(); i++) { ProtocolText::ResultsetRow row_packet(data_types, chunk.getColumns(), i); From 348b13af8cfe736103fa2952ef2af16d0ceaf3f4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 26 Dec 2019 12:13:14 +0300 Subject: [PATCH 6/6] Disable processors by default. --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 4060a8b6527..5a8af895610 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -367,7 +367,7 @@ struct Settings : public SettingsCollection M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.", 0) \ \ - M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \ + M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \ \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \