From adfe966c465e46acf77d387231ace671fe302aa3 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 11 Nov 2020 12:30:32 +0300 Subject: [PATCH 01/45] Support HTTP proxy and HTTPS S3 endpoint configuration. --- src/IO/S3/PocoHTTPClient.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 49ccb6dc1b3..00cc5f497dc 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -168,12 +168,16 @@ void PocoHTTPClient::makeRequestInternal( auto request_configuration = per_request_configuration(request); if (!request_configuration.proxyHost.empty()) + { + /// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS. + bool use_tunnel = request_configuration.proxyScheme == Aws::Http::Scheme::HTTP && poco_uri.getScheme() == "https"; session->setProxy( request_configuration.proxyHost, request_configuration.proxyPort, Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme), - false /// Disable proxy tunneling by default + use_tunnel ); + } Poco::Net::HTTPRequest poco_request(Poco::Net::HTTPRequest::HTTP_1_1); From 766883a0e381b2db352dc6de5170749f573cae04 Mon Sep 17 00:00:00 2001 From: satanson Date: Fri, 13 Nov 2020 00:47:59 +0800 Subject: [PATCH 02/45] A very fast O(logn) repeat impl --- src/Functions/repeat.cpp | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index 63af62af9af..347402a6ac8 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -124,12 +124,39 @@ struct RepeatImpl } private: + // A very fast repeat implementation, only invoke memcpy for O(log(n)) times. + // as the calling times decreases, more data will be copied for each memcpy, thus + // SIMD optimization will be more efficient. static void process(const UInt8 * src, UInt8 * dst, UInt64 size, UInt64 repeat_time) { - for (UInt64 i = 0; i < repeat_time; ++i) + if (BOOST_UNLIKELY(repeat_time <= 0)) { - memcpy(dst, src, size - 1); - dst += size - 1; + *dst = 0; + return; + } + + size -= 1; + UInt64 k = 0; + UInt64 last_bit = repeat_time & 1; + repeat_time >>= 1; + + const UInt8 * dst_hdr = dst; + memcpy(dst, src, size); + dst += size; + + while (repeat_time > 0) + { + UInt64 cpy_size = size * (1 << k); + memcpy(dst, dst_hdr, cpy_size); + dst += cpy_size; + if (last_bit) + { + memcpy(dst, dst_hdr, cpy_size); + dst += cpy_size; + } + k += 1; + last_bit = repeat_time & 1; + repeat_time >>= 1; } *dst = 0; } From 3dd4f2fb1e37f21c8244ede8280655034e642125 Mon Sep 17 00:00:00 2001 From: satanson Date: Fri, 13 Nov 2020 15:21:08 +0800 Subject: [PATCH 03/45] use unlikely instead of BOOST_UNLIKELY --- src/Functions/repeat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index 347402a6ac8..d3ae646df16 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -129,7 +129,7 @@ private: // SIMD optimization will be more efficient. static void process(const UInt8 * src, UInt8 * dst, UInt64 size, UInt64 repeat_time) { - if (BOOST_UNLIKELY(repeat_time <= 0)) + if (unlikely(repeat_time <= 0)) { *dst = 0; return; From 39fa68842074e4746648ca9b58f549bc3e55c534 Mon Sep 17 00:00:00 2001 From: satanson Date: Sat, 14 Nov 2020 22:33:08 +0800 Subject: [PATCH 04/45] use 1ULL instead of 1 in shift operation --- src/Functions/repeat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index d3ae646df16..059c6c0fc0b 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -146,7 +146,7 @@ private: while (repeat_time > 0) { - UInt64 cpy_size = size * (1 << k); + UInt64 cpy_size = size * (1ULL << k); memcpy(dst, dst_hdr, cpy_size); dst += cpy_size; if (last_bit) From 35231662b30d7ae1f4ed7b093735599a21b37d64 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 16 Nov 2020 21:50:25 +0300 Subject: [PATCH 05/45] Improve performance of AggregatingMergeTree w/ SimpleAggregateFunction(String) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While reading from AggregatingMergeTree with SimpleAggregateFunction(String) in primary key and optimize_aggregation_in_order perf top shows: Samples: 1M of event 'cycles', 4000 Hz, Event count (approx.): 287759760270 lost: 0/0 drop: 0/0 Children Self Shared Object Symbol + 12.64% 11.39% clickhouse [.] memcpy + 9.08% 0.23% [unknown] [.] 0000000000000000 + 8.45% 8.40% clickhouse [.] ProfileEvents::increment # <-- this, and in debug it has not 0.08x overhead, but 5.8x overhead + 7.68% 7.67% clickhouse [.] LZ4_compress_fast_extState + 5.29% 5.22% clickhouse [.] DB::IAggregateFunctionHelper >::addFree The reason is obvious, ProfileEvents is atomic counters (and also they are nested):
``` Samples: 7M of event 'cycles', 4000 Hz, Event count (approx.): 450726149337 ProfileEvents::increment /usr/bin/clickhouse [Percent: local period] Percent│ │ │ │ Disassembly of section .text: │ │ 00000000078d8900 : │ ProfileEvents::increment(unsigned long, unsigned long): 0.17 │ push %rbp 0.00 │ mov %rsi,%rbp 0.04 │ push %rbx 0.20 │ mov %rdi,%rbx 0.17 │ sub $0x8,%rsp 0.26 │ → callq DB::CurrentThread::getProfileEvents │ ProfileEvents::Counters::increment(unsigned long, unsigned long): 0.00 │ lea 0x0(,%rbx,8),%rdi 0.05 │ nop │ unsigned long std::__1::__cxx_atomic_fetch_add(std::__1::__cxx_atomic_base_impl*, unsigned long, std::__1::memory_order): 1.02 │ mov (%rax),%rdx 97.04 │ lock add %rbp,(%rdx,%rdi,1) │ ProfileEvents::Counters::increment(unsigned long, unsigned long): 0.21 │ mov 0x10(%rax),%rax 0.04 │ test %rax,%rax 0.00 │ → jne 78d8920 │ ProfileEvents::increment(unsigned long, unsigned long): 0.38 │ add $0x8,%rsp 0.00 │ pop %rbx 0.04 │ pop %rbp 0.38 │ ← retq ```
These ProfileEvents was ArenaAllocChunks (it shows ~1.5M events per second), and the reason is that the table has SimpleAggregateFunction(String) in PK, which requires Arena. But most of the time there Arena wasn't even used, so avoid this cost by re-creating Arena only if it was "used" (i.e. has new chunks). Another possibility is to avoid populating Arena::head in ctor, but this will make the Arena code more complex, so for now this was preferred. Also as a long-term solution it worth looking at implementing them via RCU (to move the extra overhead out from the write code path into read side). --- .../Algorithms/AggregatingSortedAlgorithm.cpp | 20 ++++++++++++++++-- .../Algorithms/AggregatingSortedAlgorithm.h | 1 + ..._tree_simple_aggregate_function_string.xml | 21 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp index b834ed82729..6b4746b1320 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp @@ -195,7 +195,14 @@ AggregatingSortedAlgorithm::AggregatingMergedData::AggregatingMergedData( MutableColumns columns_, UInt64 max_block_size_, ColumnsDefinition & def_) : MergedData(std::move(columns_), false, max_block_size_), def(def_) { - initAggregateDescription(); + initAggregateDescription(); + + /// Just to make startGroup() simpler. + if (def.allocates_memory_in_arena) + { + arena = std::make_unique(); + arena_size = arena->size(); + } } void AggregatingSortedAlgorithm::AggregatingMergedData::startGroup(const ColumnRawPtrs & raw_columns, size_t row) @@ -212,8 +219,17 @@ void AggregatingSortedAlgorithm::AggregatingMergedData::startGroup(const ColumnR for (auto & desc : def.columns_to_simple_aggregate) desc.createState(); - if (def.allocates_memory_in_arena) + /// If and only if: + /// - arena is required (i.e. SimpleAggregateFunction(any, String) in PK) + /// - arena was used since otherwise it may be too costly to increment atomic counters inside Arena. + /// i.e. SELECT with SimpleAggregateFunction(String) in PK and lots of groups + /// may produce ~1.5M of ArenaAllocChunks atomic increments, + /// while LOCK is too costly for CPU (~10% overhead here). + if (def.allocates_memory_in_arena && arena->size() > arena_size) + { arena = std::make_unique(); + arena_size = arena->size(); + } is_group_started = true; } diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h index da4ec876b69..e572ed7d526 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.h @@ -73,6 +73,7 @@ private: /// Memory pool for SimpleAggregateFunction /// (only when allocates_memory_in_arena == true). std::unique_ptr arena; + size_t arena_size = 0; bool is_group_started = false; diff --git a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml new file mode 100644 index 00000000000..d62f8d8e088 --- /dev/null +++ b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml @@ -0,0 +1,21 @@ + + + CREATE TABLE bench + ENGINE = AggregatingMergeTree() + ORDER BY key + SETTINGS index_granularity = 8192 + AS + SELECT CAST(reinterpretAsString(number), 'SimpleAggregateFunction(any, String)') AS key + FROM numbers_mt(toUInt64(10e6)) + + + + SELECT * + FROM bench + GROUP BY key + SETTINGS optimize_aggregation_in_order = 1, max_threads = 16 + FORMAT Null + + + DROP TABLE IF EXISTS bench + From a3116d5614a0a1403c85e68095ff69caabbead50 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 21 Nov 2020 12:08:59 +0300 Subject: [PATCH 06/45] Tune aggregating_merge_tree_simple_aggregate_function_string to make it faster --- ...gregating_merge_tree_simple_aggregate_function_string.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml index d62f8d8e088..c7c005d2093 100644 --- a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml +++ b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml @@ -6,9 +6,12 @@ SETTINGS index_granularity = 8192 AS SELECT CAST(reinterpretAsString(number), 'SimpleAggregateFunction(any, String)') AS key - FROM numbers_mt(toUInt64(10e6)) + FROM numbers_mt(toUInt64(5e6)) + SETTINGS max_insert_threads = 16 + SYSTEM STOP MERGES + SELECT * FROM bench From 99343bc1270ffa69706b6ee58a12bf083da8dcb6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 11:28:25 +0300 Subject: [PATCH 07/45] Add constants to ClientInfo::Interface for MySQL and PostgreSQL protocols. --- src/Interpreters/ClientInfo.h | 2 ++ src/Server/MySQLHandler.cpp | 1 + src/Server/PostgreSQLHandler.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 234b6329ff4..c66903a38cf 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -26,6 +26,8 @@ public: TCP = 1, HTTP = 2, GRPC = 3, + MYSQL = 4, + POSTGRESQL = 5, }; enum class HTTPMethod : uint8_t diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 957930ad43b..89cd055a848 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -87,6 +87,7 @@ MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & so void MySQLHandler::run() { connection_context.makeSessionContext(); + connection_context.getClientInfo().interface = ClientInfo::Interface::MYSQL; connection_context.setDefaultFormat("MySQLWire"); in = std::make_shared(socket()); diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 0c5c8ffe407..2bce5abcd11 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -50,6 +50,7 @@ void PostgreSQLHandler::changeIO(Poco::Net::StreamSocket & socket) void PostgreSQLHandler::run() { connection_context.makeSessionContext(); + connection_context.getClientInfo().interface = ClientInfo::Interface::POSTGRESQL; connection_context.setDefaultFormat("PostgreSQLWire"); try From 4cc6594cb82892ea19eac4000a22a309ed9781e7 Mon Sep 17 00:00:00 2001 From: felixxdu Date: Thu, 26 Nov 2020 17:37:42 +0800 Subject: [PATCH 08/45] Support configure Ssystem log table's ttl in config.xml --- programs/server/config.xml | 3 +++ src/Interpreters/SystemLog.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/programs/server/config.xml b/programs/server/config.xml index cef191a71f1..cd38901da59 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -589,6 +589,9 @@ toStartOfHour(event_time) --> toYYYYMM(event_date) + + 30 From 6590fdfdd27cb22411374d1e126364aac28d39ae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 30 Nov 2020 11:44:01 +0300 Subject: [PATCH 18/45] Fix logging --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 4184c012aca..9719df9726f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1140,11 +1140,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { const char * format_str = "Not executing log entry {} for part {}" " because {} merges with TTL already executing, maximum {}."; - LOG_DEBUG(log, format_str, entry.znode_name, - entry.new_part_name, total_merges_with_ttl, + LOG_DEBUG(log, format_str, + entry.znode_name, entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); - out_postpone_reason = fmt::format(format_str, entry.new_part_name, total_merges_with_ttl, + out_postpone_reason = fmt::format(format_str, + entry.znode_name, entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); return false; } From d9dd9341ce7c88f8ae64214a22e8d80704160338 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 30 Nov 2020 12:11:17 +0300 Subject: [PATCH 19/45] Fix the OpenTelemetry test --- src/Interpreters/OpenTelemetrySpanLog.cpp | 10 ---------- src/Interpreters/ThreadStatusExt.cpp | 5 ----- tests/queries/shell_config.sh | 4 +++- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index de82f8a4d01..e1df145cf51 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -90,11 +90,6 @@ OpenTelemetrySpanHolder::OpenTelemetrySpanHolder(const std::string & _operation_ start_time_us = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); -#ifndef NDEBUG - attribute_names.push_back("clickhouse.start.stacktrace"); - attribute_values.push_back(StackTrace().toString()); -#endif - thread.thread_trace_context.span_id = span_id; } @@ -130,11 +125,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() return; } -#ifndef NDEBUG - attribute_names.push_back("clickhouse.end.stacktrace"); - attribute_values.push_back(StackTrace().toString()); -#endif - auto log = context->getOpenTelemetrySpanLog(); if (!log) { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index d698e8d103b..f6ee2c571d1 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -356,11 +356,6 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) span.attribute_names.push_back("clickhouse.thread_id"); span.attribute_values.push_back(thread_id); -#ifndef NDEBUG - span.attribute_names.push_back("clickhouse.end.stacktrace"); - span.attribute_values.push_back(StackTrace().toString()); -#endif - opentelemetry_span_log->add(span); } diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 9cfec2c2a28..88ff59c5084 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -70,7 +70,9 @@ export CLICKHOUSE_PORT_INTERSERVER=${CLICKHOUSE_PORT_INTERSERVER:="9009"} export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_INTERSERVER}/"} export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} -export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} +# The queries in CI are prone to sudden delays, and we often don't check for curl +# errors, so it makes sense to set a relatively generous timeout. +export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="60"} export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} From 8fd0810142aae494e4152c2b16b2ae61284f6913 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 30 Nov 2020 12:31:30 +0300 Subject: [PATCH 20/45] Update aggregating_merge_tree_simple_aggregate_function_string.xml `system stop merges` w/o table name has global effect, so the rest of the tests is affected. Also `optimize` is more suitable here so that the end result is the same every time. --- .../aggregating_merge_tree_simple_aggregate_function_string.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml index c7c005d2093..c12f26ad595 100644 --- a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml +++ b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml @@ -10,7 +10,7 @@ SETTINGS max_insert_threads = 16 - SYSTEM STOP MERGES + OPTIMIZE TABLE bench SELECT * From 5b9d9376c3b4dcbf0bf7b68dbbdd15f2b9bb76be Mon Sep 17 00:00:00 2001 From: taichong Date: Mon, 30 Nov 2020 16:38:32 +0800 Subject: [PATCH 21/45] add integration test: ClickHouse killed while insert for MaterializeMySQL ENGINE --- .../materialize_with_ddl.py | 25 +++++++++++++++++++ .../test_materialize_mysql_database/test.py | 8 ++++++ 2 files changed, 33 insertions(+) diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 7e8688fab0a..311b76d5b9f 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -652,3 +652,28 @@ def mysql_kill_sync_thread_restore_test(clickhouse_node, mysql_node, service_nam mysql_node.query("DROP DATABASE test_database") +def clickhouse_killed_while_insert(clickhouse_node, mysql_node, service_name): + mysql_node.query("CREATE DATABASE kill_clickhouse_while_insert") + mysql_node.query("CREATE TABLE kill_clickhouse_while_insert.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + clickhouse_node.query("CREATE DATABASE kill_clickhouse_while_insert ENGINE = MaterializeMySQL('{}:3306', 'kill_clickhouse_while_insert', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "SHOW TABLES FROM kill_clickhouse_while_insert FORMAT TSV", 'test\n') + + def insert(num): + for i in range(num): + query = "INSERT INTO kill_clickhouse_while_insert.test VALUES({v});".format( v = i + 1 ) + mysql_node.query(query) + + t = threading.Thread(target=insert, args=(1000,)) + t.start() + + # TODO: add clickhouse_node.restart_clickhouse(20, kill=False) test + clickhouse_node.restart_clickhouse(20, kill=True) + t.join() + + result = mysql_node.query_and_get_data("SELECT COUNT(1) FROM kill_clickhouse_while_insert.test") + for row in result: + res = str(row[0]) + '\n' + check_query(clickhouse_node, "SELECT count() FROM kill_clickhouse_while_insert.test FORMAT TSV", res) + + mysql_node.query("DROP DATABASE kill_clickhouse_while_insert") + clickhouse_node.query("DROP DATABASE kill_clickhouse_while_insert") diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index 5d23cddcf69..fb481d3cec1 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -210,3 +210,11 @@ def test_mysql_kill_sync_thread_restore_5_7(started_cluster, started_mysql_5_7): def test_mysql_kill_sync_thread_restore_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.mysql_kill_sync_thread_restore_test(clickhouse_node, started_mysql_8_0, "mysql8_0") + + +def test_clickhouse_killed_while_insert_5_7(started_cluster, started_mysql_5_7): + materialize_with_ddl.clickhouse_killed_while_insert(clickhouse_node, started_mysql_5_7, "mysql1") + +def test_clickhouse_killed_while_insert_8_0(started_cluster, started_mysql_8_0): + materialize_with_ddl.clickhouse_killed_while_insert(clickhouse_node, started_mysql_8_0, "mysql8_0") + From 1037a994a8ea0e9a7710150a39558d4b86d3b0de Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 30 Nov 2020 15:53:37 +0300 Subject: [PATCH 22/45] done --- tests/queries/0_stateless/01322_ttest_scipy.reference | 1 - tests/queries/0_stateless/arcadia_skip_list.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 tests/queries/0_stateless/01322_ttest_scipy.reference diff --git a/tests/queries/0_stateless/01322_ttest_scipy.reference b/tests/queries/0_stateless/01322_ttest_scipy.reference deleted file mode 100644 index 587579af915..00000000000 --- a/tests/queries/0_stateless/01322_ttest_scipy.reference +++ /dev/null @@ -1 +0,0 @@ -Ok. diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 40959ac2606..086017ec038 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -174,3 +174,5 @@ 01584_distributed_buffer_cannot_find_column 01018_ip_dictionary 00976_ttl_with_old_parts +01558_ttest_scipy +01561_mann_whitney_scipy From 234c5caa084652009df1f582fb85300e323c76cc Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 30 Nov 2020 17:02:03 +0300 Subject: [PATCH 23/45] Update CHANGELOG.md --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355c664664d..c722e4a1ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * Remove `ANALYZE` and `AST` queries, and make the setting `enable_debug_queries` obsolete since now it is the part of full featured `EXPLAIN` query. [#16536](https://github.com/ClickHouse/ClickHouse/pull/16536) ([Ivan](https://github.com/abyss7)). * Aggregate functions `boundingRatio`, `rankCorr`, `retention`, `timeSeriesGroupSum`, `timeSeriesGroupRateSum`, `windowFunnel` were erroneously made case-insensitive. Now their names are made case sensitive as designed. Only functions that are specified in SQL standard or made for compatibility with other DBMS or functions similar to those should be case-insensitive. [#16407](https://github.com/ClickHouse/ClickHouse/pull/16407) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Make `rankCorr` function return nan on insufficient data https://github.com/ClickHouse/ClickHouse/issues/16124. [#16135](https://github.com/ClickHouse/ClickHouse/pull/16135) ([hexiaoting](https://github.com/hexiaoting)). +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). #### New Feature @@ -154,6 +155,7 @@ * Change default value of `format_regexp_escaping_rule` setting (it's related to `Regexp` format) to `Raw` (it means - read whole subpattern as a value) to make the behaviour more like to what users expect. [#15426](https://github.com/ClickHouse/ClickHouse/pull/15426) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Add support for nested multiline comments `/* comment /* comment */ */` in SQL. This conforms to the SQL standard. [#14655](https://github.com/ClickHouse/ClickHouse/pull/14655) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Added MergeTree settings (`max_replicated_merges_with_ttl_in_queue` and `max_number_of_merges_with_ttl_in_pool`) to control the number of merges with TTL in the background pool and replicated queue. This change breaks compatibility with older versions only if you use delete TTL. Otherwise, replication will stay compatible. You can avoid incompatibility issues if you update all shard replicas at once or execute `SYSTEM STOP TTL MERGES` until you finish the update of all replicas. If you'll get an incompatible entry in the replication queue, first of all, execute `SYSTEM STOP TTL MERGES` and after `ALTER TABLE ... DETACH PARTITION ...` the partition where incompatible TTL merge was assigned. Attach it back on a single replica. [#14490](https://github.com/ClickHouse/ClickHouse/pull/14490) ([alesapin](https://github.com/alesapin)). +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). #### New Feature @@ -438,6 +440,10 @@ ### ClickHouse release v20.9.2.20, 2020-09-22 +#### Backward Incompatible Change + +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). + #### New Feature * Added column transformers `EXCEPT`, `REPLACE`, `APPLY`, which can be applied to the list of selected columns (after `*` or `COLUMNS(...)`). For example, you can write `SELECT * EXCEPT(URL) REPLACE(number + 1 AS number)`. Another example: `select * apply(length) apply(max) from wide_string_table` to find out the maxium length of all string columns. [#14233](https://github.com/ClickHouse/ClickHouse/pull/14233) ([Amos Bird](https://github.com/amosbird)). @@ -621,6 +627,7 @@ * Now `OPTIMIZE FINAL` query doesn't recalculate TTL for parts that were added before TTL was created. Use `ALTER TABLE ... MATERIALIZE TTL` once to calculate them, after that `OPTIMIZE FINAL` will evaluate TTL's properly. This behavior never worked for replicated tables. [#14220](https://github.com/ClickHouse/ClickHouse/pull/14220) ([alesapin](https://github.com/alesapin)). * Extend `parallel_distributed_insert_select` setting, adding an option to run `INSERT` into local table. The setting changes type from `Bool` to `UInt64`, so the values `false` and `true` are no longer supported. If you have these values in server configuration, the server will not start. Please replace them with `0` and `1`, respectively. [#14060](https://github.com/ClickHouse/ClickHouse/pull/14060) ([Azat Khuzhin](https://github.com/azat)). * Remove support for the `ODBCDriver` input/output format. This was a deprecated format once used for communication with the ClickHouse ODBC driver, now long superseded by the `ODBCDriver2` format. Resolves [#13629](https://github.com/ClickHouse/ClickHouse/issues/13629). [#13847](https://github.com/ClickHouse/ClickHouse/pull/13847) ([hexiaoting](https://github.com/hexiaoting)). +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). #### New Feature @@ -765,6 +772,7 @@ * The function `groupArrayMoving*` was not working for distributed queries. It's result was calculated within incorrect data type (without promotion to the largest type). The function `groupArrayMovingAvg` was returning integer number that was inconsistent with the `avg` function. This fixes [#12568](https://github.com/ClickHouse/ClickHouse/issues/12568). [#12622](https://github.com/ClickHouse/ClickHouse/pull/12622) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Add sanity check for MergeTree settings. If the settings are incorrect, the server will refuse to start or to create a table, printing detailed explanation to the user. [#13153](https://github.com/ClickHouse/ClickHouse/pull/13153) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Protect from the cases when user may set `background_pool_size` to value lower than `number_of_free_entries_in_pool_to_execute_mutation` or `number_of_free_entries_in_pool_to_lower_max_size_of_merge`. In these cases ALTERs won't work or the maximum size of merge will be too limited. It will throw exception explaining what to do. This closes [#10897](https://github.com/ClickHouse/ClickHouse/issues/10897). [#12728](https://github.com/ClickHouse/ClickHouse/pull/12728) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). #### New Feature @@ -951,6 +959,10 @@ ### ClickHouse release v20.6.3.28-stable +#### Backward Incompatible Change + +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). + #### New Feature * Added an initial implementation of `EXPLAIN` query. Syntax: `EXPLAIN SELECT ...`. This fixes [#1118](https://github.com/ClickHouse/ClickHouse/issues/1118). [#11873](https://github.com/ClickHouse/ClickHouse/pull/11873) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). @@ -1139,6 +1151,7 @@ * Update `zstd` to 1.4.4. It has some minor improvements in performance and compression ratio. If you run replicas with different versions of ClickHouse you may see reasonable error messages `Data after merge is not byte-identical to data on another replicas.` with explanation. These messages are Ok and you should not worry. This change is backward compatible but we list it here in changelog in case you will wonder about these messages. [#10663](https://github.com/ClickHouse/ClickHouse/pull/10663) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Added a check for meaningless codecs and a setting `allow_suspicious_codecs` to control this check. This closes [#4966](https://github.com/ClickHouse/ClickHouse/issues/4966). [#10645](https://github.com/ClickHouse/ClickHouse/pull/10645) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Several Kafka setting changes their defaults. See [#11388](https://github.com/ClickHouse/ClickHouse/pull/11388). +* When upgrading from versions older than 20.5, if rolling update is performed and cluster contains both versions 20.5 or greater and less than 20.5, if ClickHouse nodes with old versions are restarted and old version has been started up in presence of newer versions, it may lead to `Part ... intersects previous part` errors. To prevent this error, first install newer clickhouse-server packages on all cluster nodes and then do restarts (so, when clickhouse-server is restarted, it will start up with the new version). #### New Feature From fe00f704248a1176129220fe5134eeb1b0f041b5 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Mon, 30 Nov 2020 18:09:46 +0300 Subject: [PATCH 24/45] DOCSUP-4194: Updated CREATE TABLE statement with PRIMARY KEY section (#17213) * Section added * Syntax corrected Co-authored-by: Olga Revyakina --- .../sql-reference/statements/create/table.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 82326bf51cf..e9952fc76fd 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -29,6 +29,8 @@ A column description is `name type` in the simplest case. Example: `RegionID UIn Expressions can also be defined for default values (see below). +If necessary, primary key can be specified, with one or more key expressions. + ### With a Schema Similar to Other Table {#with-a-schema-similar-to-other-table} ``` sql @@ -97,6 +99,34 @@ If you add a new column to a table but later change its default expression, the It is not possible to set default values for elements in nested data structures. +## Primary Key {#primary-key} + +You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: + +- inside the column list + +``` sql +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ..., + PRIMARY KEY(expr1[, expr2,...])] +) +ENGINE = engine; +``` + +- outside the column list + +``` sql +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ... +) +ENGINE = engine +PRIMARY KEY(expr1[, expr2,...]); +``` + +You can't combine both ways in one query. + ## Constraints {#constraints} Along with columns descriptions constraints could be defined: From fe7c3cb865c8c504502766d5f4620cdf31dd7259 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 30 Nov 2020 18:31:02 +0300 Subject: [PATCH 25/45] fix race on is_circular_replicated --- src/Interpreters/DDLWorker.cpp | 6 +++--- src/Interpreters/DDLWorker.h | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 1ba1fc0cb0d..57bbf95bdbe 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -186,6 +186,7 @@ struct DDLTask Cluster::Address address_in_cluster; size_t host_shard_num; size_t host_replica_num; + bool is_circular_replicated = false; /// Stage 3.3: execute query ExecutionStatus execution_status; @@ -594,7 +595,7 @@ void DDLWorker::parseQueryAndResolveHost(DDLTask & task) * To distinguish one replica from another on the same node, * every shard is placed into separate database. * */ - is_circular_replicated = true; + task.is_circular_replicated = true; auto * query_with_table = dynamic_cast(task.query.get()); if (!query_with_table || query_with_table->database.empty()) { @@ -770,7 +771,6 @@ void DDLWorker::processTask(DDLTask & task) { try { - is_circular_replicated = false; parseQueryAndResolveHost(task); ASTPtr rewritten_ast = task.query_on_cluster->getRewrittenASTWithoutOnCluster(task.address_in_cluster.default_database); @@ -787,7 +787,7 @@ void DDLWorker::processTask(DDLTask & task) storage = DatabaseCatalog::instance().tryGetTable(table_id, context); } - if (storage && taskShouldBeExecutedOnLeader(rewritten_ast, storage) && !is_circular_replicated) + if (storage && taskShouldBeExecutedOnLeader(rewritten_ast, storage) && !task.is_circular_replicated) tryExecuteQueryOnLeaderReplica(task, storage, rewritten_query, task.entry_path, zookeeper); else tryExecuteQuery(rewritten_query, task, task.execution_status); diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 75d0ce3f5a4..7dd9c38e9da 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -104,7 +104,6 @@ private: void attachToThreadGroup(); private: - std::atomic is_circular_replicated = false; Context context; Poco::Logger * log; From 9e995d97568a1752c91bca11e0174fbea6bb3bbe Mon Sep 17 00:00:00 2001 From: spongedc Date: Mon, 30 Nov 2020 23:32:17 +0800 Subject: [PATCH 26/45] 1.make ttl option more generic 2. add check if engine specified for system log --- programs/server/config.xml | 11 ++++++++--- src/Interpreters/SystemLog.cpp | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 8193d1f5c79..5c6cf5b30fd 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -581,7 +581,7 @@ system query_log
toYYYYMM(event_date) - - 30 + event_date + INTERVAL 30 DAY DELETE + event_date + INTERVAL 30 DAY DELETE + --> +[Оригинальная статья](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/)