From 0f20952f2ce98329ed64c567484272905bcfc6d9 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 10 Feb 2018 17:21:54 +0800 Subject: [PATCH 01/90] ISSUES-1885 UTF8 countCodePoints use simd --- dbms/src/Common/UTF8Helpers.h | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index 1ce31426e85..ba1685c89f3 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -3,6 +3,9 @@ #include #include +#if __SSE2__ +#include +#endif namespace DB { @@ -49,9 +52,37 @@ inline size_t seqLength(const UInt8 first_octet) inline size_t countCodePoints(const UInt8 * data, size_t size) { size_t res = 0; + const auto end = data + size; - /// TODO SIMD implementation looks quite simple. - for (auto end = data + size; data < end; ++data) /// Skip UTF-8 continuation bytes. +#if __SSE2__ + const auto bytes_sse = sizeof(__m128i); + const auto src_end_sse = (data + size) - (size % bytes_sse); + + const auto upper_bound = _mm_set1_epi8(0x7F + 1); + const auto lower_bound = _mm_set1_epi8(0xC0 - 1); + + for (; data < src_end_sse;) + { + UInt8 mem_res[16] = {0}; + auto sse_res = _mm_set1_epi8(0); + + for (int i = 0; i < 0XFF && data < src_end_sse; ++i, data += bytes_sse) + { + const auto chars = _mm_loadu_si128(reinterpret_cast(data)); + sse_res = _mm_add_epi8(sse_res, + _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound), + _mm_cmpgt_epi8(chars, lower_bound))); + } + + _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), sse_res); + + for (auto count : mem_res) + res += count; + } + +#endif + + for (; data < end; ++data) /// Skip UTF-8 continuation bytes. res += (*data <= 0x7F || *data >= 0xC0); return res; From 73077e02459fb1a6da866aa785c21c4fba2d5d01 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sun, 11 Feb 2018 22:38:51 +0800 Subject: [PATCH 02/90] ISSUES-1885 fix failed test --- dbms/src/Common/UTF8Helpers.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index ba1685c89f3..c2f57b8107a 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -2,6 +2,7 @@ #include #include +#include #if __SSE2__ #include @@ -58,23 +59,27 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) const auto bytes_sse = sizeof(__m128i); const auto src_end_sse = (data + size) - (size % bytes_sse); + const auto fill_sse = _mm_set1_epi8(0xFF); const auto upper_bound = _mm_set1_epi8(0x7F + 1); const auto lower_bound = _mm_set1_epi8(0xC0 - 1); for (; data < src_end_sse;) { - UInt8 mem_res[16] = {0}; + UInt16 mem_res[8] = {0}; auto sse_res = _mm_set1_epi8(0); - for (int i = 0; i < 0XFF && data < src_end_sse; ++i, data += bytes_sse) + for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse) { const auto chars = _mm_loadu_si128(reinterpret_cast(data)); - sse_res = _mm_add_epi8(sse_res, - _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound), - _mm_cmpgt_epi8(chars, lower_bound))); + + const auto chars_cmpgt_res = _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound), + _mm_cmpgt_epi8(chars, lower_bound)); + + sse_res = _mm_add_epi8(sse_res, _mm_add_epi8(_mm_xor_si128(chars_cmpgt_res, fill_sse), _mm_set1_epi8(1))); } - _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), sse_res); + auto horizontal_add = _mm_sad_epu8(sse_res, _mm_set1_epi8(0)); + _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add); for (auto count : mem_res) res += count; From 5f9292ac30056d23425399b451df9f24a14f09f1 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 12 Feb 2018 15:27:51 +0800 Subject: [PATCH 03/90] ISSUES-1885 resolve some opinions --- dbms/src/Common/UTF8Helpers.h | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index c2f57b8107a..71ce6e4af51 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -2,12 +2,15 @@ #include #include -#include #if __SSE2__ #include #endif +#if __SSE4_1__ +#include +#endif + namespace DB { @@ -55,34 +58,39 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) size_t res = 0; const auto end = data + size; -#if __SSE2__ +#if __SSE2__ && __SSE4_1__ const auto bytes_sse = sizeof(__m128i); const auto src_end_sse = (data + size) - (size % bytes_sse); - const auto fill_sse = _mm_set1_epi8(0xFF); - const auto upper_bound = _mm_set1_epi8(0x7F + 1); - const auto lower_bound = _mm_set1_epi8(0xC0 - 1); + const auto one_sse = _mm_set1_epi8(1); + const auto zero_sse = _mm_set1_epi8(0); + + const auto align_sse = _mm_set1_epi8(0xFF - 0xC0); + const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0) + 1); for (; data < src_end_sse;) { - UInt16 mem_res[8] = {0}; auto sse_res = _mm_set1_epi8(0); for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse) { const auto chars = _mm_loadu_si128(reinterpret_cast(data)); - const auto chars_cmpgt_res = _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound), - _mm_cmpgt_epi8(chars, lower_bound)); + ///Align to zero for the solve two case + const auto align_res = _mm_add_epi8(chars, align_sse); + ///Because _mm_cmmpgt_epid8 return 0xFF if true + const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, _mm_cmpgt_epi8(align_res, upper_bound)); - sse_res = _mm_add_epi8(sse_res, _mm_add_epi8(_mm_xor_si128(chars_cmpgt_res, fill_sse), _mm_set1_epi8(1))); + sse_res = _mm_add_epi8(sse_res, choose_res); } - auto horizontal_add = _mm_sad_epu8(sse_res, _mm_set1_epi8(0)); + UInt16 mem_res[8] = {0}; + auto horizontal_add = _mm_sad_epu8(sse_res, zero_sse); _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add); - for (auto count : mem_res) - res += count; + /// Hack,Because only bytes MSB to LSB + res += mem_res[0]; + res += mem_res[4]; } #endif From acb5b99195171b104c5a04c918e3b12f23517dd1 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Mon, 12 Feb 2018 16:05:10 +0800 Subject: [PATCH 04/90] ISSUES-1885 fix align bound --- dbms/src/Common/UTF8Helpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index 71ce6e4af51..c1aa9930eaa 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -65,8 +65,8 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) const auto one_sse = _mm_set1_epi8(1); const auto zero_sse = _mm_set1_epi8(0); - const auto align_sse = _mm_set1_epi8(0xFF - 0xC0); - const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0) + 1); + const auto align_sse = _mm_set1_epi8(0xFF - 0xC0 + 1); + const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0 + 1) + 1); for (; data < src_end_sse;) { From 0d8730c687f2ebaa67a3a2b3bd0ca0faaf2e82ec Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 13 Feb 2018 13:46:16 +0800 Subject: [PATCH 05/90] ISSUES-1885 fix signed problem --- dbms/src/Common/UTF8Helpers.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index c1aa9930eaa..698ff369684 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -63,10 +63,10 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) const auto src_end_sse = (data + size) - (size % bytes_sse); const auto one_sse = _mm_set1_epi8(1); - const auto zero_sse = _mm_set1_epi8(0); + const auto zero_sse = _mm_setzero_si128(); - const auto align_sse = _mm_set1_epi8(0xFF - 0xC0 + 1); - const auto upper_bound = _mm_set1_epi8(0x7F + (0xFF - 0xC0 + 1) + 1); + const auto align_sse = _mm_set1_epi8(0x40); + const auto upper_bound = _mm_set1_epi8(0xBF); for (; data < src_end_sse;) { @@ -77,11 +77,12 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) const auto chars = _mm_loadu_si128(reinterpret_cast(data)); ///Align to zero for the solve two case - const auto align_res = _mm_add_epi8(chars, align_sse); - ///Because _mm_cmmpgt_epid8 return 0xFF if true - const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, _mm_cmpgt_epi8(align_res, upper_bound)); + const auto align_res = _mm_adds_epu8(chars, align_sse); + const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res); + ///Because _mm_cmpeq_epi8 return 0xFF if true + const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, less_than_and_equals); - sse_res = _mm_add_epi8(sse_res, choose_res); + sse_res = _mm_adds_epu8(sse_res, choose_res); } UInt16 mem_res[8] = {0}; From 6033096fd1d6aabd4fe6dafd1c8c52c1a16ae2e0 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 13 Feb 2018 16:12:20 +0800 Subject: [PATCH 06/90] ISSUES-1885 add performance test --- .../functions_length/functions_length.xml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 dbms/tests/performance/functions_length/functions_length.xml diff --git a/dbms/tests/performance/functions_length/functions_length.xml b/dbms/tests/performance/functions_length/functions_length.xml new file mode 100644 index 00000000000..d285cd2422f --- /dev/null +++ b/dbms/tests/performance/functions_length/functions_length.xml @@ -0,0 +1,38 @@ + + functions_length + once + + + + 10000 + + + 5000 + 20000 + + + + + + + + + + string + + materialize('') + materialize('Hello, world') + toString(number) + reinterpretAsString(number) + materialize('中文测试字符串') + materialize('https://github.com/yandex/ClickHouse/pull/1882') + materialize('https://zh.wikipedia.org/wiki/%E4%B8%AD%E6%97%A5%E9%9F%93%E7%B5%B1%E4%B8%80%E8%A1%A8%E6%84%8F%E6%96%87%E5%AD%97%E6%93%B4%E5%B1%95%E5%8D%80F') + concat('中文测试字符串 ', toString(number), ' Привет, мир!') + concat(concat('中文测试字符串 ', toString(number), ' Привет, мир!') AS x, x, x, x, x, x, x, x, x, x) + convertCharset(concat(reinterpretAsString(rand64(1)), reinterpretAsString(rand64(2)), reinterpretAsString(rand64(3)), reinterpretAsString(rand64(4)), reinterpretAsString(rand64(5)), reinterpretAsString(rand64(6)), reinterpretAsString(rand64(7)), reinterpretAsString(rand64(8)), reinterpretAsString(rand64(9)), reinterpretAsString(rand64(10))), 'UTF-16', 'UTF-8') + + + + + SELECT count() FROM system.numbers WHERE NOT ignore(lengthUTF8({string})) + From eeae014a42affed560781499625c6ed6f705cb99 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Wed, 14 Feb 2018 08:11:12 +0800 Subject: [PATCH 07/90] ISSUES-1885 use movemask & popcnt --- dbms/src/Common/UTF8Helpers.h | 36 +++++++---------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/dbms/src/Common/UTF8Helpers.h b/dbms/src/Common/UTF8Helpers.h index 698ff369684..0237b6f036c 100644 --- a/dbms/src/Common/UTF8Helpers.h +++ b/dbms/src/Common/UTF8Helpers.h @@ -7,10 +7,6 @@ #include #endif -#if __SSE4_1__ -#include -#endif - namespace DB { @@ -58,40 +54,22 @@ inline size_t countCodePoints(const UInt8 * data, size_t size) size_t res = 0; const auto end = data + size; -#if __SSE2__ && __SSE4_1__ +#if __SSE2__ const auto bytes_sse = sizeof(__m128i); const auto src_end_sse = (data + size) - (size % bytes_sse); - const auto one_sse = _mm_set1_epi8(1); - const auto zero_sse = _mm_setzero_si128(); - const auto align_sse = _mm_set1_epi8(0x40); const auto upper_bound = _mm_set1_epi8(0xBF); - for (; data < src_end_sse;) + for (; data < src_end_sse; data += bytes_sse) { - auto sse_res = _mm_set1_epi8(0); + const auto chars = _mm_loadu_si128(reinterpret_cast(data)); - for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse) - { - const auto chars = _mm_loadu_si128(reinterpret_cast(data)); + ///Align to zero for the solve two case + const auto align_res = _mm_adds_epu8(chars, align_sse); + const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res); - ///Align to zero for the solve two case - const auto align_res = _mm_adds_epu8(chars, align_sse); - const auto less_than_and_equals = _mm_cmpeq_epi8(_mm_min_epu8(align_res, upper_bound), align_res); - ///Because _mm_cmpeq_epi8 return 0xFF if true - const auto choose_res = _mm_blendv_epi8(zero_sse, one_sse, less_than_and_equals); - - sse_res = _mm_adds_epu8(sse_res, choose_res); - } - - UInt16 mem_res[8] = {0}; - auto horizontal_add = _mm_sad_epu8(sse_res, zero_sse); - _mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add); - - /// Hack,Because only bytes MSB to LSB - res += mem_res[0]; - res += mem_res[4]; + res += __builtin_popcount(_mm_movemask_epi8(less_than_and_equals)); } #endif From 0b26189fabfa5e3fce522c53fb0895031360a3be Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Sat, 24 Feb 2018 14:06:53 +0200 Subject: [PATCH 08/90] prototype: show server group name in client --- dbms/src/Client/Connection.cpp | 12 ++++++++++++ dbms/src/Client/Connection.h | 2 ++ dbms/src/Core/Defines.h | 1 + dbms/src/Server/Client.cpp | 5 ++++- dbms/src/Server/TCPHandler.cpp | 4 ++++ dbms/src/Server/config.xml | 2 +- 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 3c6c8d75d0e..9b57c1b39b2 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -148,6 +148,10 @@ void Connection::receiveHello() { readStringBinary(server_timezone, *in); } + if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME) + { + readStringBinary(server_group_name, *in); + } } else if (packet_type == Protocol::Server::Exception) receiveException()->rethrow(); @@ -203,6 +207,14 @@ const String & Connection::getServerTimezone() return server_timezone; } +const String & Connection::getServerGroupname() +{ + if (!connected) + connect(); + + return server_group_name; +} + void Connection::forceConnected() { if (!connected) diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index c9b401c91ee..06f8e94e29c 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -134,6 +134,7 @@ public: void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision); const String & getServerTimezone(); + const String & getServerGroupname(); /// For log and exception messages. const String & getDescription() const; @@ -213,6 +214,7 @@ private: UInt64 server_version_minor = 0; UInt64 server_revision = 0; String server_timezone; + String server_group_name; std::unique_ptr socket; std::shared_ptr in; diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 8b69d83d69e..180d2a0dc2b 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -60,6 +60,7 @@ #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 +#define DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME 54355 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 238be504aac..1189dd4d6bc 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -153,6 +153,7 @@ private: /// If the last query resulted in exception. bool got_exception = false; String server_version; + String server_group_name; Stopwatch watch; @@ -425,6 +426,7 @@ private: connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision); server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); + server_group_name = connection->getServerGroupname(); if (is_interactive) { std::cout << "Connected to " << server_name @@ -450,7 +452,8 @@ private: { String query; String prev_query; - while (char * line_ = readline(query.empty() ? ":) " : ":-] ")) + String prompt = server_group_name.length() ? "[" + server_group_name + "] :) " : ":) "; + while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] ")) { String line = line_; free(line_); diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 7dadeef966b..70d16d6a16b 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -508,6 +508,10 @@ void TCPHandler::sendHello() { writeStringBinary(DateLUT::instance().getTimeZone(), *out); } + if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME) + { + writeStringBinary(server.config().getString("group_name"), *out); + } out->next(); } diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml index e0b4ea563ac..3fe4f580f40 100644 --- a/dbms/src/Server/config.xml +++ b/dbms/src/Server/config.xml @@ -8,7 +8,7 @@ 10 - + staging 8123 9000 From 2216f9e0cccbed113744d475c13087ade37ff2c9 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Sat, 24 Feb 2018 14:36:19 +0200 Subject: [PATCH 09/90] fix function name --- dbms/src/Client/Connection.cpp | 2 +- dbms/src/Client/Connection.h | 2 +- dbms/src/Server/Client.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 9b57c1b39b2..4e5ee88c782 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -207,7 +207,7 @@ const String & Connection::getServerTimezone() return server_timezone; } -const String & Connection::getServerGroupname() +const String & Connection::getServerGroupName() { if (!connected) connect(); diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index 06f8e94e29c..0caf6ead9f2 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -134,7 +134,7 @@ public: void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision); const String & getServerTimezone(); - const String & getServerGroupname(); + const String & getServerGroupName(); /// For log and exception messages. const String & getDescription() const; diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 1189dd4d6bc..f644da2f3ea 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -426,7 +426,7 @@ private: connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision); server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); - server_group_name = connection->getServerGroupname(); + server_group_name = connection->getServerGroupName(); if (is_interactive) { std::cout << "Connected to " << server_name From d3b6aafa097c3c772531ea30b07bd9348bf43856 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 26 Feb 2018 08:49:17 +0200 Subject: [PATCH 10/90] Rename group_name to display_name --- dbms/src/Client/Connection.cpp | 8 ++++---- dbms/src/Client/Connection.h | 4 ++-- dbms/src/Core/Defines.h | 2 +- dbms/src/Server/Client.cpp | 6 +++--- dbms/src/Server/TCPHandler.cpp | 5 +++-- dbms/src/Server/config.xml | 2 +- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 4e5ee88c782..0960ac8e77d 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -148,9 +148,9 @@ void Connection::receiveHello() { readStringBinary(server_timezone, *in); } - if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME) + if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) { - readStringBinary(server_group_name, *in); + readStringBinary(server_display_name, *in); } } else if (packet_type == Protocol::Server::Exception) @@ -207,12 +207,12 @@ const String & Connection::getServerTimezone() return server_timezone; } -const String & Connection::getServerGroupName() +const String & Connection::getServerDisplayName() { if (!connected) connect(); - return server_group_name; + return server_display_name; } void Connection::forceConnected() diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index 0caf6ead9f2..cac188f12c8 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -134,7 +134,7 @@ public: void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision); const String & getServerTimezone(); - const String & getServerGroupName(); + const String & getServerDisplayName(); /// For log and exception messages. const String & getDescription() const; @@ -214,7 +214,7 @@ private: UInt64 server_version_minor = 0; UInt64 server_revision = 0; String server_timezone; - String server_group_name; + String server_display_name; std::unique_ptr socket; std::shared_ptr in; diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 180d2a0dc2b..1cc14a62155 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -60,7 +60,7 @@ #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 -#define DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME 54355 +#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54355 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index f644da2f3ea..4760c8d4874 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -153,7 +153,7 @@ private: /// If the last query resulted in exception. bool got_exception = false; String server_version; - String server_group_name; + String server_display_name; Stopwatch watch; @@ -426,7 +426,7 @@ private: connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision); server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); - server_group_name = connection->getServerGroupName(); + server_display_name = connection->getServerDisplayName(); if (is_interactive) { std::cout << "Connected to " << server_name @@ -452,7 +452,7 @@ private: { String query; String prev_query; - String prompt = server_group_name.length() ? "[" + server_group_name + "] :) " : ":) "; + String prompt = server_display_name.length() ? "[" + server_display_name + "] :) " : ":) "; while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] ")) { String line = line_; diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 70d16d6a16b..e6ac7b5d5d6 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -31,6 +31,7 @@ #include "TCPHandler.h" #include +#include namespace DB @@ -508,9 +509,9 @@ void TCPHandler::sendHello() { writeStringBinary(DateLUT::instance().getTimeZone(), *out); } - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_GROUP_NAME) + if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) { - writeStringBinary(server.config().getString("group_name"), *out); + writeStringBinary(server.config().getString("display_name", getFQDNOrHostName()), *out); } out->next(); } diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml index 3fe4f580f40..4c1afb53639 100644 --- a/dbms/src/Server/config.xml +++ b/dbms/src/Server/config.xml @@ -8,7 +8,7 @@ 10 - staging + 8123 9000 From f15b55f9295b6422ece51bad2c0e0fccf4e5fd6e Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 26 Feb 2018 15:24:06 +0200 Subject: [PATCH 11/90] prototype: custom prompt --- dbms/src/Server/Client.cpp | 59 ++++++++++++++++++++++++++- dbms/src/Server/clickhouse-client.xml | 4 ++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 4760c8d4874..989a83a0030 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include #include @@ -138,6 +140,8 @@ private: String current_profile; + String prompt_by_server_display_name = "%display_name% :)"; + /// Path to a file containing command history. String history_file; @@ -335,6 +339,18 @@ private: } } + Strings keys; + config().keys("prompt_by_server_display_name", keys); + + for (const String & key : keys) + { + if (server_display_name.find(key) != std::string::npos) + { + prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key); + break; + } + } + if (is_interactive) { if (print_time_to_stderr) @@ -447,13 +463,52 @@ private: return select(1, &fds, 0, 0, &timeout) == 1; } + const String prompt() const + { + std::map colors = { + { "red", "31" }, + { "green", "32" }, + { "yellow", "33" }, + { "blue", "34" }, + { "magenta", "35" }, + { "cyan", "36" }, + }; + + String pattern = prompt_by_server_display_name; + + for (const auto & pair: colors) + { + String name = static_cast(pair.first); + String code = static_cast(pair.second); + + boost::replace_all(pattern, "[" + name + "]", "\33[1;" + code + "m"); + boost::replace_all(pattern, "[/" + name + "]", "\033[0m"); + } + + std::map environment = { + {"host", config().getString("host", "localhost")}, + {"display_name", server_display_name}, + {"user", config().getString("user", "default")}, + {"database", config().getString("database", "default")}, + }; + + for (const auto & pair: environment) + { + String key = static_cast(pair.first); + String value = static_cast(pair.second); + + boost::replace_all(pattern, "%" + key + "%", value); + } + + return pattern; + } void loop() { String query; String prev_query; - String prompt = server_display_name.length() ? "[" + server_display_name + "] :) " : ":) "; - while (char * line_ = readline(query.empty() ? prompt.c_str() : ":-] ")) + + while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] ")) { String line = line_; free(line_); diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml index c19606c518b..e5ec8318e94 100644 --- a/dbms/src/Server/clickhouse-client.xml +++ b/dbms/src/Server/clickhouse-client.xml @@ -13,4 +13,8 @@ + + %display_name% [green]:)[/green] + %display_name% [red]:)[/red] + From 213caffddeda54f92392a058a36df10816a1d42d Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 26 Feb 2018 15:48:07 +0200 Subject: [PATCH 12/90] add default section to client config --- dbms/src/Server/Client.cpp | 4 +++- dbms/src/Server/clickhouse-client.xml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 989a83a0030..229167763e9 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -140,7 +140,7 @@ private: String current_profile; - String prompt_by_server_display_name = "%display_name% :)"; + String prompt_by_server_display_name; /// Path to a file containing command history. String history_file; @@ -342,6 +342,8 @@ private: Strings keys; config().keys("prompt_by_server_display_name", keys); + prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "%display_name% :)"); + for (const String & key : keys) { if (server_display_name.find(key) != std::string::npos) diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml index e5ec8318e94..6b569f02bc6 100644 --- a/dbms/src/Server/clickhouse-client.xml +++ b/dbms/src/Server/clickhouse-client.xml @@ -14,6 +14,7 @@ + %display_name% :) %display_name% [green]:)[/green] %display_name% [red]:)[/red] From 390ce9428b2754a956e2e288f87d7755aa3c856a Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Tue, 27 Feb 2018 09:47:15 +0200 Subject: [PATCH 13/90] add background colors --- dbms/src/Server/Client.cpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 229167763e9..a13dd195eff 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -468,12 +468,43 @@ private: const String prompt() const { std::map colors = { + /* Foreground */ + { "black", "30" }, { "red", "31" }, { "green", "32" }, { "yellow", "33" }, { "blue", "34" }, { "magenta", "35" }, { "cyan", "36" }, + { "white", "97" }, + + { "light gray", "37" }, + { "dark gray", "90" }, + { "light red", "91" }, + { "light green", "92" }, + { "light yellow", "93" }, + { "light blue", "94" }, + { "light magenta", "95" }, + { "light cyan", "96" }, + + /* Background*/ + { "background black", "40" }, + { "background red", "41" }, + { "background green", "42" }, + { "background yellow", "43" }, + { "background blue", "44" }, + { "background magenta", "45" }, + { "background cyan", "46" }, + { "background white", "107" }, + + { "background light gray", "47" }, + { "background dark gray", "100" }, + { "background light red", "101" }, + { "background light green", "102" }, + { "background light yellow", "103" }, + { "background light blue", "104" }, + { "background light magenta", "105" }, + { "background light cyan", "106" }, }; String pattern = prompt_by_server_display_name; @@ -489,6 +520,7 @@ private: std::map environment = { {"host", config().getString("host", "localhost")}, + {"port", config().getString("port", "9000")}, {"display_name", server_display_name}, {"user", config().getString("user", "default")}, {"database", config().getString("database", "default")}, From 053b3f21a02a9ca35e2ec05b56961bbcb6d605fc Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Tue, 27 Feb 2018 09:57:18 +0200 Subject: [PATCH 14/90] move 'colors' from function body --- dbms/src/Server/Client.cpp | 78 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index a13dd195eff..741d2effd98 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -103,7 +103,45 @@ private: "учшеж", "йгшеж", "дщпщгеж", "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй" }; + std::map colors = { + /* Foreground */ + { "black", "30" }, + { "red", "31" }, + { "green", "32" }, + { "yellow", "33" }, + { "blue", "34" }, + { "magenta", "35" }, + { "cyan", "36" }, + { "white", "97" }, + { "light gray", "37" }, + { "dark gray", "90" }, + { "light red", "91" }, + { "light green", "92" }, + { "light yellow", "93" }, + { "light blue", "94" }, + { "light magenta", "95" }, + { "light cyan", "96" }, + + /* Background*/ + { "background black", "40" }, + { "background red", "41" }, + { "background green", "42" }, + { "background yellow", "43" }, + { "background blue", "44" }, + { "background magenta", "45" }, + { "background cyan", "46" }, + { "background white", "107" }, + + { "background light gray", "47" }, + { "background dark gray", "100" }, + { "background light red", "101" }, + { "background light green", "102" }, + { "background light yellow", "103" }, + { "background light blue", "104" }, + { "background light magenta", "105" }, + { "background light cyan", "106" }, + }; bool is_interactive = true; /// Use either readline interface or batch mode. bool need_render_progress = true; /// Render query execution progress. bool echo_queries = false; /// Print queries before execution in batch mode. @@ -467,46 +505,6 @@ private: const String prompt() const { - std::map colors = { - /* Foreground */ - { "black", "30" }, - { "red", "31" }, - { "green", "32" }, - { "yellow", "33" }, - { "blue", "34" }, - { "magenta", "35" }, - { "cyan", "36" }, - { "white", "97" }, - - { "light gray", "37" }, - { "dark gray", "90" }, - { "light red", "91" }, - { "light green", "92" }, - { "light yellow", "93" }, - { "light blue", "94" }, - { "light magenta", "95" }, - { "light cyan", "96" }, - - /* Background*/ - { "background black", "40" }, - { "background red", "41" }, - { "background green", "42" }, - { "background yellow", "43" }, - { "background blue", "44" }, - { "background magenta", "45" }, - { "background cyan", "46" }, - { "background white", "107" }, - - { "background light gray", "47" }, - { "background dark gray", "100" }, - { "background light red", "101" }, - { "background light green", "102" }, - { "background light yellow", "103" }, - { "background light blue", "104" }, - { "background light magenta", "105" }, - { "background light cyan", "106" }, - }; - String pattern = prompt_by_server_display_name; for (const auto & pair: colors) From bba059145299d7015ad1087282f79d86d44fdc9d Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 5 Mar 2018 12:20:23 +0200 Subject: [PATCH 15/90] add X-ClickHouse-Display-Name header to HTTPHandler --- dbms/src/Server/Client.cpp | 13 +++++-------- dbms/src/Server/HTTPHandler.cpp | 5 +++-- dbms/src/Server/HTTPHandler.h | 1 + 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 741d2effd98..bf08072a20d 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -507,10 +507,10 @@ private: { String pattern = prompt_by_server_display_name; - for (const auto & pair: colors) + for (const auto & [key, value]: colors) { - String name = static_cast(pair.first); - String code = static_cast(pair.second); + String name = static_cast(key); + String code = static_cast(value); boost::replace_all(pattern, "[" + name + "]", "\33[1;" + code + "m"); boost::replace_all(pattern, "[/" + name + "]", "\033[0m"); @@ -524,12 +524,9 @@ private: {"database", config().getString("database", "default")}, }; - for (const auto & pair: environment) + for (const auto & [key, value]: environment) { - String key = static_cast(pair.first); - String value = static_cast(pair.second); - - boost::replace_all(pattern, "%" + key + "%", value); + boost::replace_all(pattern, "%" + static_cast(key) + "%", static_cast(value)); } return pattern; diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp index 555d4b5f3ba..3d99257f70a 100644 --- a/dbms/src/Server/HTTPHandler.cpp +++ b/dbms/src/Server/HTTPHandler.cpp @@ -12,7 +12,7 @@ #include #include #include - +#include #include #include #include @@ -198,6 +198,7 @@ HTTPHandler::HTTPHandler(IServer & server_) : server(server_) , log(&Logger::get("HTTPHandler")) { + display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -631,7 +632,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne try { response.setContentType("text/plain; charset=UTF-8"); - + response.set("X-ClickHouse-Display-Name", display_name); /// For keep-alive to work. if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/src/Server/HTTPHandler.h index b0b7164852e..255da56c8d9 100644 --- a/dbms/src/Server/HTTPHandler.h +++ b/dbms/src/Server/HTTPHandler.h @@ -54,6 +54,7 @@ private: IServer & server; Poco::Logger * log; + String display_name; CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; From 3c09dacb5f1e049ffdaaddd624d18252a72e198a Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 5 Mar 2018 14:05:48 +0200 Subject: [PATCH 16/90] fix test --- dbms/tests/queries/0_stateless/00501_http_head.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh index c15b8679fab..7fb675e5b8a 100755 --- a/dbms/tests/queries/0_stateless/00501_http_head.sh +++ b/dbms/tests/queries/0_stateless/00501_http_head.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=SELECT%201"; - ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" + ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Display-Name:" if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}?query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then echo FAIL From 5e55a605f09d593b59d32a5fd354c0a21eba17fe Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 5 Mar 2018 14:18:53 +0200 Subject: [PATCH 17/90] add bold font --- dbms/src/Server/Client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index bf08072a20d..e47c4b5b5ce 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -104,6 +104,7 @@ private: "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй" }; std::map colors = { + { "bold", "1" }, /* Foreground */ { "black", "30" }, { "red", "31" }, From 27a9ce8dd74a64dcfcd4a90805cd7d752f985cbf Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 5 Mar 2018 16:47:10 +0200 Subject: [PATCH 18/90] Display host if display name is empty --- dbms/src/Server/Client.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index e47c4b5b5ce..a7394fa7c7a 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -484,6 +484,12 @@ private: server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); server_display_name = connection->getServerDisplayName(); + + if (server_display_name.length() == 0) + { + server_display_name = config().getString("host", "localhost"); + } + if (is_interactive) { std::cout << "Connected to " << server_name From 950a7f83cfdfd675558f6607e413278f214a1896 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Mar 2018 23:54:36 +0300 Subject: [PATCH 19/90] Update Defines.h --- dbms/src/Core/Defines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 1cc14a62155..f860bd8c71c 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -60,7 +60,7 @@ #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 -#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54355 +#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54358 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 From fdcc88b5a28464b46333b84ffbbd70e9c18dc3a3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Mar 2018 23:56:15 +0300 Subject: [PATCH 20/90] Update Client.cpp --- dbms/src/Server/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index a7394fa7c7a..fff59cd6e4e 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -124,7 +124,7 @@ private: { "light magenta", "95" }, { "light cyan", "96" }, - /* Background*/ + /* Background */ { "background black", "40" }, { "background red", "41" }, { "background green", "42" }, From 052079b73a49852765e5a03d482f14d9b18775a2 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 8 Mar 2018 09:36:58 +0200 Subject: [PATCH 21/90] Use standard markup '{}' like macros. Add coments. --- dbms/src/Server/Client.cpp | 9 +++------ dbms/src/Server/HTTPHandler.cpp | 4 ++-- dbms/src/Server/HTTPHandler.h | 4 +++- dbms/src/Server/TCPHandler.cpp | 3 +-- dbms/src/Server/TCPHandler.h | 6 +++++- dbms/src/Server/clickhouse-client.xml | 15 ++++++++++++--- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index a7394fa7c7a..82e15d9d302 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -381,7 +381,7 @@ private: Strings keys; config().keys("prompt_by_server_display_name", keys); - prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "%display_name% :)"); + prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) "); for (const String & key : keys) { @@ -514,11 +514,8 @@ private: { String pattern = prompt_by_server_display_name; - for (const auto & [key, value]: colors) + for (const auto & [name, code]: colors) { - String name = static_cast(key); - String code = static_cast(value); - boost::replace_all(pattern, "[" + name + "]", "\33[1;" + code + "m"); boost::replace_all(pattern, "[/" + name + "]", "\033[0m"); } @@ -533,7 +530,7 @@ private: for (const auto & [key, value]: environment) { - boost::replace_all(pattern, "%" + static_cast(key) + "%", static_cast(value)); + boost::replace_all(pattern, "{" + key + "}", value); } return pattern; diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp index 3d99257f70a..539573fc8fb 100644 --- a/dbms/src/Server/HTTPHandler.cpp +++ b/dbms/src/Server/HTTPHandler.cpp @@ -198,7 +198,7 @@ HTTPHandler::HTTPHandler(IServer & server_) : server(server_) , log(&Logger::get("HTTPHandler")) { - display_name = server.config().getString("display_name", getFQDNOrHostName()); + server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -632,7 +632,7 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne try { response.setContentType("text/plain; charset=UTF-8"); - response.set("X-ClickHouse-Display-Name", display_name); + response.set("X-ClickHouse-Server-Display-Name", server_display_name); /// For keep-alive to work. if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/src/Server/HTTPHandler.h index 255da56c8d9..72606afa827 100644 --- a/dbms/src/Server/HTTPHandler.h +++ b/dbms/src/Server/HTTPHandler.h @@ -54,7 +54,9 @@ private: IServer & server; Poco::Logger * log; - String display_name; + + /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. + String server_display_name; CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index e6ac7b5d5d6..729916738c3 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -31,7 +31,6 @@ #include "TCPHandler.h" #include -#include namespace DB @@ -511,7 +510,7 @@ void TCPHandler::sendHello() } if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) { - writeStringBinary(server.config().getString("display_name", getFQDNOrHostName()), *out); + writeStringBinary(server_display_name, *out); } out->next(); } diff --git a/dbms/src/Server/TCPHandler.h b/dbms/src/Server/TCPHandler.h index 444d8eb4990..f53850ee487 100644 --- a/dbms/src/Server/TCPHandler.h +++ b/dbms/src/Server/TCPHandler.h @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -81,6 +82,7 @@ public: , connection_context(server.context()) , query_context(server.context()) { + server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } void run(); @@ -111,7 +113,9 @@ private: QueryState state; CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection}; - + + /// It is the name of the server that will be sent to the client. + String server_display_name; void runImpl(); diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml index 6b569f02bc6..ecd19d6c623 100644 --- a/dbms/src/Server/clickhouse-client.xml +++ b/dbms/src/Server/clickhouse-client.xml @@ -13,9 +13,18 @@ + - %display_name% :) - %display_name% [green]:)[/green] - %display_name% [red]:)[/red] + {display_name} :) + {display_name} [green]:)[/green + {display_name} [red]:)[/red] From 8c4a7b774efc7e352347c0415c20e4774b5508b0 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 8 Mar 2018 10:55:27 +0200 Subject: [PATCH 22/90] Removed BB-Style macros for colors. Preparing prompt pattern once and replace current database in the cycle only. --- dbms/src/Server/Client.cpp | 93 ++++++------------- dbms/src/Server/clickhouse-client.xml | 5 +- .../queries/0_stateless/00501_http_head.sh | 2 +- 3 files changed, 33 insertions(+), 67 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index f490b61b12a..959dfe9a199 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -103,46 +103,6 @@ private: "учшеж", "йгшеж", "дщпщгеж", "q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй" }; - std::map colors = { - { "bold", "1" }, - /* Foreground */ - { "black", "30" }, - { "red", "31" }, - { "green", "32" }, - { "yellow", "33" }, - { "blue", "34" }, - { "magenta", "35" }, - { "cyan", "36" }, - { "white", "97" }, - - { "light gray", "37" }, - { "dark gray", "90" }, - { "light red", "91" }, - { "light green", "92" }, - { "light yellow", "93" }, - { "light blue", "94" }, - { "light magenta", "95" }, - { "light cyan", "96" }, - - /* Background */ - { "background black", "40" }, - { "background red", "41" }, - { "background green", "42" }, - { "background yellow", "43" }, - { "background blue", "44" }, - { "background magenta", "45" }, - { "background cyan", "46" }, - { "background white", "107" }, - - { "background light gray", "47" }, - { "background dark gray", "100" }, - { "background light red", "101" }, - { "background light green", "102" }, - { "background light yellow", "103" }, - { "background light blue", "104" }, - { "background light magenta", "105" }, - { "background light cyan", "106" }, - }; bool is_interactive = true; /// Use either readline interface or batch mode. bool need_render_progress = true; /// Render query execution progress. bool echo_queries = false; /// Print queries before execution in batch mode. @@ -379,10 +339,11 @@ private: } Strings keys; - config().keys("prompt_by_server_display_name", keys); prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) "); + config().keys("prompt_by_server_display_name", keys); + for (const String & key : keys) { if (server_display_name.find(key) != std::string::npos) @@ -392,6 +353,30 @@ private: } } + std::map terminalCharacters = { + { "\\e[", "\e[" }, + { "\\33[", "\33[" }, + { "\\033[", "\033["}, + { "\\x1B[", "\x1B["}, + }; + + for (const auto & [key, value]: terminalCharacters) + { + boost::replace_all(prompt_by_server_display_name, key, value); + } + + std::map environment = { + {"host", config().getString("host", "localhost")}, + {"port", config().getString("port", "9000")}, + {"user", config().getString("user", "default")}, + {"display_name", server_display_name}, + }; + + for (const auto & [key, value]: environment) + { + boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value); + } + if (is_interactive) { if (print_time_to_stderr) @@ -483,9 +468,8 @@ private: connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision); server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); - server_display_name = connection->getServerDisplayName(); - - if (server_display_name.length() == 0) + + if (server_display_name = connection->getServerDisplayName(); server_display_name.length() == 0) { server_display_name = config().getString("host", "localhost"); } @@ -513,26 +497,7 @@ private: const String prompt() const { String pattern = prompt_by_server_display_name; - - for (const auto & [name, code]: colors) - { - boost::replace_all(pattern, "[" + name + "]", "\33[1;" + code + "m"); - boost::replace_all(pattern, "[/" + name + "]", "\033[0m"); - } - - std::map environment = { - {"host", config().getString("host", "localhost")}, - {"port", config().getString("port", "9000")}, - {"display_name", server_display_name}, - {"user", config().getString("user", "default")}, - {"database", config().getString("database", "default")}, - }; - - for (const auto & [key, value]: environment) - { - boost::replace_all(pattern, "{" + key + "}", value); - } - + boost::replace_all(pattern, "{database}", config().getString("database", "default")); return pattern; } diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/src/Server/clickhouse-client.xml index ecd19d6c623..3cd05d7fec9 100644 --- a/dbms/src/Server/clickhouse-client.xml +++ b/dbms/src/Server/clickhouse-client.xml @@ -21,10 +21,11 @@ {user} {database} {display_name} + Terminal colors: https://misc.flogisoft.com/bash/tip_colors_and_formatting --> {display_name} :) - {display_name} [green]:)[/green - {display_name} [red]:)[/red] + {display_name} \e[1;32m:)\e[0m + {display_name} \e[1;31m:)\e[0m diff --git a/dbms/tests/queries/0_stateless/00501_http_head.sh b/dbms/tests/queries/0_stateless/00501_http_head.sh index 7fb675e5b8a..578e27da751 100755 --- a/dbms/tests/queries/0_stateless/00501_http_head.sh +++ b/dbms/tests/queries/0_stateless/00501_http_head.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh ( ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=SELECT%201"; - ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Display-Name:" + ${CLICKHOUSE_CURL} -s --head "${CLICKHOUSE_URL}?query=select+*+from+system.numbers+limit+1000000" ) | grep -v "Date:" | grep -v "X-ClickHouse-Server-Display-Name:" if [[ `${CLICKHOUSE_CURL} -sS -X POST -I "${CLICKHOUSE_URL}?query=SELECT+1" | grep -c '411 Length Required'` -ne 1 ]]; then echo FAIL From 808bfef3453fe016a4f560867e4b4d31b38409ae Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Mon, 12 Mar 2018 15:23:15 +0200 Subject: [PATCH 23/90] inline prompt function --- dbms/src/Server/Client.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 959dfe9a199..53752f1d173 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -346,7 +346,7 @@ private: for (const String & key : keys) { - if (server_display_name.find(key) != std::string::npos) + if (key != "default" && server_display_name.find(key) != std::string::npos) { prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key); break; @@ -494,11 +494,9 @@ private: return select(1, &fds, 0, 0, &timeout) == 1; } - const String prompt() const + inline const String prompt() const { - String pattern = prompt_by_server_display_name; - boost::replace_all(pattern, "{database}", config().getString("database", "default")); - return pattern; + return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default")); } void loop() From 2b3e897aa1b75b292c9f325322002038af4e34a5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 12 Mar 2018 17:14:56 +0300 Subject: [PATCH 24/90] added SHOW CREATE DATABASE query [#CLICKHOUSE-3590] --- dbms/src/Databases/DatabaseOrdinary.cpp | 17 ++++++-- dbms/src/Databases/IDatabase.h | 3 +- dbms/src/Interpreters/InterpreterFactory.cpp | 6 ++- .../InterpreterShowCreateQuery.cpp | 2 +- .../Parsers/ParserTablePropertiesQuery.cpp | 39 +++++++++++++------ dbms/src/Parsers/TablePropertiesQueriesASTs.h | 22 +++++++++-- 6 files changed, 69 insertions(+), 20 deletions(-) diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 0021f47c4c7..b0719649094 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -45,6 +45,12 @@ namespace detail { return base_path + (endsWith(base_path, "/") ? "" : "/") + escapeForFileName(table_name) + ".sql"; } + + String getDatabaseMetadataPath(const String & base_path) + { + return (endsWith(base_path, "/") ? base_path.substr(0, base_path.size() - 1) : base_path) + ".sql"; + } + } static void loadTable( @@ -332,16 +338,21 @@ void DatabaseOrdinary::removeTable( static ASTPtr getCreateQueryImpl(const String & path, const String & table_name) { - String table_metadata_path = detail::getTableMetadataPath(path, table_name); + String metadata_path; + + if (table_name.empty()) + metadata_path = detail::getDatabaseMetadataPath(path); + else + metadata_path = detail::getTableMetadataPath(path, table_name); String query; { - ReadBufferFromFile in(table_metadata_path, 4096); + ReadBufferFromFile in(metadata_path, 4096); readStringUntilEOF(query, in); } ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + table_metadata_path); + return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path); } diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index 74cbe0b2202..8d62c0e3d45 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -124,7 +124,8 @@ public: const Context & context, const String & name) = 0; - /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata. + /// Get the CREATE TABLE query for the table or CREATE DATABASE query for database if name is empty. + /// It can also provide information for detached tables for which there is metadata. virtual ASTPtr getCreateQuery( const Context & context, const String & name) const = 0; diff --git a/dbms/src/Interpreters/InterpreterFactory.cpp b/dbms/src/Interpreters/InterpreterFactory.cpp index 9d30c586129..2e43efbb12e 100644 --- a/dbms/src/Interpreters/InterpreterFactory.cpp +++ b/dbms/src/Interpreters/InterpreterFactory.cpp @@ -107,7 +107,11 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } - else if (typeid_cast(query.get())) + else if (typeid_cast(query.get())) + { + return std::make_unique(query, context); + } + else if (typeid_cast(query.get())) { return std::make_unique(query, context); } diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index d4bd116bcf7..705b9bb4120 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -39,7 +39,7 @@ Block InterpreterShowCreateQuery::getSampleBlock() BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() { - const ASTShowCreateQuery & ast = typeid_cast(*query_ptr); + const auto & ast = dynamic_cast(*query_ptr); if (ast.temporary && !ast.database.empty()) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); diff --git a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp index 570597b0fa6..45e92f9e181 100644 --- a/dbms/src/Parsers/ParserTablePropertiesQuery.cpp +++ b/dbms/src/Parsers/ParserTablePropertiesQuery.cpp @@ -19,6 +19,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ParserKeyword s_desc("DESC"); ParserKeyword s_show("SHOW"); ParserKeyword s_create("CREATE"); + ParserKeyword s_database("DATABASE"); ParserKeyword s_table("TABLE"); ParserToken s_dot(TokenType::Dot); ParserIdentifier name_p; @@ -27,6 +28,8 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ASTPtr table; std::shared_ptr query; + bool parse_only_database_name = false; + if (s_exists.ignore(pos, expected)) { query = std::make_shared(); @@ -36,26 +39,40 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!s_create.ignore(pos, expected)) return false; - query = std::make_shared(); + if (s_database.ignore(pos, expected)) + { + parse_only_database_name = true; + query = std::make_shared(); + } + else + query = std::make_shared(); } else { return false; } - if (s_temporary.ignore(pos, expected)) - query->temporary = true; - - s_table.ignore(pos, expected); - - if (!name_p.parse(pos, table, expected)) - return false; - - if (s_dot.ignore(pos, expected)) + if (parse_only_database_name) { - database = table; + if (!name_p.parse(pos, database, expected)) + return false; + } + else + { + if (s_temporary.ignore(pos, expected)) + query->temporary = true; + + s_table.ignore(pos, expected); + if (!name_p.parse(pos, table, expected)) return false; + + if (s_dot.ignore(pos, expected)) + { + database = table; + if (!name_p.parse(pos, table, expected)) + return false; + } } if (database) diff --git a/dbms/src/Parsers/TablePropertiesQueriesASTs.h b/dbms/src/Parsers/TablePropertiesQueriesASTs.h index 312b112f494..15a1d397fa4 100644 --- a/dbms/src/Parsers/TablePropertiesQueriesASTs.h +++ b/dbms/src/Parsers/TablePropertiesQueriesASTs.h @@ -12,12 +12,18 @@ struct ASTExistsQueryIDAndQueryNames static constexpr auto Query = "EXISTS TABLE"; }; -struct ASTShowCreateQueryIDAndQueryNames +struct ASTShowCreateTableQueryIDAndQueryNames { - static constexpr auto ID = "ShowCreateQuery"; + static constexpr auto ID = "ShowCreateTableQuery"; static constexpr auto Query = "SHOW CREATE TABLE"; }; +struct ASTShowCreateDatabaseQueryIDAndQueryNames +{ + static constexpr auto ID = "ShowCreateDatabaseQuery"; + static constexpr auto Query = "SHOW CREATE DATABASE"; +}; + struct ASTDescribeQueryExistsQueryIDAndQueryNames { static constexpr auto ID = "DescribeQuery"; @@ -25,7 +31,17 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames }; using ASTExistsQuery = ASTQueryWithTableAndOutputImpl; -using ASTShowCreateQuery = ASTQueryWithTableAndOutputImpl; +using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl; + +class ASTShowCreateDatabaseQuery : public ASTQueryWithTableAndOutputImpl +{ +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << ASTShowCreateDatabaseQueryIDAndQueryNames::Query + << " " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); + } +}; class ASTDescribeQuery : public ASTQueryWithOutput { From 5a07569c2a57f95197dd8af42232bf08553cdae6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 12 Mar 2018 18:21:17 +0300 Subject: [PATCH 25/90] added test [#CLICKHOUSE-3590] --- .../queries/0_stateless/00604_show_create_database.reference | 1 + dbms/tests/queries/0_stateless/00604_show_create_database.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.reference create mode 100644 dbms/tests/queries/0_stateless/00604_show_create_database.sql diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.reference b/dbms/tests/queries/0_stateless/00604_show_create_database.reference new file mode 100644 index 00000000000..ab242ee47fd --- /dev/null +++ b/dbms/tests/queries/0_stateless/00604_show_create_database.reference @@ -0,0 +1 @@ +CREATE DATABASE test ENGINE = Ordinary diff --git a/dbms/tests/queries/0_stateless/00604_show_create_database.sql b/dbms/tests/queries/0_stateless/00604_show_create_database.sql new file mode 100644 index 00000000000..b5671b3a984 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00604_show_create_database.sql @@ -0,0 +1,2 @@ +create database if not exists test; +show create database test; From cda51eb920ce3adbb96a15bd01de6bb8b44627bb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Mar 2018 15:31:58 +0300 Subject: [PATCH 26/90] added fake create query for default database [#CLICKHOUSE-3590] --- dbms/src/Databases/DatabaseOrdinary.cpp | 9 +++++++-- dbms/src/Databases/DatabaseOrdinary.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index b0719649094..73152387cf6 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -336,16 +336,21 @@ void DatabaseOrdinary::removeTable( } -static ASTPtr getCreateQueryImpl(const String & path, const String & table_name) +ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String & table_name) const { String metadata_path; + String query; if (table_name.empty()) + { metadata_path = detail::getDatabaseMetadataPath(path); + if (!Poco::File(metadata_path).exists()) + query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary"; + } else metadata_path = detail::getTableMetadataPath(path, table_name); - String query; + if (query.empty()) { ReadBufferFromFile in(metadata_path, 4096); readStringUntilEOF(query, in); diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index dbd44586c74..4cd77c85895 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -68,6 +68,7 @@ public: private: void startupTables(ThreadPool * thread_pool); + ASTPtr getCreateQueryImpl(const String & path, const String & table_name) const; }; } From d6e9d08d5ef49b82e210c42f401d8c0aad7d43e5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Mar 2018 16:28:32 +0300 Subject: [PATCH 27/90] getCreateTableQuery and getCreateDatabaseQuery [#CLICKHOUSE-3590] --- dbms/src/Databases/DatabaseDictionary.cpp | 36 ++++++++- dbms/src/Databases/DatabaseDictionary.h | 4 +- dbms/src/Databases/DatabaseMemory.cpp | 7 +- dbms/src/Databases/DatabaseMemory.h | 8 +- dbms/src/Databases/DatabaseOrdinary.cpp | 76 ++++++++++++------- dbms/src/Databases/DatabaseOrdinary.h | 5 +- dbms/src/Databases/IDatabase.h | 8 +- dbms/src/Interpreters/Context.cpp | 15 +++- dbms/src/Interpreters/Context.h | 5 +- .../Interpreters/InterpreterCreateQuery.cpp | 4 +- .../InterpreterShowCreateQuery.cpp | 9 ++- dbms/src/Storages/StorageDictionary.cpp | 1 - dbms/src/Storages/StorageDictionary.h | 40 +++++----- .../Storages/System/StorageSystemTables.cpp | 2 +- 14 files changed, 149 insertions(+), 71 deletions(-) diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 8a10535fd09..6c14312c2e3 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -3,6 +3,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { @@ -158,11 +163,34 @@ time_t DatabaseDictionary::getTableMetadataModificationTime( return static_cast(0); } -ASTPtr DatabaseDictionary::getCreateQuery( - const Context &, - const String &) const +ASTPtr DatabaseDictionary::getCreateTableQuery( + const Context & context, + const String & table_name) const { - throw Exception("There is no CREATE TABLE query for DatabaseDictionary tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); + String query; + { + WriteBufferFromString buffer(query); + + auto dictionary = context.getExternalDictionaries().getDictionary(table_name); + auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure()); + buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " ("; + buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end()); + buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")"; + } + + ParserCreateQuery parser; + return parseQuery(parser, query.data(), query.data() + query.size(), ""); +} + +ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const +{ + String query; + { + WriteBufferFromString buffer(query); + buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary"; + } + ParserCreateQuery parser; + return parseQuery(parser, query.data(), query.data() + query.size(), ""); } void DatabaseDictionary::shutdown() diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 1308ea20a40..f62bfc3e1c3 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -89,10 +89,12 @@ public: const Context & context, const String & table_name) override; - ASTPtr getCreateQuery( + ASTPtr getCreateTableQuery( const Context & context, const String & table_name) const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; + void shutdown() override; void drop() override; }; diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 66b23f07ca6..b50fb78468e 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -121,13 +121,18 @@ time_t DatabaseMemory::getTableMetadataModificationTime( return static_cast(0); } -ASTPtr DatabaseMemory::getCreateQuery( +ASTPtr DatabaseMemory::getCreateTableQuery( const Context &, const String &) const { throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); } +ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const +{ + throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); +} + void DatabaseMemory::shutdown() { /// You can not hold a lock during shutdown. diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index be095ad3755..dbf37acb5f6 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -80,9 +80,11 @@ public: const Context & context, const String & table_name) override; - ASTPtr getCreateQuery( - const Context & context, - const String & table_name) const override; + ASTPtr getCreateTableQuery( + const Context & context, + const String & table_name) const override; + + ASTPtr getCreateDatabaseQuery(const Context & context) const override; void shutdown() override; void drop() override; diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 73152387cf6..1f09c118666 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -335,22 +335,13 @@ void DatabaseOrdinary::removeTable( } } - -ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String & table_name) const +static ASTPtr getQueryFromMetadata(const String & metadata_path) { - String metadata_path; + if (!Poco::File(metadata_path).exists()) + return nullptr; + String query; - if (table_name.empty()) - { - metadata_path = detail::getDatabaseMetadataPath(path); - if (!Poco::File(metadata_path).exists()) - query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary"; - } - else - metadata_path = detail::getTableMetadataPath(path, table_name); - - if (query.empty()) { ReadBufferFromFile in(metadata_path, 4096); readStringUntilEOF(query, in); @@ -360,6 +351,20 @@ ASTPtr DatabaseOrdinary::getCreateQueryImpl(const String & path, const String & return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path); } +static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database) +{ + ASTPtr ast = getQueryFromMetadata(metadata_path); + + if (ast) + { + ASTCreateQuery & ast_create_query = typeid_cast(*ast); + ast_create_query.attach = false; + ast_create_query.database = database; + } + + return ast; +} + void DatabaseOrdinary::renameTable( const Context & context, @@ -394,7 +399,9 @@ void DatabaseOrdinary::renameTable( throw Exception{e}; } - ASTPtr ast = getCreateQueryImpl(metadata_path, table_name); + ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(metadata_path, table_name)); + if (!ast) + throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST); ASTCreateQuery & ast_create_query = typeid_cast(*ast); ast_create_query.table = to_table_name; @@ -422,27 +429,40 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime( } -ASTPtr DatabaseOrdinary::getCreateQuery( +ASTPtr DatabaseOrdinary::getCreateTableQuery( const Context & context, const String & table_name) const { ASTPtr ast; - try - { - ast = getCreateQueryImpl(metadata_path, table_name); - } - catch (const Exception & e) - { - /// Handle system.* tables for which there are no table.sql files - if (e.code() == ErrorCodes::FILE_DOESNT_EXIST && tryGetTable(context, table_name) != nullptr) - throw Exception("There is no CREATE TABLE query for table " + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); - throw; + auto table_metadata_path = detail::getTableMetadataPath(metadata_path, table_name); + ast = getCreateQueryFromMetadata(table_metadata_path, name); + if (!ast) + { + /// Handle system.* tables for which there are no table.sql files. + auto msg = tryGetTable(context, table_name) + ? "There is no CREATE TABLE query for table " + : "There is no metadata file for table "; + + throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); } - ASTCreateQuery & ast_create_query = typeid_cast(*ast); - ast_create_query.attach = false; - ast_create_query.database = name; + return ast; +} + +ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const +{ + ASTPtr ast; + + auto database_metadata_path = detail::getDatabaseMetadataPath(metadata_path); + ast = getCreateQueryFromMetadata(database_metadata_path, name); + if (!ast) + { + /// Handle databases (such as default) for which there are no database.sql files. + String query = "CREATE DATABASE " + backQuoteIfNeed(name) + " ENGINE = Ordinary"; + ParserCreateQuery parser; + ast = parseQuery(parser, query.data(), query.data() + query.size(), ""); + } return ast; } diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index 4cd77c85895..9d8d2458df6 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -55,10 +55,12 @@ public: const Context & context, const String & table_name) override; - ASTPtr getCreateQuery( + ASTPtr getCreateTableQuery( const Context & context, const String & table_name) const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; + String getDataPath() const override; String getMetadataPath() const override; String getTableMetadataPath(const String & table_name) const override; @@ -68,7 +70,6 @@ public: private: void startupTables(ThreadPool * thread_pool); - ASTPtr getCreateQueryImpl(const String & path, const String & table_name) const; }; } diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index 8d62c0e3d45..01c8970901d 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -124,12 +124,14 @@ public: const Context & context, const String & name) = 0; - /// Get the CREATE TABLE query for the table or CREATE DATABASE query for database if name is empty. - /// It can also provide information for detached tables for which there is metadata. - virtual ASTPtr getCreateQuery( + /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata. + virtual ASTPtr getCreateTableQuery( const Context & context, const String & name) const = 0; + /// Get the CREATE DATABASE query for current database. + virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0; + /// Returns path for persistent data storage if the database supports it, empty string otherwise virtual String getDataPath() const { return {}; } /// Returns metadata path if the database supports it, empty string otherwise diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 6fddcbc2f05..e3e0e268f98 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -910,17 +910,17 @@ DatabasePtr Context::detachDatabase(const String & database_name) } -ASTPtr Context::getCreateQuery(const String & database_name, const String & table_name) const +ASTPtr Context::getCreateTableQuery(const String & database_name, const String & table_name) const { auto lock = getLock(); String db = resolveDatabase(database_name, current_database); assertDatabaseExists(db); - return shared->databases[db]->getCreateQuery(*this, table_name); + return shared->databases[db]->getCreateTableQuery(*this, table_name); } -ASTPtr Context::getCreateExternalQuery(const String & table_name) const +ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { TableAndCreateASTs::const_iterator jt = external_tables.find(table_name); if (external_tables.end() == jt) @@ -929,6 +929,15 @@ ASTPtr Context::getCreateExternalQuery(const String & table_name) const return jt->second.second; } +ASTPtr Context::getCreateDatabaseQuery(const String & database_name) const +{ + auto lock = getLock(); + + String db = resolveDatabase(database_name, current_database); + assertDatabaseExists(db); + + return shared->databases[db]->getCreateDatabaseQuery(*this); +} Settings Context::getSettings() const { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index bab3be07add..f1e44705f3a 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -241,8 +241,9 @@ public: UInt16 getTCPPort() const; /// Get query for the CREATE table. - ASTPtr getCreateQuery(const String & database_name, const String & table_name) const; - ASTPtr getCreateExternalQuery(const String & table_name) const; + ASTPtr getCreateTableQuery(const String & database_name, const String & table_name) const; + ASTPtr getCreateExternalTableQuery(const String & table_name) const; + ASTPtr getCreateDatabaseQuery(const String & database_name) const; const DatabasePtr getDatabase(const String & database_name) const; DatabasePtr getDatabase(const String & database_name); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 2b6d333d1f6..347d6208e70 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -432,7 +432,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database; String as_table_name = create.as_table; - ASTPtr as_create_ptr = context.getCreateQuery(as_database_name, as_table_name); + ASTPtr as_create_ptr = context.getCreateTableQuery(as_database_name, as_table_name); const auto & as_create = typeid_cast(*as_create_ptr); if (as_create.is_view) @@ -461,7 +461,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.attach && !create.storage && !create.columns) { // Table SQL definition is available even if the table is detached - auto query = context.getCreateQuery(database_name, table_name); + auto query = context.getCreateTableQuery(database_name, table_name); auto & as_create = typeid_cast(*query); create = as_create; // Copy the saved create query, but use ATTACH instead of CREATE create.attach = true; diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index 705b9bb4120..a4c3bd7224c 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -44,8 +44,13 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() if (ast.temporary && !ast.database.empty()) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); - ASTPtr create_query = (ast.temporary ? context.getCreateExternalQuery(ast.table) : - context.getCreateQuery(ast.database, ast.table)); + ASTPtr create_query; + if (ast.temporary) + create_query = context.getCreateExternalTableQuery(ast.table); + else if (ast.table.empty()) + create_query = context.getCreateDatabaseQuery(ast.database); + else + create_query = context.getCreateTableQuery(ast.database, ast.table); if (!create_query && ast.temporary) throw Exception("Unable to show the create query of " + ast.table + ". Maybe it was created by the system.", ErrorCodes::THERE_IS_NO_QUERY); diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 2bb69748c6d..89d59e11b38 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -89,7 +89,6 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction } } - void registerStorageDictionary(StorageFactory & factory) { factory.registerStorage("Dictionary", [](const StorageFactory::Arguments & args) diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 0f21373f546..079c65ff134 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace Poco @@ -33,6 +35,26 @@ public: void drop() override {} static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); + template + static std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) + { + std::string description; + { + WriteBufferFromString buffer(description); + bool first = true; + for (; begin != end; ++begin) + { + if (!first) + buffer << ", "; + first = false; + + buffer << begin->name << ' ' << begin->type->getName(); + } + } + + return description; + } + private: using Ptr = MultiVersion::Version; @@ -42,24 +64,6 @@ private: void checkNamesAndTypesCompatibleWithDictionary(const DictionaryStructure & dictionary_structure) const; - template - std::string generateNamesAndTypesDescription(ForwardIterator begin, ForwardIterator end) const - { - if (begin == end) - { - return ""; - } - std::string description; - for (; begin != end; ++begin) - { - description += ", "; - description += begin->name; - description += ' '; - description += begin->type->getName(); - } - return description.substr(2, description.size()); - } - protected: StorageDictionary(const String & table_name_, const NamesAndTypesList & columns_, diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 852392a63d1..493750d5dfa 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -204,7 +204,7 @@ BlockInputStreams StorageSystemTables::read( try { - ast = database->getCreateQuery(context, table_name); + ast = database->getCreateTableQuery(context, table_name); } catch (const Exception & e) { From 51301d38cd81b0b1757163023af92cf7275ea390 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 14 Mar 2018 22:56:44 +0300 Subject: [PATCH 28/90] added tryGetCreateTableQuery [#CLICKHOUSE-3590] --- dbms/src/Databases/DatabaseDictionary.cpp | 5 +++++ dbms/src/Databases/DatabaseDictionary.h | 4 ++++ dbms/src/Databases/DatabaseMemory.cpp | 7 +++++++ dbms/src/Databases/DatabaseMemory.h | 4 ++++ dbms/src/Databases/DatabaseOrdinary.cpp | 21 +++++++++++++++---- dbms/src/Databases/DatabaseOrdinary.h | 5 +++++ dbms/src/Databases/IDatabase.h | 5 +++++ .../Storages/System/StorageSystemTables.cpp | 12 +---------- 8 files changed, 48 insertions(+), 15 deletions(-) diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 6c14312c2e3..c68c801e4cb 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -182,6 +182,11 @@ ASTPtr DatabaseDictionary::getCreateTableQuery( return parseQuery(parser, query.data(), query.data() + query.size(), ""); } +ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const +{ + return getCreateTableQuery(context, table_name); +} + ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const { String query; diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index f62bfc3e1c3..e2a036362b0 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -93,6 +93,10 @@ public: const Context & context, const String & table_name) const override; + ASTPtr tryGetCreateTableQuery( + const Context & context, + const String & table_name) const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; void shutdown() override; diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index b50fb78468e..8eacc7f856f 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -128,6 +128,13 @@ ASTPtr DatabaseMemory::getCreateTableQuery( throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); } +ASTPtr DatabaseMemory::tryGetCreateTableQuery( + const Context &, + const String &) const +{ + throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); +} + ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const { throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index dbf37acb5f6..1b5f0a20c62 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -84,6 +84,10 @@ public: const Context & context, const String & table_name) const override; + ASTPtr tryGetCreateTableQuery( + const Context & context, + const String & table_name) const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; void shutdown() override; diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 1f09c118666..4f63834d396 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -429,9 +429,7 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime( } -ASTPtr DatabaseOrdinary::getCreateTableQuery( - const Context & context, - const String & table_name) const +ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const { ASTPtr ast; @@ -440,7 +438,12 @@ ASTPtr DatabaseOrdinary::getCreateTableQuery( if (!ast) { /// Handle system.* tables for which there are no table.sql files. - auto msg = tryGetTable(context, table_name) + bool has_table = tryGetTable(context, table_name) != nullptr; + + if (has_table && try_get) + return nullptr; + + auto msg = has_table ? "There is no CREATE TABLE query for table " : "There is no metadata file for table "; @@ -450,6 +453,16 @@ ASTPtr DatabaseOrdinary::getCreateTableQuery( return ast; } +ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const +{ + return getCreateTableQueryImpl(context, table_name, false); +} + +ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const +{ + return getCreateTableQueryImpl(context, table_name, true); +} + ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const { ASTPtr ast; diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index 9d8d2458df6..cae6686d5ba 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -59,6 +59,10 @@ public: const Context & context, const String & table_name) const override; + ASTPtr tryGetCreateTableQuery( + const Context & context, + const String & table_name) const override; + ASTPtr getCreateDatabaseQuery(const Context & context) const override; String getDataPath() const override; @@ -70,6 +74,7 @@ public: private: void startupTables(ThreadPool * thread_pool); + ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const; }; } diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index 01c8970901d..6b84f3fa165 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -129,6 +129,11 @@ public: const Context & context, const String & name) const = 0; + /// Like getCreateTableQuery, but don't throw for existing tables without metadata. + virtual ASTPtr tryGetCreateTableQuery( + const Context & context, + const String & name) const = 0; + /// Get the CREATE DATABASE query for current database. virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0; diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 493750d5dfa..77700c2a256 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -200,17 +200,7 @@ BlockInputStreams StorageSystemTables::read( if (has_create_table_query || has_engine_full) { - ASTPtr ast; - - try - { - ast = database->getCreateTableQuery(context, table_name); - } - catch (const Exception & e) - { - if (e.code() != ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY) - throw; - } + ASTPtr ast = database->tryGetCreateTableQuery(context, table_name); if (has_create_table_query) res_columns[j++]->insert(ast ? queryToString(ast) : ""); From e41b653f1444b89edd56c9df3544a76aa61edc89 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Mon, 19 Mar 2018 21:26:52 +0300 Subject: [PATCH 29/90] Parallelized block split in sync Distributed insert. [#CLICKHOUSE-2] --- .../DistributedBlockOutputStream.cpp | 108 ++++++++++++------ .../DistributedBlockOutputStream.h | 32 ++++-- 2 files changed, 92 insertions(+), 48 deletions(-) diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 607eb9efb67..98416437497 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -89,7 +89,7 @@ void DistributedBlockOutputStream::writeAsync(const Block & block) return writeSplitAsync(block); writeAsyncImpl(block); - ++blocks_inserted; + ++inserted_blocks; } @@ -100,17 +100,17 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription() buffer << "Insertion status:\n"; for (auto & shard_jobs : per_shard_jobs) - for (JobInfo & job : shard_jobs) + for (JobReplica & job : shard_jobs.replicas_jobs) { buffer << "Wrote " << job.blocks_written << " blocks and " << job.rows_written << " rows" << " on shard " << job.shard_index << " replica " << job.replica_index << ", " << addresses[job.shard_index][job.replica_index].readableString(); /// Performance statistics - if (job.bloks_started > 0) + if (job.blocks_started > 0) { - buffer << " (average " << job.elapsed_time_ms / job.bloks_started << " ms per block"; - if (job.bloks_started > 1) + buffer << " (average " << job.elapsed_time_ms / job.blocks_started << " ms per block"; + if (job.blocks_started > 1) buffer << ", the slowest block " << job.max_elapsed_time_for_block_ms << " ms"; buffer << ")"; } @@ -122,10 +122,11 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription() } -void DistributedBlockOutputStream::initWritingJobs() +void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) { const auto & addresses_with_failovers = cluster->getShardsAddresses(); const auto & shards_info = cluster->getShardsInfo(); + size_t num_shards = shards_info.size(); remote_jobs_count = 0; local_jobs_count = 0; @@ -145,7 +146,7 @@ void DistributedBlockOutputStream::initWritingJobs() { if (!replicas[replica_index].is_local) { - shard_jobs.emplace_back(shard_index, replica_index, false); + shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block); ++remote_jobs_count; if (shard_info.hasInternalReplication()) @@ -156,9 +157,12 @@ void DistributedBlockOutputStream::initWritingJobs() if (shard_info.isLocal()) { - shard_jobs.emplace_back(shard_index, 0, true); + shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block); ++local_jobs_count; } + + if (num_shards > 1) + shard_jobs.shard_current_block_permuation.reserve(first_block.rows()); } } @@ -184,15 +188,15 @@ void DistributedBlockOutputStream::waitForJobs() } -ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobInfo & job) +ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobReplica & job, const Block & current_block) { auto memory_tracker = current_memory_tracker; - return [this, memory_tracker, &job]() + return [this, memory_tracker, &job, ¤t_block]() { SCOPE_EXIT({++finished_jobs_count;}); Stopwatch watch; - ++job.bloks_started; + ++job.blocks_started; SCOPE_EXIT({ UInt64 elapsed_time_for_block_ms = watch.elapsedMilliseconds(); @@ -207,8 +211,30 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp } const auto & shard_info = cluster->getShardsInfo()[job.shard_index]; + size_t num_shards = cluster->getShardsInfo().size(); + auto & shard_job = per_shard_jobs[job.shard_index]; const auto & addresses = cluster->getShardsAddresses(); - Block & block = current_blocks.at(job.shard_index); + + /// Generate current shard block + if (num_shards > 1) + { + auto & shard_permutation = shard_job.shard_current_block_permuation; + size_t num_shard_rows = shard_permutation.size(); + + for (size_t j = 0; j < current_block.columns(); ++j) + { + auto & src_column = current_block.getByPosition(j).column; + auto & dst_column = job.current_shard_block.getByPosition(j).column; + + /// Zero permutation size has special meaning in IColumn::permute + if (num_shard_rows) + dst_column = src_column->permute(shard_permutation, num_shard_rows); + else + dst_column = src_column->cloneEmpty(); + } + } + + const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; if (!job.is_local_job) { @@ -217,7 +243,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp if (shard_info.hasInternalReplication()) { /// Skip replica_index in case of internal replication - if (per_shard_jobs[job.shard_index].size() != 1) + if (shard_job.replicas_jobs.size() != 1) throw Exception("There are several writing job for an automatically replicated shard", ErrorCodes::LOGICAL_ERROR); /// TODO: it make sense to rewrite skip_unavailable_shards and max_parallel_replicas here @@ -248,7 +274,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp } CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; - job.stream->write(block); + job.stream->write(shard_block); } else { @@ -265,21 +291,25 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp size_t num_repetitions = shard_info.getLocalNodeCount(); for (size_t i = 0; i < num_repetitions; ++i) - job.stream->write(block); + job.stream->write(shard_block); } job.blocks_written += 1; - job.rows_written += block.rows(); + job.rows_written += shard_block.rows(); }; } void DistributedBlockOutputStream::writeSync(const Block & block) { + const auto & shards_info = cluster->getShardsInfo(); + size_t num_shards = shards_info.size(); + if (!pool) { /// Deferred initialization. Only for sync insertion. - initWritingJobs(); + initWritingJobs(block); + pool.emplace(remote_jobs_count + local_jobs_count); query_string = queryToString(query_ast); @@ -292,14 +322,23 @@ void DistributedBlockOutputStream::writeSync(const Block & block) watch.restart(); } - const auto & shards_info = cluster->getShardsInfo(); - current_blocks = shards_info.size() > 1 ? splitBlock(block) : Blocks({block}); + if (num_shards > 1) + { + auto current_selector = createSelector(block); + + /// Prepare row numbers for each shard + for (size_t shard_index : ext::range(0, num_shards)) + per_shard_jobs[shard_index].shard_current_block_permuation.resize(0); + + for (size_t i = 0; i < block.rows(); ++i) + per_shard_jobs[current_selector[i]].shard_current_block_permuation.push_back(i); + } /// Run jobs in parallel for each block and wait them finished_jobs_count = 0; - for (size_t shard_index : ext::range(0, current_blocks.size())) - for (JobInfo & job : per_shard_jobs.at(shard_index)) - pool->schedule(runWritingJob(job)); + for (size_t shard_index : ext::range(0, shards_info.size())) + for (JobReplica & job : per_shard_jobs[shard_index].replicas_jobs) + pool->schedule(runWritingJob(job, block)); try { @@ -311,7 +350,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block) throw; } - ++blocks_inserted; + inserted_blocks += 1; + inserted_rows += block.rows(); } @@ -321,7 +361,7 @@ void DistributedBlockOutputStream::writeSuffix() { finished_jobs_count = 0; for (auto & shard_jobs : per_shard_jobs) - for (JobInfo & job : shard_jobs) + for (JobReplica & job : shard_jobs.replicas_jobs) { if (job.stream) pool->schedule([&job] () { job.stream->writeSuffix(); }); @@ -338,17 +378,19 @@ void DistributedBlockOutputStream::writeSuffix() } double elapsed = watch.elapsedSeconds(); - LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << blocks_inserted << " blocks" - << " (average " << std::fixed << std::setprecision(1) << elapsed / blocks_inserted * 1000 << " ms. per block)" + LOG_DEBUG(log, "It took " << std::fixed << std::setprecision(1) << elapsed << " sec. to insert " << inserted_blocks << " blocks" + << ", " << std::fixed << std::setprecision(1) << inserted_rows / elapsed << " rows per second" << ". " << getCurrentStateDescription()); } } -IColumn::Selector DistributedBlockOutputStream::createSelector(Block block) +IColumn::Selector DistributedBlockOutputStream::createSelector(const Block & source_block) { - storage.getShardingKeyExpr()->execute(block); - const auto & key_column = block.getByName(storage.getShardingKeyColumnName()); + Block current_block_with_sharding_key_expr = source_block; + storage.getShardingKeyExpr()->execute(current_block_with_sharding_key_expr); + + const auto & key_column = current_block_with_sharding_key_expr.getByName(storage.getShardingKeyColumnName()); const auto & slot_to_shard = cluster->getSlotToShard(); #define CREATE_FOR_TYPE(TYPE) \ @@ -372,12 +414,6 @@ IColumn::Selector DistributedBlockOutputStream::createSelector(Block block) Blocks DistributedBlockOutputStream::splitBlock(const Block & block) { - const auto num_cols = block.columns(); - /// cache column pointers for later reuse - std::vector columns(num_cols); - for (size_t i = 0; i < columns.size(); ++i) - columns[i] = block.safeGetByPosition(i).column.get(); - auto selector = createSelector(block); /// Split block to num_shard smaller block, using 'selector'. @@ -409,7 +445,7 @@ void DistributedBlockOutputStream::writeSplitAsync(const Block & block) if (splitted_blocks[shard_idx].rows()) writeAsyncImpl(splitted_blocks[shard_idx], shard_idx); - ++blocks_inserted; + ++inserted_blocks; } diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h index eed38adf417..0b37d51c856 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -44,7 +44,7 @@ public: private: - IColumn::Selector createSelector(Block block); + IColumn::Selector createSelector(const Block & source_block); void writeAsync(const Block & block); @@ -65,10 +65,10 @@ private: /// Performs synchronous insertion to remote nodes. If timeout_exceeded flag was set, throws. void writeSync(const Block & block); - void initWritingJobs(); + void initWritingJobs(const Block & first_block); - struct JobInfo; - ThreadPool::Job runWritingJob(JobInfo & job); + struct JobReplica; + ThreadPool::Job runWritingJob(JobReplica & job, const Block & current_block); void waitForJobs(); @@ -80,7 +80,8 @@ private: ASTPtr query_ast; ClusterPtr cluster; const Settings & settings; - size_t blocks_inserted = 0; + size_t inserted_blocks = 0; + size_t inserted_rows = 0; bool insert_sync; @@ -91,16 +92,18 @@ private: ThrottlerPtr throttler; String query_string; - struct JobInfo + struct JobReplica { - JobInfo() = default; - JobInfo(size_t shard_index, size_t replica_index, bool is_local_job) - : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job) {} + JobReplica() = default; + JobReplica(size_t shard_index, size_t replica_index, bool is_local_job, const Block & sample_block) + : shard_index(shard_index), replica_index(replica_index), is_local_job(is_local_job), current_shard_block(sample_block.cloneEmpty()) {} size_t shard_index = 0; size_t replica_index = 0; bool is_local_job = false; + Block current_shard_block; + ConnectionPool::Entry connection_entry; std::unique_ptr local_context; BlockOutputStreamPtr stream; @@ -108,13 +111,18 @@ private: UInt64 blocks_written = 0; UInt64 rows_written = 0; - UInt64 bloks_started = 0; + UInt64 blocks_started = 0; UInt64 elapsed_time_ms = 0; UInt64 max_elapsed_time_for_block_ms = 0; }; - std::vector> per_shard_jobs; - Blocks current_blocks; + struct JobShard + { + std::list replicas_jobs; + IColumn::Permutation shard_current_block_permuation; + }; + + std::vector per_shard_jobs; size_t remote_jobs_count = 0; size_t local_jobs_count = 0; From 1a0833d0e7759a85965781f28ecbfb68469059b1 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Mon, 19 Mar 2018 21:30:49 +0300 Subject: [PATCH 30/90] Fixed small race condition in ClusterCopier. [#CLICKHOUSE-2] --- dbms/src/Server/ClusterCopier.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp index 4233ce70e3f..3bcbad95d76 100644 --- a/dbms/src/Server/ClusterCopier.cpp +++ b/dbms/src/Server/ClusterCopier.cpp @@ -1259,8 +1259,10 @@ protected: } /// Remove the locking node - cleaner_holder.reset(); - zookeeper->remove(is_dirty_flag_path); + zkutil::Ops ops; + ops.emplace_back(new zkutil::Op::Remove(dirt_cleaner_path, -1)); + ops.emplace_back(new zkutil::Op::Remove(is_dirty_flag_path, -1)); + zookeeper->multi(ops); LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); return true; From a399c17f0196c7ae0326676f2a00b0d74cb1a8e4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 20 Mar 2018 13:58:16 +0300 Subject: [PATCH 31/90] && - qualified IColumn::mutate --- dbms/src/Columns/FilterDescription.cpp | 3 ++- dbms/src/Columns/IColumn.h | 4 ++-- dbms/src/Common/COWPtr.h | 18 ++++++++++++++---- dbms/src/Core/Block.cpp | 2 +- dbms/src/Core/Block.h | 2 +- dbms/src/DataStreams/ColumnGathererStream.cpp | 5 ++--- .../MergingSortedBlockInputStream.cpp | 2 +- dbms/src/DataStreams/SquashingTransform.cpp | 2 +- .../ComplexKeyHashedDictionary.cpp | 2 +- .../Dictionaries/ExternalResultDescription.cpp | 2 +- dbms/src/Dictionaries/FlatDictionary.cpp | 2 +- dbms/src/Dictionaries/HashedDictionary.cpp | 2 +- dbms/src/Functions/FunctionsConditional.h | 7 ++++--- dbms/src/Functions/IFunction.cpp | 2 +- dbms/src/Interpreters/NullableUtils.cpp | 2 +- dbms/src/Storages/StorageBuffer.cpp | 4 ++-- 16 files changed, 36 insertions(+), 25 deletions(-) diff --git a/dbms/src/Columns/FilterDescription.cpp b/dbms/src/Columns/FilterDescription.cpp index 1d29cd97435..3d3e3f5ffea 100644 --- a/dbms/src/Columns/FilterDescription.cpp +++ b/dbms/src/Columns/FilterDescription.cpp @@ -57,7 +57,8 @@ FilterDescription::FilterDescription(const IColumn & column) if (const ColumnNullable * nullable_column = typeid_cast(&column)) { - MutableColumnPtr mutable_holder = nullable_column->getNestedColumn().mutate(); + ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); + MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate(); ColumnUInt8 * concrete_column = typeid_cast(mutable_holder.get()); if (!concrete_column) diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 9fd98e6f701..3a733e4a025 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -247,10 +247,10 @@ public: virtual void forEachSubcolumn(ColumnCallback) {} - MutablePtr mutate() const + MutablePtr mutate() const && { MutablePtr res = COWPtr::mutate(); - res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = subcolumn->mutate(); }); + res->forEachSubcolumn([](Ptr & subcolumn) { subcolumn = (*std::move(subcolumn)).mutate(); }); return res; } diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h index 9f7cf21ad63..39c912f8abc 100644 --- a/dbms/src/Common/COWPtr.h +++ b/dbms/src/Common/COWPtr.h @@ -80,12 +80,22 @@ private: Derived * derived() { return static_cast(this); } const Derived * derived() const { return static_cast(this); } + template + class IntrusivePtr : public boost::intrusive_ptr + { + public: + using boost::intrusive_ptr::intrusive_ptr; + + T & operator*() const & { return boost::intrusive_ptr::operator*(); } + T && operator*() const && { return const_cast::type &&>(*boost::intrusive_ptr::get()); } + }; + protected: template - class mutable_ptr : public boost::intrusive_ptr + class mutable_ptr : public IntrusivePtr { private: - using Base = boost::intrusive_ptr; + using Base = IntrusivePtr; template friend class COWPtr; template friend class COWPtrHelper; @@ -114,10 +124,10 @@ public: protected: template - class immutable_ptr : public boost::intrusive_ptr + class immutable_ptr : public IntrusivePtr { private: - using Base = boost::intrusive_ptr; + using Base = IntrusivePtr; template friend class COWPtr; template friend class COWPtrHelper; diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index c969e3256da..1cae4ba252e 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -312,7 +312,7 @@ MutableColumns Block::mutateColumns() const size_t num_columns = data.size(); MutableColumns columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - columns[i] = data[i].column ? data[i].column->mutate() : data[i].type->createColumn(); + columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn(); return columns; } diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index 7c836e49532..ea4a4c00cef 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -109,7 +109,7 @@ public: /** Get empty columns with the same types as in block. */ MutableColumns cloneEmptyColumns() const; - /** Get columns from block for mutation. */ + /** Get columns from block for mutation. Columns in block will be nullptr. */ MutableColumns mutateColumns() const; /** Replace columns in a block */ diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp index ba4107f7e8b..baa19ae9130 100644 --- a/dbms/src/DataStreams/ColumnGathererStream.cpp +++ b/dbms/src/DataStreams/ColumnGathererStream.cpp @@ -76,10 +76,9 @@ Block ColumnGathererStream::readImpl() return Block(); output_block = Block{column.cloneEmpty()}; - MutableColumnPtr output_column = output_block.getByPosition(0).column->mutate(); + MutableColumnPtr output_column = (*std::move(output_block.getByPosition(0).column)).mutate(); output_column->gather(*this); - if (!output_column->empty()) - output_block.getByPosition(0).column = std::move(output_column); + output_block.getByPosition(0).column = std::move(output_column); return output_block; } diff --git a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp index f93197ca123..b60cc072b47 100644 --- a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -221,7 +221,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std:: throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR); for (size_t i = 0; i < num_columns; ++i) - merged_columns[i] = source_blocks[source_num]->getByPosition(i).column->mutate(); + merged_columns[i] = (*std::move(source_blocks[source_num]->getByPosition(i).column)).mutate(); // std::cerr << "copied columns\n"; diff --git a/dbms/src/DataStreams/SquashingTransform.cpp b/dbms/src/DataStreams/SquashingTransform.cpp index 9e8f590f171..d31ff6f8063 100644 --- a/dbms/src/DataStreams/SquashingTransform.cpp +++ b/dbms/src/DataStreams/SquashingTransform.cpp @@ -62,7 +62,7 @@ void SquashingTransform::append(Block && block) for (size_t i = 0; i < columns; ++i) { - MutableColumnPtr mutable_column = accumulated_block.getByPosition(i).column->mutate(); + MutableColumnPtr mutable_column = (*std::move(accumulated_block.getByPosition(i).column)).mutate(); mutable_column->insertRangeFrom(*block.getByPosition(i).column, 0, rows); accumulated_block.getByPosition(i).column = std::move(mutable_column); } diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp index 0f8c75ebd89..eaaeeab4e0a 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp @@ -289,7 +289,7 @@ void ComplexKeyHashedDictionary::updateData() for (const auto attribute_idx : ext::range(0, keys_size + attributes_size)) { const IColumn & update_column = *block.getByPosition(attribute_idx).column.get(); - MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate(); + MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable(); saved_column->insertRangeFrom(update_column, 0, update_column.size()); } } diff --git a/dbms/src/Dictionaries/ExternalResultDescription.cpp b/dbms/src/Dictionaries/ExternalResultDescription.cpp index 4c05fc32555..5eca329e5f2 100644 --- a/dbms/src/Dictionaries/ExternalResultDescription.cpp +++ b/dbms/src/Dictionaries/ExternalResultDescription.cpp @@ -66,7 +66,7 @@ void ExternalResultDescription::init(const Block & sample_block_) /// If default value for column was not provided, use default from data type. if (sample_columns.back()->empty()) { - MutableColumnPtr mutable_column = sample_columns.back()->mutate(); + MutableColumnPtr mutable_column = (*std::move(sample_columns.back())).mutate(); column.type->insertDefaultInto(*mutable_column); sample_columns.back() = std::move(mutable_column); } diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp index 0fc477725be..e1341907649 100644 --- a/dbms/src/Dictionaries/FlatDictionary.cpp +++ b/dbms/src/Dictionaries/FlatDictionary.cpp @@ -316,7 +316,7 @@ void FlatDictionary::updateData() for (const auto attribute_idx : ext::range(0, attributes.size() + 1)) { const IColumn & update_column = *block.getByPosition(attribute_idx).column.get(); - MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate(); + MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable(); saved_column->insertRangeFrom(update_column, 0, update_column.size()); } } diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp index c0c7f1a8a17..09880cdbeb5 100644 --- a/dbms/src/Dictionaries/HashedDictionary.cpp +++ b/dbms/src/Dictionaries/HashedDictionary.cpp @@ -310,7 +310,7 @@ void HashedDictionary::updateData() for (const auto attribute_idx : ext::range(0, attributes.size() + 1)) { const IColumn & update_column = *block.getByPosition(attribute_idx).column.get(); - MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->mutate(); + MutableColumnPtr saved_column = saved_block->getByPosition(attribute_idx).column->assumeMutable(); saved_column->insertRangeFrom(update_column, 0, update_column.size()); } } diff --git a/dbms/src/Functions/FunctionsConditional.h b/dbms/src/Functions/FunctionsConditional.h index 841295f0643..27567237928 100644 --- a/dbms/src/Functions/FunctionsConditional.h +++ b/dbms/src/Functions/FunctionsConditional.h @@ -592,7 +592,7 @@ private: const ColumnPtr & result_column = temporary_block.getByPosition(3).column; if (result_column->isColumnNullable()) { - MutableColumnPtr mutable_result_column = result_column->mutate(); + MutableColumnPtr mutable_result_column = (*std::move(result_column)).mutate(); static_cast(*mutable_result_column).applyNullMap(static_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(mutable_result_column); return true; @@ -744,7 +744,7 @@ private: { if (arg_else.column->isColumnNullable()) { - auto result_column = arg_else.column->mutate(); + auto result_column = (*std::move(arg_else.column)).mutate(); static_cast(*result_column).applyNullMap(static_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } @@ -785,7 +785,8 @@ private: if (arg_then.column->isColumnNullable()) { - auto result_column = arg_then.column->mutate(); + auto arg_then_column = arg_then.column; + auto result_column = (*std::move(arg_then_column)).mutate(); static_cast(*result_column).applyNegatedNullMap(static_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index ba05bc34e9d..12e8dfabbd8 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -63,7 +63,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, Block & block, const ColumnNumbe } else { - MutableColumnPtr mutable_result_null_map_column = result_null_map_column->mutate(); + MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); NullMap & result_null_map = static_cast(*mutable_result_null_map_column).getData(); const NullMap & src_null_map = static_cast(*null_map_column).getData(); diff --git a/dbms/src/Interpreters/NullableUtils.cpp b/dbms/src/Interpreters/NullableUtils.cpp index 972c548e67e..dcb0657aa9d 100644 --- a/dbms/src/Interpreters/NullableUtils.cpp +++ b/dbms/src/Interpreters/NullableUtils.cpp @@ -31,7 +31,7 @@ void extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ColumnPtr & nul } else { - MutableColumnPtr mutable_null_map_holder = null_map_holder->mutate(); + MutableColumnPtr mutable_null_map_holder = (*std::move(null_map_holder)).mutate(); PaddedPODArray & mutable_null_map = static_cast(*mutable_null_map_holder).getData(); const PaddedPODArray & other_null_map = column_nullable.getNullMapData(); diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 5c59ffa7c40..8688d191b9a 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -164,7 +164,7 @@ static void appendBlock(const Block & from, Block & to) for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { const IColumn & col_from = *from.getByPosition(column_no).column.get(); - MutableColumnPtr col_to = to.getByPosition(column_no).column->mutate(); + MutableColumnPtr col_to = (*std::move(to.getByPosition(column_no).column)).mutate(); col_to->insertRangeFrom(col_from, 0, rows); @@ -183,7 +183,7 @@ static void appendBlock(const Block & from, Block & to) { ColumnPtr & col_to = to.getByPosition(column_no).column; if (col_to->size() != old_rows) - col_to = col_to->mutate()->cut(0, old_rows); + col_to = (*std::move(col_to)).mutate()->cut(0, old_rows); } } catch (...) From 1f4db58d533cc4708a9a9a6e965e9d9c27d86166 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 20 Mar 2018 20:01:13 +0300 Subject: [PATCH 32/90] fix test --- dbms/src/DataStreams/ColumnGathererStream.cpp | 5 +++-- dbms/src/Functions/FunctionsConditional.h | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataStreams/ColumnGathererStream.cpp b/dbms/src/DataStreams/ColumnGathererStream.cpp index baa19ae9130..5ce00ea8417 100644 --- a/dbms/src/DataStreams/ColumnGathererStream.cpp +++ b/dbms/src/DataStreams/ColumnGathererStream.cpp @@ -76,9 +76,10 @@ Block ColumnGathererStream::readImpl() return Block(); output_block = Block{column.cloneEmpty()}; - MutableColumnPtr output_column = (*std::move(output_block.getByPosition(0).column)).mutate(); + MutableColumnPtr output_column = output_block.getByPosition(0).column->assumeMutable(); output_column->gather(*this); - output_block.getByPosition(0).column = std::move(output_column); + if (!output_column->empty()) + output_block.getByPosition(0).column = std::move(output_column); return output_block; } diff --git a/dbms/src/Functions/FunctionsConditional.h b/dbms/src/Functions/FunctionsConditional.h index 27567237928..7c88bca6264 100644 --- a/dbms/src/Functions/FunctionsConditional.h +++ b/dbms/src/Functions/FunctionsConditional.h @@ -744,7 +744,8 @@ private: { if (arg_else.column->isColumnNullable()) { - auto result_column = (*std::move(arg_else.column)).mutate(); + auto arg_else_column = arg_else.column; + auto result_column = (*std::move(arg_else_column)).mutate(); static_cast(*result_column).applyNullMap(static_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } From 476d4b339c257d1d458237808e429420eefa0249 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 20 Mar 2018 17:17:09 +0300 Subject: [PATCH 33/90] Immutable construction for columns --- dbms/src/Columns/ColumnAggregateFunction.cpp | 6 +- dbms/src/Columns/ColumnAggregateFunction.h | 6 +- dbms/src/Columns/ColumnArray.cpp | 56 ++++++++++--------- dbms/src/Columns/ColumnArray.h | 46 +++++++++------ dbms/src/Columns/ColumnConst.cpp | 8 +-- dbms/src/Columns/ColumnConst.h | 11 ++-- dbms/src/Columns/ColumnFixedString.cpp | 6 +- dbms/src/Columns/ColumnFixedString.h | 6 +- dbms/src/Columns/ColumnFunction.cpp | 8 +-- dbms/src/Columns/ColumnFunction.h | 8 +-- dbms/src/Columns/ColumnNullable.cpp | 14 ++--- dbms/src/Columns/ColumnNullable.h | 15 +++-- dbms/src/Columns/ColumnString.cpp | 6 +- dbms/src/Columns/ColumnString.h | 6 +- dbms/src/Columns/ColumnTuple.cpp | 33 ++++++++--- dbms/src/Columns/ColumnTuple.h | 10 ++-- dbms/src/Columns/ColumnVector.cpp | 6 +- dbms/src/Columns/ColumnVector.h | 6 +- dbms/src/Columns/IColumn.h | 12 ++-- dbms/src/Columns/IColumnDummy.h | 6 +- dbms/src/Common/COWPtr.h | 42 +++++++++++++- .../MergingSortedBlockInputStream.cpp | 2 +- .../SummingSortedBlockInputStream.cpp | 6 +- dbms/src/DataTypes/DataTypeTuple.cpp | 4 +- dbms/src/Functions/FunctionsHigherOrder.h | 2 +- .../Functions/GatherUtils/createArraySink.cpp | 3 +- .../Storages/MergeTree/MergeTreeReader.cpp | 3 +- 27 files changed, 210 insertions(+), 127 deletions(-) diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp index c901dac36f1..b989c007e56 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.cpp +++ b/dbms/src/Columns/ColumnAggregateFunction.cpp @@ -113,7 +113,7 @@ void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start } -MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const +ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_size_hint) const { size_t size = getData().size(); if (size != filter.size()) @@ -140,7 +140,7 @@ MutableColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t } -MutableColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const { size_t size = getData().size(); @@ -325,7 +325,7 @@ void ColumnAggregateFunction::popBack(size_t n) data.resize_assume_reserved(new_size); } -MutableColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const +ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const { size_t size = data.size(); if (size != offsets.size()) diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h index a9ec5ed486c..852d76130d5 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.h +++ b/dbms/src/Columns/ColumnAggregateFunction.h @@ -152,11 +152,11 @@ public: void popBack(size_t n) override; - MutableColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override; + ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; - MutableColumnPtr replicate(const Offsets & offsets) const override; + ColumnPtr replicate(const Offsets & offsets) const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override; diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 4674c57eef6..a943b46eb49 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -31,8 +31,8 @@ namespace ErrorCodes } -ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column) - : data(nested_column), offsets(offsets_column) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) + : data(std::move(nested_column)), offsets(std::move(offsets_column)) { if (!typeid_cast(offsets_column.get())) throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN); @@ -43,8 +43,8 @@ ColumnArray::ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offs */ } -ColumnArray::ColumnArray(const ColumnPtr & nested_column) - : data(nested_column) +ColumnArray::ColumnArray(MutableColumnPtr && nested_column) + : data(std::move(nested_column)) { if (!data->empty()) throw Exception("Not empty data passed to ColumnArray, but no offsets passed", ErrorCodes::ILLEGAL_COLUMN); @@ -317,7 +317,7 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const } -MutableColumnPtr ColumnArray::convertToFullColumnIfConst() const +ColumnPtr ColumnArray::convertToFullColumnIfConst() const { ColumnPtr new_data; @@ -391,7 +391,7 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng } -MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const { if (typeid_cast(data.get())) return filterNumber(filt, result_size_hint); if (typeid_cast(data.get())) return filterNumber(filt, result_size_hint); @@ -410,7 +410,7 @@ MutableColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hi } template -MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const { if (getOffsets().size() == 0) return ColumnArray::create(data); @@ -424,7 +424,7 @@ MutableColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_s return std::move(res); } -MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_size_hint) const { size_t col_size = getOffsets().size(); if (col_size != filt.size()) @@ -492,7 +492,7 @@ MutableColumnPtr ColumnArray::filterString(const Filter & filt, ssize_t result_s return std::move(res); } -MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const { size_t size = getOffsets().size(); if (size != filt.size()) @@ -537,7 +537,7 @@ MutableColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_ return std::move(res); } -MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const { if (getOffsets().size() == 0) return ColumnArray::create(data); @@ -560,7 +560,7 @@ MutableColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result filtered_offsets); } -MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const { if (getOffsets().size() == 0) return ColumnArray::create(data); @@ -576,7 +576,8 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si Columns temporary_arrays(tuple_size); for (size_t i = 0; i < tuple_size; ++i) - temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).filter(filt, result_size_hint); + temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable()) + .filter(filt, result_size_hint); Columns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) @@ -588,7 +589,7 @@ MutableColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_si } -MutableColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const { size_t size = getOffsets().size(); @@ -652,7 +653,7 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h } -MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const { if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); if (typeid_cast(data.get())) return replicateNumber(replicate_offsets); @@ -673,7 +674,7 @@ MutableColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const template -MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const { size_t col_size = size(); if (col_size != replicate_offsets.size()) @@ -721,7 +722,7 @@ MutableColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) } -MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const { size_t col_size = size(); if (col_size != replicate_offsets.size()) @@ -796,7 +797,7 @@ MutableColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) } -MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const { size_t col_size = size(); if (col_size != replicate_offsets.size()) @@ -834,7 +835,7 @@ MutableColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) } -MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const { size_t col_size = size(); if (col_size != replicate_offsets.size()) @@ -860,25 +861,27 @@ MutableColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets } -MutableColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateNullable(const Offsets & replicate_offsets) const { const ColumnNullable & nullable = static_cast(*data); /// Make temporary arrays for each components of Nullable. Then replicate them independently and collect back to result. /// NOTE Offsets are calculated twice and it is redundant. - auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets); - auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr(), getOffsetsPtr()).replicate(replicate_offsets); + auto array_of_nested = ColumnArray(nullable.getNestedColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()) + .replicate(replicate_offsets); + auto array_of_null_map = ColumnArray(nullable.getNullMapColumnPtr()->assumeMutable(), getOffsetsPtr()->assumeMutable()) + .replicate(replicate_offsets); return ColumnArray::create( ColumnNullable::create( - static_cast(*array_of_nested).getDataPtr(), - static_cast(*array_of_null_map).getDataPtr()), - static_cast(*array_of_nested).getOffsetsPtr()); + static_cast(*array_of_nested).getDataPtr(), + static_cast(*array_of_null_map).getDataPtr()), + static_cast(*array_of_nested).getOffsetsPtr()); } -MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const +ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const { const ColumnTuple & tuple = static_cast(*data); @@ -891,7 +894,8 @@ MutableColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) Columns temporary_arrays(tuple_size); for (size_t i = 0; i < tuple_size; ++i) - temporary_arrays[i] = ColumnArray(tuple.getColumns()[i], getOffsetsPtr()).replicate(replicate_offsets); + temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable()) + .replicate(replicate_offsets); Columns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index d1f951eedb7..a2564b20f98 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -24,10 +24,22 @@ private: friend class COWPtrHelper; /** Create an array column with specified values and offsets. */ - ColumnArray(const ColumnPtr & nested_column, const ColumnPtr & offsets_column); + ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ - ColumnArray(const ColumnPtr & nested_column); + ColumnArray(MutableColumnPtr && nested_column); + + using Ptr = COWPtrHelper::Ptr; + + static Ptr createImmutable(const ColumnPtr & nested_column, const ColumnPtr & offsets_column) + { + return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable()); + } + + static Ptr createImmutable(const ColumnPtr & nested_column) + { + return ColumnArray::create(nested_column->assumeMutable()); + } ColumnArray(const ColumnArray &) = default; @@ -51,15 +63,15 @@ public: void insertFrom(const IColumn & src_, size_t n) override; void insertDefault() override; void popBack(size_t n) override; - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; - MutableColumnPtr replicate(const Offsets & replicate_offsets) const override; - MutableColumnPtr convertToFullColumnIfConst() const override; + ColumnPtr replicate(const Offsets & replicate_offsets) const override; + ColumnPtr convertToFullColumnIfConst() const override; void getExtremes(Field & min, Field & max) const override; bool hasEqualOffsets(const ColumnArray & other) const; @@ -110,33 +122,33 @@ private: /// Multiply values if the nested column is ColumnVector. template - MutableColumnPtr replicateNumber(const Offsets & replicate_offsets) const; + ColumnPtr replicateNumber(const Offsets & replicate_offsets) const; /// Multiply the values if the nested column is ColumnString. The code is too complicated. - MutableColumnPtr replicateString(const Offsets & replicate_offsets) const; + ColumnPtr replicateString(const Offsets & replicate_offsets) const; /** Non-constant arrays of constant values are quite rare. * Most functions can not work with them, and does not create such columns as a result. * An exception is the function `replicate`(see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions. * Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst). */ - MutableColumnPtr replicateConst(const Offsets & replicate_offsets) const; + ColumnPtr replicateConst(const Offsets & replicate_offsets) const; /** The following is done by simply replicating of nested columns. */ - MutableColumnPtr replicateTuple(const Offsets & replicate_offsets) const; - MutableColumnPtr replicateNullable(const Offsets & replicate_offsets) const; - MutableColumnPtr replicateGeneric(const Offsets & replicate_offsets) const; + ColumnPtr replicateTuple(const Offsets & replicate_offsets) const; + ColumnPtr replicateNullable(const Offsets & replicate_offsets) const; + ColumnPtr replicateGeneric(const Offsets & replicate_offsets) const; /// Specializations for the filter function. template - MutableColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const; + ColumnPtr filterNumber(const Filter & filt, ssize_t result_size_hint) const; - MutableColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const; - MutableColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const; - MutableColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; - MutableColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; + ColumnPtr filterString(const Filter & filt, ssize_t result_size_hint) const; + ColumnPtr filterTuple(const Filter & filt, ssize_t result_size_hint) const; + ColumnPtr filterNullable(const Filter & filt, ssize_t result_size_hint) const; + ColumnPtr filterGeneric(const Filter & filt, ssize_t result_size_hint) const; }; diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp index 53e43547108..325160e5fc5 100644 --- a/dbms/src/Columns/ColumnConst.cpp +++ b/dbms/src/Columns/ColumnConst.cpp @@ -25,12 +25,12 @@ ColumnConst::ColumnConst(const ColumnPtr & data_, size_t s) ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); } -MutableColumnPtr ColumnConst::convertToFullColumn() const +ColumnPtr ColumnConst::convertToFullColumn() const { return data->replicate(Offsets(1, s)); } -MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const +ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const { if (s != filt.size()) throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" + toString(s) + ")", @@ -39,7 +39,7 @@ MutableColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_ return ColumnConst::create(data, countBytesInFilter(filt)); } -MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const +ColumnPtr ColumnConst::replicate(const Offsets & offsets) const { if (s != offsets.size()) throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" + toString(s) + ")", @@ -49,7 +49,7 @@ MutableColumnPtr ColumnConst::replicate(const Offsets & offsets) const return ColumnConst::create(data, replicated_size); } -MutableColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const { if (limit == 0) limit = s; diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index 0ff94d6ec4b..89f4c488940 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -29,9 +29,9 @@ private: ColumnConst(const ColumnConst & src) = default; public: - MutableColumnPtr convertToFullColumn() const; + ColumnPtr convertToFullColumn() const; - MutableColumnPtr convertToFullColumnIfConst() const override + ColumnPtr convertToFullColumnIfConst() const override { return convertToFullColumn(); } @@ -133,6 +133,7 @@ public: const char * deserializeAndInsertFromArena(const char * pos) override { + data = (*std::move(data)).mutate(); auto & mutable_data = data->assumeMutableRef(); auto res = mutable_data.deserializeAndInsertFromArena(pos); mutable_data.popBack(1); @@ -145,9 +146,9 @@ public: data->updateHashWithValue(0, hash); } - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr replicate(const Offsets & offsets) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr replicate(const Offsets & offsets) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override; size_t byteSize() const override diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 61e2fbdd7d6..2da3c28e50a 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -153,7 +153,7 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_ memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n); } -MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const { size_t col_size = size(); if (col_size != filt.size()) @@ -230,7 +230,7 @@ MutableColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t return std::move(res); } -MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) const { size_t col_size = size(); @@ -258,7 +258,7 @@ MutableColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t lim return std::move(res); } -MutableColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const +ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const { size_t col_size = size(); if (col_size != offsets.size()) diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index e5ed788aaf0..cd465a1814d 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -104,11 +104,11 @@ public: void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; - MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; - MutableColumnPtr replicate(const Offsets & offsets) const override; + ColumnPtr replicate(const Offsets & offsets) const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override { diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp index 498bcbc26b3..4675e7915f7 100644 --- a/dbms/src/Columns/ColumnFunction.cpp +++ b/dbms/src/Columns/ColumnFunction.cpp @@ -26,7 +26,7 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const return ColumnFunction::create(size, function, capture); } -MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const +ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const { if (size_ != offsets.size()) throw Exception("Size of offsets (" + toString(offsets.size()) + ") doesn't match size of column (" @@ -40,7 +40,7 @@ MutableColumnPtr ColumnFunction::replicate(const Offsets & offsets) const return ColumnFunction::create(replicated_size, function, capture); } -MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const +ColumnPtr ColumnFunction::cut(size_t start, size_t length) const { ColumnsWithTypeAndName capture = captured_columns; for (auto & column : capture) @@ -49,7 +49,7 @@ MutableColumnPtr ColumnFunction::cut(size_t start, size_t length) const return ColumnFunction::create(length, function, capture); } -MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const { if (size_ != filt.size()) throw Exception("Size of filter (" + toString(filt.size()) + ") doesn't match size of column (" @@ -68,7 +68,7 @@ MutableColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size return ColumnFunction::create(filtered_size, function, capture); } -MutableColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnFunction::permute(const Permutation & perm, size_t limit) const { if (limit == 0) limit = size_; diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h index 087086302aa..d10459175cc 100644 --- a/dbms/src/Columns/ColumnFunction.h +++ b/dbms/src/Columns/ColumnFunction.h @@ -29,10 +29,10 @@ public: size_t size() const override { return size_; } - MutableColumnPtr cut(size_t start, size_t length) const override; - MutableColumnPtr replicate(const Offsets & offsets) const override; - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr cut(size_t start, size_t length) const override; + ColumnPtr replicate(const Offsets & offsets) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; void insertDefault() override; void popBack(size_t n) override; std::vector scatter(IColumn::ColumnIndex num_columns, diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 59f225c962f..557a57b3c6d 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -18,8 +18,8 @@ namespace ErrorCodes } -ColumnNullable::ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_) - : nested_column{nested_column_}, null_map{null_map_} +ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_) + : nested_column(std::move(nested_column_)), null_map(std::move(null_map_)) { /// ColumnNullable cannot have constant nested column. But constant argument could be passed. Materialize it. if (ColumnPtr nested_column_materialized = getNestedColumn().convertToFullColumnIfConst()) @@ -44,7 +44,7 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const { - ColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size); + MutableColumnPtr new_nested_col = getNestedColumn().cloneResized(new_size); auto new_null_map = ColumnUInt8::create(); if (new_size > 0) @@ -59,7 +59,7 @@ MutableColumnPtr ColumnNullable::cloneResized(size_t new_size) const memset(&new_null_map->getData()[count], 1, new_size - count); } - return ColumnNullable::create(new_nested_col, std::move(new_null_map)); + return ColumnNullable::create(std::move(new_nested_col), std::move(new_null_map)); } @@ -152,14 +152,14 @@ void ColumnNullable::popBack(size_t n) getNullMapColumn().popBack(n); } -MutableColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnNullable::filter(const Filter & filt, ssize_t result_size_hint) const { ColumnPtr filtered_data = getNestedColumn().filter(filt, result_size_hint); ColumnPtr filtered_null_map = getNullMapColumn().filter(filt, result_size_hint); return ColumnNullable::create(filtered_data, filtered_null_map); } -MutableColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnNullable::permute(const Permutation & perm, size_t limit) const { ColumnPtr permuted_data = getNestedColumn().permute(perm, limit); ColumnPtr permuted_null_map = getNullMapColumn().permute(perm, limit); @@ -384,7 +384,7 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const } -MutableColumnPtr ColumnNullable::replicate(const Offsets & offsets) const +ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const { ColumnPtr replicated_data = getNestedColumn().replicate(offsets); ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets); diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index 6eff31379ff..bc082bbcb8c 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -23,9 +23,16 @@ class ColumnNullable final : public COWPtrHelper private: friend class COWPtrHelper; - ColumnNullable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_); + ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_); ColumnNullable(const ColumnNullable &) = default; + using Ptr = COWPtrHelper::Ptr; + + static Ptr createImmutable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_) + { + return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable()); + } + public: const char * getFamilyName() const override { return "Nullable"; } std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; } @@ -50,14 +57,14 @@ public: } void popBack(size_t n) override; - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override; void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override; void reserve(size_t n) override; size_t byteSize() const override; size_t allocatedBytes() const override; - MutableColumnPtr replicate(const Offsets & replicate_offsets) const override; + ColumnPtr replicate(const Offsets & replicate_offsets) const override; void updateHashWithValue(size_t n, SipHash & hash) const override; void getExtremes(Field & min, Field & max) const override; diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 654a56590d1..ecb3bc6f62e 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -97,7 +97,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len } -MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_hint) const { if (offsets.size() == 0) return ColumnString::create(); @@ -112,7 +112,7 @@ MutableColumnPtr ColumnString::filter(const Filter & filt, ssize_t result_size_h } -MutableColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnString::permute(const Permutation & perm, size_t limit) const { size_t size = offsets.size(); @@ -208,7 +208,7 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio } -MutableColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const +ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const { size_t col_size = size(); if (col_size != replicate_offsets.size()) diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index cd60eedfa43..c37fca3ab44 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -206,9 +206,9 @@ public: void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; void insertDefault() override { @@ -239,7 +239,7 @@ public: /// Sorting with respect of collation. void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const; - MutableColumnPtr replicate(const Offsets & replicate_offsets) const override; + ColumnPtr replicate(const Offsets & replicate_offsets) const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override { diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index 4bb250aa4c9..4bfb6a30c2a 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -31,21 +31,38 @@ std::string ColumnTuple::getName() const return res.str(); } -ColumnTuple::ColumnTuple(const Columns & columns) : columns(columns) +ColumnTuple::ColumnTuple(MutableColumns && mutable_columns) +{ + columns.reserve(mutable_columns.size()); + for (auto & column : mutable_columns) + { + if (column->isColumnConst()) + throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN}; + + columns.push_back(std::move(column)); + } +} + +ColumnTuple::Ptr ColumnTuple::createImmutable(const Columns & columns) { for (const auto & column : columns) if (column->isColumnConst()) throw Exception{"ColumnTuple cannot have ColumnConst as its element", ErrorCodes::ILLEGAL_COLUMN}; + + auto column_tuple = ColumnTuple::create(MutableColumns()); + column_tuple->columns = columns; + + return std::move(column_tuple); } MutableColumnPtr ColumnTuple::cloneEmpty() const { const size_t tuple_size = columns.size(); - Columns new_columns(tuple_size); + MutableColumns new_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) new_columns[i] = columns[i]->cloneEmpty(); - return ColumnTuple::create(new_columns); + return ColumnTuple::create(std::move(new_columns)); } Field ColumnTuple::operator[](size_t n) const @@ -140,7 +157,7 @@ void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t leng start, length); } -MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hint) const { const size_t tuple_size = columns.size(); Columns new_columns(tuple_size); @@ -151,7 +168,7 @@ MutableColumnPtr ColumnTuple::filter(const Filter & filt, ssize_t result_size_hi return ColumnTuple::create(new_columns); } -MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const +ColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) const { const size_t tuple_size = columns.size(); Columns new_columns(tuple_size); @@ -162,7 +179,7 @@ MutableColumnPtr ColumnTuple::permute(const Permutation & perm, size_t limit) co return ColumnTuple::create(new_columns); } -MutableColumnPtr ColumnTuple::replicate(const Offsets & offsets) const +ColumnPtr ColumnTuple::replicate(const Offsets & offsets) const { const size_t tuple_size = columns.size(); Columns new_columns(tuple_size); @@ -185,10 +202,10 @@ MutableColumns ColumnTuple::scatter(ColumnIndex num_columns, const Selector & se for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx) { - Columns new_columns(tuple_size); + MutableColumns new_columns(tuple_size); for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx) new_columns[tuple_element_idx] = std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]); - res[scattered_idx] = ColumnTuple::create(new_columns); + res[scattered_idx] = ColumnTuple::create(std::move(new_columns)); } return res; diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index c2ca2e43ab7..752a649ba28 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -22,9 +22,11 @@ private: template struct Less; - ColumnTuple(const Columns & columns); + ColumnTuple(MutableColumns && columns); ColumnTuple(const ColumnTuple &) = default; + static Ptr createImmutable(const Columns & columns); + public: std::string getName() const override; const char * getFamilyName() const override { return "Tuple"; } @@ -49,9 +51,9 @@ public: const char * deserializeAndInsertFromArena(const char * pos) override; void updateHashWithValue(size_t n, SipHash & hash) const override; void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; - MutableColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override; - MutableColumnPtr replicate(const Offsets & offsets) const override; + ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr permute(const Permutation & perm, size_t limit) const override; + ColumnPtr replicate(const Offsets & offsets) const override; MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override; void gather(ColumnGathererStream & gatherer_stream) override; int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override; diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index b1a3e55cb86..3509c14c3aa 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -146,7 +146,7 @@ void ColumnVector::insertRangeFrom(const IColumn & src, size_t start, size_t } template -MutableColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const +ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const { size_t size = data.size(); if (size != filt.size()) @@ -210,7 +210,7 @@ MutableColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t r } template -MutableColumnPtr ColumnVector::permute(const IColumn::Permutation & perm, size_t limit) const +ColumnPtr ColumnVector::permute(const IColumn::Permutation & perm, size_t limit) const { size_t size = data.size(); @@ -231,7 +231,7 @@ MutableColumnPtr ColumnVector::permute(const IColumn::Permutation & perm, siz } template -MutableColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const +ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { size_t size = data.size(); if (size != offsets.size()) diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 0b75a0c3dfc..5ce33e82028 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -243,11 +243,11 @@ public: void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; - MutableColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; + ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; - MutableColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override; + ColumnPtr permute(const IColumn::Permutation & perm, size_t limit) const override; - MutableColumnPtr replicate(const IColumn::Offsets & offsets) const override; + ColumnPtr replicate(const IColumn::Offsets & offsets) const override; void getExtremes(Field & min, Field & max) const override; diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 3a733e4a025..c546f315c0c 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -45,7 +45,7 @@ public: /** If column isn't constant, returns nullptr (or itself). * If column is constant, transforms constant to full column (if column type allows such tranform) and return it. */ - virtual MutablePtr convertToFullColumnIfConst() const { return {}; } + virtual Ptr convertToFullColumnIfConst() const { return {}; } /// Creates empty column with the same type. virtual MutablePtr cloneEmpty() const { return cloneResized(0); } @@ -104,11 +104,11 @@ public: /// Removes all elements outside of specified range. /// Is used in LIMIT operation, for example. - virtual MutablePtr cut(size_t start, size_t length) const + virtual Ptr cut(size_t start, size_t length) const { MutablePtr res = cloneEmpty(); res->insertRangeFrom(*this, start, length); - return res; + return std::move(res); } /// Appends new value at the end of column (column's size is increased by 1). @@ -171,12 +171,12 @@ public: * otherwise (i.e. < 0), makes reserve() using size of source column. */ using Filter = PaddedPODArray; - virtual MutablePtr filter(const Filter & filt, ssize_t result_size_hint) const = 0; + virtual Ptr filter(const Filter & filt, ssize_t result_size_hint) const = 0; /// Permutes elements using specified permutation. Is used in sortings. /// limit - if it isn't 0, puts only first limit elements in the result. using Permutation = PaddedPODArray; - virtual MutablePtr permute(const Permutation & perm, size_t limit) const = 0; + virtual Ptr permute(const Permutation & perm, size_t limit) const = 0; /** Compares (*this)[n] and rhs[m]. * Returns negative number, 0, or positive number (*this)[n] is less, equal, greater than rhs[m] respectively. @@ -205,7 +205,7 @@ public: */ using Offset = UInt64; using Offsets = PaddedPODArray; - virtual MutablePtr replicate(const Offsets & offsets) const = 0; + virtual Ptr replicate(const Offsets & offsets) const = 0; /** Split column to smaller columns. Each value goes to column index, selected by corresponding element of 'selector'. * Selector must contain values from 0 to num_columns - 1. diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h index dbf9daea796..c580a2b0cb1 100644 --- a/dbms/src/Columns/IColumnDummy.h +++ b/dbms/src/Columns/IColumnDummy.h @@ -74,12 +74,12 @@ public: s += length; } - MutableColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override + ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override { return cloneDummy(countBytesInFilter(filt)); } - MutableColumnPtr permute(const Permutation & perm, size_t limit) const override + ColumnPtr permute(const Permutation & perm, size_t limit) const override { if (s != perm.size()) throw Exception("Size of permutation doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); @@ -94,7 +94,7 @@ public: res[i] = i; } - MutableColumnPtr replicate(const Offsets & offsets) const override + ColumnPtr replicate(const Offsets & offsets) const override { if (s != offsets.size()) throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h index 39c912f8abc..a7fce08ae13 100644 --- a/dbms/src/Common/COWPtr.h +++ b/dbms/src/Common/COWPtr.h @@ -226,15 +226,53 @@ private: Derived * derived() { return static_cast(this); } const Derived * derived() const { return static_cast(this); } + template + struct HasCreateImmutable + { + private: + template + static constexpr auto check(decltype(std::declval().createImmutable(std::declval()... )) *) -> std::true_type; + + template + static constexpr std::false_type check(...); + + typedef decltype(check(nullptr)) type; + + public: + static constexpr bool value = type::value; + }; + + template + struct IsConstructable + { + private: + template + static constexpr auto check(decltype(T(std::declval()... )) *) -> std::true_type; + + template + static constexpr std::false_type check(...); + + typedef decltype(check(nullptr)) type; + + public: + static constexpr bool value = type::value; + }; + public: using Ptr = typename Base::template immutable_ptr; using MutablePtr = typename Base::template mutable_ptr; template - static MutablePtr create(Args &&... args) { return MutablePtr(new Derived(std::forward(args)...)); } + static auto create(Args &&... args) + { + if constexpr (HasCreateImmutable::value && !IsConstructable::value) + return Derived::createImmutable(std::forward(args)...); + else + return MutablePtr(new Derived(std::forward(args)...)); + } template - static MutablePtr create(std::initializer_list && arg) { return create(std::forward>(arg)); } + static auto create(std::initializer_list && arg) { return create(std::forward>(arg)); } typename Base::MutablePtr clone() const override { return typename Base::MutablePtr(new Derived(*derived())); } }; diff --git a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp index b60cc072b47..be90a00e4b9 100644 --- a/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -233,7 +233,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, std:: for (size_t i = 0; i < num_columns; ++i) { auto & column = merged_columns[i]; - column = column->cut(0, merged_rows); + column = (*column->cut(0, merged_rows)).mutate(); } cancel(false); diff --git a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp index 1102258d348..66817f23037 100644 --- a/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp +++ b/dbms/src/DataStreams/SummingSortedBlockInputStream.cpp @@ -259,11 +259,11 @@ Block SummingSortedBlockInputStream::readImpl() if (checkDataType(desc.function->getReturnType().get())) { size_t tuple_size = desc.column_numbers.size(); - Columns tuple_columns(tuple_size); + MutableColumns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) - tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column; + tuple_columns[i] = header.safeGetByPosition(desc.column_numbers[i]).column->assumeMutable(); - desc.merged_column = ColumnTuple::create(tuple_columns); + desc.merged_column = ColumnTuple::create(std::move(tuple_columns)); } else desc.merged_column = header.safeGetByPosition(desc.column_numbers[0]).column->cloneEmpty(); diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 6f7d692af5f..2da9f0f6ecf 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -329,10 +329,10 @@ void DataTypeTuple::deserializeBinaryBulkWithMultipleStreams( MutableColumnPtr DataTypeTuple::createColumn() const { size_t size = elems.size(); - Columns tuple_columns(size); + MutableColumns tuple_columns(size); for (size_t i = 0; i < size; ++i) tuple_columns[i] = elems[i]->createColumn(); - return ColumnTuple::create(tuple_columns); + return ColumnTuple::create(std::move(tuple_columns)); } Field DataTypeTuple::getDefault() const diff --git a/dbms/src/Functions/FunctionsHigherOrder.h b/dbms/src/Functions/FunctionsHigherOrder.h index c63dce364b9..47722370777 100644 --- a/dbms/src/Functions/FunctionsHigherOrder.h +++ b/dbms/src/Functions/FunctionsHigherOrder.h @@ -936,7 +936,7 @@ public: } /// Put all the necessary columns multiplied by the sizes of arrays into the block. - auto replicated_column_function_ptr = column_function->replicate(column_first_array->getOffsets()); + auto replicated_column_function_ptr = (*column_function->replicate(column_first_array->getOffsets())).mutate(); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); diff --git a/dbms/src/Functions/GatherUtils/createArraySink.cpp b/dbms/src/Functions/GatherUtils/createArraySink.cpp index b97bf476919..e4097195e4c 100644 --- a/dbms/src/Functions/GatherUtils/createArraySink.cpp +++ b/dbms/src/Functions/GatherUtils/createArraySink.cpp @@ -41,7 +41,8 @@ std::unique_ptr createArraySink(ColumnArray & col, size_t column_siz using Creator = ApplyTypeListForClass::Type; if (auto column_nullable = typeid_cast(&col.getData())) { - auto column = ColumnArray::create(column_nullable->getNestedColumnPtr(), col.getOffsetsPtr()); + auto column = ColumnArray::create(column_nullable->getNestedColumnPtr()->assumeMutable(), + col.getOffsetsPtr()->assumeMutable()); return Creator::create(*column, &column_nullable->getNullMapData(), column_size); } return Creator::create(col, nullptr, column_size); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index e172760b17f..a69b7ca57e5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -111,7 +111,8 @@ size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t /// share offsets in all elements of nested structure if (!append) - column = ColumnArray::create(type_arr->getNestedType()->createColumn(), it_inserted.first->second); + column = ColumnArray::create(type_arr->getNestedType()->createColumn(), + it_inserted.first->second)->assumeMutable(); } try From 5c91eecb2b0900846ea0d8b8030ef2d60c43fb3e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 21 Mar 2018 21:43:17 +0300 Subject: [PATCH 34/90] fix test --- dbms/src/Columns/ColumnConst.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index 89f4c488940..2e4a692451f 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -133,7 +133,6 @@ public: const char * deserializeAndInsertFromArena(const char * pos) override { - data = (*std::move(data)).mutate(); auto & mutable_data = data->assumeMutableRef(); auto res = mutable_data.deserializeAndInsertFromArena(pos); mutable_data.popBack(1); From f4b742e9fb4ef02208b6702b1edd13bcb2e1aace Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 22 Mar 2018 18:37:24 +0300 Subject: [PATCH 35/90] Fixed segfault in case of bad argument types for some aggregate functions [#CLICKHOUSE-2] --- dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp | 5 ++++- .../AggregateFunctionMaxIntersections.cpp | 7 ++++++- dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 5fc341b3095..eca854a031b 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -70,7 +70,10 @@ AggregateFunctionPtr AggregateFunctionFactory::get( return combinator->transformAggregateFunction(nested_function, argument_types, parameters); } - return getImpl(name, argument_types, parameters, recursion_level); + auto res = getImpl(name, argument_types, parameters, recursion_level); + if (!res) + throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR); + return res; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp index e81dfe98259..9f63d399fe7 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionMaxIntersections.cpp @@ -16,7 +16,12 @@ namespace assertBinary(name, argument_types); assertNoParameters(name, parameters); - return AggregateFunctionPtr{createWithNumericType(*argument_types[0], kind, argument_types)}; + AggregateFunctionPtr res(createWithNumericType(*argument_types[0], kind, argument_types)); + if (!res) + throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName() + + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return res; } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp index 9aedeaabe24..7cad7c35092 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp @@ -37,7 +37,11 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con values_types.push_back(array_type->getNestedType()); } - return AggregateFunctionPtr(createWithNumericType(*keys_type, keys_type, std::move(values_types))); + AggregateFunctionPtr res(createWithNumericType(*keys_type, keys_type, std::move(values_types))); + if (!res) + throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return res; } } From 97624d7d78f8b0232552ad666573ab884cc898a3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Mar 2018 22:43:00 +0300 Subject: [PATCH 36/90] fix reading with evaluated defaults [#CLICKHOUSE-3663] --- .../Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp | 8 ++++---- dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 7 +++---- dbms/src/Storages/MergeTree/MergeTreeReader.h | 6 +++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp index b5095e79118..0cd903038ed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp @@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() } progressImpl({ res.rows(), res.bytes() }); - pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); /// Compute the expression in PREWHERE. prewhere_actions->execute(res); @@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() /// as a result of reading components of Nested data structures with no data in filesystem. /// We must fill these arrays to filter them correctly. - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true); /// Filter the columns related to PREWHERE using pre_filter, /// other columns - using post_filter. @@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (task->size_predictor) task->size_predictor->update(res); - reader->fillMissingColumns(res, task->ordered_names, true); + reader->fillMissingColumns(res, task->ordered_names, true, res.rows()); res.checkNumberOfRows(); } @@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() return res; progressImpl({ res.rows(), res.bytes() }); - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); } return res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index e172760b17f..37d41749420 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -413,7 +413,8 @@ static bool arrayHasNoElementsRead(const IColumn & column) } -void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder) +void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder, + size_t rows, bool never_evaluate_defaults) { if (!res) throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR); @@ -444,8 +445,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name bool should_evaluate_defaults = false; bool should_sort = always_reorder; - size_t rows = res.rows(); - /// insert default values only for columns without default expressions for (const auto & requested_column : columns) { @@ -497,7 +496,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name } /// evaluate defaulted columns if necessary - if (should_evaluate_defaults) + if (!never_evaluate_defaults && should_evaluate_defaults) evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); /// sort columns to ensure consistent order among all blocks diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index c2cd086d343..b76cf19ff93 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -44,7 +44,11 @@ public: /// Add columns from ordered_names that are not present in the block. /// Missing columns are added in the order specified by ordered_names. /// If at least one column was added, reorders all columns in the block according to ordered_names. - void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder); + /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size. + /// This function may be used in order to create default columns in nested structures. In that case block may has + /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used. + void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder, + size_t rows, bool never_evaluate_defaults = false); private: class Stream From 74211b5bf8e6e26cd948e42e9b1ef3ba78f179b6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Mar 2018 22:54:21 +0300 Subject: [PATCH 37/90] added test [#CLICKHOUSE-3663] --- .../0_stateless/00609_prewhere_and_default.reference | 0 .../0_stateless/00609_prewhere_and_default.sql | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql new file mode 100644 index 00000000000..915f0c85823 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql @@ -0,0 +1,12 @@ +drop table if exists test.table; +create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192; +insert into test.table select number, number / 8192 from system.numbers limit 100000; +alter table test.table add column def UInt64 default val + 1; +select * from test.table prewhere val > 2 format Null; + +drop table if exists test.table; +create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192; +insert into test.table select number, number / 8192 from system.numbers limit 100000; +alter table test.table add column def UInt64; +select * from test.table prewhere val > 2 format Null; + From cbfd862fa3df628aa30804107924e083cab8c566 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Mar 2018 22:43:00 +0300 Subject: [PATCH 38/90] fix reading with evaluated defaults [#CLICKHOUSE-3663] --- .../Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp | 8 ++++---- dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 7 +++---- dbms/src/Storages/MergeTree/MergeTreeReader.h | 6 +++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp index b5095e79118..0cd903038ed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp @@ -216,7 +216,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() } progressImpl({ res.rows(), res.bytes() }); - pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + pre_reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); /// Compute the expression in PREWHERE. prewhere_actions->execute(res); @@ -396,7 +396,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() /// as a result of reading components of Nested data structures with no data in filesystem. /// We must fill these arrays to filter them correctly. - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, post_filter.size(), true); /// Filter the columns related to PREWHERE using pre_filter, /// other columns - using post_filter. @@ -422,7 +422,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (task->size_predictor) task->size_predictor->update(res); - reader->fillMissingColumns(res, task->ordered_names, true); + reader->fillMissingColumns(res, task->ordered_names, true, res.rows()); res.checkNumberOfRows(); } @@ -464,7 +464,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() return res; progressImpl({ res.rows(), res.bytes() }); - reader->fillMissingColumns(res, task->ordered_names, task->should_reorder); + reader->fillMissingColumns(res, task->ordered_names, task->should_reorder, res.rows()); } return res; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index e172760b17f..37d41749420 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -413,7 +413,8 @@ static bool arrayHasNoElementsRead(const IColumn & column) } -void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder) +void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_names, bool always_reorder, + size_t rows, bool never_evaluate_defaults) { if (!res) throw Exception("Empty block passed to fillMissingColumns", ErrorCodes::LOGICAL_ERROR); @@ -444,8 +445,6 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name bool should_evaluate_defaults = false; bool should_sort = always_reorder; - size_t rows = res.rows(); - /// insert default values only for columns without default expressions for (const auto & requested_column : columns) { @@ -497,7 +496,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name } /// evaluate defaulted columns if necessary - if (should_evaluate_defaults) + if (!never_evaluate_defaults && should_evaluate_defaults) evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); /// sort columns to ensure consistent order among all blocks diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index c2cd086d343..b76cf19ff93 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -44,7 +44,11 @@ public: /// Add columns from ordered_names that are not present in the block. /// Missing columns are added in the order specified by ordered_names. /// If at least one column was added, reorders all columns in the block according to ordered_names. - void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder); + /// Rows is the number of rows will be used to create default columns. It's res.rows() when all columns in block has the same size. + /// This function may be used in order to create default columns in nested structures. In that case block may has + /// columns with different size, and default columns can't be evaluated, so never_evaluate_defaults = false is used. + void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder, + size_t rows, bool never_evaluate_defaults = false); private: class Stream From 6f60e6ee4f3a2808e73527e3a7fe879c8f95bdb7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 22 Mar 2018 22:54:21 +0300 Subject: [PATCH 39/90] added test [#CLICKHOUSE-3663] --- .../0_stateless/00609_prewhere_and_default.reference | 0 .../0_stateless/00609_prewhere_and_default.sql | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference create mode 100644 dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql new file mode 100644 index 00000000000..915f0c85823 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00609_prewhere_and_default.sql @@ -0,0 +1,12 @@ +drop table if exists test.table; +create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192; +insert into test.table select number, number / 8192 from system.numbers limit 100000; +alter table test.table add column def UInt64 default val + 1; +select * from test.table prewhere val > 2 format Null; + +drop table if exists test.table; +create table test.table (key UInt64, val UInt64) engine = MergeTree order by key settings index_granularity=8192; +insert into test.table select number, number / 8192 from system.numbers limit 100000; +alter table test.table add column def UInt64; +select * from test.table prewhere val > 2 format Null; + From 7c2a99c9b8f3bf4ff3e25eea308c5b0e45db0123 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 21 Mar 2018 23:21:34 +0300 Subject: [PATCH 40/90] Introduced MinimalisticDataPartChecksums. [#CLICKHOUSE-3660] --- .../Storages/MergeTree/MergeTreeDataPart.cpp | 283 +----------- .../Storages/MergeTree/MergeTreeDataPart.h | 70 +-- .../MergeTree/MergeTreeDataPartChecksum.cpp | 434 ++++++++++++++++++ .../MergeTree/MergeTreeDataPartChecksum.h | 119 +++++ .../ReplicatedMergeTreeBlockOutputStream.cpp | 2 +- 5 files changed, 579 insertions(+), 329 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 8ab7699a03b..79f3ddb523f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -30,273 +30,15 @@ namespace DB namespace ErrorCodes { - extern const int CHECKSUM_DOESNT_MATCH; extern const int FILE_DOESNT_EXIST; extern const int NO_FILE_IN_DATA_PART; extern const int EXPECTED_END_OF_FILE; - extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; extern const int CORRUPTED_DATA; - extern const int FORMAT_VERSION_TOO_OLD; - extern const int UNKNOWN_FORMAT; - extern const int UNEXPECTED_FILE_IN_DATA_PART; extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; } -void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const -{ - if (is_compressed && have_uncompressed) - { - if (!rhs.is_compressed) - throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH); - if (rhs.uncompressed_size != uncompressed_size) - throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - if (rhs.uncompressed_hash != uncompressed_hash) - throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH); - return; - } - if (rhs.file_size != file_size) - throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - if (rhs.file_hash != file_hash) - throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH); -} - -void MergeTreeDataPartChecksum::checkSize(const String & path) const -{ - Poco::File file(path); - if (!file.exists()) - throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); - UInt64 size = file.getSize(); - if (size != file_size) - throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size), - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); -} - - -void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const -{ - for (const auto & it : rhs.files) - { - const String & name = it.first; - - if (!files.count(name)) - throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART); - } - - for (const auto & it : files) - { - const String & name = it.first; - - auto jt = rhs.files.find(name); - if (jt == rhs.files.end()) - throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART); - - it.second.checkEqual(jt->second, have_uncompressed, name); - } -} - -void MergeTreeDataPartChecksums::checkSizes(const String & path) const -{ - for (const auto & it : files) - { - const String & name = it.first; - it.second.checkSize(path + name); - } -} - -bool MergeTreeDataPartChecksums::read(ReadBuffer & in) -{ - files.clear(); - - assertString("checksums format version: ", in); - int format_version; - readText(format_version, in); - assertChar('\n', in); - - switch (format_version) - { - case 1: - return false; - case 2: - return read_v2(in); - case 3: - return read_v3(in); - case 4: - return read_v4(in); - default: - throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT); - } -} - -bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in) -{ - size_t count; - - readText(count, in); - assertString(" files:\n", in); - - for (size_t i = 0; i < count; ++i) - { - String name; - Checksum sum; - - readString(name, in); - assertString("\n\tsize: ", in); - readText(sum.file_size, in); - assertString("\n\thash: ", in); - readText(sum.file_hash.first, in); - assertString(" ", in); - readText(sum.file_hash.second, in); - assertString("\n\tcompressed: ", in); - readText(sum.is_compressed, in); - if (sum.is_compressed) - { - assertString("\n\tuncompressed size: ", in); - readText(sum.uncompressed_size, in); - assertString("\n\tuncompressed hash: ", in); - readText(sum.uncompressed_hash.first, in); - assertString(" ", in); - readText(sum.uncompressed_hash.second, in); - } - assertChar('\n', in); - - files.insert(std::make_pair(name, sum)); - } - - return true; -} - -bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in) -{ - size_t count; - - readVarUInt(count, in); - - for (size_t i = 0; i < count; ++i) - { - String name; - Checksum sum; - - readBinary(name, in); - readVarUInt(sum.file_size, in); - readPODBinary(sum.file_hash, in); - readBinary(sum.is_compressed, in); - - if (sum.is_compressed) - { - readVarUInt(sum.uncompressed_size, in); - readPODBinary(sum.uncompressed_hash, in); - } - - files.emplace(std::move(name), sum); - } - - return true; -} - -bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from) -{ - CompressedReadBuffer in{from}; - return read_v3(in); -} - -void MergeTreeDataPartChecksums::write(WriteBuffer & to) const -{ - writeString("checksums format version: 4\n", to); - - CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16}; - writeVarUInt(files.size(), out); - - for (const auto & it : files) - { - const String & name = it.first; - const Checksum & sum = it.second; - - writeBinary(name, out); - writeVarUInt(sum.file_size, out); - writePODBinary(sum.file_hash, out); - writeBinary(sum.is_compressed, out); - - if (sum.is_compressed) - { - writeVarUInt(sum.uncompressed_size, out); - writePODBinary(sum.uncompressed_hash, out); - } - } -} - -void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash) -{ - files[file_name] = Checksum(file_size, file_hash); -} - -void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums) -{ - for (auto & checksum : rhs_checksums.files) - files[std::move(checksum.first)] = std::move(checksum.second); - - rhs_checksums.files.clear(); -} - -/// Checksum computed from the set of control sums of .bin files. -void MergeTreeDataPartChecksums::summaryDataChecksum(SipHash & hash) const -{ - /// We use fact that iteration is in deterministic (lexicographical) order. - for (const auto & it : files) - { - const String & name = it.first; - const Checksum & sum = it.second; - - if (!endsWith(name, ".bin")) - continue; - - UInt64 len = name.size(); - hash.update(len); - hash.update(name.data(), len); - hash.update(sum.uncompressed_size); - hash.update(sum.uncompressed_hash); - } -} - -String MergeTreeDataPartChecksums::toString() const -{ - WriteBufferFromOwnString out; - write(out); - return out.str(); -} - -MergeTreeDataPartChecksums MergeTreeDataPartChecksums::parse(const String & s) -{ - ReadBufferFromString in(s); - MergeTreeDataPartChecksums res; - if (!res.read(in)) - throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD); - assertEOF(in); - return res; -} - -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const -{ - if (checksums.empty()) - return nullptr; - - const auto & files = checksums.files; - const auto file_name = escapeForFileName(name) + ext; - auto it = files.find(file_name); - - return (it == files.end()) ? nullptr : &it->second; -} - -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const -{ - return tryGetChecksum(name, ".bin"); -} - -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const -{ - return tryGetChecksum(name, ".mrk"); -} - static ReadBufferFromFile openForReading(const String & path) { return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); @@ -928,6 +670,29 @@ bool MergeTreeDataPart::hasColumnFiles(const String & column) const } +const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const +{ + if (checksums.empty()) + return nullptr; + + const auto & files = checksums.files; + const auto file_name = escapeForFileName(name) + ext; + auto it = files.find(file_name); + + return (it == files.end()) ? nullptr : &it->second; +} + +const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const +{ + return tryGetChecksum(name, ".bin"); +} + +const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const +{ + return tryGetChecksum(name, ".mrk"); +} + + UInt64 MergeTreeDataPart::getIndexSizeInBytes() const { UInt64 res = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index 95bed39eaec..a617dfedacd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -5,82 +5,14 @@ #include #include #include +#include #include #include -class SipHash; - - namespace DB { - -/// Checksum of one file. -struct MergeTreeDataPartChecksum -{ - using uint128 = CityHash_v1_0_2::uint128; - - UInt64 file_size {}; - uint128 file_hash {}; - - bool is_compressed = false; - UInt64 uncompressed_size {}; - uint128 uncompressed_hash {}; - - MergeTreeDataPartChecksum() {} - MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {} - MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_) - : file_size(file_size_), file_hash(file_hash_), is_compressed(true), - uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {} - - void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const; - void checkSize(const String & path) const; -}; - - -/** Checksums of all non-temporary files. - * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method. - */ -struct MergeTreeDataPartChecksums -{ - using Checksum = MergeTreeDataPartChecksum; - - /// The order is important. - using FileChecksums = std::map; - FileChecksums files; - - void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash); - - void add(MergeTreeDataPartChecksums && rhs_checksums); - - /// Checks that the set of columns and their checksums are the same. If not, throws an exception. - /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. Otherwise, it compares only the checksums of the files. - void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const; - - /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum. - void checkSizes(const String & path) const; - - /// Serializes and deserializes in human readable form. - bool read(ReadBuffer & in); /// Returns false if the checksum is too old. - bool read_v2(ReadBuffer & in); - bool read_v3(ReadBuffer & in); - bool read_v4(ReadBuffer & in); - void write(WriteBuffer & out) const; - - bool empty() const - { - return files.empty(); - } - - /// Checksum from the set of checksums of .bin files. - void summaryDataChecksum(SipHash & hash) const; - - String toString() const; - static MergeTreeDataPartChecksums parse(const String & s); -}; - - class MergeTreeData; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp new file mode 100644 index 00000000000..b252830d0d1 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -0,0 +1,434 @@ +#include "MergeTreeDataPartChecksum.h" +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + + +namespace ErrorCodes +{ + extern const int CHECKSUM_DOESNT_MATCH; + extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; + extern const int FORMAT_VERSION_TOO_OLD; + extern const int FILE_DOESNT_EXIST; + extern const int UNEXPECTED_FILE_IN_DATA_PART; + extern const int UNKNOWN_FORMAT; + extern const int NO_FILE_IN_DATA_PART; +} + + +void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const +{ + if (is_compressed && have_uncompressed) + { + if (!rhs.is_compressed) + throw Exception("No uncompressed checksum for file " + name, ErrorCodes::CHECKSUM_DOESNT_MATCH); + if (rhs.uncompressed_size != uncompressed_size) + throw Exception("Unexpected uncompressed size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + if (rhs.uncompressed_hash != uncompressed_hash) + throw Exception("Checksum mismatch for uncompressed file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH); + return; + } + if (rhs.file_size != file_size) + throw Exception("Unexpected size of file " + name + " in data part", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + if (rhs.file_hash != file_hash) + throw Exception("Checksum mismatch for file " + name + " in data part", ErrorCodes::CHECKSUM_DOESNT_MATCH); +} + +void MergeTreeDataPartChecksum::checkSize(const String & path) const +{ + Poco::File file(path); + if (!file.exists()) + throw Exception(path + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); + UInt64 size = file.getSize(); + if (size != file_size) + throw Exception(path + " has unexpected size: " + toString(size) + " instead of " + toString(file_size), + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); +} + + +void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const +{ + for (const auto & it : rhs.files) + { + const String & name = it.first; + + if (!files.count(name)) + throw Exception("Unexpected file " + name + " in data part", ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART); + } + + for (const auto & it : files) + { + const String & name = it.first; + + auto jt = rhs.files.find(name); + if (jt == rhs.files.end()) + throw Exception("No file " + name + " in data part", ErrorCodes::NO_FILE_IN_DATA_PART); + + it.second.checkEqual(jt->second, have_uncompressed, name); + } +} + +void MergeTreeDataPartChecksums::checkSizes(const String & path) const +{ + for (const auto & it : files) + { + const String & name = it.first; + it.second.checkSize(path + name); + } +} + +bool MergeTreeDataPartChecksums::read(ReadBuffer & in, size_t format_version) +{ + switch (format_version) + { + case 1: + return false; + case 2: + return read_v2(in); + case 3: + return read_v3(in); + case 4: + return read_v4(in); + default: + throw Exception("Bad checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT); + } +} + +bool MergeTreeDataPartChecksums::read(ReadBuffer & in) +{ + files.clear(); + + assertString("checksums format version: ", in); + size_t format_version; + readText(format_version, in); + assertChar('\n', in); + + read(in, format_version); + return true; +} + +bool MergeTreeDataPartChecksums::read_v2(ReadBuffer & in) +{ + size_t count; + + readText(count, in); + assertString(" files:\n", in); + + for (size_t i = 0; i < count; ++i) + { + String name; + Checksum sum; + + readString(name, in); + assertString("\n\tsize: ", in); + readText(sum.file_size, in); + assertString("\n\thash: ", in); + readText(sum.file_hash.first, in); + assertString(" ", in); + readText(sum.file_hash.second, in); + assertString("\n\tcompressed: ", in); + readText(sum.is_compressed, in); + if (sum.is_compressed) + { + assertString("\n\tuncompressed size: ", in); + readText(sum.uncompressed_size, in); + assertString("\n\tuncompressed hash: ", in); + readText(sum.uncompressed_hash.first, in); + assertString(" ", in); + readText(sum.uncompressed_hash.second, in); + } + assertChar('\n', in); + + files.insert(std::make_pair(name, sum)); + } + + return true; +} + +bool MergeTreeDataPartChecksums::read_v3(ReadBuffer & in) +{ + size_t count; + + readVarUInt(count, in); + + for (size_t i = 0; i < count; ++i) + { + String name; + Checksum sum; + + readBinary(name, in); + readVarUInt(sum.file_size, in); + readPODBinary(sum.file_hash, in); + readBinary(sum.is_compressed, in); + + if (sum.is_compressed) + { + readVarUInt(sum.uncompressed_size, in); + readPODBinary(sum.uncompressed_hash, in); + } + + files.emplace(std::move(name), sum); + } + + return true; +} + +bool MergeTreeDataPartChecksums::read_v4(ReadBuffer & from) +{ + CompressedReadBuffer in{from}; + return read_v3(in); +} + +void MergeTreeDataPartChecksums::write(WriteBuffer & to) const +{ + writeString("checksums format version: 4\n", to); + + CompressedWriteBuffer out{to, CompressionSettings(CompressionMethod::LZ4), 1 << 16}; + + writeVarUInt(files.size(), out); + + for (const auto & it : files) + { + const String & name = it.first; + const Checksum & sum = it.second; + + writeBinary(name, out); + writeVarUInt(sum.file_size, out); + writePODBinary(sum.file_hash, out); + writeBinary(sum.is_compressed, out); + + if (sum.is_compressed) + { + writeVarUInt(sum.uncompressed_size, out); + writePODBinary(sum.uncompressed_hash, out); + } + } +} + +void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_size, MergeTreeDataPartChecksum::uint128 file_hash) +{ + files[file_name] = Checksum(file_size, file_hash); +} + +void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums) +{ + for (auto & checksum : rhs_checksums.files) + files[std::move(checksum.first)] = std::move(checksum.second); + + rhs_checksums.files.clear(); +} + +/// Checksum computed from the set of control sums of .bin files. +void MergeTreeDataPartChecksums::computeTotalChecksumDataOnly(SipHash & hash) const +{ + /// We use fact that iteration is in deterministic (lexicographical) order. + for (const auto & it : files) + { + const String & name = it.first; + const Checksum & sum = it.second; + + if (!endsWith(name, ".bin")) + continue; + + UInt64 len = name.size(); + hash.update(len); + hash.update(name.data(), len); + hash.update(sum.uncompressed_size); + hash.update(sum.uncompressed_hash); + } +} + +String MergeTreeDataPartChecksums::toString() const +{ + WriteBufferFromOwnString out; + write(out); + return out.str(); +} + +MergeTreeDataPartChecksums MergeTreeDataPartChecksums::deserializeFrom(const String & s) +{ + ReadBufferFromString in(s); + MergeTreeDataPartChecksums res; + if (!res.read(in)) + throw Exception("Checksums format is too old", ErrorCodes::FORMAT_VERSION_TOO_OLD); + assertEOF(in); + return res; +} + +MinimalisticDataPartChecksums::MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums_) +{ + full_checksums = std::make_unique(full_checksums_); + computeTotalChecksums(*full_checksums); +} + +void MinimalisticDataPartChecksums::serialize(WriteBuffer & to) const +{ + writeString("checksums format version: 5\n", to); + + writeVarUInt(num_compressed_files, to); + writeVarUInt(num_uncompressed_files, to); + + writePODBinary(hash_of_all_files, to); + writePODBinary(hash_of_uncompressed_files, to); + writePODBinary(uncompressed_hash_of_compressed_files, to); +} + +String MinimalisticDataPartChecksums::getSerializedString() +{ + WriteBufferFromOwnString wb; + serialize(wb); + return wb.str(); +} + +bool MinimalisticDataPartChecksums::deserialize(ReadBuffer & in) +{ + assertString("checksums format version: ", in); + size_t format_version; + readText(format_version, in); + assertChar('\n', in); + + if (format_version < MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS) + { + auto full_checksums_ptr = std::make_unique(); + if (!full_checksums_ptr->read(in, format_version)) + return false; + + computeTotalChecksums(*full_checksums_ptr); + full_checksums = std::move(full_checksums_ptr); + return true; + } + + if (format_version > MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS) + throw Exception("Unknown checksums format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT); + + readVarUInt(num_compressed_files, in); + readVarUInt(num_uncompressed_files, in); + + readPODBinary(hash_of_all_files, in); + readPODBinary(hash_of_uncompressed_files, in); + readPODBinary(uncompressed_hash_of_compressed_files, in); + + return true; +} + +void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums) +{ + num_compressed_files = 0; + num_uncompressed_files = 0; + + SipHash hash_of_all_files_; + SipHash hash_of_uncompressed_files_; + SipHash uncompressed_hash_of_compressed_files_; + + auto update_hash = [] (SipHash & hash, const std::string & data) + { + UInt64 len = data.size(); + hash.update(len); + hash.update(data.data(), len); + }; + + for (const auto & elem : full_checksums.files) + { + const String & name = elem.first; + const auto & checksum = elem.second; + + update_hash(hash_of_all_files_, name); + hash_of_all_files_.update(checksum.file_hash); + + if (!checksum.is_compressed) + { + ++num_uncompressed_files; + update_hash(hash_of_uncompressed_files_, name); + hash_of_uncompressed_files_.update(checksum.file_hash); + } + else + { + ++num_compressed_files; + update_hash(uncompressed_hash_of_compressed_files_, name); + uncompressed_hash_of_compressed_files_.update(checksum.uncompressed_hash); + } + } + + auto get_hash = [] (SipHash & hash, uint128 & data) + { + hash.get128(data.first, data.second); + }; + + get_hash(hash_of_all_files_, hash_of_all_files); + get_hash(hash_of_uncompressed_files_, hash_of_uncompressed_files); + get_hash(uncompressed_hash_of_compressed_files_, uncompressed_hash_of_compressed_files); +} + +String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic) +{ + if (!minimalistic) + return full_checksums.toString(); + + MinimalisticDataPartChecksums checksums; + checksums.computeTotalChecksums(full_checksums); + return checksums.getSerializedString(); +} + +void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) +{ + if (full_checksums && rhs.full_checksums) + full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files); + + checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files); +} + +void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) +{ + if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files) + { + std::stringstream error_msg; + error_msg << "Different number of files: " << rhs.num_compressed_files << " compressed (expected " << num_compressed_files << ")" + << " and " << rhs.num_uncompressed_files << " uncompressed ones (expected " << num_uncompressed_files << ")"; + + throw Exception(error_msg.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + } + + Strings errors; + + if (hash_of_uncompressed_files != rhs.hash_of_uncompressed_files) + errors.emplace_back("hash of uncompressed files doesn't match"); + + if (check_uncompressed_hash_in_compressed_files) + { + if (uncompressed_hash_of_compressed_files != rhs.uncompressed_hash_of_compressed_files) + errors.emplace_back("uncompressed hash of compressed files doesn't match"); + } + else + { + if (hash_of_all_files != rhs.hash_of_all_files) + errors.emplace_back("total hash of all files doesn't match"); + } + + if (!errors.empty()) + { + String error_msg = "Checksums of parts don't match: " + errors.front(); + for (size_t i = 1; i < errors.size(); ++i) + error_msg += ", " + errors[i]; + + throw Exception(error_msg, ErrorCodes::CHECKSUM_DOESNT_MATCH); + } +} + +MinimalisticDataPartChecksums MinimalisticDataPartChecksums::deserializeFrom(const String & s) +{ + MinimalisticDataPartChecksums res; + ReadBufferFromString rb(s); + res.deserialize(rb); + return res; +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h new file mode 100644 index 00000000000..ac2e034b55e --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -0,0 +1,119 @@ +#pragma once +#include +#include +#include +#include +#include +#include + + +class SipHash; + + +namespace DB +{ + +/// Checksum of one file. +struct MergeTreeDataPartChecksum +{ + using uint128 = CityHash_v1_0_2::uint128; + + UInt64 file_size {}; + uint128 file_hash {}; + + bool is_compressed = false; + UInt64 uncompressed_size {}; + uint128 uncompressed_hash {}; + + MergeTreeDataPartChecksum() {} + MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {} + MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_) + : file_size(file_size_), file_hash(file_hash_), is_compressed(true), + uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {} + + void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const; + void checkSize(const String & path) const; +}; + + +/** Checksums of all non-temporary files. + * For compressed files, the check sum and the size of the decompressed data are stored to not depend on the compression method. + */ +struct MergeTreeDataPartChecksums +{ + using Checksum = MergeTreeDataPartChecksum; + + /// The order is important. + using FileChecksums = std::map; + FileChecksums files; + + void addFile(const String & file_name, UInt64 file_size, Checksum::uint128 file_hash); + + void add(MergeTreeDataPartChecksums && rhs_checksums); + + bool empty() const + { + return files.empty(); + } + + /// Checks that the set of columns and their checksums are the same. If not, throws an exception. + /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. + /// Otherwise, it compares only the checksums of the files. + void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const; + + /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum. + void checkSizes(const String & path) const; + + /// Returns false if the checksum is too old. + bool read(ReadBuffer & in); + /// Assume that header with version (the first line) is read + bool read(ReadBuffer & in, size_t format_version); + bool read_v2(ReadBuffer & in); + bool read_v3(ReadBuffer & in); + bool read_v4(ReadBuffer & in); + + void write(WriteBuffer & out) const; + String toString() const; + + /// Checksum from the set of checksums of .bin files (for deduplication). + void computeTotalChecksumDataOnly(SipHash & hash) const; + + static MergeTreeDataPartChecksums deserializeFrom(const String & s); +}; + + +/// A kind of MergeTreeDataPartChecksums intended to be stored in ZooKeeper (to save its RAM) +/// MinimalisticDataPartChecksums and MergeTreeDataPartChecksums hasve the same serialization format +/// for versions less MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS. +struct MinimalisticDataPartChecksums +{ + UInt64 num_compressed_files = 0; + UInt64 num_uncompressed_files = 0; + + using uint128 = MergeTreeDataPartChecksum::uint128; + uint128 hash_of_all_files {}; + uint128 hash_of_uncompressed_files {}; + uint128 uncompressed_hash_of_compressed_files {}; + + /// Is set only for old formats + std::unique_ptr full_checksums; + + static constexpr size_t MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS = 5; + + MinimalisticDataPartChecksums() = default; + explicit MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums); + void computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums); + + bool deserialize(ReadBuffer & in); + static MinimalisticDataPartChecksums deserializeFrom(const String & s); + + void serialize(WriteBuffer & to) const; + String getSerializedString(); + static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic); + + void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files); + void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files); +}; + + +} diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 1d5ec76c652..14020e7b9d2 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -133,7 +133,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) if (deduplicate) { SipHash hash; - part->checksums.summaryDataChecksum(hash); + part->checksums.computeTotalChecksumDataOnly(hash); union { char bytes[16]; From fe8dfd303807dfae4d11cffe7966110e2a2db787 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Thu, 22 Mar 2018 02:30:20 +0300 Subject: [PATCH 41/90] Add minimalistic checksums. [#CLICKHOUSE-3660] --- .../MergeTree/MergeTreeDataPartChecksum.cpp | 24 ++- .../MergeTree/MergeTreeDataPartChecksum.h | 6 +- .../Storages/MergeTree/MergeTreeSettings.h | 6 + .../ReplicatedMergeTreeAlterThread.cpp | 4 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 165 +++++++++++------- .../src/Storages/StorageReplicatedMergeTree.h | 12 +- ...eper_different_checksums_formats.reference | 9 + ..._zookeeper_different_checksums_formats.sql | 27 +++ 10 files changed, 183 insertions(+), 74 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference create mode 100644 dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index b252830d0d1..b9be8e3d9f0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -247,7 +247,7 @@ void MergeTreeDataPartChecksums::computeTotalChecksumDataOnly(SipHash & hash) co } } -String MergeTreeDataPartChecksums::toString() const +String MergeTreeDataPartChecksums::getSerializedString() const { WriteBufferFromOwnString out; write(out); @@ -264,10 +264,12 @@ MergeTreeDataPartChecksums MergeTreeDataPartChecksums::deserializeFrom(const Str return res; } -MinimalisticDataPartChecksums::MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums_) +bool MergeTreeDataPartChecksums::isBadChecksumsErrorCode(int code) { - full_checksums = std::make_unique(full_checksums_); - computeTotalChecksums(*full_checksums); + return code == ErrorCodes::CHECKSUM_DOESNT_MATCH + || code == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART + || code == ErrorCodes::NO_FILE_IN_DATA_PART + || code == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART; } void MinimalisticDataPartChecksums::serialize(WriteBuffer & to) const @@ -371,7 +373,7 @@ void MinimalisticDataPartChecksums::computeTotalChecksums(const MergeTreeDataPar String MinimalisticDataPartChecksums::getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic) { if (!minimalistic) - return full_checksums.toString(); + return full_checksums.getSerializedString(); MinimalisticDataPartChecksums checksums; checksums.computeTotalChecksums(full_checksums); @@ -383,9 +385,21 @@ void MinimalisticDataPartChecksums::checkEqual(const MinimalisticDataPartChecksu if (full_checksums && rhs.full_checksums) full_checksums->checkEqual(*rhs.full_checksums, check_uncompressed_hash_in_compressed_files); + // If full checksums were checked, check total checksums just in case checkEqualImpl(rhs, check_uncompressed_hash_in_compressed_files); } +void MinimalisticDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) +{ + if (full_checksums) + full_checksums->checkEqual(rhs, check_uncompressed_hash_in_compressed_files); + + // If full checksums were checked, check total checksums just in case + MinimalisticDataPartChecksums rhs_minimalistic; + rhs_minimalistic.computeTotalChecksums(rhs); + checkEqualImpl(rhs_minimalistic, check_uncompressed_hash_in_compressed_files); +} + void MinimalisticDataPartChecksums::checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files) { if (num_compressed_files != rhs.num_compressed_files || num_uncompressed_files != rhs.num_uncompressed_files) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index ac2e034b55e..797c8929b58 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -61,6 +61,8 @@ struct MergeTreeDataPartChecksums /// Otherwise, it compares only the checksums of the files. void checkEqual(const MergeTreeDataPartChecksums & rhs, bool have_uncompressed) const; + static bool isBadChecksumsErrorCode(int code); + /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum. void checkSizes(const String & path) const; @@ -73,11 +75,11 @@ struct MergeTreeDataPartChecksums bool read_v4(ReadBuffer & in); void write(WriteBuffer & out) const; - String toString() const; /// Checksum from the set of checksums of .bin files (for deduplication). void computeTotalChecksumDataOnly(SipHash & hash) const; + String getSerializedString() const; static MergeTreeDataPartChecksums deserializeFrom(const String & s); }; @@ -101,7 +103,6 @@ struct MinimalisticDataPartChecksums static constexpr size_t MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS = 5; MinimalisticDataPartChecksums() = default; - explicit MinimalisticDataPartChecksums(const MergeTreeDataPartChecksums & full_checksums); void computeTotalChecksums(const MergeTreeDataPartChecksums & full_checksums); bool deserialize(ReadBuffer & in); @@ -112,6 +113,7 @@ struct MinimalisticDataPartChecksums static String getSerializedString(const MergeTreeDataPartChecksums & full_checksums, bool minimalistic); void checkEqual(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files); + void checkEqual(const MergeTreeDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files); void checkEqualImpl(const MinimalisticDataPartChecksums & rhs, bool check_uncompressed_hash_in_compressed_files); }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index f91ca880a66..424c634afaf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -131,6 +131,12 @@ struct MergeTreeSettings * for backward compatibility. \ */ \ M(SettingBool, compatibility_allow_sampling_expression_not_in_primary_key, false) \ + \ + /** Use small format (dozens bytes) for part checksums in ZooKeeper \ + * instead of ordinary ones (dozens KB). \ + * Before enabling check that all replicas support new format. \ + */ \ + M(SettingBool, use_minimalistic_checksums_in_zookeeper, false) /// Settings that should not change after the creation of a table. #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \ diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index 72d7cd91e3a..f0513c04f9e 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -144,7 +144,9 @@ void ReplicatedMergeTreeAlterThread::run() ops.emplace_back(std::make_shared( storage.replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1)); ops.emplace_back(std::make_shared( - storage.replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1)); + storage.replica_path + "/parts/" + part->name + "/checksums", + storage.getChecksumsForZooKeeper(transaction->getNewChecksums()), + -1)); try { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 14020e7b9d2..52bc1c4be3f 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -306,7 +306,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo zkutil::CreateMode::Persistent)); ops.emplace_back(std::make_shared( storage.replica_path + "/parts/" + part->name + "/checksums", - part->checksums.toString(), + storage.getChecksumsForZooKeeper(part->checksums), acl, zkutil::CreateMode::Persistent)); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index f64826f6266..85e58f4551b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -237,7 +237,7 @@ void ReplicatedMergeTreePartCheckThread::checkPart(const String & part_name) try { - auto zk_checksums = MergeTreeData::DataPart::Checksums::parse( + auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom( zookeeper->get(storage.replica_path + "/parts/" + part_name + "/checksums")); zk_checksums.checkEqual(part->checksums, true); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 70326cddd4a..006c2fd63a1 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -838,7 +838,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) LOG_ERROR(log, "Adding unexpected local part to ZooKeeper: " << part->name); zkutil::Ops ops; - checkPartAndAddToZooKeeper(part, ops); + checkPartChecksumsAndAddCommitOps(zookeeper, part, ops); zookeeper->multi(ops); } @@ -878,11 +878,9 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) } -void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper( - const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name) +void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper, + const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String part_name, NameSet * absent_replicas_paths) { - auto zookeeper = getZooKeeper(); - if (part_name.empty()) part_name = part->name; @@ -892,24 +890,34 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper( Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); std::shuffle(replicas.begin(), replicas.end(), rng); String expected_columns_str = part->columns.toString(); + bool has_been_alredy_added = false; for (const String & replica : replicas) { zkutil::Stat stat_before, stat_after; + String current_part_path = zookeeper_path + "/replicas/" + replica + "/parts/" + part_name; + String columns_str; - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", columns_str, &stat_before)) + if (!zookeeper->tryGet(current_part_path + "/columns", columns_str, &stat_before)) + { + if (absent_replicas_paths) + absent_replicas_paths->emplace(current_part_path); + continue; + } + if (columns_str != expected_columns_str) { LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica << " because columns are different"); continue; } + String checksums_str; /// Let's check that the node's version with the columns did not change while we were reading the checksums. /// This ensures that the columns and the checksum refer to the same data. - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/checksums", checksums_str) || - !zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/parts/" + part_name + "/columns", &stat_after) || + if (!zookeeper->tryGet(current_part_path + "/checksums", checksums_str) || + !zookeeper->exists(current_part_path + "/columns", &stat_after) || stat_before.version != stat_after.version) { LOG_INFO(log, "Not checking checksums of part " << part_name << " with replica " << replica @@ -917,36 +925,90 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper( continue; } - auto checksums = MergeTreeData::DataPart::Checksums::parse(checksums_str); - checksums.checkEqual(part->checksums, true); + auto zk_checksums = MinimalisticDataPartChecksums::deserializeFrom(checksums_str); + zk_checksums.checkEqual(part->checksums, true); + + if (replica == replica_name) + has_been_alredy_added = true; } - if (zookeeper->exists(replica_path + "/parts/" + part_name)) + if (!has_been_alredy_added) { - LOG_ERROR(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists"); - return; + auto acl = zookeeper->getDefaultACL(); + String part_path = replica_path + "/parts/" + part_name; + + ops.emplace_back(std::make_shared( + zookeeper_path + "/columns", expected_columns_version)); + ops.emplace_back(std::make_shared( + part_path, "", acl, zkutil::CreateMode::Persistent)); + ops.emplace_back(std::make_shared( + part_path + "/columns", part->columns.toString(), acl, zkutil::CreateMode::Persistent)); + ops.emplace_back(std::make_shared( + part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), acl, zkutil::CreateMode::Persistent)); } + else + { + LOG_WARNING(log, "checkPartAndAddToZooKeeper: node " << replica_path + "/parts/" + part_name << " already exists." + << " Will not commit any nodes."); + } +} - auto acl = zookeeper->getDefaultACL(); +MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction, + const MergeTreeData::DataPartPtr & part) +{ + auto zookeeper = getZooKeeper(); - ops.emplace_back(std::make_shared( - zookeeper_path + "/columns", - expected_columns_version)); - ops.emplace_back(std::make_shared( - replica_path + "/parts/" + part_name, - "", - acl, - zkutil::CreateMode::Persistent)); - ops.emplace_back(std::make_shared( - replica_path + "/parts/" + part_name + "/columns", - part->columns.toString(), - acl, - zkutil::CreateMode::Persistent)); - ops.emplace_back(std::make_shared( - replica_path + "/parts/" + part_name + "/checksums", - part->checksums.toString(), - acl, - zkutil::CreateMode::Persistent)); + while (true) + { + zkutil::Ops ops; + NameSet absent_part_paths_on_replicas; + + /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`. + checkPartChecksumsAndAddCommitOps(zookeeper, part, ops, part->name, &absent_part_paths_on_replicas); + + /// Do not commit if the part is obsolete, we have just briefly checked its checksums + if (transaction.isEmpty()) + return {}; + + /// Will check that the part did not suddenly appear on skipped replicas + if (!absent_part_paths_on_replicas.empty()) + { + zkutil::Ops new_ops; + for (const String & part_path : absent_part_paths_on_replicas) + { + new_ops.emplace_back(std::make_shared(part_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); + new_ops.emplace_back(std::make_shared(part_path, -1)); + } + + /// Add check ops at the beginning + new_ops.insert(new_ops.end(), ops.begin(), ops.end()); + ops = std::move(new_ops); + } + + try + { + zookeeper->multi(ops); + return transaction.commit(); + } + catch (zkutil::KeeperMultiException & e) + { + size_t num_check_ops = 2 * absent_part_paths_on_replicas.size(); + size_t failed_op_index = zkutil::getFailedOpIndex(e.info.op_results, e.info.code); + + if (failed_op_index < num_check_ops && e.info.code == ZNODEEXISTS) + { + LOG_INFO(log, "The part " << e.info.getFailedOp().describe() << " on a replica suddenly appeared, will recheck checksums"); + } + else + throw; + } + } +} + +String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) +{ + return MinimalisticDataPartChecksums::getSerializedString(checksums, + static_cast(data.settings.use_minimalistic_checksums_in_zookeeper)); } @@ -1171,22 +1233,18 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre part = merger.mergePartsToTemporaryPart( future_merged_part, *merge_entry, aio_threshold, entry.create_time, reserved_space.get(), entry.deduplicate); - zkutil::Ops ops; + merger.renameMergedTemporaryPart(part, parts, &transaction); try { - /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`. - checkPartAndAddToZooKeeper(part, ops, entry.new_part_name); + checkPartChecksumsAndCommit(transaction, part); } catch (const Exception & e) { - if (e.code() == ErrorCodes::CHECKSUM_DOESNT_MATCH - || e.code() == ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART - || e.code() == ErrorCodes::NO_FILE_IN_DATA_PART - || e.code() == ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART) + if (MergeTreeDataPartChecksums::isBadChecksumsErrorCode(e.code())) { do_fetch = true; - part->remove(); + transaction.rollback(); ProfileEvents::increment(ProfileEvents::DataAfterMergeDiffersFromReplica); @@ -1210,15 +1268,6 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre throw; } - merger.renameMergedTemporaryPart(part, parts, &transaction); - - /// Do not commit if the part is obsolete - if (!transaction.isEmpty()) - { - getZooKeeper()->multi(ops); /// After long merge, get fresh ZK handle, because previous session may be expired. - transaction.commit(); - } - /** Removing old chunks from ZK and from the disk is delayed - see ReplicatedMergeTreeCleanupThread, clearOldParts. */ @@ -1526,7 +1575,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & ops.emplace_back(std::make_shared( replica_path + "/parts/" + part->name + "/columns", transaction->getNewColumns().toString(), -1)); ops.emplace_back(std::make_shared( - replica_path + "/parts/" + part->name + "/checksums", transaction->getNewChecksums().toString(), -1)); + replica_path + "/parts/" + part->name + "/checksums", getChecksumsForZooKeeper(transaction->getNewChecksums()), -1)); zookeeper->multi(ops); @@ -2269,24 +2318,15 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (!to_detached) { - zkutil::Ops ops; + MergeTreeData::Transaction transaction; + data.renameTempPartAndReplace(part, nullptr, &transaction); /** NOTE * Here, an error occurs if ALTER occurred with a change in the column type or column deletion, * and the part on remote server has not yet been modified. * After a while, one of the following attempts to make `fetchPart` succeed. */ - checkPartAndAddToZooKeeper(part, ops, part_name); - - MergeTreeData::Transaction transaction; - data.renameTempPartAndReplace(part, nullptr, &transaction); - - /// Do not commit if the part is obsolete - if (!transaction.isEmpty()) - { - getZooKeeper()->multi(ops); - replaced_parts = transaction.commit(); - } + replaced_parts = checkPartChecksumsAndCommit(transaction, part); /** If a quorum is tracked for this part, you must update it. * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. @@ -3731,5 +3771,4 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( LOG_TRACE(log, "Deleted " << to_delete_futures.size() << " deduplication block IDs in partition ID " << partition_id); } - } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 4a2193b05db..b16e2538fe5 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -241,6 +241,9 @@ private: zkutil::LeaderElectionPtr leader_election; + /// When activated, replica is initialized and startup() method could exit + Poco::Event startup_event; + /// Do I need to complete background threads (except restarting_thread)? std::atomic shutdown_called {false}; Poco::Event shutdown_event; @@ -317,7 +320,14 @@ private: * Adds actions to `ops` that add data about the part into ZooKeeper. * Call under TableStructureLock. */ - void checkPartAndAddToZooKeeper(const MergeTreeData::DataPartPtr & part, zkutil::Ops & ops, String name_override = ""); + void checkPartChecksumsAndAddCommitOps(const zkutil::ZooKeeperPtr & zookeeper, const MergeTreeData::DataPartPtr & part, + zkutil::Ops & ops, String part_name = "", NameSet * absent_replicas_paths = nullptr); + + String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums); + + /// Accepts a PreComitted part, atomically checks its checksums with ones on other replicas and commit the part + MergeTreeData::DataPartsVector checkPartChecksumsAndCommit(MergeTreeData::Transaction & transaction, + const MergeTreeData::DataPartPtr & part); /// Adds actions to `ops` that remove a part from ZooKeeper. void removePartFromZooKeeper(const String & part_name, zkutil::Ops & ops); diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference new file mode 100644 index 00000000000..da05f197b7e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.reference @@ -0,0 +1,9 @@ +1 +1 +0 [] +1 [] +0 [] +1 [] +DETACH +0 [] +1 [] diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql new file mode 100644 index 00000000000..988d4fbd1b3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS test.table_old; +DROP TABLE IF EXISTS test.table_new; + +CREATE TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=0; +CREATE TABLE test.table_new (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'new') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1; + +-- Workaround to wait registration in leader election to correctly calculate number of active replicas to be able make quorum INSERT +OPTIMIZE TABLE test.table_new; + +SET insert_quorum=2; +INSERT INTO test.table_old VALUES (0, []); +SELECT value LIKE '%checksums format version: 4%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/old/parts/all_0_0_0' AND name = 'checksums'; + +INSERT INTO test.table_new VALUES (1, []); +SELECT value LIKE '%checksums format version: 5%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/new/parts/all_1_1_0' AND name = 'checksums'; + +OPTIMIZE TABLE test.table_old; +SELECT * FROM test.table_old ORDER BY k; +SELECT * FROM test.table_new ORDER BY k; + +SELECT 'DETACH'; +DETACH TABLE test.table_old; +ATTACH TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1; +SELECT * FROM test.table_old ORDER BY k; + +DROP TABLE IF EXISTS test.table_old; +DROP TABLE IF EXISTS test.table_new; \ No newline at end of file From 883de7a2a6c28304ee5248bb37c0c4bfc1c8243d Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Thu, 22 Mar 2018 22:46:01 +0300 Subject: [PATCH 42/90] startup() waits for initialization of ReplicatedMergeTreeRestartingThread. [#CLICKHOUSE-2] --- .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp | 7 +++++++ dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 +++ .../00611_zookeeper_different_checksums_formats.sql | 3 --- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index f6686c273ce..f8ea02cff9b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -94,16 +94,22 @@ void ReplicatedMergeTreeRestartingThread::run() /// The exception when you try to zookeeper_init usually happens if DNS does not work. We will try to do it again. tryLogCurrentException(__PRETTY_FUNCTION__); + if (first_time) + storage.startup_event.set(); wakeup_event.tryWait(retry_period_ms); continue; } if (!need_stop && !tryStartup()) { + if (first_time) + storage.startup_event.set(); wakeup_event.tryWait(retry_period_ms); continue; } + if (first_time) + storage.startup_event.set(); break; } @@ -150,6 +156,7 @@ void ReplicatedMergeTreeRestartingThread::run() } catch (...) { + storage.startup_event.set(); tryLogCurrentException(__PRETTY_FUNCTION__); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 006c2fd63a1..f11d9fe28b8 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2386,6 +2386,9 @@ void StorageReplicatedMergeTree::startup() /// In this thread replica will be activated. restarting_thread = std::make_unique(*this); + + /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attmept to do it + startup_event.wait(); } diff --git a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql index 988d4fbd1b3..9076485cfa1 100644 --- a/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql +++ b/dbms/tests/queries/0_stateless/00611_zookeeper_different_checksums_formats.sql @@ -4,9 +4,6 @@ DROP TABLE IF EXISTS test.table_new; CREATE TABLE test.table_old (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'old') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=0; CREATE TABLE test.table_new (k UInt64, d Array(String)) ENGINE = ReplicatedMergeTree('/clickhouse/test/tables/checksums_test', 'new') ORDER BY k SETTINGS use_minimalistic_checksums_in_zookeeper=1; --- Workaround to wait registration in leader election to correctly calculate number of active replicas to be able make quorum INSERT -OPTIMIZE TABLE test.table_new; - SET insert_quorum=2; INSERT INTO test.table_old VALUES (0, []); SELECT value LIKE '%checksums format version: 4%' FROM system.zookeeper WHERE path='/clickhouse/test/tables/checksums_test/replicas/old/parts/all_0_0_0' AND name = 'checksums'; From a75d5525e7fbf7de59804b701c983b5fd3aada28 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Mar 2018 00:20:20 +0300 Subject: [PATCH 43/90] Update MergeTreeDataPartChecksum.h --- dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 797c8929b58..2471d0ef681 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -86,7 +86,7 @@ struct MergeTreeDataPartChecksums /// A kind of MergeTreeDataPartChecksums intended to be stored in ZooKeeper (to save its RAM) /// MinimalisticDataPartChecksums and MergeTreeDataPartChecksums hasve the same serialization format -/// for versions less MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS. +/// for versions less than MINIMAL_VERSION_WITH_MINIMALISTIC_CHECKSUMS. struct MinimalisticDataPartChecksums { UInt64 num_compressed_files = 0; From 62492450180fa37d018a0f454066765b125580b3 Mon Sep 17 00:00:00 2001 From: robot-metrika-test Date: Fri, 23 Mar 2018 01:11:43 +0300 Subject: [PATCH 44/90] Auto version update to [54371] --- dbms/cmake/version.cmake | 4 ++-- debian/changelog | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index ad5ad09cce2..990c638bf46 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,6 +1,6 @@ # This strings autochanged from release_lib.sh: -set(VERSION_DESCRIBE v1.1.54370-testing) -set(VERSION_REVISION 54370) +set(VERSION_DESCRIBE v1.1.54371-testing) +set(VERSION_REVISION 54371) # end of autochange set (VERSION_MAJOR 1) diff --git a/debian/changelog b/debian/changelog index 75502e3cc96..2ff28b59456 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (1.1.54370) unstable; urgency=low +clickhouse (1.1.54371) unstable; urgency=low * Modified source code - -- Fri, 16 Mar 2018 12:01:50 +0300 + -- Fri, 23 Mar 2018 01:11:42 +0300 From 2f468f1ae1345e52567a291df33e4ab6ac9f810c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 23 Mar 2018 13:50:20 +0300 Subject: [PATCH 45/90] fix test [#CLICKHOUSE-3663] --- dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 37d41749420..8f29c93511d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -500,7 +500,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); /// sort columns to ensure consistent order among all blocks - if (should_sort) + if (!never_evaluate_defaults && should_sort) { Block ordered_block; From c76d1a008d27d4917bf90fd2413863700d94d095 Mon Sep 17 00:00:00 2001 From: Tobias Adamson Date: Fri, 23 Mar 2018 22:13:09 +0800 Subject: [PATCH 46/90] Make sure to split the topics args list on ',' --- dbms/src/Storages/StorageKafka.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index bfe72aa016e..d82b12a6fb2 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -574,9 +575,12 @@ void registerStorageKafka(StorageFactory & factory) throw Exception("Number of consumers must be a positive integer", ErrorCodes::BAD_ARGUMENTS); } - // Parse topic list and consumer group + // Parse topic list Names topics; - topics.push_back(static_cast(*engine_args[1]).value.safeGet()); + String topicArg = static_cast(*engine_args[1]).value.safeGet(); + boost::split(topics, topics , [](char c){return c == ',';}); + + // Parse consumer group String group = static_cast(*engine_args[2]).value.safeGet(); // Parse format from string From b87cfd2924a608b64c1d295c1e94ecdf6a323d38 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 22 Mar 2018 20:32:11 +0300 Subject: [PATCH 47/90] Add tests --- dbms/tests/queries/0_stateless/00534_filimonov.data | 3 +++ .../0_stateless/00534_long_functions_bad_arguments7.sh | 2 +- .../0_stateless/00534_long_functions_bad_arguments8.sh | 10 ++++++++++ .../0_stateless/00534_long_functions_bad_arguments9.sh | 10 ++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh create mode 100755 dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh diff --git a/dbms/tests/queries/0_stateless/00534_filimonov.data b/dbms/tests/queries/0_stateless/00534_filimonov.data index 53c64592d0a..2dd470403c0 100644 --- a/dbms/tests/queries/0_stateless/00534_filimonov.data +++ b/dbms/tests/queries/0_stateless/00534_filimonov.data @@ -427,3 +427,6 @@ SELECT COVAR_POPArray([CAST( 0 AS Int8)],arrayPopBack([CAST( 0 AS Int8)])); SELECT COVAR_SAMPArray([CAST( 0 AS Int8)],arrayPopBack([CAST( 0 AS Int8)])); SELECT medianTimingWeightedArray([CAST( 0 AS Int8)],arrayPopBack([CAST( 0 AS Int8)])); SELECT quantilesDeterministicArray([CAST( 0 AS Int8)],arrayPopBack([CAST( 0 AS Int32)])); + +SELECT maxIntersections([], []) +SELECT sumMap([], []) diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh index 78b9afafe55..2cc422cdf9b 100755 --- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh +++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/00534_long_functions_bad_arguments.lib -test_variant 'SELECT $_('');' +test_variant 'SELECT $_(\'\');' diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh new file mode 100755 index 00000000000..f884a926c3b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments8.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +. $CURDIR/00534_long_functions_bad_arguments.lib + +test_variant 'SELECT $_(NULL, NULL);' +test_variant 'SELECT $_([], []);' +test_variant 'SELECT $_([NULL],[NULL]);' diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh new file mode 100755 index 00000000000..bc1ceda33fa --- /dev/null +++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments9.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +. $CURDIR/00534_long_functions_bad_arguments.lib + +test_variant 'SELECT $_(NULL, NULL, NULL);' +test_variant 'SELECT $_([], [], []);' +test_variant 'SELECT $_([NULL], [NULL], [NULL]);' From 1619189ed5dbbec429fcd18d4e3893d2158722d7 Mon Sep 17 00:00:00 2001 From: Artemeey Date: Tue, 20 Mar 2018 11:41:26 +0300 Subject: [PATCH 48/90] Update settings.md --- docs/ru/operations/settings/settings.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 624dadcf096..92e9c2cccae 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -340,6 +340,10 @@ ClickHouse применяет настройку в том случае, ког ## strict_insert_defaults +``` +Настройка удалена в версии 1.1.54362 +``` + Строгое присвоение значений по умолчанию при добавлении данных. Если при выполнении запроса [INSERT](../../query_language/queries.md#queries-insert) данные для столбца не заданы, то ClickHouse присваивает полям значения по умолчанию. Значения по умолчанию определяются свойством `DEFAULT` для каждого столбца в настройках таблицы. Если для столбца не определен `DEFAULT`, то когда: From 2590e3b0a6f68448d3481d3fe5214cdd0e61ea3d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Mar 2018 18:37:37 +0300 Subject: [PATCH 49/90] Update settings.md --- docs/ru/operations/settings/settings.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 92e9c2cccae..62d183234a4 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -335,19 +335,3 @@ ClickHouse применяет настройку в том случае, ког ## output_format_json_quote_64bit_integers Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек. - - - -## strict_insert_defaults - -``` -Настройка удалена в версии 1.1.54362 -``` - -Строгое присвоение значений по умолчанию при добавлении данных. - -Если при выполнении запроса [INSERT](../../query_language/queries.md#queries-insert) данные для столбца не заданы, то ClickHouse присваивает полям значения по умолчанию. Значения по умолчанию определяются свойством `DEFAULT` для каждого столбца в настройках таблицы. Если для столбца не определен `DEFAULT`, то когда: - -- `strict_insert_defaults=0` - полям столбца присваиваются нули и пустые строки. -- `strict_insert_defaults=1` - ClickHouse генерирует исключение и обязывает пользователя передать данные в столбец. - From 723141c19b92b9f4e72e9c430c0ad6cb28146414 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Mar 2018 18:39:33 +0300 Subject: [PATCH 50/90] Removed setting "strict_insert_defaults" from the en docs #2079 --- docs/en/operations/settings/settings.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a87328cb06c..e006f302c68 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -338,15 +338,3 @@ It works for JSONEachRow and TSKV formats. ## output_format_json_quote_64bit_integers If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes. - - - -## strict_insert_defaults - -Strictly assigns default values when adding data. - -If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert) query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column: - -- `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings. -- `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column. - From a6c2e43bd25016105857c6558bc52e35ac6efeb0 Mon Sep 17 00:00:00 2001 From: Alexander Marshalov <_@marshalov.org> Date: Fri, 23 Mar 2018 00:31:25 +0700 Subject: [PATCH 51/90] The lost information in the Kafka engine documentation has been restored, modified and translated (from commit: https://github.com/yandex/ClickHouse/pull/1654/files#diff-c9ba89e2414c83b5dcc579233c5c80b5) --- docs/en/table_engines/kafka.md | 23 ++++++++++++++++++++++- docs/ru/table_engines/kafka.md | 26 ++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md index 55d2027309d..85943e44fc5 100644 --- a/docs/en/table_engines/kafka.md +++ b/docs/en/table_engines/kafka.md @@ -9,7 +9,7 @@ Kafka lets you: - Process streams as they become available. ``` -Kafka(broker_list, topic_list, group_name, format[, schema]) +Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers]) ``` Parameters: @@ -19,6 +19,7 @@ Parameters: - `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. - `format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the section "Formats". - `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root ` schema.capnp:Message` object. +- `num_consumers` - Number of created consumers per engine. By default `1`. Create more consumers if the throughput of a single consumer is insufficient. The total number of consumers shouldn't exceed the number of partitions in given topic, as there can be at most 1 consumers assigned to any single partition. Example: @@ -77,3 +78,23 @@ To stop receiving topic data or to change the conversion logic, detach the mater If you want to change the target table by using `ALTER` materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view. + +## Configuration + +Similarly to GraphiteMergeTree, Kafka engine supports extended configuration through the ClickHouse config file. There are two configuration keys you can use - global (`kafka`), and per-topic (`kafka_topic_*`). The global configuration is applied first, then per-topic configuration (if exists). + +```xml + + + cgrp + smallest + + + + + 250 + 100000 + +``` + +See [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for the list of possible configuration options. Use underscores instead of dots in the ClickHouse configuration, for example `check.crcs=true` would correspond to `true`. diff --git a/docs/ru/table_engines/kafka.md b/docs/ru/table_engines/kafka.md index 127820a8ef2..8225192e337 100644 --- a/docs/ru/table_engines/kafka.md +++ b/docs/ru/table_engines/kafka.md @@ -9,7 +9,7 @@ Kafka позволяет: - Обрабатывать потоки по мере их появления. ``` -Kafka(broker_list, topic_list, group_name, format[, schema]) +Kafka(broker_list, topic_list, group_name, format[, schema, num_consumers]) ``` Параметры: @@ -19,6 +19,7 @@ Kafka(broker_list, topic_list, group_name, format[, schema]) - `group_name` - Группа потребителя Kafka (`group1`). Отступы для чтения отслеживаются для каждой группы отдельно. Если необходимо, чтобы сообщения не повторялись на кластере, используйте везде одно имя группы. - `format` - Формат сообщений. Имеет те же обозначения, что выдает SQL-выражение `FORMAT`, например, `JSONEachRow`. Подробнее смотрите в разделе "Форматы". - `schema` - Опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. +- `num_consumers` - Количество потребителей (consumer) на таблицу. По умолчанию `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя. Пример: @@ -75,4 +76,25 @@ Kafka(broker_list, topic_list, group_name, format[, schema]) ATTACH MATERIALIZED VIEW consumer; ``` -Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления. \ No newline at end of file +Если необходимо изменить целевую таблицу с помощью `ALTER`, то материализованное представление рекомендуется отключить, чтобы избежать несостыковки между целевой таблицей и данными от представления. + + +## Конфигурация + +Аналогично GraphiteMergeTree, движок Kafka поддерживает расширенную конфигурацию с помощью конфигурационного файла ClickHouse. Существует два конфигурационных ключа, которые можно использовать - глобальный (`kafka`) и по топикам (`kafka_topic_*`). Сначала применяется глобальная конфигурация, затем конфигурация по топикам (если она существует). + +```xml + + + cgrp + smallest + + + + + 250 + 100000 + +``` + +В документе [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) можно увидеть список возможных опций конфигурации. Используйте подчёркивания (`_`) вместо точек в конфигурации ClickHouse, например, `check.crcs=true` будет соответствовать `true`. \ No newline at end of file From 06d71a9493dd2a831e297323242c0278a11c34fb Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 3 May 2018 08:30:24 +0800 Subject: [PATCH 52/90] ISSUES-218 add test for distributed & case when then --- ...609_distributed_with_case_when_then.reference | 6 ++++++ .../00609_distributed_with_case_when_then.sql | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference create mode 100644 dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference new file mode 100644 index 00000000000..c5bb9a19a34 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.reference @@ -0,0 +1,6 @@ +hello +world +2 +hello +world +2 diff --git a/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql new file mode 100644 index 00000000000..8d7568415a2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00609_distributed_with_case_when_then.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test.mergetree; +DROP TABLE IF EXISTS test.distributed; + +CREATE TABLE test.mergetree (x UInt64, s String) ENGINE = MergeTree ORDER BY x; +INSERT INTO test.mergetree VALUES (1, 'hello'), (2, 'world'); + +SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE 'unknow' END FROM test.mergetree; +SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE 'unknow' END FROM test.mergetree); + +CREATE TABLE test.distributed AS test.mergetree ENGINE = Distributed(test_shard_localhost, test, mergetree); + +SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE 'unknow' END FROM test.distributed; +SELECT count() AS cnt FROM (SELECT CASE x WHEN 1 THEN 'hello' WHEN 2 THEN 'world' ELSE 'unknow' END FROM test.distributed); + +DROP TABLE test.mergetree; +DROP TABLE test.distributed; From caf83a650e88f5c1ac3da33dfc9409c5314f4554 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 23 Mar 2018 09:05:14 -0700 Subject: [PATCH 53/90] libs: compiling with msvc2017 (#1996) * Allow use bundled *ssl library * fix * Add submodule * Fixes * fix * fixes * fixes * fix * fix * update poco * fix warnings * fix * fix * Build fixes * Build fixes * fix * fix * fix * fix * fix * fix * fix * add bat * no zookeeper in dbms * update boost * fixes * fixes * fix * fix * fix * fix * try fix * try fix * fix * fix * fix * fix * fix * fix * fix * fix * Better * fix * dh verbose * fix * dh verbose * fix * clean * Update LocalDate.h * Update LocalDateTime.h --- cmake/find_poco.cmake | 3 ++- contrib/CMakeLists.txt | 20 ++++++++++++------- contrib/boost | 2 +- contrib/libcityhash/src/config.h | 4 ++++ contrib/libfarmhash/CMakeLists.txt | 4 ++++ contrib/libtcmalloc/src/config.h | 4 ++++ dbms/CMakeLists.txt | 5 ----- dbms/src/Common/AIO.h | 2 +- dbms/src/Common/ShellCommand.cpp | 3 +-- dbms/src/Common/ZooKeeper/CMakeLists.txt | 6 +++++- dbms/src/Common/ZooKeeper/ZooKeeper.h | 2 +- .../ZooKeeper/tests/zkutil_test_commands.cpp | 2 +- dbms/src/Common/localBackup.cpp | 5 +---- dbms/src/Common/randomSeed.cpp | 4 +--- dbms/src/Common/tests/compact_array.cpp | 4 +--- dbms/src/Core/Defines.h | 14 +++++++++++-- dbms/src/Core/Field.h | 12 +++++------ .../DataStreams/PrettyBlockOutputStream.cpp | 3 +-- dbms/src/IO/ReadBufferAIO.cpp | 2 +- dbms/src/IO/ReadBufferAIO.h | 3 +-- dbms/src/IO/ReadBufferFromFileDescriptor.cpp | 4 ---- dbms/src/IO/ReadBufferFromFileDescriptor.h | 3 +-- dbms/src/IO/WriteBufferAIO.cpp | 2 +- dbms/src/IO/WriteBufferAIO.h | 2 +- dbms/src/IO/WriteBufferFromFileDescriptor.cpp | 2 +- dbms/src/IO/createReadBufferFromFileBase.cpp | 6 +++--- dbms/src/IO/createWriteBufferFromFileBase.cpp | 6 +++--- dbms/src/IO/tests/read_buffer_aio.cpp | 5 ++--- dbms/src/Interpreters/ClientInfo.cpp | 3 +-- dbms/src/Interpreters/tests/users.cpp | 4 +--- dbms/src/Server/Benchmark.cpp | 13 +----------- dbms/src/Server/Client.cpp | 6 +----- dbms/src/Server/PerformanceTest.cpp | 7 +------ .../tests/remove_symlink_directory.cpp | 3 +-- debian/.pbuilderrc | 2 +- debian/rules | 2 +- libs/libcommon/cmake/find_gperftools.cmake | 2 +- libs/libcommon/include/common/DateLUT.h | 5 ++++- libs/libcommon/include/common/LocalDate.h | 2 +- libs/libcommon/include/common/LocalDateTime.h | 2 +- libs/libcommon/include/common/likely.h | 9 +++++++-- libs/libcommon/include/common/mremap.h | 3 +++ libs/libcommon/include/port/ssize_t.h | 8 ++++++++ libs/libcommon/include/port/unistd.h | 11 ++++++++++ libs/libcommon/src/DateLUT.cpp | 8 ++++---- libs/libcommon/src/mremap.cpp | 11 ++++++++-- libs/libdaemon/include/daemon/BaseDaemon.h | 2 +- libs/libpocoext/src/ThreadNumber.cpp | 10 ++++------ utils/build/build_msvc2017.bat | 14 +++++++++++++ utils/compressor/zstd_test.cpp | 3 ++- utils/iotest/iotest.cpp | 9 +-------- utils/iotest/iotest_aio.cpp | 10 ++-------- utils/iotest/iotest_nonblock.cpp | 8 +------- 53 files changed, 155 insertions(+), 136 deletions(-) create mode 100644 libs/libcommon/include/port/ssize_t.h create mode 100644 libs/libcommon/include/port/unistd.h create mode 100644 utils/build/build_msvc2017.bat diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index 9051d63ae47..315e27b798f 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -49,7 +49,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/") endif () - if (OPENSSL_FOUND) + # TODO! fix internal ssl + if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY) set (Poco_NetSSL_FOUND 1) set (Poco_NetSSL_LIBRARY PocoNetSSL) set (Poco_Crypto_LIBRARY PocoCrypto) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e74e9a5b7db..9015f7e7d36 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -100,13 +100,17 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY) mark_as_advanced (ZLIB_INCLUDE_DIR) if (USE_INTERNAL_SSL_LIBRARY) - add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY}) - set (WITH_BUNDLED_SSL 1) + if (MAKE_STATIC_LIBRARIES) + add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY}) + set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "") + else () + set (WITH_SSL 0 CACHE INTERNAL "") + endif () endif () add_subdirectory (librdkafka) - if (USE_INTERNAL_SSL_LIBRARY) + if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES) target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR}) endif () target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR}) @@ -127,16 +131,18 @@ endif () if (USE_INTERNAL_POCO_LIBRARY) - set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1) set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (_save ${ENABLE_TESTS}) set (ENABLE_TESTS 0) set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1) if (USE_INTERNAL_SSL_LIBRARY) - set (DISABLE_INTERNAL_OPENSSL 1) - set (ENABLE_NETSSL 0) # TODO! - set (ENABLE_CRYPTO 0) # TODO! + set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "") + set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO! + set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO! + endif () + if (MSVC) + set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail) endif () add_subdirectory (poco) unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB) diff --git a/contrib/boost b/contrib/boost index eb5943711e8..5121cc9d037 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit eb5943711e88d1008583e6ae3720a5489313d02e +Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded diff --git a/contrib/libcityhash/src/config.h b/contrib/libcityhash/src/config.h index c873732dd87..61c75fca818 100644 --- a/contrib/libcityhash/src/config.h +++ b/contrib/libcityhash/src/config.h @@ -5,7 +5,11 @@ /* #undef AC_APPLE_UNIVERSAL_BUILD */ /* Define to 1 if the compiler supports __builtin_expect. */ +#if _MSC_VER +#define HAVE_BUILTIN_EXPECT 0 +#else #define HAVE_BUILTIN_EXPECT 1 +#endif /* Define to 1 if you have the header file. */ #define HAVE_DLFCN_H 1 diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt index 7781028094d..16471da3f2f 100644 --- a/contrib/libfarmhash/CMakeLists.txt +++ b/contrib/libfarmhash/CMakeLists.txt @@ -2,4 +2,8 @@ add_library(farmhash farmhash.cc farmhash.h) +if (MSVC) + target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1) +endif () + target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/contrib/libtcmalloc/src/config.h b/contrib/libtcmalloc/src/config.h index 3338309073d..9f9a7a259e5 100644 --- a/contrib/libtcmalloc/src/config.h +++ b/contrib/libtcmalloc/src/config.h @@ -13,7 +13,11 @@ /* #undef ENABLE_SIZED_DELETE */ /* Define to 1 if compiler supports __builtin_expect */ +#if _MSC_VER +#define HAVE_BUILTIN_EXPECT 0 +#else #define HAVE_BUILTIN_EXPECT 1 +#endif /* Define to 1 if compiler supports __builtin_stack_pointer */ /* #undef HAVE_BUILTIN_STACK_POINTER */ diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 965b9b2bfe1..d740ce5a45c 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -131,7 +131,6 @@ target_link_libraries (clickhouse_common_io ${LINK_LIBRARIES_ONLY_ON_X86_64} ${LZ4_LIBRARY} ${ZSTD_LIBRARY} - ${ZOOKEEPER_LIBRARY} ${DOUBLE_CONVERSION_LIBRARIES} ${Poco_Net_LIBRARY} ${Poco_Data_LIBRARY} @@ -158,10 +157,6 @@ if (NOT USE_INTERNAL_RE2_LIBRARY) target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR}) endif () -if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY) - target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR}) -endif () - if (NOT USE_INTERNAL_BOOST_LIBRARY) target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) endif () diff --git a/dbms/src/Common/AIO.h b/dbms/src/Common/AIO.h index e7287c85d3d..254a7f02ecb 100644 --- a/dbms/src/Common/AIO.h +++ b/dbms/src/Common/AIO.h @@ -1,6 +1,6 @@ #pragma once -#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)) #include #include diff --git a/dbms/src/Common/ShellCommand.cpp b/dbms/src/Common/ShellCommand.cpp index b3765874ac4..c9e625810ca 100644 --- a/dbms/src/Common/ShellCommand.cpp +++ b/dbms/src/Common/ShellCommand.cpp @@ -1,13 +1,12 @@ #include #include -#include #include #include - #include #include #include #include +#include namespace DB diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt index 9bd72b6fa54..6a02d5733b1 100644 --- a/dbms/src/Common/ZooKeeper/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt @@ -5,7 +5,11 @@ add_headers_and_sources(clickhouse_common_zookeeper .) add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources}) -target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io) +if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY) + target_include_directories (clickhouse_common_zookeeper BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR}) +endif () + +target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io ${ZOOKEEPER_LIBRARY}) if (ENABLE_TESTS) add_subdirectory (tests) diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.h b/dbms/src/Common/ZooKeeper/ZooKeeper.h index 52f1968eba6..dcfb5e9393d 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.h +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.h @@ -8,10 +8,10 @@ #include #include #include -#include #include #include #include +#include namespace ProfileEvents diff --git a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp index 3c9aed5113d..9b73f79ad7d 100644 --- a/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp +++ b/dbms/src/Common/ZooKeeper/tests/zkutil_test_commands.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include using namespace zkutil; diff --git a/dbms/src/Common/localBackup.cpp b/dbms/src/Common/localBackup.cpp index 3b93e0ee452..eb1f76d8c60 100644 --- a/dbms/src/Common/localBackup.cpp +++ b/dbms/src/Common/localBackup.cpp @@ -1,13 +1,10 @@ #include -#include - #include #include - #include #include - #include +#include namespace DB diff --git a/dbms/src/Common/randomSeed.cpp b/dbms/src/Common/randomSeed.cpp index 62925f55dd3..7618c41a020 100644 --- a/dbms/src/Common/randomSeed.cpp +++ b/dbms/src/Common/randomSeed.cpp @@ -1,12 +1,10 @@ #include -#include +#include #include - #include #include #include #include - #ifdef __APPLE__ #include #endif diff --git a/dbms/src/Common/tests/compact_array.cpp b/dbms/src/Common/tests/compact_array.cpp index 3beb8b6db9a..14c6b819c37 100644 --- a/dbms/src/Common/tests/compact_array.cpp +++ b/dbms/src/Common/tests/compact_array.cpp @@ -7,15 +7,13 @@ #include #include #include - #include - #include #include #include #include -#include #include +#include namespace fs = boost::filesystem; diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 8b69d83d69e..db722cbf91a 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -74,8 +74,18 @@ #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1 -#define ALWAYS_INLINE __attribute__((__always_inline__)) -#define NO_INLINE __attribute__((__noinline__)) +// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html + +#if defined(_MSC_VER) + #define ALWAYS_INLINE __forceinline + #define NO_INLINE static __declspec(noinline) + #define MAY_ALIAS +#else + #define ALWAYS_INLINE __attribute__((__always_inline__)) + #define NO_INLINE __attribute__((__noinline__)) + #define MAY_ALIAS __attribute__((__may_alias__)) +#endif + #define PLATFORM_NOT_SUPPORTED "The only supported platforms are x86_64 and AArch64 (work in progress)" diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index f4c2f1ade86..795bc4b7070 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -198,14 +198,14 @@ public: template T & get() { using TWithoutRef = std::remove_reference_t; - TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + TWithoutRef * MAY_ALIAS ptr = reinterpret_cast(&storage); return *ptr; }; template const T & get() const { using TWithoutRef = std::remove_reference_t; - const TWithoutRef * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + const TWithoutRef * MAY_ALIAS ptr = reinterpret_cast(&storage); return *ptr; }; @@ -340,7 +340,7 @@ private: void createConcrete(T && x) { using JustT = std::decay_t; - JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + JustT * MAY_ALIAS ptr = reinterpret_cast(&storage); new (ptr) JustT(std::forward(x)); which = TypeToEnum::value; } @@ -350,7 +350,7 @@ private: void assignConcrete(T && x) { using JustT = std::decay_t; - JustT * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + JustT * MAY_ALIAS ptr = reinterpret_cast(&storage); *ptr = std::forward(x); } @@ -398,7 +398,7 @@ private: void create(const char * data, size_t size) { - String * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + String * MAY_ALIAS ptr = reinterpret_cast(&storage); new (ptr) String(data, size); which = Types::String; } @@ -434,7 +434,7 @@ private: template void destroy() { - T * __attribute__((__may_alias__)) ptr = reinterpret_cast(&storage); + T * MAY_ALIAS ptr = reinterpret_cast(&storage); ptr->~T(); } }; diff --git a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp index 58b61664b7c..2bb919d4806 100644 --- a/dbms/src/DataStreams/PrettyBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PrettyBlockOutputStream.cpp @@ -1,6 +1,5 @@ #include -#include - +#include #include #include #include diff --git a/dbms/src/IO/ReadBufferAIO.cpp b/dbms/src/IO/ReadBufferAIO.cpp index c4680b0e409..b9f760b51da 100644 --- a/dbms/src/IO/ReadBufferAIO.cpp +++ b/dbms/src/IO/ReadBufferAIO.cpp @@ -1,4 +1,4 @@ -#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)) #include #include diff --git a/dbms/src/IO/ReadBufferAIO.h b/dbms/src/IO/ReadBufferAIO.h index ccbd7330851..9f8451338ac 100644 --- a/dbms/src/IO/ReadBufferAIO.h +++ b/dbms/src/IO/ReadBufferAIO.h @@ -1,6 +1,6 @@ #pragma once -#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)) #include #include @@ -8,7 +8,6 @@ #include #include #include - #include #include #include diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp index eced314ff40..d2acffaf2e7 100644 --- a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp @@ -1,14 +1,10 @@ #include #include - #include - #include #include - #include #include - #include #include diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.h b/dbms/src/IO/ReadBufferFromFileDescriptor.h index 1331878912e..55065931148 100644 --- a/dbms/src/IO/ReadBufferFromFileDescriptor.h +++ b/dbms/src/IO/ReadBufferFromFileDescriptor.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include #include #include diff --git a/dbms/src/IO/WriteBufferAIO.cpp b/dbms/src/IO/WriteBufferAIO.cpp index ae1253df263..087717b452d 100644 --- a/dbms/src/IO/WriteBufferAIO.cpp +++ b/dbms/src/IO/WriteBufferAIO.cpp @@ -1,4 +1,4 @@ -#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)) #include #include diff --git a/dbms/src/IO/WriteBufferAIO.h b/dbms/src/IO/WriteBufferAIO.h index 0942934e87d..3c99eb40eed 100644 --- a/dbms/src/IO/WriteBufferAIO.h +++ b/dbms/src/IO/WriteBufferAIO.h @@ -1,6 +1,6 @@ #pragma once -#if !(defined(__FreeBSD__) || defined(__APPLE__)) +#if !(defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)) #include #include diff --git a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp index 8f6f2b1b126..d22fe7229b8 100644 --- a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/dbms/src/IO/createReadBufferFromFileBase.cpp b/dbms/src/IO/createReadBufferFromFileBase.cpp index bae7a737d36..beb73eda861 100644 --- a/dbms/src/IO/createReadBufferFromFileBase.cpp +++ b/dbms/src/IO/createReadBufferFromFileBase.cpp @@ -1,6 +1,6 @@ #include #include -#if !defined(__APPLE__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER) #include #endif #include @@ -14,7 +14,7 @@ namespace ProfileEvents namespace DB { -#if defined(__APPLE__) || defined(__FreeBSD__) +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER) namespace ErrorCodes { extern const int NOT_IMPLEMENTED; @@ -31,7 +31,7 @@ std::unique_ptr createReadBufferFromFileBase(const std:: } else { -#if !defined(__APPLE__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER) ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO); return std::make_unique(filename_, buffer_size_, flags_, existing_memory_); #else diff --git a/dbms/src/IO/createWriteBufferFromFileBase.cpp b/dbms/src/IO/createWriteBufferFromFileBase.cpp index 94d16d1d41b..de06e2eb1cb 100644 --- a/dbms/src/IO/createWriteBufferFromFileBase.cpp +++ b/dbms/src/IO/createWriteBufferFromFileBase.cpp @@ -1,6 +1,6 @@ #include #include -#if !defined(__APPLE__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER) #include #endif #include @@ -15,7 +15,7 @@ namespace ProfileEvents namespace DB { -#if defined(__APPLE__) || defined(__FreeBSD__) +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(_MSC_VER) namespace ErrorCodes { extern const int NOT_IMPLEMENTED; @@ -33,7 +33,7 @@ WriteBufferFromFileBase * createWriteBufferFromFileBase(const std::string & file } else { -#if !defined(__APPLE__) && !defined(__FreeBSD__) +#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(_MSC_VER) ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO); return new WriteBufferAIO(filename_, buffer_size_, flags_, mode, existing_memory_); #else diff --git a/dbms/src/IO/tests/read_buffer_aio.cpp b/dbms/src/IO/tests/read_buffer_aio.cpp index b6352d7a54a..2f641d0cd3c 100644 --- a/dbms/src/IO/tests/read_buffer_aio.cpp +++ b/dbms/src/IO/tests/read_buffer_aio.cpp @@ -1,14 +1,13 @@ #include #include - #include - #include #include #include #include #include -#include +#include + namespace { diff --git a/dbms/src/Interpreters/ClientInfo.cpp b/dbms/src/Interpreters/ClientInfo.cpp index 40ebabc60df..06fd5e9c589 100644 --- a/dbms/src/Interpreters/ClientInfo.cpp +++ b/dbms/src/Interpreters/ClientInfo.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -8,6 +6,7 @@ #include #include #include +#include namespace DB diff --git a/dbms/src/Interpreters/tests/users.cpp b/dbms/src/Interpreters/tests/users.cpp index 5da4ed26134..f4905398262 100644 --- a/dbms/src/Interpreters/tests/users.cpp +++ b/dbms/src/Interpreters/tests/users.cpp @@ -1,8 +1,6 @@ #include #include - #include - #include #include #include @@ -10,8 +8,8 @@ #include #include #include -#include #include +#include namespace { diff --git a/dbms/src/Server/Benchmark.cpp b/dbms/src/Server/Benchmark.cpp index 78a19a398a1..ee4d2dd8802 100644 --- a/dbms/src/Server/Benchmark.cpp +++ b/dbms/src/Server/Benchmark.cpp @@ -1,31 +1,24 @@ -#include +#include #include #include #include #include - #include #include #include #include #include - #include #include - #include #include #include #include - #include - #include - #include #include #include - #include #include #include @@ -33,13 +26,9 @@ #include #include #include - #include - #include - #include - #include "InterruptListener.h" diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 1c838a99569..a20662ad053 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -1,8 +1,7 @@ -#include +#include #include #include #include - #include #include #include @@ -10,13 +9,10 @@ #include #include #include - #include #include - #include #include - #include #include #include diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/src/Server/PerformanceTest.cpp index 81f679cd61e..7c9d4436f3c 100644 --- a/dbms/src/Server/PerformanceTest.cpp +++ b/dbms/src/Server/PerformanceTest.cpp @@ -3,14 +3,11 @@ #include #include #include -#include - +#include #include #include #include - #include - #include #include #include @@ -27,13 +24,11 @@ #include #include #include - #include #include #include #include #include - #include "InterruptListener.h" /** Tests launcher for ClickHouse. diff --git a/dbms/src/Storages/tests/remove_symlink_directory.cpp b/dbms/src/Storages/tests/remove_symlink_directory.cpp index 16b2a1b4750..59011c557ee 100644 --- a/dbms/src/Storages/tests/remove_symlink_directory.cpp +++ b/dbms/src/Storages/tests/remove_symlink_directory.cpp @@ -1,6 +1,5 @@ -#include +#include #include - #include #include #include diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc index e3704760ec1..ba1cdb2c324 100644 --- a/debian/.pbuilderrc +++ b/debian/.pbuilderrc @@ -188,7 +188,7 @@ export DEB_BUILD_OPTIONS=parallel=`nproc` # Floating bug with permissions: [ -n "$CCACHEDIR" ] && sudo mkdir -p $CCACHEDIR -[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR +[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR || true # chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR diff --git a/debian/rules b/debian/rules index 8f3818c0478..7990db33b1c 100755 --- a/debian/rules +++ b/debian/rules @@ -2,7 +2,7 @@ # -*- makefile -*- # Uncomment this to turn on verbose mode. -#export DH_VERBOSE=1 +export DH_VERBOSE=1 # -pie only for static mode export DEB_BUILD_MAINT_OPTIONS=hardening=+all,-pie diff --git a/libs/libcommon/cmake/find_gperftools.cmake b/libs/libcommon/cmake/find_gperftools.cmake index b9fcc40c0c6..a311f2ba74e 100644 --- a/libs/libcommon/cmake/find_gperftools.cmake +++ b/libs/libcommon/cmake/find_gperftools.cmake @@ -4,7 +4,7 @@ else () option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED}) endif () -if (ARCH_FREEBSD) +if (ARCH_FREEBSD OR MSVC) option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" OFF) else () option (ENABLE_TCMALLOC "Set to TRUE to enable tcmalloc" ON) diff --git a/libs/libcommon/include/common/DateLUT.h b/libs/libcommon/include/common/DateLUT.h index c86b4010108..75a7457d101 100644 --- a/libs/libcommon/include/common/DateLUT.h +++ b/libs/libcommon/include/common/DateLUT.h @@ -2,7 +2,6 @@ #include #include - #include #include #include @@ -10,8 +9,12 @@ // Also defined in Core/Defines.h #if !defined(ALWAYS_INLINE) +#if defined(_MSC_VER) + #define ALWAYS_INLINE __forceinline +#else #define ALWAYS_INLINE __attribute__((__always_inline__)) #endif +#endif /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone. diff --git a/libs/libcommon/include/common/LocalDate.h b/libs/libcommon/include/common/LocalDate.h index 93d045786e9..8eddc6f9115 100644 --- a/libs/libcommon/include/common/LocalDate.h +++ b/libs/libcommon/include/common/LocalDate.h @@ -18,7 +18,7 @@ * * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047) */ -class __attribute__ ((__packed__)) LocalDate +class LocalDate { private: unsigned short m_year; diff --git a/libs/libcommon/include/common/LocalDateTime.h b/libs/libcommon/include/common/LocalDateTime.h index 133bef46723..d0c208edadc 100644 --- a/libs/libcommon/include/common/LocalDateTime.h +++ b/libs/libcommon/include/common/LocalDateTime.h @@ -17,7 +17,7 @@ * * packed - for memcmp to work naturally (but because m_year is 2 bytes, on little endian, comparison is correct only before year 2047) */ -class __attribute__ ((__packed__)) LocalDateTime +class LocalDateTime { private: unsigned short m_year; diff --git a/libs/libcommon/include/common/likely.h b/libs/libcommon/include/common/likely.h index 7a0a98fcdc0..787d46c0e67 100644 --- a/libs/libcommon/include/common/likely.h +++ b/libs/libcommon/include/common/likely.h @@ -1,4 +1,9 @@ #pragma once -#define likely(x) (__builtin_expect(!!(x), 1)) -#define unlikely(x) (__builtin_expect(!!(x), 0)) +#if _MSC_VER +#define likely(x) (x) +#define unlikely(x) (x) +#else +#define likely(x) (__builtin_expect(!!(x), 1)) +#define unlikely(x) (__builtin_expect(!!(x), 0)) +#endif diff --git a/libs/libcommon/include/common/mremap.h b/libs/libcommon/include/common/mremap.h index a6a841adc52..edb62070247 100644 --- a/libs/libcommon/include/common/mremap.h +++ b/libs/libcommon/include/common/mremap.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#if !_MSC_VER #include +#endif #if defined(MREMAP_MAYMOVE) // we already have implementation (linux) diff --git a/libs/libcommon/include/port/ssize_t.h b/libs/libcommon/include/port/ssize_t.h new file mode 100644 index 00000000000..62199bbe1f0 --- /dev/null +++ b/libs/libcommon/include/port/ssize_t.h @@ -0,0 +1,8 @@ +#pragma once + +#ifdef _MSC_VER + #include + typedef SSIZE_T ssize_t; +#else + #include +#endif diff --git a/libs/libcommon/include/port/unistd.h b/libs/libcommon/include/port/unistd.h new file mode 100644 index 00000000000..2c08bd0bcd2 --- /dev/null +++ b/libs/libcommon/include/port/unistd.h @@ -0,0 +1,11 @@ +#pragma once + +// https://stackoverflow.com/questions/341817/is-there-a-replacement-for-unistd-h-for-windows-visual-c + +#ifdef _MSC_VER + #include +#else + #include +#endif + +#include "ssize_t.h" diff --git a/libs/libcommon/src/DateLUT.cpp b/libs/libcommon/src/DateLUT.cpp index 40634698dbc..22c50a9f694 100644 --- a/libs/libcommon/src/DateLUT.cpp +++ b/libs/libcommon/src/DateLUT.cpp @@ -56,13 +56,13 @@ std::string determineDefaultTimeZone() /// then the relative path is the time zone id. fs::path relative_path = tz_file_path.lexically_relative(tz_database_path); if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".") - return relative_path.native(); + return relative_path.string(); /// The file is not inside the tz_database_dir, so we hope that it was copied and /// try to find the file with exact same contents in the database. size_t tzfile_size = fs::file_size(tz_file_path); - Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.native()); + Poco::SHA1Engine::Digest tzfile_sha1 = calcSHA1(tz_file_path.string()); fs::recursive_directory_iterator begin(tz_database_path); fs::recursive_directory_iterator end; @@ -80,8 +80,8 @@ std::string determineDefaultTimeZone() if (candidate_it->status().type() != fs::regular_file || path.filename() == "localtime") continue; - if (fs::file_size(path) == tzfile_size && calcSHA1(path.native()) == tzfile_sha1) - return path.lexically_relative(tz_database_path).native(); + if (fs::file_size(path) == tzfile_size && calcSHA1(path.string()) == tzfile_sha1) + return path.lexically_relative(tz_database_path).string(); } } catch (const Poco::Exception & ex) diff --git a/libs/libcommon/src/mremap.cpp b/libs/libcommon/src/mremap.cpp index e01acc8d5d8..5d1ba5287bc 100644 --- a/libs/libcommon/src/mremap.cpp +++ b/libs/libcommon/src/mremap.cpp @@ -1,9 +1,8 @@ +#include #include #include #include -#include #include -#include #if defined(MREMAP_FIXED) // we already have implementation (linux) @@ -22,18 +21,26 @@ void * mremap( return nullptr; } +#if _MSC_VER + void * new_address = ::operator new(new_size); +#else void * new_address = mmap(nullptr, new_size, mmap_prot, mmap_flags, mmap_fd, mmap_offset); if (MAP_FAILED == new_address) { return MAP_FAILED; } +#endif memcpy(new_address, old_address, old_size); +#if _MSC_VER + delete old_address; +#else if (munmap(old_address, old_size)) { abort(); } +#endif return new_address; } diff --git a/libs/libdaemon/include/daemon/BaseDaemon.h b/libs/libdaemon/include/daemon/BaseDaemon.h index 20c38469d16..112502e06f6 100644 --- a/libs/libdaemon/include/daemon/BaseDaemon.h +++ b/libs/libdaemon/include/daemon/BaseDaemon.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/libs/libpocoext/src/ThreadNumber.cpp b/libs/libpocoext/src/ThreadNumber.cpp index 266627d3168..6458ea8c221 100644 --- a/libs/libpocoext/src/ThreadNumber.cpp +++ b/libs/libpocoext/src/ThreadNumber.cpp @@ -1,16 +1,14 @@ -#include -#include - #include - +#include +#include static thread_local unsigned thread_number = 0; -static unsigned threads = 0; +static std::atomic_uint threads{0}; unsigned Poco::ThreadNumber::get() { if (unlikely(thread_number == 0)) - thread_number = __sync_add_and_fetch(&threads, 1); + thread_number = ++threads; return thread_number; } diff --git a/utils/build/build_msvc2017.bat b/utils/build/build_msvc2017.bat new file mode 100644 index 00000000000..880802999ab --- /dev/null +++ b/utils/build/build_msvc2017.bat @@ -0,0 +1,14 @@ + +:: WINDOWS BUILD NOT SUPPORTED! +:: Script only for development + +cd ../.. +git clone --recursive https://github.com/madler/zlib contrib/zlib +md build +cd build + +:: Stage 1: try build client +cmake .. -G "Visual Studio 15 2017 Win64" -DENABLE_CLICKHOUSE_ALL=0 -DENABLE_CLICKHOUSE_CLIENT=1 > cmake.log +cmake --build . --target clickhouse -- /m > build.log +:: Stage 2: try build minimal server +:: Stage 3: enable all possible features (ssl, ...) diff --git a/utils/compressor/zstd_test.cpp b/utils/compressor/zstd_test.cpp index 6a8b9cb5be6..01016830e7d 100644 --- a/utils/compressor/zstd_test.cpp +++ b/utils/compressor/zstd_test.cpp @@ -1,7 +1,8 @@ -#include +#include #include #include #include +#include int main(int argc, char ** argv) diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp index 439672fc25c..a4849638e48 100644 --- a/utils/iotest/iotest.cpp +++ b/utils/iotest/iotest.cpp @@ -1,28 +1,21 @@ #include -#include +#include #include #include - #include #include #include #include #include - #include #include #include - #include #include - #include #include - #include - #include - #ifdef __APPLE__ #include #endif diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp index f86b4e76e3e..bfab8249d19 100644 --- a/utils/iotest/iotest_aio.cpp +++ b/utils/iotest/iotest_aio.cpp @@ -1,28 +1,21 @@ #include -#include +#include #include #include - #include #include #include - #include #include #include - #include - #include #include - #include - #include #if !defined(__APPLE__) && !defined(__FreeBSD__) #include #endif - #include #include #include @@ -33,6 +26,7 @@ #endif #include + using DB::throwFromErrno; inline int io_setup(unsigned nr, aio_context_t *ctxp) diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp index fdd25860825..8f1ad435bf0 100644 --- a/utils/iotest/iotest_nonblock.cpp +++ b/utils/iotest/iotest_nonblock.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -7,24 +7,18 @@ #include #endif #include - #include #include #include - #include #include - #include #include #include - #include #include - #include #include - #ifdef __APPLE__ #include #endif From 7ba82eb359a3b37ef54d819bc54df407222c98f1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 23 Mar 2018 19:18:36 +0300 Subject: [PATCH 54/90] Update Defines.h --- dbms/src/Core/Defines.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index f860bd8c71c..569799bd956 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -60,7 +60,7 @@ #define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060 #define DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 -#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54358 +#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 From 7cdb6b5de12ec05cc973b615cfdfd668bd30e8cf Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 23 Mar 2018 19:25:02 +0300 Subject: [PATCH 55/90] Fix test --- .../queries/0_stateless/00534_long_functions_bad_arguments7.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh index 2cc422cdf9b..af34af94458 100755 --- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh +++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments7.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/00534_long_functions_bad_arguments.lib -test_variant 'SELECT $_(\'\');' +test_variant "SELECT \$_('');" From e7a2f10e2a681627779f8d41aa1fb880ddbcc805 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 23 Mar 2018 19:40:15 +0300 Subject: [PATCH 56/90] Fix test build --- utils/compressor/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index ae4333337db..ca4dd334fb5 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -7,7 +7,7 @@ set_target_properties(util-clickhouse-compressor PROPERTIES OUTPUT_NAME "clickho #install (TARGETS util-clickhouse-compressor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-compressor) add_executable (zstd_test zstd_test.cpp) -target_link_libraries (zstd_test ${ZSTD_LIBRARY} Threads::Threads) +target_link_libraries (zstd_test ${ZSTD_LIBRARY} common Threads::Threads) if (NOT USE_INTERNAL_ZSTD_LIBRARY) target_include_directories (zstd_test BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) endif () From 046346645f5d7039100ec15db11b887d5be7d7f3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Mar 2018 19:55:35 +0300 Subject: [PATCH 57/90] Minor modifications #1959 --- dbms/src/IO/ReadHelpers.h | 2 ++ dbms/src/Server/Client.cpp | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 875515149b6..2d3b405675f 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -52,6 +52,8 @@ inline char parseEscapeSequence(char c) return '\a'; case 'b': return '\b'; + case 'e': + return '\x1B'; /// \e escape sequence is non standard for C and C++ but supported by gcc and clang. case 'f': return '\f'; case 'n': diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 721df841820..d16a5de3b03 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -340,38 +341,37 @@ private: Strings keys; - prompt_by_server_display_name = config().getString("prompt_by_server_display_name.default", "{display_name} :) "); + prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", "{display_name} :) "); config().keys("prompt_by_server_display_name", keys); for (const String & key : keys) { - if (key != "default" && server_display_name.find(key) != std::string::npos) + if (key != "default" && server_display_name.find(key) != std::string::npos) { - prompt_by_server_display_name = config().getString("prompt_by_server_display_name." + key); + prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name." + key); break; } } - std::map terminalCharacters = { - { "\\e[", "\e[" }, - { "\\33[", "\33[" }, - { "\\033[", "\033["}, - { "\\x1B[", "\x1B["}, - }; - - for (const auto & [key, value]: terminalCharacters) + /// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color. { - boost::replace_all(prompt_by_server_display_name, key, value); + String unescaped_prompt_by_server_display_name; + ReadBufferFromString in(prompt_by_server_display_name); + readEscapedString(unescaped_prompt_by_server_display_name, in); + prompt_by_server_display_name = std::move(unescaped_prompt_by_server_display_name); } - std::map environment = { + /// Prompt may contain the following substitutions in a form of {name}. + std::map environment = + { {"host", config().getString("host", "localhost")}, {"port", config().getString("port", "9000")}, {"user", config().getString("user", "default")}, {"display_name", server_display_name}, }; + /// Quite suboptimal. for (const auto & [key, value]: environment) { boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value); From f5896d35b4ce975400b67ac62b3fb20a14f02c52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Mar 2018 21:20:54 +0300 Subject: [PATCH 58/90] Removed duplicate code that could lead to wrong behaviour #1959 --- dbms/src/Server/Client.cpp | 91 +++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index d16a5de3b03..47b632aa28e 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -169,6 +169,49 @@ private: std::list external_tables; + struct ConnectionParameters + { + String host; + UInt16 port; + String default_database; + String user; + String password; + Protocol::Encryption security; + Protocol::Compression compression; + ConnectionTimeouts timeouts; + + ConnectionParameters() {} + + ConnectionParameters(const Poco::Util::AbstractConfiguration & config) + { + bool is_secure = config.getBool("secure", false); + security = is_secure + ? Protocol::Encryption::Enable + : Protocol::Encryption::Disable; + + host = config.getString("host", "localhost"); + port = config.getInt("port", + config.getInt(is_secure ? "tcp_secure_port" : "tcp_port", + is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); + + default_database = config.getString("database", ""); + user = config.getString("user", ""); + password = config.getString("password", ""); + + compression = config.getBool("compression", true) + ? Protocol::Compression::Enable + : Protocol::Compression::Disable; + + timeouts = ConnectionTimeouts( + Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0)); + } + }; + + ConnectionParameters connection_parameters; + + void initialize(Poco::Util::Application & self) { Poco::Util::Application::initialize(self); @@ -310,6 +353,7 @@ private: echo_queries = config().getBool("echo", false); } + connection_parameters = ConnectionParameters(config()); connect(); /// Initialize DateLUT here to avoid counting time spent here as query execution time. @@ -363,19 +407,17 @@ private: } /// Prompt may contain the following substitutions in a form of {name}. - std::map environment = + std::map prompt_substitutions { - {"host", config().getString("host", "localhost")}, - {"port", config().getString("port", "9000")}, - {"user", config().getString("user", "default")}, + {"host", connection_parameters.host}, + {"port", toString(connection_parameters.port)}, + {"user", connection_parameters.user}, {"display_name", server_display_name}, }; /// Quite suboptimal. - for (const auto & [key, value]: environment) - { + for (const auto & [key, value]: prompt_substitutions) boost::replace_all(prompt_by_server_display_name, "{" + key + "}", value); - } if (is_interactive) { @@ -425,34 +467,23 @@ private: void connect() { - auto encryption = config().getBool("ssl", false) - ? Protocol::Encryption::Enable - : Protocol::Encryption::Disable; - - String host = config().getString("host", "localhost"); - UInt16 port = config().getInt("port", config().getInt(static_cast(encryption) ? "tcp_ssl_port" : "tcp_port", static_cast(encryption) ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); - String default_database = config().getString("database", ""); - String user = config().getString("user", ""); - String password = config().getString("password", ""); - - auto compression = config().getBool("compression", true) - ? Protocol::Compression::Enable - : Protocol::Compression::Disable; - if (is_interactive) std::cout << "Connecting to " - << (!default_database.empty() ? "database " + default_database + " at " : "") - << host << ":" << port - << (!user.empty() ? " as user " + user : "") + << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " : "") + << connection_parameters.host << ":" << connection_parameters.port + << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; - ConnectionTimeouts timeouts( - Poco::Timespan(config().getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), - Poco::Timespan(config().getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), - Poco::Timespan(config().getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0)); - connection = std::make_unique( - host, port, default_database, user, password, timeouts, "client", compression, encryption); + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + connection_parameters.timeouts, + "client", + connection_parameters.compression, + connection_parameters.security); String server_name; UInt64 server_version_major = 0; From cc2c1f5171b4f4586189d130461dd52daa2eb5f2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 21 Mar 2018 22:39:14 +0300 Subject: [PATCH 59/90] removed createImmutable --- dbms/src/Columns/ColumnArray.cpp | 2 +- dbms/src/Columns/ColumnArray.h | 32 +++++++++++++---------- dbms/src/Columns/ColumnNullable.h | 13 +++++++--- dbms/src/Columns/ColumnTuple.cpp | 2 +- dbms/src/Columns/ColumnTuple.h | 13 +++++++--- dbms/src/Columns/IColumn.h | 12 +++++++++ dbms/src/Common/COWPtr.h | 42 ++----------------------------- 7 files changed, 54 insertions(+), 62 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index a943b46eb49..010569e40ad 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -34,7 +34,7 @@ namespace ErrorCodes ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column) : data(std::move(nested_column)), offsets(std::move(offsets_column)) { - if (!typeid_cast(offsets_column.get())) + if (!typeid_cast(offsets.get())) throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::ILLEGAL_COLUMN); /** NOTE diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index a2564b20f98..ce7744b1fc1 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -27,23 +27,29 @@ private: ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column); /** Create an empty column of arrays with the type of values as in the column `nested_column` */ - ColumnArray(MutableColumnPtr && nested_column); - - using Ptr = COWPtrHelper::Ptr; - - static Ptr createImmutable(const ColumnPtr & nested_column, const ColumnPtr & offsets_column) - { - return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable()); - } - - static Ptr createImmutable(const ColumnPtr & nested_column) - { - return ColumnArray::create(nested_column->assumeMutable()); - } + explicit ColumnArray(MutableColumnPtr && nested_column); ColumnArray(const ColumnArray &) = default; public: + /** Create immutable column using immutable arguments. This arguments may be shared with other columns. + * Use IColumn::mutate in order to make mutable column and mutate shared nested columns. + */ + using Base = COWPtrHelper; + + static Ptr create(const ColumnPtr & nested_column, const ColumnPtr & offsets_column) + { + return ColumnArray::create(nested_column->assumeMutable(), offsets_column->assumeMutable()); + } + + static Ptr create(const ColumnPtr & nested_column) + { + return ColumnArray::create(nested_column->assumeMutable()); + } + + template ::value>::type> + static MutablePtr create(Args &&... args) { return Base::create(std::forward(args)...); } + /** On the index i there is an offset to the beginning of the i + 1 -th element. */ using ColumnOffsets = ColumnVector; diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index bc082bbcb8c..4ac0f87b8da 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -26,14 +26,19 @@ private: ColumnNullable(MutableColumnPtr && nested_column_, MutableColumnPtr && null_map_); ColumnNullable(const ColumnNullable &) = default; - using Ptr = COWPtrHelper::Ptr; - - static Ptr createImmutable(const ColumnPtr & nested_column_, const ColumnPtr & null_map_) +public: + /** Create immutable column using immutable arguments. This arguments may be shared with other columns. + * Use IColumn::mutate in order to make mutable column and mutate shared nested columns. + */ + using Base = COWPtrHelper; + static Ptr create(const ColumnPtr & nested_column_, const ColumnPtr & null_map_) { return ColumnNullable::create(nested_column_->assumeMutable(), null_map_->assumeMutable()); } -public: + template ::value>::type> + static MutablePtr create(Args &&... args) { return Base::create(std::forward(args)...); } + const char * getFamilyName() const override { return "Nullable"; } std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; } MutableColumnPtr cloneResized(size_t size) const override; diff --git a/dbms/src/Columns/ColumnTuple.cpp b/dbms/src/Columns/ColumnTuple.cpp index 4bfb6a30c2a..558b1685276 100644 --- a/dbms/src/Columns/ColumnTuple.cpp +++ b/dbms/src/Columns/ColumnTuple.cpp @@ -43,7 +43,7 @@ ColumnTuple::ColumnTuple(MutableColumns && mutable_columns) } } -ColumnTuple::Ptr ColumnTuple::createImmutable(const Columns & columns) +ColumnTuple::Ptr ColumnTuple::create(const Columns & columns) { for (const auto & column : columns) if (column->isColumnConst()) diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index 752a649ba28..a91cb210212 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -22,12 +22,19 @@ private: template struct Less; - ColumnTuple(MutableColumns && columns); + explicit ColumnTuple(MutableColumns && columns); ColumnTuple(const ColumnTuple &) = default; - static Ptr createImmutable(const Columns & columns); - public: + /** Create immutable column using immutable arguments. This arguments may be shared with other columns. + * Use IColumn::mutate in order to make mutable column and mutate shared nested columns. + */ + using Base = COWPtrHelper; + static Ptr create(const Columns & columns); + + template ::value>::type> + static MutablePtr create(Arg && arg) { return Base::create(std::forward(arg)); } + std::string getName() const override; const char * getFamilyName() const override { return "Tuple"; } diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index c546f315c0c..40577a11d3f 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -360,4 +360,16 @@ using MutableColumns = std::vector; using ColumnRawPtrs = std::vector; //using MutableColumnRawPtrs = std::vector; +template +struct IsMutableColumns; + +template +struct IsMutableColumns +{ + static const bool value = std::is_assignable::value && IsMutableColumns::value; +}; + +template <> +struct IsMutableColumns<> { static const bool value = true; }; + } diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h index a7fce08ae13..39c912f8abc 100644 --- a/dbms/src/Common/COWPtr.h +++ b/dbms/src/Common/COWPtr.h @@ -226,53 +226,15 @@ private: Derived * derived() { return static_cast(this); } const Derived * derived() const { return static_cast(this); } - template - struct HasCreateImmutable - { - private: - template - static constexpr auto check(decltype(std::declval().createImmutable(std::declval()... )) *) -> std::true_type; - - template - static constexpr std::false_type check(...); - - typedef decltype(check(nullptr)) type; - - public: - static constexpr bool value = type::value; - }; - - template - struct IsConstructable - { - private: - template - static constexpr auto check(decltype(T(std::declval()... )) *) -> std::true_type; - - template - static constexpr std::false_type check(...); - - typedef decltype(check(nullptr)) type; - - public: - static constexpr bool value = type::value; - }; - public: using Ptr = typename Base::template immutable_ptr; using MutablePtr = typename Base::template mutable_ptr; template - static auto create(Args &&... args) - { - if constexpr (HasCreateImmutable::value && !IsConstructable::value) - return Derived::createImmutable(std::forward(args)...); - else - return MutablePtr(new Derived(std::forward(args)...)); - } + static MutablePtr create(Args &&... args) { return MutablePtr(new Derived(std::forward(args)...)); } template - static auto create(std::initializer_list && arg) { return create(std::forward>(arg)); } + static MutablePtr create(std::initializer_list && arg) { return create(std::forward>(arg)); } typename Base::MutablePtr clone() const override { return typename Base::MutablePtr(new Derived(*derived())); } }; From 405ad39cd42afd76b70538789f5f8ad8c49f0d3b Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 23 Mar 2018 22:45:33 +0300 Subject: [PATCH 60/90] Fix test --- .../0_stateless/00416_pocopatch_progress_in_http_headers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 12f20a956ad..8e762515da1 100755 --- a/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/dbms/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # This test will fail with external poco (progress not supported) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse|^[0-9]' +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM system.numbers LIMIT 10' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default From dd06b39e5fd52fee48852e2bcae4f9ee7e5db962 Mon Sep 17 00:00:00 2001 From: Tobias Adamson Date: Sat, 24 Mar 2018 09:16:33 +0800 Subject: [PATCH 61/90] Fix rename of variable, duplication of wrong variable name --- dbms/src/Storages/StorageKafka.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index d82b12a6fb2..de6ffeaff05 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -578,7 +578,7 @@ void registerStorageKafka(StorageFactory & factory) // Parse topic list Names topics; String topicArg = static_cast(*engine_args[1]).value.safeGet(); - boost::split(topics, topics , [](char c){return c == ',';}); + boost::split(topics, topicArg , [](char c){return c == ',';}); // Parse consumer group String group = static_cast(*engine_args[2]).value.safeGet(); From 0af09ae0c7a07b281d76a0787820f67c3da98f24 Mon Sep 17 00:00:00 2001 From: Tobias Adamson Date: Sat, 24 Mar 2018 14:51:35 +0800 Subject: [PATCH 62/90] Add trim to topic names to be a bit more lenient if config is 'topic1, topic2' --- dbms/src/Storages/StorageKafka.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index de6ffeaff05..4ec2307ef13 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -579,7 +580,9 @@ void registerStorageKafka(StorageFactory & factory) Names topics; String topicArg = static_cast(*engine_args[1]).value.safeGet(); boost::split(topics, topicArg , [](char c){return c == ',';}); - + for(String & topic : topics) { + boost::trim(topic); + } // Parse consumer group String group = static_cast(*engine_args[2]).value.safeGet(); From d406454f392ae3ca9ef92a42d4b005070d483d7e Mon Sep 17 00:00:00 2001 From: elBroom Date: Sun, 25 Mar 2018 02:22:58 +0300 Subject: [PATCH 63/90] Added query_id to client --- dbms/src/Server/Client.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 47239c97c6c..3f923209c5d 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -731,7 +731,8 @@ private: /// Process the query that doesn't require transfering data blocks to the server. void processOrdinaryQuery() { - connection->sendQuery(query, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); + String query_id = config().getString("query_id", ""); + connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); sendExternalTables(); receiveResult(); } @@ -740,6 +741,7 @@ private: /// Process the query that requires transfering data blocks to the server. void processInsertQuery() { + String query_id = config().getString("query_id", ""); /// Send part of query without data, because data will be sent separately. const ASTInsertQuery & parsed_insert_query = typeid_cast(*parsed_query); String query_without_data = parsed_insert_query.data @@ -749,7 +751,7 @@ private: if (!parsed_insert_query.data && (is_interactive || (stdin_is_not_tty && std_in.eof()))) throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); - connection->sendQuery(query_without_data, "", QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); + connection->sendQuery(query_without_data, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); sendExternalTables(); /// Receive description of table structure. @@ -1286,6 +1288,7 @@ public: ("ssl,s", "ssl") ("user,u", boost::program_options::value(), "user") ("password", boost::program_options::value(), "password") + ("query_id", boost::program_options::value(), "query_id") ("query,q", boost::program_options::value(), "query") ("database,d", boost::program_options::value(), "database") ("pager", boost::program_options::value(), "pager") @@ -1377,6 +1380,8 @@ public: config().setString("config-file", options["config-file"].as()); if (options.count("host") && !options["host"].defaulted()) config().setString("host", options["host"].as()); + if (options.count("query_id")) + config().setString("query_id", options["query_id"].as()); if (options.count("query")) config().setString("query", options["query"].as()); if (options.count("database")) From b80f5ed8750ed2ac83f45b4db70fc34ba8e0db7c Mon Sep 17 00:00:00 2001 From: elBroom Date: Sun, 25 Mar 2018 02:50:30 +0300 Subject: [PATCH 64/90] ignoring query_id in interactive mode --- dbms/src/Server/Client.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 3f923209c5d..1a8d97af25a 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -111,6 +111,7 @@ private: winsize terminal_size {}; /// Terminal size is needed to render progress bar. std::unique_ptr connection; /// Connection to DB. + String query_id; /// Current query_id. String query; /// Current query. String format; /// Query results output format. @@ -336,6 +337,7 @@ private: if (is_interactive) { + query_id = ""; if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); @@ -370,6 +372,7 @@ private: } else { + query_id = config().getString("query_id", ""); nonInteractive(); if (last_exception) @@ -731,7 +734,6 @@ private: /// Process the query that doesn't require transfering data blocks to the server. void processOrdinaryQuery() { - String query_id = config().getString("query_id", ""); connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true); sendExternalTables(); receiveResult(); @@ -741,7 +743,6 @@ private: /// Process the query that requires transfering data blocks to the server. void processInsertQuery() { - String query_id = config().getString("query_id", ""); /// Send part of query without data, because data will be sent separately. const ASTInsertQuery & parsed_insert_query = typeid_cast(*parsed_query); String query_without_data = parsed_insert_query.data From a011a466aba86bc37bcccddeb70eec83ea802e4a Mon Sep 17 00:00:00 2001 From: BayoNet Date: Sun, 25 Mar 2018 05:04:22 +0300 Subject: [PATCH 65/90] English translation update. --- docs/en/agg_functions/combinators.md | 0 docs/en/agg_functions/index.md | 1 + docs/en/agg_functions/parametric_functions.md | 0 docs/en/agg_functions/reference.md | 34 ++-- docs/en/data_types/array.md | 0 docs/en/data_types/boolean.md | 0 docs/en/data_types/date.md | 2 +- docs/en/data_types/datetime.md | 0 docs/en/data_types/enum.md | 0 docs/en/data_types/fixedstring.md | 0 docs/en/data_types/float.md | 10 +- docs/en/data_types/index.md | 5 +- docs/en/data_types/int_uint.md | 0 .../aggregatefunction.md | 0 .../nested_data_structures/index.md | 0 .../nested_data_structures/nested.md | 0 .../special_data_types/expression.md | 0 .../en/data_types/special_data_types/index.md | 0 docs/en/data_types/special_data_types/set.md | 0 docs/en/data_types/string.md | 0 docs/en/data_types/tuple.md | 0 docs/en/development/style.md | 65 +++---- docs/en/dicts/external_dicts.md | 5 +- docs/en/dicts/external_dicts_dict.md | 9 +- docs/en/dicts/external_dicts_dict_layout.md | 99 ++--------- docs/en/dicts/external_dicts_dict_lifetime.md | 6 +- docs/en/dicts/external_dicts_dict_sources.md | 6 +- .../en/dicts/external_dicts_dict_structure.md | 14 +- docs/en/dicts/index.md | 0 docs/en/dicts/internal_dicts.md | 0 docs/en/formats/capnproto.md | 52 +++--- docs/en/formats/csv.md | 0 docs/en/formats/csvwithnames.md | 0 docs/en/formats/index.md | 1 + docs/en/formats/json.md | 0 docs/en/formats/jsoncompact.md | 2 +- docs/en/formats/jsoneachrow.md | 0 docs/en/formats/native.md | 0 docs/en/formats/null.md | 0 docs/en/formats/pretty.md | 0 docs/en/formats/prettycompact.md | 0 docs/en/formats/prettycompactmonoblock.md | 0 docs/en/formats/prettynoescapes.md | 0 docs/en/formats/prettyspace.md | 0 docs/en/formats/rowbinary.md | 2 +- docs/en/formats/tabseparated.md | 0 docs/en/formats/tabseparatedraw.md | 0 docs/en/formats/tabseparatedwithnames.md | 0 .../formats/tabseparatedwithnamesandtypes.md | 0 docs/en/formats/tskv.md | 0 docs/en/formats/values.md | 2 +- docs/en/formats/vertical.md | 0 docs/en/formats/xml.md | 2 +- docs/en/functions/arithmetic_functions.md | 0 docs/en/functions/array_functions.md | 12 +- docs/en/functions/array_join.md | 1 - docs/en/functions/bit_functions.md | 1 - docs/en/functions/comparison_functions.md | 2 +- docs/en/functions/conditional_functions.md | 0 docs/en/functions/date_time_functions.md | 4 + docs/en/functions/encoding_functions.md | 0 docs/en/functions/ext_dict_functions.md | 8 +- docs/en/functions/hash_functions.md | 0 docs/en/functions/higher_order_functions.md | 3 +- docs/en/functions/in_functions.md | 0 docs/en/functions/index.md | 0 docs/en/functions/ip_address_functions.md | 0 docs/en/functions/json_functions.md | 9 +- docs/en/functions/logical_functions.md | 1 - docs/en/functions/math_functions.md | 1 - docs/en/functions/other_functions.md | 5 +- docs/en/functions/random_functions.md | 0 docs/en/functions/rounding_functions.md | 0 .../functions/splitting_merging_functions.md | 0 docs/en/functions/string_functions.md | 0 docs/en/functions/string_replace_functions.md | 1 - docs/en/functions/string_search_functions.md | 0 .../en/functions/type_conversion_functions.md | 0 docs/en/functions/url_functions.md | 0 docs/en/functions/ym_dict_functions.md | 12 +- .../example_datasets/amplab_benchmark.md | 0 .../example_datasets/criteo.md | 4 +- .../example_datasets/nyc_taxi.md | 15 +- .../example_datasets/ontime.md | 0 .../example_datasets/star_schema.md | 3 +- .../example_datasets/wikistat.md | 2 +- docs/en/getting_started/index.md | 14 +- docs/en/index.md | 2 +- docs/en/interfaces/cli.md | 7 +- docs/en/interfaces/http_interface.md | 16 +- docs/en/interfaces/index.md | 3 +- docs/en/interfaces/jdbc.md | 0 docs/en/interfaces/tcp.md | 0 .../third-party_client_libraries.md | 2 +- docs/en/interfaces/third-party_gui.md | 0 docs/en/introduction/distinctive_features.md | 20 +-- .../features_considered_disadvantages.md | 0 docs/en/introduction/index.md | 1 + docs/en/introduction/performance.md | 0 .../introduction/possible_silly_questions.md | 4 +- docs/en/introduction/ya_metrika_task.md | 4 +- docs/en/operations/access_rights.md | 59 +++---- docs/en/operations/configuration_files.md | 2 +- docs/en/operations/index.md | 3 +- docs/en/operations/quotas.md | 35 ++-- docs/en/operations/server_settings/index.md | 1 + .../en/operations/server_settings/settings.md | 37 ++-- docs/en/operations/settings/index.md | 4 +- .../operations/settings/query_complexity.md | 0 docs/en/operations/settings/settings.md | 18 +- .../operations/settings/settings_profiles.md | 2 +- docs/en/operations/tips.md | 5 +- docs/en/operators/index.md | 4 +- docs/en/query_language/index.md | 1 + docs/en/query_language/queries.md | 53 +++--- docs/en/query_language/syntax.md | 2 +- docs/en/roadmap.md | 70 ++++---- docs/en/system_tables/index.md | 2 +- .../system.asynchronous_metrics.md | 0 docs/en/system_tables/system.clusters.md | 17 +- docs/en/system_tables/system.columns.md | 1 - docs/en/system_tables/system.databases.md | 0 docs/en/system_tables/system.dictionaries.md | 26 +-- docs/en/system_tables/system.events.md | 0 docs/en/system_tables/system.functions.md | 2 +- docs/en/system_tables/system.merges.md | 1 - docs/en/system_tables/system.numbers.md | 0 docs/en/system_tables/system.numbers_mt.md | 0 docs/en/system_tables/system.one.md | 0 docs/en/system_tables/system.parts.md | 0 docs/en/system_tables/system.processes.md | 6 +- docs/en/system_tables/system.replicas.md | 26 +-- docs/en/system_tables/system.settings.md | 4 +- docs/en/system_tables/system.tables.md | 3 +- docs/en/system_tables/system.zookeeper.md | 1 - docs/en/table_engines/aggregatingmergetree.md | 31 ++-- docs/en/table_engines/buffer.md | 0 docs/en/table_engines/collapsingmergetree.md | 0 .../table_engines/custom_partitioning_key.md | 14 +- docs/en/table_engines/dictionary.md | 106 ++++++++++++ docs/en/table_engines/distributed.md | 46 ++--- docs/en/table_engines/external_data.md | 0 docs/en/table_engines/file.md | 0 docs/en/table_engines/graphitemergetree.md | 5 +- docs/en/table_engines/index.md | 0 docs/en/table_engines/join.md | 0 docs/en/table_engines/kafka.md | 158 +++++++++--------- docs/en/table_engines/log.md | 0 docs/en/table_engines/materializedview.md | 2 +- docs/en/table_engines/memory.md | 0 docs/en/table_engines/merge.md | 0 docs/en/table_engines/mergetree.md | 2 +- docs/en/table_engines/null.md | 0 docs/en/table_engines/replacingmergetree.md | 0 docs/en/table_engines/replication.md | 2 +- docs/en/table_engines/set.md | 0 docs/en/table_engines/summingmergetree.md | 0 docs/en/table_engines/tinylog.md | 0 docs/en/table_engines/view.md | 0 docs/en/table_functions/index.md | 0 docs/en/table_functions/merge.md | 0 docs/en/table_functions/remote.md | 2 +- docs/en/utils/clickhouse-copier.md | 120 ++++++------- docs/en/utils/clickhouse-local.md | 6 +- docs/en/utils/index.md | 7 +- docs/mkdocs_en.yml | 2 + docs/ru/dicts/external_dicts_dict_layout.md | 18 +- 167 files changed, 727 insertions(+), 653 deletions(-) mode change 100644 => 100755 docs/en/agg_functions/combinators.md mode change 100644 => 100755 docs/en/agg_functions/index.md mode change 100644 => 100755 docs/en/agg_functions/parametric_functions.md mode change 100644 => 100755 docs/en/agg_functions/reference.md mode change 100644 => 100755 docs/en/data_types/array.md mode change 100644 => 100755 docs/en/data_types/boolean.md mode change 100644 => 100755 docs/en/data_types/date.md mode change 100644 => 100755 docs/en/data_types/datetime.md mode change 100644 => 100755 docs/en/data_types/enum.md mode change 100644 => 100755 docs/en/data_types/fixedstring.md mode change 100644 => 100755 docs/en/data_types/float.md mode change 100644 => 100755 docs/en/data_types/index.md mode change 100644 => 100755 docs/en/data_types/int_uint.md mode change 100644 => 100755 docs/en/data_types/nested_data_structures/aggregatefunction.md mode change 100644 => 100755 docs/en/data_types/nested_data_structures/index.md mode change 100644 => 100755 docs/en/data_types/nested_data_structures/nested.md mode change 100644 => 100755 docs/en/data_types/special_data_types/expression.md mode change 100644 => 100755 docs/en/data_types/special_data_types/index.md mode change 100644 => 100755 docs/en/data_types/special_data_types/set.md mode change 100644 => 100755 docs/en/data_types/string.md mode change 100644 => 100755 docs/en/data_types/tuple.md mode change 100644 => 100755 docs/en/development/style.md mode change 100644 => 100755 docs/en/dicts/external_dicts.md mode change 100644 => 100755 docs/en/dicts/external_dicts_dict.md mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_layout.md mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_lifetime.md mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_sources.md mode change 100644 => 100755 docs/en/dicts/external_dicts_dict_structure.md mode change 100644 => 100755 docs/en/dicts/index.md mode change 100644 => 100755 docs/en/dicts/internal_dicts.md mode change 100644 => 100755 docs/en/formats/capnproto.md mode change 100644 => 100755 docs/en/formats/csv.md mode change 100644 => 100755 docs/en/formats/csvwithnames.md mode change 100644 => 100755 docs/en/formats/index.md mode change 100644 => 100755 docs/en/formats/json.md mode change 100644 => 100755 docs/en/formats/jsoncompact.md mode change 100644 => 100755 docs/en/formats/jsoneachrow.md mode change 100644 => 100755 docs/en/formats/native.md mode change 100644 => 100755 docs/en/formats/null.md mode change 100644 => 100755 docs/en/formats/pretty.md mode change 100644 => 100755 docs/en/formats/prettycompact.md mode change 100644 => 100755 docs/en/formats/prettycompactmonoblock.md mode change 100644 => 100755 docs/en/formats/prettynoescapes.md mode change 100644 => 100755 docs/en/formats/prettyspace.md mode change 100644 => 100755 docs/en/formats/rowbinary.md mode change 100644 => 100755 docs/en/formats/tabseparated.md mode change 100644 => 100755 docs/en/formats/tabseparatedraw.md mode change 100644 => 100755 docs/en/formats/tabseparatedwithnames.md mode change 100644 => 100755 docs/en/formats/tabseparatedwithnamesandtypes.md mode change 100644 => 100755 docs/en/formats/tskv.md mode change 100644 => 100755 docs/en/formats/values.md mode change 100644 => 100755 docs/en/formats/vertical.md mode change 100644 => 100755 docs/en/formats/xml.md mode change 100644 => 100755 docs/en/functions/arithmetic_functions.md mode change 100644 => 100755 docs/en/functions/array_functions.md mode change 100644 => 100755 docs/en/functions/array_join.md mode change 100644 => 100755 docs/en/functions/bit_functions.md mode change 100644 => 100755 docs/en/functions/comparison_functions.md mode change 100644 => 100755 docs/en/functions/conditional_functions.md mode change 100644 => 100755 docs/en/functions/date_time_functions.md mode change 100644 => 100755 docs/en/functions/encoding_functions.md mode change 100644 => 100755 docs/en/functions/ext_dict_functions.md mode change 100644 => 100755 docs/en/functions/hash_functions.md mode change 100644 => 100755 docs/en/functions/higher_order_functions.md mode change 100644 => 100755 docs/en/functions/in_functions.md mode change 100644 => 100755 docs/en/functions/index.md mode change 100644 => 100755 docs/en/functions/ip_address_functions.md mode change 100644 => 100755 docs/en/functions/json_functions.md mode change 100644 => 100755 docs/en/functions/logical_functions.md mode change 100644 => 100755 docs/en/functions/math_functions.md mode change 100644 => 100755 docs/en/functions/other_functions.md mode change 100644 => 100755 docs/en/functions/random_functions.md mode change 100644 => 100755 docs/en/functions/rounding_functions.md mode change 100644 => 100755 docs/en/functions/splitting_merging_functions.md mode change 100644 => 100755 docs/en/functions/string_functions.md mode change 100644 => 100755 docs/en/functions/string_replace_functions.md mode change 100644 => 100755 docs/en/functions/string_search_functions.md mode change 100644 => 100755 docs/en/functions/type_conversion_functions.md mode change 100644 => 100755 docs/en/functions/url_functions.md mode change 100644 => 100755 docs/en/functions/ym_dict_functions.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/amplab_benchmark.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/criteo.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/nyc_taxi.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/ontime.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/star_schema.md mode change 100644 => 100755 docs/en/getting_started/example_datasets/wikistat.md mode change 100644 => 100755 docs/en/getting_started/index.md mode change 100644 => 100755 docs/en/index.md mode change 100644 => 100755 docs/en/interfaces/cli.md mode change 100644 => 100755 docs/en/interfaces/http_interface.md mode change 100644 => 100755 docs/en/interfaces/index.md mode change 100644 => 100755 docs/en/interfaces/jdbc.md mode change 100644 => 100755 docs/en/interfaces/tcp.md mode change 100644 => 100755 docs/en/interfaces/third-party_client_libraries.md mode change 100644 => 100755 docs/en/interfaces/third-party_gui.md mode change 100644 => 100755 docs/en/introduction/distinctive_features.md mode change 100644 => 100755 docs/en/introduction/features_considered_disadvantages.md mode change 100644 => 100755 docs/en/introduction/index.md mode change 100644 => 100755 docs/en/introduction/performance.md mode change 100644 => 100755 docs/en/introduction/possible_silly_questions.md mode change 100644 => 100755 docs/en/introduction/ya_metrika_task.md mode change 100644 => 100755 docs/en/operations/access_rights.md mode change 100644 => 100755 docs/en/operations/configuration_files.md mode change 100644 => 100755 docs/en/operations/index.md mode change 100644 => 100755 docs/en/operations/quotas.md mode change 100644 => 100755 docs/en/operations/server_settings/index.md mode change 100644 => 100755 docs/en/operations/server_settings/settings.md mode change 100644 => 100755 docs/en/operations/settings/index.md mode change 100644 => 100755 docs/en/operations/settings/query_complexity.md mode change 100644 => 100755 docs/en/operations/settings/settings.md mode change 100644 => 100755 docs/en/operations/settings/settings_profiles.md mode change 100644 => 100755 docs/en/operations/tips.md mode change 100644 => 100755 docs/en/operators/index.md mode change 100644 => 100755 docs/en/query_language/index.md mode change 100644 => 100755 docs/en/query_language/queries.md mode change 100644 => 100755 docs/en/query_language/syntax.md mode change 100644 => 100755 docs/en/roadmap.md mode change 100644 => 100755 docs/en/system_tables/index.md mode change 100644 => 100755 docs/en/system_tables/system.asynchronous_metrics.md mode change 100644 => 100755 docs/en/system_tables/system.clusters.md mode change 100644 => 100755 docs/en/system_tables/system.columns.md mode change 100644 => 100755 docs/en/system_tables/system.databases.md mode change 100644 => 100755 docs/en/system_tables/system.dictionaries.md mode change 100644 => 100755 docs/en/system_tables/system.events.md mode change 100644 => 100755 docs/en/system_tables/system.functions.md mode change 100644 => 100755 docs/en/system_tables/system.merges.md mode change 100644 => 100755 docs/en/system_tables/system.numbers.md mode change 100644 => 100755 docs/en/system_tables/system.numbers_mt.md mode change 100644 => 100755 docs/en/system_tables/system.one.md mode change 100644 => 100755 docs/en/system_tables/system.parts.md mode change 100644 => 100755 docs/en/system_tables/system.processes.md mode change 100644 => 100755 docs/en/system_tables/system.replicas.md mode change 100644 => 100755 docs/en/system_tables/system.settings.md mode change 100644 => 100755 docs/en/system_tables/system.tables.md mode change 100644 => 100755 docs/en/system_tables/system.zookeeper.md mode change 100644 => 100755 docs/en/table_engines/aggregatingmergetree.md mode change 100644 => 100755 docs/en/table_engines/buffer.md mode change 100644 => 100755 docs/en/table_engines/collapsingmergetree.md mode change 100644 => 100755 docs/en/table_engines/custom_partitioning_key.md create mode 100755 docs/en/table_engines/dictionary.md mode change 100644 => 100755 docs/en/table_engines/distributed.md mode change 100644 => 100755 docs/en/table_engines/external_data.md mode change 100644 => 100755 docs/en/table_engines/file.md mode change 100644 => 100755 docs/en/table_engines/graphitemergetree.md mode change 100644 => 100755 docs/en/table_engines/index.md mode change 100644 => 100755 docs/en/table_engines/join.md mode change 100644 => 100755 docs/en/table_engines/kafka.md mode change 100644 => 100755 docs/en/table_engines/log.md mode change 100644 => 100755 docs/en/table_engines/materializedview.md mode change 100644 => 100755 docs/en/table_engines/memory.md mode change 100644 => 100755 docs/en/table_engines/merge.md mode change 100644 => 100755 docs/en/table_engines/mergetree.md mode change 100644 => 100755 docs/en/table_engines/null.md mode change 100644 => 100755 docs/en/table_engines/replacingmergetree.md mode change 100644 => 100755 docs/en/table_engines/replication.md mode change 100644 => 100755 docs/en/table_engines/set.md mode change 100644 => 100755 docs/en/table_engines/summingmergetree.md mode change 100644 => 100755 docs/en/table_engines/tinylog.md mode change 100644 => 100755 docs/en/table_engines/view.md mode change 100644 => 100755 docs/en/table_functions/index.md mode change 100644 => 100755 docs/en/table_functions/merge.md mode change 100644 => 100755 docs/en/table_functions/remote.md mode change 100644 => 100755 docs/en/utils/clickhouse-copier.md mode change 100644 => 100755 docs/en/utils/clickhouse-local.md mode change 100644 => 100755 docs/en/utils/index.md diff --git a/docs/en/agg_functions/combinators.md b/docs/en/agg_functions/combinators.md old mode 100644 new mode 100755 diff --git a/docs/en/agg_functions/index.md b/docs/en/agg_functions/index.md old mode 100644 new mode 100755 index e87bf4ff833..3864f7271c4 --- a/docs/en/agg_functions/index.md +++ b/docs/en/agg_functions/index.md @@ -8,3 +8,4 @@ ClickHouse also supports: - [Parametric aggregate functions](parametric_functions.md#aggregate_functions_parametric), which accept other parameters in addition to columns. - [Combinators](combinators.md#aggregate_functions_combinators), which change the behavior of aggregate functions. + diff --git a/docs/en/agg_functions/parametric_functions.md b/docs/en/agg_functions/parametric_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md old mode 100644 new mode 100755 index 90ff9da58e7..0eb896e4664 --- a/docs/en/agg_functions/reference.md +++ b/docs/en/agg_functions/reference.md @@ -19,7 +19,7 @@ In some cases, you can rely on the order of execution. This applies to cases whe When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function. -## anyHeavy(x) +## anyHeavy Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic. @@ -39,7 +39,6 @@ Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets SELECT anyHeavy(AirlineID) AS res FROM ontime ``` - ``` ┌───res─┐ │ 19690 │ @@ -125,11 +124,11 @@ The result is always Float64. Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments. Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65536. -This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions). +This algorithm is also very accurate for data sets with low cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions). The result is determinate (it doesn't depend on the order of query processing). -This function provides excellent accuracy even for data sets with huge cardinality (10B+ elements) and is recommended for use by default. +This function provides excellent accuracy even for data sets with extremely high cardinality (over 10 billion elements). It is recommended for default use. ## uniqCombined(x) @@ -139,16 +138,16 @@ A combination of three algorithms is used: array, hash table and [HyperLogLog](h The result is determinate (it doesn't depend on the order of query processing). -The `uniqCombined` function is a good default choice for calculating the number of different values, but the following should be considered: for data sets with large cardinality (200M+) error of estimate will only grow and for data sets with huge cardinality(1B+ elements) it returns result with high inaccuracy. +The `uniqCombined` function is a good default choice for calculating the number of different values, but keep in mind that the estimation error will increase for high-cardinality data sets (200M+ elements), and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). ## uniqHLL12(x) Uses the [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm to approximate the number of different values of the argument. -212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. Result is not very accurate (error up to ~10%) for data sets of small cardinality(<10K elements), but for data sets with large cardinality (10K - 100M) result is quite accurate (error up to ~1.6%) and after that error of estimate will only grow and for data sets with huge cardinality (1B+ elements) it returns result with high inaccuracy. +212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). The result is determinate (it doesn't depend on the order of query processing). -This function is not recommended for use, and in most cases, use the `uniq` or `uniqCombined` function. +We don't recommend using this function. In most cases, use the `uniq` or `uniqCombined` function. ## uniqExact(x) @@ -170,7 +169,7 @@ In some cases, you can still rely on the order of execution. This applies to cas -## groupArrayInsertAt(x) +## groupArrayInsertAt Inserts a value into the array in the specified position. @@ -236,8 +235,8 @@ For its purpose (calculating quantiles of page loading times), using this functi ## quantileTimingWeighted(level)(x, weight) -Differs from the 'quantileTiming' function in that it has a second argument, "weights". Weight is a non-negative integer. -The result is calculated as if the 'x' value were passed 'weight' number of times to the 'quantileTiming' function. +Differs from the `quantileTiming` function in that it has a second argument, "weights". Weight is a non-negative integer. +The result is calculated as if the `x` value were passed `weight` number of times to the `quantileTiming` function. ## quantileExact(level)(x) @@ -257,7 +256,7 @@ The performance of the function is lower than for ` quantile`, ` quantileTiming` The result depends on the order of running the query, and is nondeterministic. -## median(x) +## median All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical. @@ -275,7 +274,7 @@ Returns `Float64`. When `n <= 1`, returns `+∞`. ## varPop(x) -Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. +Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. @@ -287,30 +286,33 @@ The result is equal to the square root of `varSamp(x)`. The result is equal to the square root of `varPop(x)`. -## topK(N)(column) +## topK Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves). Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf). +``` +topK(N)(column) +``` + This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values. We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`. **Arguments** -- 'N' – The number of values. +- 'N' is the number of values. - ' x ' – The column. **Example** -Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select the three most frequently occurring values in the `AirlineID` column. +Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime) data set and select the three most frequently occurring values in the `AirlineID` column. ```sql SELECT topK(3)(AirlineID) AS res FROM ontime ``` - ``` ┌─res─────────────────┐ │ [19393,19790,19805] │ diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md old mode 100644 new mode 100755 index 355e5555bfa..cb179c0d8c4 --- a/docs/en/data_types/date.md +++ b/docs/en/data_types/date.md @@ -1,6 +1,6 @@ # Date -Date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2038, but it may be expanded to 2106). +A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105). The minimum value is output as 0000-00-00. The date is stored without the time zone. diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md old mode 100644 new mode 100755 index c6d12999604..9d5cc2c01bb --- a/docs/en/data_types/float.md +++ b/docs/en/data_types/float.md @@ -4,8 +4,8 @@ Types are equivalent to types of C: -- `Float32` - `float`; -- `Float64` - ` double`. +- `Float32` - `float` +- `Float64` - ` double` We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds. @@ -24,9 +24,7 @@ SELECT 1 - 0.9 ``` - The result of the calculation depends on the calculation method (the processor type and architecture of the computer system). - - Floating-point calculations might result in numbers such as infinity (`Inf`) and "not-a-number" (`NaN`). This should be taken into account when processing the results of calculations. - - When reading floating point numbers from rows, the result might not be the nearest machine-representable number. ## NaN and Inf @@ -44,6 +42,7 @@ SELECT 0.5 / 0 │ inf │ └────────────────┘ ``` + - `-Inf` – Negative infinity. ```sql @@ -55,6 +54,7 @@ SELECT -0.5 / 0 │ -inf │ └─────────────────┘ ``` + - `NaN` – Not a number. ``` @@ -67,5 +67,5 @@ SELECT 0 / 0 └──────────────┘ ``` - See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by). +See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by). diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md old mode 100644 new mode 100755 index c17b51c08a2..4008a872161 --- a/docs/en/data_types/index.md +++ b/docs/en/data_types/index.md @@ -2,6 +2,7 @@ # Data types -ClickHouse table fields can contain data of different types. +ClickHouse can store various types of data in table cells. + +This section describes the supported data types and special considerations when using and/or implementing them, if any. -The topic contains descriptions of data types supported and specificity of their usage of implementation if exists. \ No newline at end of file diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.md b/docs/en/data_types/nested_data_structures/aggregatefunction.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/special_data_types/expression.md b/docs/en/data_types/special_data_types/expression.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/special_data_types/index.md b/docs/en/data_types/special_data_types/index.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/special_data_types/set.md b/docs/en/data_types/special_data_types/set.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/string.md b/docs/en/data_types/string.md old mode 100644 new mode 100755 diff --git a/docs/en/data_types/tuple.md b/docs/en/data_types/tuple.md old mode 100644 new mode 100755 diff --git a/docs/en/development/style.md b/docs/en/development/style.md old mode 100644 new mode 100755 index ef6490187c8..700fede5373 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -4,7 +4,7 @@ 1. The following are recommendations, not requirements. 2. If you are editing code, it makes sense to follow the formatting of the existing code. -3. Code style is needed for consistency. Consistency makes it easier to read the code. and it also makes it easier to search the code. +3. Code style is needed for consistency. Consistency makes it easier to read the code, and it also makes it easier to search the code. 4. Many of the rules do not have logical reasons; they are dictated by established practices. ## Formatting @@ -93,25 +93,25 @@ 14. In classes and structures, public, private, and protected are written on the same level as the class/struct, but all other internal elements should be deeper. ```cpp - template > - class MultiVersion - { - public: - /// The specific version of the object to use. - using Version = Ptr; - ... - } + template +class MultiVersion +{ +public: + /// Version of object for usage. shared_ptr manage lifetime of version. + using Version = std::shared_ptr; + ... +} ``` 15. If the same namespace is used for the entire file, and there isn't anything else significant, an offset is not necessary inside namespace. 16. If the block for if, for, while... expressions consists of a single statement, you don't need to use curly brackets. Place the statement on a separate line, instead. The same is true for a nested if, for, while... statement. But if the inner statement contains curly brackets or else, the external block should be written in curly brackets. - ```cpp - /// Finish write. - for (auto & stream : streams) - stream.second->finalize(); - ``` + ```cpp + /// Finish write. +for (auto & stream : streams) + stream.second->finalize(); + ``` 17. There should be any spaces at the ends of lines. @@ -218,11 +218,11 @@ */ void executeQuery( ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) - WriteBuffer & ostr, /// Where to write the result - Context & context, /// DB, tables, data types, engines, functions, aggregate functions... - BlockInputStreamPtr & query_plan, /// A description of query processing can be included here - QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to - ) + WriteBuffer & ostr, /// Where to write the result + Context & context, /// DB, tables, data types, engines, functions, aggregate functions... + BlockInputStreamPtr & query_plan, /// A description of query processing can be included here + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to + ) ``` 4. Comments should be written in English only. @@ -252,7 +252,7 @@ */ ``` - (Example taken from: [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) + (the example is borrowed from the resource [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) 7. Do not write garbage comments (author, creation date ..) at the beginning of each file. @@ -497,7 +497,15 @@ This is not recommended, but it is allowed. You can create a separate code block inside a single function in order to make certain variables local, so that the destructors are called when exiting the block. ```cpp - Block block = data.in->read();{ std::lock_guard lock(mutex); data.ready = true; data.block = block;}ready_any.set(); + Block block = data.in->read(); + + { + std::lock_guard lock(mutex); + data.ready = true; + data.block = block; + } + + ready_any.set(); ``` 7. Multithreading. @@ -560,13 +568,12 @@ This is not recommended, but it is allowed. ```cpp using AggregateFunctionPtr = std::shared_ptr; - - /** Creates an aggregate function by name. - */ + + /** Creates an aggregate function by name. */ class AggregateFunctionFactory { public: - AggregateFunctionFactory(); + AggregateFunctionFactory(); AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; ``` @@ -591,10 +598,10 @@ This is not recommended, but it is allowed. If later you’ll need to delay initialization, you can add a default constructor that will create an invalid object. Or, for a small number of objects, you can use shared_ptr/unique_ptr. ```cpp - Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); - - /// For delayed initialization - Loader() {} + Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); + + /// For delayed initialization + Loader() {} ``` 17. Virtual functions. diff --git a/docs/en/dicts/external_dicts.md b/docs/en/dicts/external_dicts.md old mode 100644 new mode 100755 index b99b02bbf57..a6af84a313f --- a/docs/en/dicts/external_dicts.md +++ b/docs/en/dicts/external_dicts.md @@ -21,12 +21,11 @@ The dictionary config file has the following format: /etc/metrika.xml - - + - + ... diff --git a/docs/en/dicts/external_dicts_dict.md b/docs/en/dicts/external_dicts_dict.md old mode 100644 new mode 100755 index 4133b036e1f..6d2f4128704 --- a/docs/en/dicts/external_dicts_dict.md +++ b/docs/en/dicts/external_dicts_dict.md @@ -27,7 +27,8 @@ The dictionary configuration has the following structure: ``` - name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. -- [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) – Source of the dictionary. -- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) – Location of the dictionary in memory. -- [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) – Structure of the dictionary. A key and attributes that can be retrieved by this key. -- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) – How frequently to update dictionaries. +- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Source of the dictionary . +- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) — Dictionary layout in memory. +- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Structure of the dictionary . A key and attributes that can be retrieved by this key. +- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) — Frequency of dictionary updates. + diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md old mode 100644 new mode 100755 index ad635db94f5..8b7cad24b65 --- a/docs/en/dicts/external_dicts_dict_layout.md +++ b/docs/en/dicts/external_dicts_dict_layout.md @@ -2,11 +2,11 @@ # Storing dictionaries in memory -There are [many different ways](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory. +There are a [variety of ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory. -We recommend [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed. +We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed)and[complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed. -Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)" section. +Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more in the section " [cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)". There are several ways to improve dictionary performance: @@ -46,7 +46,6 @@ The configuration looks like this: - [range_hashed](#dicts-external_dicts_dict_layout-range_hashed) - [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed) - [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache) -- [ip_trie](#dicts-external_dicts_dict_layout-ip_trie) @@ -88,7 +87,7 @@ Configuration example: ### complex_key_hashed -This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). It is similar to hashed. +This type is for use with composite [keys](external_dicts_dict_structure.md/#dicts-external_dicts_dict_structure). Similar to `hashed`. Configuration example: @@ -109,18 +108,18 @@ This storage method works the same way as hashed and allows using date/time rang Example: The table contains discounts for each advertiser in the format: ``` - +---------------+---------------------+-------------------+--------+ - | advertiser id | discount start date | discount end date | amount | - +===============+=====================+===================+========+ - | 123 | 2015-01-01 | 2015-01-15 | 0.15 | - +---------------+---------------------+-------------------+--------+ - | 123 | 2015-01-16 | 2015-01-31 | 0.25 | - +---------------+---------------------+-------------------+--------+ - | 456 | 2015-01-01 | 2015-01-15 | 0.05 | - +---------------+---------------------+-------------------+--------+ ++---------------+---------------------+-------------------+--------+ +| advertiser id | discount start date | discount end date | amount | ++===============+=====================+===================+========+ +| 123 | 2015-01-01 | 2015-01-15 | 0.15 | ++---------------+---------------------+-------------------+--------+ +| 123 | 2015-01-16 | 2015-01-31 | 0.25 | ++---------------+---------------------+-------------------+--------+ +| 456 | 2015-01-01 | 2015-01-15 | 0.05 | ++---------------+---------------------+-------------------+--------+ ``` -To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure). +To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Example: @@ -197,15 +196,15 @@ This is the least effective of all the ways to store dictionaries. The speed of To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally. -Supported [sources](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP. +Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP. Example of settings: ```xml - - 1000000000 + + 1000000000 ``` @@ -227,66 +226,4 @@ Do not use ClickHouse as a source, because it is slow to process queries with ra ### complex_key_cache -This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Similar to `cache`. - - - -### ip_trie - - -The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score. - -Example: in the table there are prefixes matches to AS number and country: - -``` - +-----------------+-------+--------+ - | prefix | asn | cca2 | - +=================+=======+========+ - | 202.79.32.0/20 | 17501 | NP | - +-----------------+-------+--------+ - | 2620:0:870::/48 | 3856 | US | - +-----------------+-------+--------+ - | 2a02:6b8:1::/48 | 13238 | RU | - +-----------------+-------+--------+ - | 2001:db8::/32 | 65536 | ZZ | - +-----------------+-------+--------+ -``` - -When using such a layout, the structure should have the "key" element. - -Example: - -```xml - - - - prefix - String - - - - asn - UInt32 - - - - cca2 - String - ?? - - ... -``` - -These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported. - -For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used: - - dictGetT('dict_name', 'attr_name', tuple(ip)) - -The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format: - - dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) - -No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned. - -The data is stored currently in a bitwise trie, it has to fit in memory. +This type of storage is for use with composite [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`. diff --git a/docs/en/dicts/external_dicts_dict_lifetime.md b/docs/en/dicts/external_dicts_dict_lifetime.md old mode 100644 new mode 100755 index 6431fb3de48..52ee7a4aa78 --- a/docs/en/dicts/external_dicts_dict_lifetime.md +++ b/docs/en/dicts/external_dicts_dict_lifetime.md @@ -36,13 +36,13 @@ Example of settings: When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): > - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. -> - For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. -> - Dictionaries from other sources are updated every time by default. +- For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. +- Dictionaries from other sources are updated every time by default. For MySQL (InnoDB) and ODBC sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: > - The dictionary table must have a field that always changes when the source data is updated. -> - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. The query must be specified in the `` field in the [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) settings. +- The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `` field in the settings for the [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources). Example of settings: diff --git a/docs/en/dicts/external_dicts_dict_sources.md b/docs/en/dicts/external_dicts_dict_sources.md old mode 100644 new mode 100755 index 721302cd556..6cb4e0ea44d --- a/docs/en/dicts/external_dicts_dict_sources.md +++ b/docs/en/dicts/external_dicts_dict_sources.md @@ -80,7 +80,7 @@ Setting fields: ## HTTP(s) -Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method. +Working with an HTTP(s) server depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method. Example of settings: @@ -135,9 +135,9 @@ Installing unixODBC and the ODBC driver for PostgreSQL: Configuring `/etc/odbc.ini` (or `~/.odbc.ini`): ``` - [DEFAULT] +[DEFAULT] Driver = myconnection - + [myconnection] Description = PostgreSQL connection to my_db Driver = PostgreSQL Unicode diff --git a/docs/en/dicts/external_dicts_dict_structure.md b/docs/en/dicts/external_dicts_dict_structure.md old mode 100644 new mode 100755 index 5a6d349b350..2542af00ec6 --- a/docs/en/dicts/external_dicts_dict_structure.md +++ b/docs/en/dicts/external_dicts_dict_structure.md @@ -25,8 +25,8 @@ Overall structure: Columns are described in the structure: -- `` – [Key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key). -- `` – [Data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns. +- `` - [key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key). +- `` - [data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns. @@ -63,12 +63,10 @@ Configuration fields: ### Composite key -The key can be a `tuple` from any types of fields. The [ layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`. +The key can be a `tuple` from any types of fields. The [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`.
- -A composite key can also consist of a single element, which makes it possible to use a string as the key, for instance. - +A composite key can consist of a single element. This makes it possible to use a string as the key, for instance.
The key structure is set in the element ``. Key fields are specified in the same format as the dictionary [attributes](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). Example: @@ -119,6 +117,6 @@ Configuration fields: - `null_value` – The default value for a non-existing element. In the example, it is an empty string. - `expression` – The attribute can be an expression. The tag is not required. - `hierarchical` – Hierarchical support. Mirrored to the parent identifier. By default, ` false`. -- `injective` Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`. -- `is_object_id` - Used for query mongo documents by ObjectId +- `injective` – Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`. +- `is_object_id` – Whether the query is executed for a MongoDB document by `ObjectID`. diff --git a/docs/en/dicts/index.md b/docs/en/dicts/index.md old mode 100644 new mode 100755 diff --git a/docs/en/dicts/internal_dicts.md b/docs/en/dicts/internal_dicts.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/capnproto.md b/docs/en/formats/capnproto.md old mode 100644 new mode 100755 index 0d482e20887..918197b2bd9 --- a/docs/en/formats/capnproto.md +++ b/docs/en/formats/capnproto.md @@ -1,26 +1,26 @@ - - -# CapnProto - -Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack. - -Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query. - -```sql -SELECT SearchPhrase, count() AS c FROM test.hits - GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' -``` - -Where `schema.capnp` looks like this: - -``` -struct Message { - SearchPhrase @0 :Text; - c @1 :Uint64; -} -``` - -Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration. - -Deserialization is effective and usually doesn't increase the system load. - + + +# CapnProto + +Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack. + +Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query. + +```sql +SELECT SearchPhrase, count() AS c FROM test.hits + GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' +``` + +Where `schema.capnp` looks like this: + +``` +struct Message { + SearchPhrase @0 :Text; + c @1 :Uint64; +} +``` + +Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration. + +Deserialization is effective and usually doesn't increase the system load. + diff --git a/docs/en/formats/csv.md b/docs/en/formats/csv.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/csvwithnames.md b/docs/en/formats/csvwithnames.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/index.md b/docs/en/formats/index.md old mode 100644 new mode 100755 index 815a2d060cb..112a13ff5e5 --- a/docs/en/formats/index.md +++ b/docs/en/formats/index.md @@ -3,3 +3,4 @@ # Formats The format determines how data is returned to you after SELECTs (how it is written and formatted by the server), and how it is accepted for INSERTs (how it is read and parsed by the server). + diff --git a/docs/en/formats/json.md b/docs/en/formats/json.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/jsoncompact.md b/docs/en/formats/jsoncompact.md old mode 100644 new mode 100755 index e4ce0867bc2..d870b6dff08 --- a/docs/en/formats/jsoncompact.md +++ b/docs/en/formats/jsoncompact.md @@ -24,7 +24,7 @@ Example: ["bathroom interior design", "2166"], ["yandex", "1655"], ["spring 2014 fashion", "1549"], - ["freeform photos", "1480"] + ["freeform photo", "1480"] ], "totals": ["","8873898"], diff --git a/docs/en/formats/jsoneachrow.md b/docs/en/formats/jsoneachrow.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/native.md b/docs/en/formats/native.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/null.md b/docs/en/formats/null.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/pretty.md b/docs/en/formats/pretty.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/prettycompact.md b/docs/en/formats/prettycompact.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/prettycompactmonoblock.md b/docs/en/formats/prettycompactmonoblock.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/prettynoescapes.md b/docs/en/formats/prettynoescapes.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/prettyspace.md b/docs/en/formats/prettyspace.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/rowbinary.md b/docs/en/formats/rowbinary.md old mode 100644 new mode 100755 index bc8479332ba..aeb3df4c8a8 --- a/docs/en/formats/rowbinary.md +++ b/docs/en/formats/rowbinary.md @@ -9,5 +9,5 @@ Date is represented as a UInt16 object that contains the number of days since 19 String is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string. FixedString is represented simply as a sequence of bytes. -Arrays are represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the array elements in order. +Array is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by successive elements of the array. diff --git a/docs/en/formats/tabseparated.md b/docs/en/formats/tabseparated.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/tabseparatedraw.md b/docs/en/formats/tabseparatedraw.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/tabseparatedwithnames.md b/docs/en/formats/tabseparatedwithnames.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.md b/docs/en/formats/tabseparatedwithnamesandtypes.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/tskv.md b/docs/en/formats/tskv.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/values.md b/docs/en/formats/values.md old mode 100644 new mode 100755 index 2e929369848..a672723f33d --- a/docs/en/formats/values.md +++ b/docs/en/formats/values.md @@ -4,5 +4,5 @@ Prints every row in brackets. Rows are separated by commas. There is no comma af The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. -This is the format that is used in `INSERT INTO t VALUES ...` but you can also use it for query result. +This is the format that is used in `INSERT INTO t VALUES ...`, but you can also use it for formatting query results. diff --git a/docs/en/formats/vertical.md b/docs/en/formats/vertical.md old mode 100644 new mode 100755 diff --git a/docs/en/formats/xml.md b/docs/en/formats/xml.md old mode 100644 new mode 100755 index f91adec9356..0da55875cc3 --- a/docs/en/formats/xml.md +++ b/docs/en/formats/xml.md @@ -35,7 +35,7 @@ XML format is suitable only for output, not for parsing. Example: 1549 - freeform photos + freeform photo 1480 diff --git a/docs/en/functions/arithmetic_functions.md b/docs/en/functions/arithmetic_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/array_functions.md b/docs/en/functions/array_functions.md old mode 100644 new mode 100755 index 493a465ac82..232f6a20427 --- a/docs/en/functions/array_functions.md +++ b/docs/en/functions/array_functions.md @@ -39,7 +39,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa Returns an array of numbers from 0 to N-1. Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block. -## array(x1, ...), operator \[x1, ...\] +## array(x1, ...), оператор \[x1, ...\] Creates an array from the function arguments. The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray\*' function described above). @@ -62,7 +62,6 @@ arrayConcat(arrays) ```sql SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res ``` - ``` ┌─res───────────┐ │ [1,2,3,4,5,6] │ @@ -203,7 +202,6 @@ arrayPopBack(array) ```sql SELECT arrayPopBack([1, 2, 3]) AS res ``` - ``` ┌─res───┐ │ [1,2] │ @@ -245,14 +243,13 @@ arrayPushBack(array, single_value) **Arguments** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)". **Example** ```sql SELECT arrayPushBack(['a'], 'b') AS res ``` - ``` ┌─res───────┐ │ ['a','b'] │ @@ -270,7 +267,7 @@ arrayPushFront(array, single_value) **Arguments** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)". **Example** @@ -295,7 +292,7 @@ arraySlice(array, offset[, length]) **Arguments** - `array` – Array of data. -- `offset` – Offset from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. +- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. - `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`. **Example** @@ -303,7 +300,6 @@ arraySlice(array, offset[, length]) ```sql SELECT arraySlice([1, 2, 3, 4, 5], 2, 3) AS res ``` - ``` ┌─res─────┐ │ [2,3,4] │ diff --git a/docs/en/functions/array_join.md b/docs/en/functions/array_join.md old mode 100644 new mode 100755 index 6e18f8203c0..f94b2707f52 --- a/docs/en/functions/array_join.md +++ b/docs/en/functions/array_join.md @@ -28,4 +28,3 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src │ 3 │ Hello │ [1,2,3] │ └─────┴───────────┴─────────┘ ``` - diff --git a/docs/en/functions/bit_functions.md b/docs/en/functions/bit_functions.md old mode 100644 new mode 100755 index 523413f200a..c5a032aa5d6 --- a/docs/en/functions/bit_functions.md +++ b/docs/en/functions/bit_functions.md @@ -15,4 +15,3 @@ The result type is an integer with bits equal to the maximum bits of its argumen ## bitShiftLeft(a, b) ## bitShiftRight(a, b) - diff --git a/docs/en/functions/comparison_functions.md b/docs/en/functions/comparison_functions.md old mode 100644 new mode 100755 index e37642d42ed..9b95966ba84 --- a/docs/en/functions/comparison_functions.md +++ b/docs/en/functions/comparison_functions.md @@ -15,7 +15,7 @@ For example, you can't compare a date with a string. You have to use a function Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character. -Note: Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 > -1. This behavior changed in version 1.1.54134 and is now mathematically correct. +Note. Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 > -1. This behavior changed in version 1.1.54134 and is now mathematically correct. ## equals, a = b and a == b operator diff --git a/docs/en/functions/conditional_functions.md b/docs/en/functions/conditional_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/date_time_functions.md b/docs/en/functions/date_time_functions.md old mode 100644 new mode 100755 index da6ad9b4a7c..a7529e5f0e1 --- a/docs/en/functions/date_time_functions.md +++ b/docs/en/functions/date_time_functions.md @@ -79,6 +79,10 @@ Rounds down a date with time to the start of the minute. Rounds down a date with time to the start of the hour. +## toStartOfFifteenMinutes + +Rounds down the date with time to the start of the fifteen-minute interval. + Note: If you need to round a date with time to any other number of seconds, minutes, or hours, you can convert it into a number by using the toUInt32 function, then round the number using intDiv and multiplication, and convert it back using the toDateTime function. ## toStartOfHour diff --git a/docs/en/functions/encoding_functions.md b/docs/en/functions/encoding_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/ext_dict_functions.md b/docs/en/functions/ext_dict_functions.md old mode 100644 new mode 100755 index 949e805d9ab..002e2f55845 --- a/docs/en/functions/ext_dict_functions.md +++ b/docs/en/functions/ext_dict_functions.md @@ -18,18 +18,20 @@ For information on connecting and configuring external dictionaries, see "[Exter `dictGetT('dict_name', 'attr_name', id)` -- Get the value of the attr_name attribute from the dict_name dictionary using the 'id' key.`dict_name` and `attr_name` are constant strings.`id`must be UInt64. +- Get the value of the attr_name attribute from the dict_name dictionary using the 'id' key. +`dict_name` and `attr_name` are constant strings. +`id`must be UInt64. If there is no `id` key in the dictionary, it returns the default value specified in the dictionary description. ## dictGetTOrDefault `dictGetT('dict_name', 'attr_name', id, default)` -Similar to the functions dictGetT, but the default value is taken from the last argument of the function. +The same as the `dictGetT` functions, but the default value is taken from the function's last argument. ## dictIsIn -`dictIsIn ('dict_name', child_id, ancestor_id)` +`dictIsIn('dict_name', child_id, ancestor_id)` - For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8. diff --git a/docs/en/functions/hash_functions.md b/docs/en/functions/hash_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/higher_order_functions.md b/docs/en/functions/higher_order_functions.md old mode 100644 new mode 100755 index ab9bdc50786..15cc40dead1 --- a/docs/en/functions/higher_order_functions.md +++ b/docs/en/functions/higher_order_functions.md @@ -73,7 +73,7 @@ Returns the index of the first element in the 'arr1' array for which 'func' retu ### arrayCumSum(\[func,\] arr1, ...) -Returns the cumulative sum of the array obtained from the original application of the 'func' function to each element in the 'arr' array. +Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing. Example: @@ -86,4 +86,3 @@ SELECT arrayCumSum([1, 1, 1, 1]) AS res │ [1, 2, 3, 4] │ └──────────────┘ ``` - diff --git a/docs/en/functions/in_functions.md b/docs/en/functions/in_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/index.md b/docs/en/functions/index.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/ip_address_functions.md b/docs/en/functions/ip_address_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/json_functions.md b/docs/en/functions/json_functions.md old mode 100644 new mode 100755 index 1bf10e9cf0c..90a2ddc47dd --- a/docs/en/functions/json_functions.md +++ b/docs/en/functions/json_functions.md @@ -1,11 +1,11 @@ -# Functions for working with JSON. +# Functions for working with JSON In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done. The following assumptions are made: 1. The field name (function argument) must be a constant. -2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` +2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, но `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` 3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used. 4. The JSON doesn't have space characters outside of string literals. @@ -47,7 +47,10 @@ Parses the string in double quotes. The value is unescaped. If unescaping failed Examples: ```text -visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''visitParamExtractString('{"abc":"hello}', 'abc') = '' +visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' +visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺' +visitParamExtractString('{"abc":"\\u263"}', 'abc') = '' +visitParamExtractString('{"abc":"hello}', 'abc') = '' ``` There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8). diff --git a/docs/en/functions/logical_functions.md b/docs/en/functions/logical_functions.md old mode 100644 new mode 100755 index 4ef0fe5fd32..d396640a49d --- a/docs/en/functions/logical_functions.md +++ b/docs/en/functions/logical_functions.md @@ -11,4 +11,3 @@ Zero as an argument is considered "false," while any non-zero value is considere ## not, NOT operator ## xor - diff --git a/docs/en/functions/math_functions.md b/docs/en/functions/math_functions.md old mode 100644 new mode 100755 index d606c87a509..42e3f3e8018 --- a/docs/en/functions/math_functions.md +++ b/docs/en/functions/math_functions.md @@ -97,4 +97,3 @@ The arc tangent. ## pow(x, y) xy. - diff --git a/docs/en/functions/other_functions.md b/docs/en/functions/other_functions.md old mode 100644 new mode 100755 index befd94ecd4e..8a0063750fe --- a/docs/en/functions/other_functions.md +++ b/docs/en/functions/other_functions.md @@ -59,7 +59,8 @@ For elements in a nested data structure, the function checks for the existence o Allows building a unicode-art diagram. -`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction. +`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max. +`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction. The band is drawn with accuracy to one eighth of a symbol. @@ -137,7 +138,7 @@ Example: ```sql SELECT - transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other' AS title, + transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title, count() AS c FROM test.hits WHERE SearchEngineID != 0 diff --git a/docs/en/functions/random_functions.md b/docs/en/functions/random_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/rounding_functions.md b/docs/en/functions/rounding_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/splitting_merging_functions.md b/docs/en/functions/splitting_merging_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/string_functions.md b/docs/en/functions/string_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/string_replace_functions.md b/docs/en/functions/string_replace_functions.md old mode 100644 new mode 100755 index d3773504278..d70d8f404de --- a/docs/en/functions/string_replace_functions.md +++ b/docs/en/functions/string_replace_functions.md @@ -76,4 +76,3 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res │ here: Hello, World! │ └─────────────────────┘ ``` - diff --git a/docs/en/functions/string_search_functions.md b/docs/en/functions/string_search_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/type_conversion_functions.md b/docs/en/functions/type_conversion_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/url_functions.md b/docs/en/functions/url_functions.md old mode 100644 new mode 100755 diff --git a/docs/en/functions/ym_dict_functions.md b/docs/en/functions/ym_dict_functions.md old mode 100644 new mode 100755 index 540b5dd601a..7ba7e7012cf --- a/docs/en/functions/ym_dict_functions.md +++ b/docs/en/functions/ym_dict_functions.md @@ -21,7 +21,9 @@ All functions for working with regions have an optional argument at the end – Example: ```text -regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt +regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt +regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt +regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt ``` ### regionToCity(id[, geobase]) @@ -33,7 +35,9 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. If this regio Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'. ```sql -SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 +SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua')) +FROM system.numbers +LIMIT 15 ``` ```text @@ -61,7 +65,9 @@ SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.nu Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'. ```sql -SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 +SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua')) +FROM system.numbers +LIMIT 15 ``` ```text diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md old mode 100644 new mode 100755 diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md old mode 100644 new mode 100755 index 9b59d6e5f3d..3c60a68f430 --- a/docs/en/getting_started/example_datasets/criteo.md +++ b/docs/en/getting_started/example_datasets/criteo.md @@ -66,6 +66,8 @@ CREATE TABLE criteo Transform data from the raw log and put it in the second table: ```sql -INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;DROP TABLE criteo_log; +INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; + +DROP TABLE criteo_log; ``` diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md old mode 100644 new mode 100755 index 11ed81d1a43..a9f04f595d1 --- a/docs/en/getting_started/example_datasets/nyc_taxi.md +++ b/docs/en/getting_started/example_datasets/nyc_taxi.md @@ -1,8 +1,8 @@ -# Data about New York taxis +# New York Taxi data -## How to import raw data +## How to import the raw data -See and for description of the dataset and loading instructions. +See and for the description of the dataset and instructions for downloading. Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel). Some of the files might not download fully. Check the file sizes and re-download any that seem doubtful. @@ -301,19 +301,14 @@ SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetr Q4: ```sql -SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) -FROM trips_mergetree -GROUP BY passenger_count, year, distance -ORDER BY year, count(*) DESC +SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)FROM trips_mergetreeGROUP BY passenger_count, year, distanceORDER BY year, count(*) DESC ``` 3.593 seconds. The following server was used: -Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total, -128 GiB RAM, -8x6 TB HD on hardware RAID-5 +Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB RAM,8x6 TB HD on hardware RAID-5 Execution time is the best of three runsBut starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run. diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md old mode 100644 new mode 100755 diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md old mode 100644 new mode 100755 index 8807de3e670..664ba59f48c --- a/docs/en/getting_started/example_datasets/star_schema.md +++ b/docs/en/getting_started/example_datasets/star_schema.md @@ -1,4 +1,4 @@ -# Star scheme +# Star Schema Benchmark Compiling dbgen: @@ -82,3 +82,4 @@ Downloading data (change 'customer' to 'customerd' in the distributed version): cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV" cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" ``` + diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md old mode 100644 new mode 100755 index 6cbc3b15561..fee0a56b52c --- a/docs/en/getting_started/example_datasets/wikistat.md +++ b/docs/en/getting_started/example_datasets/wikistat.md @@ -20,7 +20,7 @@ CREATE TABLE wikistat Loading data: ```bash -for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sS "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt +for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done ``` diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md old mode 100644 new mode 100755 index 42fc1c75551..07d0d91a224 --- a/docs/en/getting_started/index.md +++ b/docs/en/getting_started/index.md @@ -16,14 +16,15 @@ The terminal must use UTF-8 encoding (the default in Ubuntu). For testing and development, the system can be installed on a single server or on a desktop computer. -### Installing from packages Debian/Ubuntu +### Installing from packages In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository: ```text -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +deb http://repo.yandex.ru/clickhouse/trusty stable main ``` +On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`. If you want to use the most recent test version, replace 'stable' with 'testing'. Then run: @@ -34,7 +35,10 @@ sudo apt-get update sudo apt-get install clickhouse-client clickhouse-server-common ``` -You can also download and install packages manually from here: +You can also download and install packages manually from here: + + + ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml'). By default, access is allowed from anywhere for the 'default' user, without a password. See 'user/default/networks'. @@ -100,7 +104,8 @@ clickhouse-client ``` The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password. -The client can be used for connecting to a remote server. Example: +The client can be used for connecting to a remote server. +Example: ```bash clickhouse-client --host=example.com @@ -132,3 +137,4 @@ SELECT 1 **Congratulations, the system works!** To continue experimenting, you can try to download from the test data sets. + diff --git a/docs/en/index.md b/docs/en/index.md old mode 100644 new mode 100755 index 586c18297a8..72efa70802b --- a/docs/en/index.md +++ b/docs/en/index.md @@ -39,7 +39,7 @@ We'll say that the following is true for the OLAP (online analytical processing) - Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. - Data is added to the DB but is not modified. - For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. -- Tables are "wide", meaning they contain a large number of columns. +- Tables are "wide," meaning they contain a large number of columns. - Queries are relatively rare (usually hundreds of queries per server or less per second). - For simple queries, latencies around 50 ms are allowed. - Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md old mode 100644 new mode 100755 index 4fd998fed66..76549b46b36 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -6,7 +6,9 @@ To work from the command line, you can use ` clickhouse-client`: $ clickhouse-client ClickHouse client version 0.0.26176. Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.26176.:) +Connected to ClickHouse server version 0.0.26176. + +:) ``` The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)". @@ -29,7 +31,6 @@ _EOF cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; ``` -In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query. By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the --multiquery parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators. Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program. @@ -64,7 +65,7 @@ The command-line client allows passing external data (external temporary tables) -## Configure +## Configuring You can pass parameters to `clickhouse-client` (all parameters have a default value) using: diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md old mode 100644 new mode 100755 index 5c989a59d65..38a70feef46 --- a/docs/en/interfaces/http_interface.md +++ b/docs/en/interfaces/http_interface.md @@ -37,7 +37,8 @@ Date: Fri, 16 Nov 2012 19:21:50 GMT 1 ``` -As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. +As you can see, curl is somewhat inconvenient in that spaces must be URL escaped. +Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. ```bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -130,15 +131,11 @@ POST 'http://localhost:8123/?query=DROP TABLE t' For successful requests that don't return a data table, an empty response body is returned. -You can use compression when transmitting data. +You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex). -For using ClickHouse internal compression format, and you will need to use the special clickhouse-compressor program to work with it (installed as a part of clickhouse-client package). If you specified 'compress=1' in the URL, the server will compress the data it sends you. If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method. -Also standard gzip-based HTTP compression can be used. To send gzip compressed POST data just add `Content-Encoding: gzip` to request headers, and gzip POST body. -To get response compressed, you need to add `Accept-Encoding: gzip` to request headers, and turn on ClickHouse setting called `enable_http_compression`. - You can use this to reduce network traffic when transmitting a large amount of data, or for creating dumps that are immediately compressed. You can use the 'database' URL parameter to specify the default database. @@ -194,11 +191,7 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812 For information about other parameters, see the section "SET". -You can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` GET parameter in HTTP request. You can use any alphanumeric string as a session_id. By default session will be timed out after 60 seconds of inactivity. You can change that by setting `default_session_timeout` in server config file, or by adding GET parameter `session_timeout`. You can also check the status of the session by using GET parameter `session_check=1`. When using sessions you can't run 2 queries with the same session_id simultaneously. - -You can get the progress of query execution in X-ClickHouse-Progress headers, by enabling setting send_progress_in_http_headers. - -Running query are not aborted automatically after closing HTTP connection. Parsing and data formatting are performed on the server side, and using the network might be ineffective. +In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases), and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective. The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query". The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas". @@ -220,3 +213,4 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa ``` Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage. + diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md old mode 100644 new mode 100755 index 3e3e3df4853..e43f4474271 --- a/docs/en/interfaces/index.md +++ b/docs/en/interfaces/index.md @@ -2,4 +2,5 @@ # Interfaces -To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program. \ No newline at end of file +To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program. + diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md old mode 100644 new mode 100755 diff --git a/docs/en/interfaces/tcp.md b/docs/en/interfaces/tcp.md old mode 100644 new mode 100755 diff --git a/docs/en/interfaces/third-party_client_libraries.md b/docs/en/interfaces/third-party_client_libraries.md old mode 100644 new mode 100755 index cc8ff1f4307..8437be23b99 --- a/docs/en/interfaces/third-party_client_libraries.md +++ b/docs/en/interfaces/third-party_client_libraries.md @@ -2,7 +2,7 @@ There are libraries for working with ClickHouse for: -- Python: +- Python - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) diff --git a/docs/en/interfaces/third-party_gui.md b/docs/en/interfaces/third-party_gui.md old mode 100644 new mode 100755 diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md old mode 100644 new mode 100755 index 3927405579f..59853b8e202 --- a/docs/en/introduction/distinctive_features.md +++ b/docs/en/introduction/distinctive_features.md @@ -1,10 +1,10 @@ # Distinctive features of ClickHouse -## True column-oriented DBMS. +## True column-oriented DBMS -In a true column-oriented DBMS, there isn't any "garbage" stored with the values. For example, constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data. +In a true column-oriented DBMS, there isn't any "garbage" stored with the values. Among other things, this means that constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data. -This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Example are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second. +This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Examples are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second. Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows creating tables and databases in runtime, loading data, and running queries without reconfiguring and restarting the server. @@ -12,15 +12,15 @@ Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows cr Some column-oriented DBMSs (InfiniDB CE and MonetDB) do not use data compression. However, data compression really improves performance. -## Disk storage of data. +## Disk storage of data Many column-oriented DBMSs (such as SAP HANA and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica. -## Parallel processing on multiple cores. +## Parallel processing on multiple cores Large queries are parallelized in a natural way. -## Distributed processing on multiple servers. +## Distributed processing on multiple servers Almost none of the columnar DBMSs listed above have support for distributed processing. In ClickHouse, data can reside on different shards. Each shard can be a group of replicas that are used for fault tolerance. The query is processed on all the shards in parallel. This is transparent for the user. @@ -30,12 +30,12 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of If you are familiar with standard SQL, we can't really talk about SQL support. All the functions have different names. However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances. -Support for JOINs. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries. +JOINs are supported. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries. Dependent subqueries are not supported. ## Vector engine -Data is not only stored by columns, but is processed by vectors – parts of columns. This allows us to achieve high CPU performance. +Data is not only stored by columns, but is processed by vectors (parts of columns). This allows us to achieve high CPU performance. ## Real-time data updates @@ -43,13 +43,13 @@ ClickHouse supports primary key tables. In order to quickly perform queries on t ## Indexes -Having a primary key allows, for example, extracting data for specific clients (Metrica counters) for a specific time range, with low latency less than several dozen milliseconds. +Having a primary key makes it possible to extract data for specific clients (for instance, Yandex.Metrica tracking tags) for a specific time range, with low latency less than several dozen milliseconds. ## Suitable for online queries This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading. In other words, in online mode. -## Support for approximated calculations. +## Support for approximated calculations 1. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles. 2. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk. diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md old mode 100644 new mode 100755 diff --git a/docs/en/introduction/index.md b/docs/en/introduction/index.md old mode 100644 new mode 100755 index e10b99d0138..3d07efe555d --- a/docs/en/introduction/index.md +++ b/docs/en/introduction/index.md @@ -1 +1,2 @@ # Introduction + diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md old mode 100644 new mode 100755 diff --git a/docs/en/introduction/possible_silly_questions.md b/docs/en/introduction/possible_silly_questions.md old mode 100644 new mode 100755 index 36363ebe247..cf7b2c48032 --- a/docs/en/introduction/possible_silly_questions.md +++ b/docs/en/introduction/possible_silly_questions.md @@ -1,8 +1,8 @@ -# Everything you were afraid to ask +# Questions you were afraid to ask ## Why not use something like MapReduce? -We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop and YT (Yandex proprietary technology). +We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop, and YT (YT is developed at Yandex for internal use). These systems aren't appropriate for online queries due to their high latency. In other words, they can't be used as the back-end for a web interface. These types of systems aren't useful for real-time data updates. diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md old mode 100644 new mode 100755 index 6a488be9b5f..10f45f061d6 --- a/docs/en/introduction/ya_metrika_task.md +++ b/docs/en/introduction/ya_metrika_task.md @@ -1,6 +1,6 @@ -# The Yandex.Metrica task +# Yandex.Metrica use case -ClickHouse currently powers [ Yandex.Metrica](https://metrika.yandex.ru/), [ the second largest platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all) for Web Analytics. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data. +ClickHouse currently powers [Yandex.Metrica](https://metrika.yandex.ru/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all). With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data. We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors. At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds. diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md old mode 100644 new mode 100755 index 9879dab9a99..1c72bf13b3e --- a/docs/en/operations/access_rights.md +++ b/docs/en/operations/access_rights.md @@ -2,14 +2,14 @@ Users and access rights are set up in the user config. This is usually `users.xml`. -Users are recorded in the 'users' section. We'll look at a fragment of the `users.xml` file: +Users are recorded in the 'users' section. Here is a fragment of the `users.xml` file: ```xml - - - + How to generate a decent password: Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + The first line has the password and the second line has the corresponding SHA256. + --> + + - - - - default - + + ::/0 + + --> + + + + default + default - - - - - - - web - default - - test + + + + + + + web + default + + test ``` You can see a declaration from two users: `default`and`web`. We added the `web` user separately. -The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#distributed_distributed) engine). +The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify the `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#table_engines-distributed) engine). The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md old mode 100644 new mode 100755 index c3122617bf1..52e9e10ffea --- a/docs/en/operations/configuration_files.md +++ b/docs/en/operations/configuration_files.md @@ -14,7 +14,7 @@ If `replace` is specified, it replaces the entire element with the specified one If ` remove` is specified, it deletes the element. -The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the config in the [include_from](server_settings/settings.md#server_settings-include_from) element. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional = "true"` attribute (for example, settings for [ macros](server_settings/settings.md#server_settings-macros)). +The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](server_settings/settings.md#server_settings-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros]()server_settings/settings.md#server_settings-macros)). Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at ` /path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element. diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md old mode 100644 new mode 100755 index eb90f937cff..0ff38af8086 --- a/docs/en/operations/index.md +++ b/docs/en/operations/index.md @@ -1 +1,2 @@ -# Operation +# Usage + diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md old mode 100644 new mode 100755 index d7b1a61ce7f..fb1238b257d --- a/docs/en/operations/quotas.md +++ b/docs/en/operations/quotas.md @@ -18,10 +18,10 @@ Let's look at the section of the 'users.xml' file that defines quotas. - + - 3600 - + 3600 + 0 0 @@ -39,21 +39,19 @@ The resource consumption calculated for each interval is output to the server lo - - 3600 + + 3600 + 1000 + 100 + 1000000000 + 100000000000 + 900 + - 1000 - 100 - 1000000000 - 100000000000 - 900 - - - - 86400 - - 10000 - 1000 + + 86400 + 10000 + 1000 5000000000 500000000000 7200 @@ -89,7 +87,7 @@ Quotas can use the "quota key" feature in order to report on resources for multi Using keys makes sense only if quota_key is transmitted by the program, not by a user. You can also write so the IP address is used as the quota key.(But keep in mind that users can change the IPv6 address fairly easily.) - --> + --> ``` @@ -98,4 +96,3 @@ The quota is assigned to users in the 'users' section of the config. See the sec For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over". When the server is restarted, quotas are reset. - diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md old mode 100644 new mode 100755 index 2293e86f5c7..208deec710c --- a/docs/en/operations/server_settings/index.md +++ b/docs/en/operations/server_settings/index.md @@ -9,3 +9,4 @@ These settings are stored in the ` config.xml` file on the ClickHouse server. Other settings are described in the "[Settings](../settings/index.md#settings)" section. Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes). + diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md old mode 100644 new mode 100755 index 8818f8ec932..e1575df2f88 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -67,7 +67,7 @@ ClickHouse checks ` min_part_size` and ` min_part_size_ratio` and processes th The default database. -Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_show_databases) query to get a list of databases. +To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/queries.md#query_language_queries_show_databases). **Example** @@ -81,7 +81,7 @@ Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_s Default settings profile. -Settings profiles are located in the file specified in the [user_config](#server_settings-users_config) parameter. +Settings profiles are located in the file specified in the parameter [user_config](#server_settings-users_config). **Example** @@ -100,7 +100,7 @@ Path: - Specify the absolute path or the path relative to the server config file. - The path can contain wildcards \* and ?. -See also "[External dictionaries](../../dicts/external_dicts.md#dicts-external_dicts)". +See also "[External dictionaries]("./../dicts/external_dicts.md#dicts-external_dicts)". **Example** @@ -130,12 +130,12 @@ The default is ` true`. ## format_schema_path -The path to the directory with the schemas for the input data, such as schemas for the [ CapnProto](../../formats/capnproto.md#format_capnproto) format. +The path to the directory with the schemes for the input data, such as schemas for the [CapnProto](../../formats/capnproto.md#format_capnproto) format. **Example** ```xml - + format_schemas/ ``` @@ -179,7 +179,7 @@ You can configure multiple `` clauses. For instance, you can use this Settings for thinning data for Graphite. -For more details, see [ GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree). +For more information, see [GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree). **Example** @@ -241,7 +241,7 @@ Opens `https://tabix.io/` when accessing ` http://localhost: http_port`. The path to the file with substitutions. -For details, see the section "[Configuration files](../configuration_files.md#configuration_files)". +For more information, see the section "[Configuration files](../configuration_files.md#configuration_files)". **Example** @@ -298,7 +298,8 @@ Restriction on hosts that requests can come from. If you want the server to answ Examples: ```xml -::1127.0.0.1 +::1 +127.0.0.1 ``` @@ -347,7 +348,7 @@ For more information, see the section "[Creating replicated tables](../../table_ ## mark_cache_size -Approximate size (in bytes) of the cache of "marks" used by [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) engines. +Approximate size (in bytes) of the cache of "marks" used by [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120. @@ -403,7 +404,7 @@ We recommend using this option in Mac OS X, since the ` getrlimit()` function re Restriction on deleting tables. -If the size of a [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. +If the size of a [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. If you still need to delete the table without restarting the ClickHouse server, create the ` /flags/force_drop_table` file and run the DROP query. @@ -439,17 +440,17 @@ For more information, see the MergeTreeSettings.h header file. SSL client/server configuration. -Support for SSL is provided by the `` libpoco`` library. The description of the interface is in the [ SSLManager.h file.](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) +Support for SSL is provided by the `` libpoco`` library. The interface is described in the file [SSLManager.h](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) Keys for server/client settings: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `` privateKeyFile`` contains the certificate. - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Acceptable values: ``none``, ``relaxed``, ``strict``, ``once``. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``none``, ``relaxed``, ``strict``, ``once``. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `` true``, `` false``. | -- cipherList - Supported OpenSSL-ciphers. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``. +- cipherList - Поддерживаемые OpenSSL-шифры. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``. - cacheSessions – Enables or disables caching sessions. Must be used in combination with ``sessionIdContext``. Acceptable values: `` true``, `` false``. - sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed ``SSL_MAX_SSL_SESSION_ID_LENGTH``. This parameter is always recommended, since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: ``${application.name}``. - sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. @@ -498,7 +499,7 @@ Keys for server/client settings: ## part_log -Logging events that are associated with the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data type. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. +Logging events that are associated with [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. Queries are logged in the ClickHouse table, not in a separate file. @@ -518,7 +519,7 @@ Use the following parameters to configure logging: - database – Name of the database. - table – Name of the table. -- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). +- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). - flush_interval_milliseconds – Interval for flushing data from memory to the disk. **Example** @@ -562,7 +563,7 @@ Use the following parameters to configure logging: - database – Name of the database. - table – Name of the table. -- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). +- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). - flush_interval_milliseconds – Interval for flushing data from memory to the disk. If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. @@ -584,7 +585,7 @@ If the table doesn't exist, ClickHouse will create it. If the structure of the q Configuration of clusters used by the Distributed table engine. -For more information, see the section "[Duplicated table engine](../../table_engines/distributed.md#table_engines-distributed)". +For more information, see the section "[Table engines/Distributed](../../table_engines/distributed.md#table_engines-distributed)". **Example** @@ -644,7 +645,7 @@ The end slash is mandatory. ## uncompressed_cache_size -Cache size (in bytes) for uncompressed data used by table engines from the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. +Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#settings-use_uncompressed_cache) is enabled. diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md old mode 100644 new mode 100755 index 0e967a4c081..0c5ca5d5171 --- a/docs/en/operations/settings/index.md +++ b/docs/en/operations/settings/index.md @@ -9,9 +9,9 @@ Ways to configure settings, in order of priority: - Settings in the server config file. - Set via user profiles. + Settings from user profiles. -- For the session. +- Session settings. Send ` SET setting=value` from the ClickHouse console client in interactive mode. Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter. diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md old mode 100644 new mode 100755 diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md old mode 100644 new mode 100755 index a87328cb06c..7d95b9609c7 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4,7 +4,7 @@ ## distributed_product_mode -Alters the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables. +Changes the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables. ClickHouse applies the configuration if the subqueries on any level have a distributed table that exists on the local server and has more than one shard. @@ -12,9 +12,9 @@ Restrictions: - Only applied for IN and JOIN subqueries. - Used only if a distributed table is used in the FROM clause. -- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote)function. +- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote) function. -Possible values: +The possible values ​​are: @@ -36,7 +36,7 @@ Disables query execution if the index can't be used by date. Works with tables in the MergeTree family. -If `force_index_by_date=1`, ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". +If `force_index_by_date=1`, ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". @@ -46,7 +46,7 @@ Disables query execution if indexing by the primary key is not possible. Works with tables in the MergeTree family. -If `force_primary_key=1`, ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". +If `force_primary_key=1`, ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about data ranges in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". @@ -158,7 +158,7 @@ Don't confuse blocks for compression (a chunk of memory consisting of bytes) and ## min_compress_block_size -For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536. +For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)" tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536. The actual size of the block, if the uncompressed data is less than 'max_compress_block_size', is no less than this value and no less than the volume of data for one mark. @@ -253,13 +253,13 @@ Yandex.Metrica uses this parameter set to 1 for implementing suggestions for seg ## schema -This parameter is useful when you are using formats that require a schema definition, such as [ Cap'n Proto](https://capnproto.org/). The value depends on the format. +This parameter is useful when you are using formats that require a schema definition, such as [Cap'n Proto](https://capnproto.org/). The value depends on the format. ## stream_flush_interval_ms -Works for tables with streaming in the case of a timeout, or when a thread generates [ max_insert_block_size](#settings-settings-max_insert_block_size) rows. +Works for tables with streaming in the case of a timeout, or when a thread generates[max_insert_block_size](#settings-settings-max_insert_block_size) rows. The default value is 7500. @@ -345,7 +345,7 @@ If the value is true, integers appear in quotes when using JSON\* Int64 and UInt Strictly assigns default values when adding data. -If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert) query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column: +If data is not specified for a column when running an [ INSERT](../../query_language/queries.md#queries-insert) query, ClickHouse assigns default values to the fields. Default values are defined in the ` DEFAULT` property for each column in the table settings. If ` DEFAULT` is not defined for a column: - `If strict_insert_defaults = 0`, column fields are assigned zeros and empty strings. - `If strict_insert_defaults = 1`, ClickHouse throws an exception and requires the user to pass data to the column. diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md old mode 100644 new mode 100755 index c978c599bd5..f1fce41ba75 --- a/docs/en/operations/settings/settings_profiles.md +++ b/docs/en/operations/settings/settings_profiles.md @@ -17,7 +17,7 @@ Example: - + 8 diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md old mode 100644 new mode 100755 index 11fc8f6da11..652698fe24c --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -105,7 +105,7 @@ Use at least a 10 GB network, if possible. 1 Gb will also work, but it will be m You are probably already using ZooKeeper for other purposes. You can use the same installation of ZooKeeper, if it isn't already overloaded. -It's best to use a fresh version of ZooKeeper – 3.5 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. With the default settings, ZooKeeper is a time bomb: @@ -174,8 +174,7 @@ dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic Java version: ```text -Java(TM) SE Runtime Environment (build 1.8.0_25-b17) -Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) +Java(TM) SE Runtime Environment (build 1.8.0_25-b17)Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) ``` JVM parameters: diff --git a/docs/en/operators/index.md b/docs/en/operators/index.md old mode 100644 new mode 100755 index 779f3bf3843..411cde34b50 --- a/docs/en/operators/index.md +++ b/docs/en/operators/index.md @@ -67,11 +67,11 @@ Groups of operators are listed in order of priority (the higher it is in the lis `NOT a` The `not(a) function.` -## Logical 'AND' operator +## Logical AND operator `a AND b` – The`and(a, b) function.` -## Logical 'OR' operator +## Logical OR operator `a OR b` – The `or(a, b) function.` diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md old mode 100644 new mode 100755 index 247d76fc6ed..769d94eb4fd --- a/docs/en/query_language/index.md +++ b/docs/en/query_language/index.md @@ -1 +1,2 @@ # Query language + diff --git a/docs/en/query_language/queries.md b/docs/en/query_language/queries.md old mode 100644 new mode 100755 index 743706a551f..d235945a646 --- a/docs/en/query_language/queries.md +++ b/docs/en/query_language/queries.md @@ -11,7 +11,6 @@ CREATE DATABASE [IF NOT EXISTS] db_name `A database` is just a directory for tables. If `IF NOT EXISTS` is included, the query won't return an error if the database already exists. - ## CREATE TABLE @@ -184,7 +183,7 @@ Deletes all tables inside the 'db' database, then deletes the 'db' database itse If `IF EXISTS` is specified, it doesn't return an error if the database doesn't exist. ```sql -DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` Deletes the table. @@ -308,8 +307,7 @@ SELECT * FROM system.parts WHERE active `active` – Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part – these parts are deleted approximately 10 minutes after merging. Another way to view a set of parts and partitions is to go into the directory with table data. -Data directory: `/var/lib/clickhouse/data/database/table/`, -where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: +Data directory: `/var/lib/clickhouse/data/database/table/`,where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: ```bash $ ls -l /var/lib/clickhouse/data/test/visits/ @@ -325,7 +323,7 @@ Here, `20140317_20140323_2_2_0` and ` 20140317_20140323_4_4_0` are the directori Let's break down the name of the first part: `20140317_20140323_2_2_0`. - `20140317` is the minimum date of the data in the chunk. -- `20140323` is the maximum date of the data in the chunk. +- `20140323` is the maximum data of the data in the chunk. - `2` is the minimum number of the data block. - `2` is the maximum number of the data block. - `0` is the chunk level (the depth of the merge tree it is formed from). @@ -452,7 +450,7 @@ See also the section "Formats". ## SHOW TABLES ```sql -SHOW [TEMPORARY] TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] +SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] ``` Displays a list of tables @@ -462,7 +460,7 @@ Displays a list of tables This query is identical to: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]`. -See the section "LIKE operator" also. +See also the section "LIKE operator". ## SHOW PROCESSLIST @@ -486,7 +484,7 @@ Prints a table containing the columns: **query** – The query itself. In INSERT queries, the data for insertion is not output. -**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. +**query_id** – The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. This query is identical to: `SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]`. @@ -499,7 +497,7 @@ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" ## SHOW CREATE TABLE ```sql -SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] ``` Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table. @@ -517,7 +515,7 @@ Nested data structures are output in "expanded" format. Each column is shown sep ## EXISTS ```sql -EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] +EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] ``` Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database. @@ -573,9 +571,9 @@ The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case - The values calculated from the `DEFAULT` expressions specified in the table definition. - Zeros and empty strings, if `DEFAULT` expressions are not defined. -If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have ` DEFAULT` defined must be listed in the query. +If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have `DEFAULT` defined must be listed in the query. -The INSERT can pass data in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: +Data can be passed to the INSERT in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: ```sql INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set @@ -974,8 +972,7 @@ All columns that are not needed for the JOIN are deleted from the subquery. There are several types of JOINs: -`INNER` or `LEFT` type: -If INNER is specified, the result will contain only those rows that have a matching row in the right table. +`INNER` or `LEFT` type:If INNER is specified, the result will contain only those rows that have a matching row in the right table. If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything. `ANY` or `ALL` stringency:If `ANY` is specified and the right table has several matching rows, only the first one found is joined. @@ -1106,7 +1103,7 @@ Example: SELECT domainWithoutWWW(URL) AS domain, count(), - any(Title) AS title -- getting the first occurred page header for each domain. + any(Title) AS title -- getting the first occurring page header for each domain. FROM hits GROUP BY domain ``` @@ -1351,7 +1348,7 @@ There are two options for IN-s with subqueries (similar to JOINs): normal `IN`
-Remember that the algorithms described below may work differently depending on the [](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting. +Remember that the algorithms described below may work differently depending on the [settings](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
@@ -1479,34 +1476,34 @@ In all other cases, we don't recommend using the asterisk, since it only gives y ## KILL QUERY ```sql -KILL QUERY - WHERE - [SYNC|ASYNC|TEST] - [FORMAT format] +KILL QUERY WHERE [SYNC|ASYNC|TEST] [FORMAT format] ``` Attempts to terminate queries currently running. -The queries to terminate are selected from the system.processes table for which `WHERE` expression is true. +The queries to terminate are selected from the system.processes table for which expression_for_system.processes is true. Examples: ```sql --- Terminates all queries with the specified query_id. KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' +``` --- Synchronously terminates all queries run by `username`. +Terminates all queries with the specified query_id. + +```sql KILL QUERY WHERE user='username' SYNC ``` +Synchronously terminates all queries run by `username`. + Readonly-users can only terminate their own requests. - -By default, the asynchronous version of queries is used (`ASYNC`), which doesn't wait for query termination. - -The synchronous version (`SYNC`) waits for all queries to be killed and displays information about each process as it terminates. +By default, the asynchronous version of queries is used (`ASYNC`), which terminates without waiting for queries to complete. +The synchronous version (`SYNC`) waits for all queries to be completed and displays information about each process as it terminates. The response contains the `kill_status` column, which can take the following values: -1. 'finished' – The query terminated successfully. +1. 'finished' – The query completed successfully. 2. 'waiting' – Waiting for the query to finish after sending it a signal to terminate. 3. The other values ​​explain why the query can't be terminated. A test query (`TEST`) only checks the user's rights and displays a list of queries to terminate. + diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md old mode 100644 new mode 100755 index e151d2ee3d9..4928f2d4a12 --- a/docs/en/query_language/syntax.md +++ b/docs/en/query_language/syntax.md @@ -46,7 +46,7 @@ There are numeric literals, string literals, and compound literals. A numeric literal tries to be parsed: -- first as a 64-bit signed number, using the 'strtoull' function. +- First as a 64-bit signed number, using the 'strtoull' function. - If unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function. - If unsuccessful, as a floating-point number using the 'strtod' function. - Otherwise, an error is returned. diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md old mode 100644 new mode 100755 index 46b08a89607..8241b0a65ae --- a/docs/en/roadmap.md +++ b/docs/en/roadmap.md @@ -3,11 +3,13 @@ ## Q1 2018 ### New functionality -- Initial support for `UPDATE` and `DELETE`. -- Multi-dimensional and nested arrays. - - It may look like this: - + +- Support for `UPDATE` and `DELETE`. + +- Multidimensional and nested arrays. + + It can look something like this: + ```sql CREATE TABLE t ( @@ -21,7 +23,7 @@ ENGINE = MergeTree ORDER BY x - External MySQL and ODBC tables. - External tables can be integrated to ClickHouse using external dictionaries. This will be an alternative and a more convenient way to do so. + External tables can be integrated into ClickHouse using external dictionaries. This new functionality is a convenient alternative to connecting external tables. ```sql SELECT ... @@ -30,66 +32,68 @@ FROM mysql('host:port', 'db', 'table', 'user', 'password')` ### Improvements -- Efficient data copy between ClickHouse clusters. +- Effective data copying between ClickHouse clusters. - Currently, it is possible to copy data using remote() function, e.g.: ` + Now you can copy data with the remote() function. For example: ` INSERT INTO t SELECT * FROM remote(...) `. - The performance of this will be improved by proper distributed execution. + This operation will have improved performance. - O_DIRECT for merges. - Should improve OS cache performance and correspondingly query performance for 'hot' queries. - + This will improve the performance of the OS cache and "hot" queries. ## Q2 2018 ### New functionality -- UPDATE/DELETE in order to comply with Europe GDPR. -- Protobuf and Parquet input/output formats. -- Create dictionaries by DDL queries. +- UPDATE/DELETE conform to the EU GDPR. +- Protobuf and Parquet input and output formats. +- Creating dictionaries using DDL queries. - Currently, it is inconvenient and confusing that dictionaries are defined in external XML files while being a part of DB schema. The new approach will fix that. + Currently, dictionaries that are part of the database schema are defined in external XML files. This is inconvenient and counter-intuitive. The new approach should fix it. + +- Integration with LDAP. -- LDAP integration. - WITH ROLLUP and WITH CUBE for GROUP BY. -- Custom encoding/compression for columns. - Currently, ClickHouse support LZ4 and ZSTD compressions for columns, and compressions settings are global (see our article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)) for more details). Column level encoding (e.g. delta encoding) and compression will allow more efficient data storage and therefore faster queries. +- Custom encoding and compression for each column individually. -- Store data at multiple disk volumes of a single server. + As of now, ClickHouse supports LZ4 and ZSTD compression of columns, and compression settings are global (see the article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Per-column compression and encoding will provide more efficient data storage, which in turn will speed up queries. - That will make it easier to extend disk system as well as use different disk systems for different DBs or tables. Currently, users have to use symlinks if DB/table needs to be stored in another volume. +- Storing data on multiple disks on the same server. + + This functionality will make it easier to extend the disk space, since different disk systems can be used for different databases or tables. Currently, users are forced to use symbolic links if the databases and tables must be stored on a different disk. ### Improvements -A lot of enhancements and fixes are planned for query execution. In particular: +Many improvements and fixes are planned for the query execution system. For example: -- Using index for ‘in (subquery)’. +- Using an index for `in (subquery)`. - Currently, index is not used for such queries resulting in lower performance. + The index is not used right now, which reduces performance. -- Predicate pushdown from ‘where’ into subqueries and Predicate pushdown for views. +- Passing predicates from `where` to subqueries, and passing predicates to views. - These two are related since view is replaced by subquery. Currently, performance of filter conditions for views is significantly degraded, views can not use primary key of the underlying table, that makes views on big tables pretty much useless. + The predicates must be passed, since the view is changed by the subquery. Performance is still low for view filters, and views can't use the primary key of the original table, which makes views useless for large tables. -- Short-circuit expressions evaluation (ternary operator, if, multiIf). +- Optimizing branching operations (ternary operator, if, multiIf). - Currently, ClickHouse evaluates all branches even if the first one needs to be returned due to logical condition result. + ClickHouse currently performs all branches, even if they aren't necessary. -- Using primary key for GROUP BY and ORDER BY. +- Using a primary key for GROUP BY and ORDER BY. - This may speed up certain types of queries since data is already partially pre-sorted. + This will speed up certain types of queries with partially sorted data. ## Q3-Q4 2018 -Longer term plans are not yet finalized. There are two major projects on the list so far. +We don't have any set plans yet, but the main projects will be: -- Resource pools for query execution. +- Resource pools for executing queries. - That will allow managing workloads more efficiently. + This will make load management more efficient. - ANSI SQL JOIN syntax. - That will make ClickHouse more friendly for numerous SQL tools. + Improve ClickHouse compatibility with many SQL tools. + diff --git a/docs/en/system_tables/index.md b/docs/en/system_tables/index.md old mode 100644 new mode 100755 index 614ce4020ec..240105a684b --- a/docs/en/system_tables/index.md +++ b/docs/en/system_tables/index.md @@ -4,5 +4,5 @@ System tables are used for implementing part of the system's functionality, and You can't delete a system table (but you can perform DETACH). System tables don't have files with data on the disk or files with metadata. The server creates all the system tables when it starts. System tables are read-only. -System tables are located in the 'system' database. +They are located in the 'system' database. diff --git a/docs/en/system_tables/system.asynchronous_metrics.md b/docs/en/system_tables/system.asynchronous_metrics.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.clusters.md b/docs/en/system_tables/system.clusters.md old mode 100644 new mode 100755 index c0bc3dd13fa..bc8dab86b3c --- a/docs/en/system_tables/system.clusters.md +++ b/docs/en/system_tables/system.clusters.md @@ -4,13 +4,12 @@ Contains information about clusters available in the config file and the servers Columns: ```text -cluster String – Cluster name. -shard_num UInt32 – Number of a shard in the cluster, starting from 1. -shard_weight UInt32 – Relative weight of a shard when writing data. -replica_num UInt32 – Number of a replica in the shard, starting from 1. -host_name String – Host name as specified in the config. -host_address String – Host's IP address obtained from DNS. -port UInt16 – The port used to access the server. -user String – The username to use for connecting to the server. +cluster String - Cluster name. +shard_num UInt32 - Number of a shard in the cluster, starting from 1. +shard_weight UInt32 - Relative weight of a shard when writing data. +replica_num UInt32 - Number of a replica in the shard, starting from 1. +host_name String - Host name as specified in the config. +host_address String - Host's IP address obtained from DNS. +port UInt16 - The port used to access the server. +user String - The username to use for connecting to the server. ``` - diff --git a/docs/en/system_tables/system.columns.md b/docs/en/system_tables/system.columns.md old mode 100644 new mode 100755 index 975b84fe9d4..bf05616fbef --- a/docs/en/system_tables/system.columns.md +++ b/docs/en/system_tables/system.columns.md @@ -11,4 +11,3 @@ type String - Column type. default_type String - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined. default_expression String - Expression for the default value, or an empty string if it is not defined. ``` - diff --git a/docs/en/system_tables/system.databases.md b/docs/en/system_tables/system.databases.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.dictionaries.md b/docs/en/system_tables/system.dictionaries.md old mode 100644 new mode 100755 index f3c9929d38e..4ef0d7707b8 --- a/docs/en/system_tables/system.dictionaries.md +++ b/docs/en/system_tables/system.dictionaries.md @@ -5,19 +5,19 @@ Contains information about external dictionaries. Columns: ```text -name String – Dictionary name. -type String – Dictionary type: Flat, Hashed, Cache. -origin String – Path to the config file where the dictionary is described.attribute. -names Array(String) – Array of attribute names provided by the dictionary. -attribute.types Array(String) – Corresponding array of attribute types provided by the dictionary. -has_hierarchy UInt8 – Whether the dictionary is hierarchical. -bytes_allocated UInt64 – The amount of RAM used by the dictionary. -hit_rate Float64 – For cache dictionaries, the percent of usage for which the value was in the cache. -element_count UInt64 – The number of items stored in the dictionary. -load_factor Float64 – The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). -creation_time DateTime – Time spent for the creation or last successful reload of the dictionary. -last_exception String – Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. -source String – Text describing the data source for the dictionary. +name String - Dictionary name. +type String - Dictionary type: Flat, Hashed, Cache. +origin String - Path to the config file where the dictionary is described. +attribute.names Array(String) - Array of attribute names provided by the dictionary. +attribute.types Array(String) - Corresponding array of attribute types provided by the dictionary. +has_hierarchy UInt8 - Whether the dictionary is hierarchical. +bytes_allocated UInt64 - The amount of RAM used by the dictionary. +hit_rate Float64 - For cache dictionaries, the percent of usage for which the value was in the cache. +element_count UInt64 - The number of items stored in the dictionary. +load_factor Float64 - The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). +creation_time DateTime - Time spent for the creation or last successful reload of the dictionary. +last_exception String - Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. +source String - Text describing the data source for the dictionary. ``` Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is. diff --git a/docs/en/system_tables/system.events.md b/docs/en/system_tables/system.events.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.functions.md b/docs/en/system_tables/system.functions.md old mode 100644 new mode 100755 index ac550acc14b..a1022a5e557 --- a/docs/en/system_tables/system.functions.md +++ b/docs/en/system_tables/system.functions.md @@ -6,6 +6,6 @@ Columns: ```text name String – Function name. -is_aggregate UInt8 – Whether it is an aggregate function. +is_aggregate UInt8 – Whether it is an aggregate function. ``` diff --git a/docs/en/system_tables/system.merges.md b/docs/en/system_tables/system.merges.md old mode 100644 new mode 100755 index 0a10e4a5a8c..59870922ea5 --- a/docs/en/system_tables/system.merges.md +++ b/docs/en/system_tables/system.merges.md @@ -18,4 +18,3 @@ rows_read UInt64 - Number of rows read. bytes_written_uncompressed UInt64 - Amount of bytes written, uncompressed. rows_written UInt64 - Number of rows written. ``` - diff --git a/docs/en/system_tables/system.numbers.md b/docs/en/system_tables/system.numbers.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.numbers_mt.md b/docs/en/system_tables/system.numbers_mt.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.one.md b/docs/en/system_tables/system.one.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.parts.md b/docs/en/system_tables/system.parts.md old mode 100644 new mode 100755 diff --git a/docs/en/system_tables/system.processes.md b/docs/en/system_tables/system.processes.md old mode 100644 new mode 100755 index b9ad6a44e81..0802e555648 --- a/docs/en/system_tables/system.processes.md +++ b/docs/en/system_tables/system.processes.md @@ -14,12 +14,12 @@ rows_read UInt64 – The number of rows read from the table. For distribu bytes_read UInt64 – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -total_rows_approx UInt64 – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. +UInt64 total_rows_approx – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. memory_usage UInt64 – Memory consumption by the query. It might not include some types of dedicated memory. -query String – The query text. For INSERT, it doesn't include the data to insert. +Query String – The query text. For INSERT, it doesn't include the data to insert. -query_id String - The query ID, if defined. +query_id – Query ID, if defined. ``` diff --git a/docs/en/system_tables/system.replicas.md b/docs/en/system_tables/system.replicas.md old mode 100644 new mode 100755 index ec1341198dc..75cd8e34340 --- a/docs/en/system_tables/system.replicas.md +++ b/docs/en/system_tables/system.replicas.md @@ -42,37 +42,41 @@ Columns: database: database name table: table name engine: table engine name + is_leader: whether the replica is the leader Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform. Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. -is_readonly: Whether the replica is in read-only mode.This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. +is_readonly: Whether the replica is in read-only mode. +This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. is_session_expired: Whether the ZK session expired. Basically, the same thing as is_readonly. -future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. +future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. -parts_to_check: The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged. +parts_to_check: The number of data parts in the queue for verification. +A part is put in the verification queue if there is suspicion that it might be damaged. -zookeeper_path: The path to the table data in ZK. +zookeeper_path: The path to the table data in ZK. replica_name: Name of the replica in ZK. Different replicas of the same table have different names. -replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path. +replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path. -columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. +columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. -queue_size: Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions. +queue_size: Size of the queue for operations waiting to be performed. +Operations include inserting blocks of data, merges, and certain other actions. Normally coincides with future_parts. -inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong. +inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong. -merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time. +merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time. The next 4 columns have a non-null value only if the ZK session is active. -log_max_index: Maximum entry number in the log of general activity. -log_pointer: Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. If log_pointer is much smaller than log_max_index, something is wrong. +log_max_index: Maximum entry number in the log of general activity. log_pointer: Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. +If log_pointer is much smaller than log_max_index, something is wrong. total_replicas: Total number of known replicas of this table. active_replicas: Number of replicas of this table that have a ZK session (the number of active replicas). diff --git a/docs/en/system_tables/system.settings.md b/docs/en/system_tables/system.settings.md old mode 100644 new mode 100755 index a055135ebcf..90a392bcc24 --- a/docs/en/system_tables/system.settings.md +++ b/docs/en/system_tables/system.settings.md @@ -6,9 +6,9 @@ I.e. used for executing the query you are using to read from the system.settings Columns: ```text -name String – Setting name. +name String – Setting name. value String – Setting value. -changed UInt8 -–Whether the setting was explicitly defined in the config or explicitly changed. +changed UInt8 - Whether the setting was explicitly defined in the config or explicitly changed. ``` Example: diff --git a/docs/en/system_tables/system.tables.md b/docs/en/system_tables/system.tables.md old mode 100644 new mode 100755 index fabddf4dbb1..5757a8ac3da --- a/docs/en/system_tables/system.tables.md +++ b/docs/en/system_tables/system.tables.md @@ -1,6 +1,7 @@ # system.tables This table contains the String columns 'database', 'name', and 'engine'. -Also, the table has three virtual columns: metadata_modification_time of type DateTime, create_table_query and engine_full of type String. +The table also contains three virtual columns: metadata_modification_time (DateTime type), create_table_query, and engine_full (String type). Each table that the server knows about is entered in the 'system.tables' table. This system table is used for implementing SHOW TABLES queries. + diff --git a/docs/en/system_tables/system.zookeeper.md b/docs/en/system_tables/system.zookeeper.md old mode 100644 new mode 100755 index c456e7d9207..46b40e7a08f --- a/docs/en/system_tables/system.zookeeper.md +++ b/docs/en/system_tables/system.zookeeper.md @@ -70,4 +70,3 @@ numChildren: 7 pzxid: 987021252247 path: /clickhouse/tables/01-08/visits/replicas ``` - diff --git a/docs/en/table_engines/aggregatingmergetree.md b/docs/en/table_engines/aggregatingmergetree.md old mode 100644 new mode 100755 index d75d8353e6d..987c102508b --- a/docs/en/table_engines/aggregatingmergetree.md +++ b/docs/en/table_engines/aggregatingmergetree.md @@ -1,10 +1,11 @@ # AggregatingMergeTree -This engine differs from MergeTree in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. +This engine differs from `MergeTree` in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. -In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely. +For this to work, it uses the `AggregateFunction` data type, as well as `-State` and `-Merge` modifiers for aggregate functions. Let's examine it more closely. + +There is an `AggregateFunction` data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. -There is an AggregateFunction data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. Examples: ```sql @@ -19,12 +20,16 @@ CREATE TABLE t This type of column stores the state of an aggregate function. To get this type of value, use aggregate functions with the `State` suffix. -Example: `uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` – in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value. -An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump. +Example: +`uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` -The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. -Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction type`. +In contrast to the corresponding `uniq` and `quantiles` functions, these functions return the state, rather than the prepared value. In other words, they return an `AggregateFunction` type value. + +An `AggregateFunction` type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The `AggregateFunction` type values are not intended for output or saving in a dump. + +The only useful thing you can do with `AggregateFunction` type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. +Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction` type. In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result. As an example, these two queries return the same result: @@ -37,15 +42,15 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP There is an ` AggregatingMergeTree` engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value. -You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data. +You can't use a normal INSERT to insert a row in a table containing `AggregateFunction` columns, because you can't explicitly define the `AggregateFunction` value. Instead, use `INSERT SELECT` with `-State` aggregate functions for inserting data. -With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. +With SELECT from an `AggregatingMergeTree` table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. -You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views. +You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views. Example: -Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table: +Create an `AggregatingMergeTree` materialized view that watches the `test.visits` table: ```sql CREATE MATERIALIZED VIEW test.basic @@ -59,13 +64,13 @@ FROM test.visits GROUP BY CounterID, StartDate; ``` -Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated: +Insert data in the `test.visits` table. Data will also be inserted in the view, where it will be aggregated: ```sql INSERT INTO test.visits ... ``` -Performing SELECT from the view using GROUP BY to finish data aggregation: +Perform `SELECT` from the view using `GROUP BY` in order to complete data aggregation: ```sql SELECT diff --git a/docs/en/table_engines/buffer.md b/docs/en/table_engines/buffer.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/collapsingmergetree.md b/docs/en/table_engines/collapsingmergetree.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/custom_partitioning_key.md b/docs/en/table_engines/custom_partitioning_key.md old mode 100644 new mode 100755 index 6a468a1137d..bfcb3c2c545 --- a/docs/en/table_engines/custom_partitioning_key.md +++ b/docs/en/table_engines/custom_partitioning_key.md @@ -2,7 +2,7 @@ # Custom partitioning key -Starting with version 1.1.54310, you can create tables in the MergeTree family with any partition expression (not only partitioning by month). +Starting with version 1.1.54310, you can create tables in the MergeTree family with any partitioning expression (not only partitioning by month). The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax: @@ -10,7 +10,7 @@ The partition key can be an expression from the table columns, or a tuple of suc ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...] ``` -For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). Example: +For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). The other engine parameters are specified in parentheses after the engine name, as previously. Example: ```sql ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) @@ -23,7 +23,7 @@ The traditional partitioning by month is expressed as `toYYYYMM(date_column)`. You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT). -After this table is created, merge will only work for data parts that have the same value for the partition expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly. +After this table is created, merge will only work for data parts that have the same value for the partitioning expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly. To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example: @@ -35,13 +35,13 @@ Deletes the partition for the current week with event type 1. The same is true f Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match). -In the `system.parts` table, the `partition` column should specify the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format. +In the `system.parts` table, the `partition` column specifies the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format. -Before: `20140317_20140323_2_2_0` (minimal data - maximal data - number of minimal block - number of maximal block - level). +Was: `20140317_20140323_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level). -After: `201403_2_2_0` (partition ID - number of minimal block - number of maximal block - level). +Now: `201403_2_2_0` (partition ID - minimum block number - maximum block number - level). The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`. -There are more examples in the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). +For more examples, see the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). diff --git a/docs/en/table_engines/dictionary.md b/docs/en/table_engines/dictionary.md new file mode 100755 index 00000000000..ae8cca90d7c --- /dev/null +++ b/docs/en/table_engines/dictionary.md @@ -0,0 +1,106 @@ + + +# Dictionary + +The `Dictionary` engine displays the dictionary data as a ClickHouse table. + +As an example, consider a dictionary of `products` with the following configuration: + +```xml + + + products + + + products
+ DSN=some-db-server + + + + 300 + 360 + + + + + + + product_id + + + title + String + + + + + +``` + +Query the dictionary data: + +```sql +select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products'; + +SELECT + name, + type, + key, + attribute.names, + attribute.types, + bytes_allocated, + element_count, + source +FROM system.dictionaries +WHERE name = 'products' +``` +``` +┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐ +│ products │ Flat │ UInt64 │ ['title'] │ ['String'] │ 23065376 │ 175032 │ ODBC: .products │ +└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ +``` + +You can use the [dictGet*](../functions/ext_dict_functions.md#ext_dict_functions) function to get the dictionary data in this format. + +This view isn't helpful when you need to get raw data, or when performing a `JOIN` operation. For these cases, you can use the `Dictionary` engine, which displays the dictionary data in a table. + +Syntax: + +``` +CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` +``` + +Usage example: + +```sql +create table products (product_id UInt64, title String) Engine = Dictionary(products); + +CREATE TABLE products +( + product_id UInt64, + title String, +) +ENGINE = Dictionary(products) +``` +``` +Ok. + +0 rows in set. Elapsed: 0.004 sec. +``` + +Take a look at what's in the table. + +```sql +select * from products limit 1; + +SELECT * +FROM products +LIMIT 1 +``` +``` +┌────product_id─┬─title───────────┐ +│ 152689 │ Некоторый товар │ +└───────────────┴─────────────────┘ + +1 rows in set. Elapsed: 0.006 sec. +``` diff --git a/docs/en/table_engines/distributed.md b/docs/en/table_engines/distributed.md old mode 100644 new mode 100755 index b8643461fbb..dd2ffe27fe5 --- a/docs/en/table_engines/distributed.md +++ b/docs/en/table_engines/distributed.md @@ -25,29 +25,29 @@ Clusters are set like this: - - 1 - - false - - example01-01-1 - 9000 - - - example01-01-2 - 9000 - - - - 2 - false - - example01-02-1 - 9000 - - - example01-02-2 - 9000 + + 1 + + false + + example01-01-1 + 9000 + + + example01-01-2 + 9000 + + + + 2 + false + + example01-02-1 + 9000 + + + example01-02-2 + 9000 diff --git a/docs/en/table_engines/external_data.md b/docs/en/table_engines/external_data.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/file.md b/docs/en/table_engines/file.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/graphitemergetree.md b/docs/en/table_engines/graphitemergetree.md old mode 100644 new mode 100755 index 6452377ac15..a4b62424954 --- a/docs/en/table_engines/graphitemergetree.md +++ b/docs/en/table_engines/graphitemergetree.md @@ -2,13 +2,13 @@ # GraphiteMergeTree -This engine is designed for rollup (thinning and aggregating/averaging) [ Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. +This engine is designed for rollup (thinning and aggregating/averaging) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. Graphite stores full data in ClickHouse, and data can be retrieved in the following ways: - Without thinning. - Using the [MergeTree](mergetree.md#table_engines-mergetree) engine. + Uses the [MergeTree](mergetree.md#table_engines-mergetree) engine. - With thinning. @@ -83,4 +83,3 @@ Example of settings: ``` - diff --git a/docs/en/table_engines/index.md b/docs/en/table_engines/index.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/join.md b/docs/en/table_engines/join.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md old mode 100644 new mode 100755 index 55d2027309d..e69c06ae9f9 --- a/docs/en/table_engines/kafka.md +++ b/docs/en/table_engines/kafka.md @@ -1,79 +1,79 @@ -# Kafka - -The engine works with [Apache Kafka](http://kafka.apache.org/). - -Kafka lets you: - -- Publish or subscribe to data flows. -- Organize fault-tolerant storage. -- Process streams as they become available. - -``` -Kafka(broker_list, topic_list, group_name, format[, schema]) -``` - -Parameters: - -- `broker_list` – A comma-separated list of brokers (`localhost:9092`). -- `topic_list` – A list of Kafka topics (`my_topic`). -- `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. -- `format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. For more information, see the section "Formats". -- `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root ` schema.capnp:Message` object. - -Example: - -```sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - SELECT * FROM queue LIMIT 5; -``` - -The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. - -Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. For more information, see [http://kafka.apache.org/intro](http://kafka.apache.org/intro). - -`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using materialized views. For this purpose, the following was done: - -1. Use the engine to create a Kafka consumer and consider it a data stream. -2. Create a table with the desired structure. -3. Create a materialized view that converts data from the engine and puts it into a previously created table. - -When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from Kafka and convert them to the required format using `SELECT` - -Example: - -```sql - CREATE TABLE queue ( - timestamp UInt64, - level String, - message String - ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); - - CREATE TABLE daily ( - day Date, - level String, - total UInt64 - ) ENGINE = SummingMergeTree(day, (day, level), 8192); - - CREATE MATERIALIZED VIEW consumer TO daily - AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total - FROM queue GROUP BY day, level; - - SELECT level, sum(total) FROM daily GROUP BY level; -``` - -To improve performance, received messages are grouped into blocks the size of [max_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). If the block wasn't formed within [ stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block. - -To stop receiving topic data or to change the conversion logic, detach the materialized view: - -``` - DETACH TABLE consumer; - ATTACH MATERIALIZED VIEW consumer; -``` - -If you want to change the target table by using `ALTER` materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view. - +# Kafka + +This engine works with [Apache Kafka](http://kafka.apache.org/). + +Kafka lets you: + +- Publish or subscribe to data flows. +- Organize fault-tolerant storage. +- Process streams as they become available. + +``` +Kafka(broker_list, topic_list, group_name, format[, schema]) +``` + +Parameters: + +- `broker_list` – A comma-separated list of brokers (`localhost:9092`). +- `topic_list` – A list of Kafka topics (`my_topic`). +- `group_name` – A group of Kafka consumers (`group1`). Reading margins are tracked for each group separately. If you don't want messages to be duplicated in the cluster, use the same group name everywhere. +- `--format` – Message format. Uses the same notation as the SQL ` FORMAT` function, such as ` JSONEachRow`. +- `schema` – An optional parameter that must be used if the format requires a schema definition. For example, [Cap'n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. + +Example: + +```sql +CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + SELECT * FROM queue LIMIT 5; +``` + +The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. + +Groups are flexible and synced on the cluster. For instance, if you have 10 topics and 5 copies of a table in a cluster, then each copy gets 2 topics. If the number of copies changes, the topics are redistributed across the copies automatically. Read more about this at [http://kafka.apache.org/intro](http://kafka.apache.org/intro). + +`SELECT` is not particularly useful for reading messages (except for debugging), because each message can be read only once. It is more practical to create real-time threads using materialized views. To do this: + +1. Use the engine to create a Kafka consumer and consider it a data stream. +2. Create a table with the desired structure. +3. Create a materialized view that converts data from the engine and puts it into a previously created table. + +When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. This allows you to continually receive messages from Kafka and convert them to the required format using `SELECT` + +Example: + +```sql +CREATE TABLE queue ( + timestamp UInt64, + level String, + message String + ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow'); + + CREATE TABLE daily ( + day Date, + level String, + total UInt64 + ) ENGINE = SummingMergeTree(day, (day, level), 8192); + + CREATE MATERIALIZED VIEW consumer TO daily + AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total + FROM queue GROUP BY day, level; + +SELECT level, sum(total) FROM daily GROUP BY level; +``` + +To improve performance, received messages are grouped into blocks the size of [max_insert_block_size](../operations/settings/settings.md#settings-settings-max_insert_block_size). If the block wasn't formed within [stream_flush_interval_ms](../operations/settings/settings.md#settings-settings_stream_flush_interval_ms) milliseconds, the data will be flushed to the table regardless of the completeness of the block. + +To stop receiving topic data or to change the conversion logic, detach the materialized view: + +``` +DETACH TABLE consumer; +ATTACH MATERIALIZED VIEW consumer; +``` + +If you want to change the target table by using ` ALTER`materialized view, we recommend disabling the material view to avoid discrepancies between the target table and the data from the view. + diff --git a/docs/en/table_engines/log.md b/docs/en/table_engines/log.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/materializedview.md b/docs/en/table_engines/materializedview.md old mode 100644 new mode 100755 index 00f70bd72bd..5e2741c6aa1 --- a/docs/en/table_engines/materializedview.md +++ b/docs/en/table_engines/materializedview.md @@ -1,4 +1,4 @@ # MaterializedView -Used for implementing materialized views (for more information, see [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. +Used for implementing materialized views (for more information, see the [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)) query. For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. diff --git a/docs/en/table_engines/memory.md b/docs/en/table_engines/memory.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/merge.md b/docs/en/table_engines/merge.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/mergetree.md b/docs/en/table_engines/mergetree.md old mode 100644 new mode 100755 index 71197f21b34..fea02e01d72 --- a/docs/en/table_engines/mergetree.md +++ b/docs/en/table_engines/mergetree.md @@ -56,7 +56,7 @@ In this example, the index can't be used: SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -To check whether ClickHouse can use the index when executing the query, use the settings [ force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date) and [ force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key). +To check whether ClickHouse can use the index when executing the query, use the settings [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)and[force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key). The index by date only allows reading those parts that contain dates from the desired range. However, a data part may contain data for many dates (up to an entire month), while within a single part the data is ordered by the primary key, which might not contain the date as the first column. Because of this, using a query with only a date condition that does not specify the primary key prefix will cause more data to be read than for a single date. diff --git a/docs/en/table_engines/null.md b/docs/en/table_engines/null.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/replacingmergetree.md b/docs/en/table_engines/replacingmergetree.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/replication.md b/docs/en/table_engines/replication.md old mode 100644 new mode 100755 index 1e58878c34e..20dd17e444f --- a/docs/en/table_engines/replication.md +++ b/docs/en/table_engines/replication.md @@ -46,7 +46,7 @@ You can specify any existing ZooKeeper cluster and the system will use a directo If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only. -ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries). +ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries). For each INSERT query (more precisely, for each inserted block of data; the INSERT query contains a single block, or per block for every max_insert_block_size = 1048576 rows), approximately ten entries are made in ZooKeeper in several transactions. This leads to slightly longer latencies for INSERT compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one INSERT per second, it doesn't create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred INSERTs per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. diff --git a/docs/en/table_engines/set.md b/docs/en/table_engines/set.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/summingmergetree.md b/docs/en/table_engines/summingmergetree.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/tinylog.md b/docs/en/table_engines/tinylog.md old mode 100644 new mode 100755 diff --git a/docs/en/table_engines/view.md b/docs/en/table_engines/view.md old mode 100644 new mode 100755 diff --git a/docs/en/table_functions/index.md b/docs/en/table_functions/index.md old mode 100644 new mode 100755 diff --git a/docs/en/table_functions/merge.md b/docs/en/table_functions/merge.md old mode 100644 new mode 100755 diff --git a/docs/en/table_functions/remote.md b/docs/en/table_functions/remote.md old mode 100644 new mode 100755 index 99b0c7bb116..e26e245207b --- a/docs/en/table_functions/remote.md +++ b/docs/en/table_functions/remote.md @@ -52,7 +52,7 @@ example01-{01..02}-1 If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets. -Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting. +Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. However, the replicas are iterated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting. Example: diff --git a/docs/en/utils/clickhouse-copier.md b/docs/en/utils/clickhouse-copier.md old mode 100644 new mode 100755 index 25d22f19222..9d15053fe06 --- a/docs/en/utils/clickhouse-copier.md +++ b/docs/en/utils/clickhouse-copier.md @@ -1,40 +1,54 @@ -# clickhouse-copier util + -The util copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed and fault-tolerant manner. +# clickhouse-copier -Configuration of copying tasks is set in special ZooKeeper node (called the `/description` node). -A ZooKeeper path to the description node is specified via `--task-path ` parameter. -So, node `/task/path/description` should contain special XML content describing copying tasks. +Copies data from the tables in one cluster to tables in another (or the same) cluster. -Simultaneously many `clickhouse-copier` processes located on any servers could execute the same task. -ZooKeeper node `/task/path/` is used by the processes to coordinate their work. -You must not add additional child nodes to `/task/path/`. +You can run multiple `clickhouse-copier` instances on different servers to perform the same job. ZooKeeper is used for syncing the processes. -Currently you are responsible for manual launching of all `cluster-copier` processes. -You can launch as many processes as you want, whenever and wherever you want. -Each process try to select the nearest available shard of source cluster and copy some part of data (partition) from it to the whole -destination cluster (with resharding). -Therefore it makes sense to launch cluster-copier processes on the source cluster nodes to reduce the network usage. +After starting, `clickhouse-copier`: -Since the workers coordinate their work via ZooKeeper, in addition to `--task-path ` you have to specify ZooKeeper -cluster configuration via `--config-file ` parameter. Example of `zookeeper.xml`: +- Connects to ZooKeeper and receives: + - Copying jobs. + - The state of the copying jobs. + +- It performs the jobs. + + Each running process chooses the "closest" shard of the source cluster and copies the data into the destination cluster, resharding the data if necessary. + +`clickhouse-copier` tracks the changes in ZooKeeper and applies them on the fly. + +To reduce network traffic, we recommend running `clickhouse-copier` on the same server where the source data is located. + +## Running clickhouse-copier + +The utility should be run manually: + +```bash +clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir +``` + +Parameters: + +- `daemon` — Starts `clickhouse-copier` in daemon mode. +- `config` — The path to the `zookeeper.xml` file with the parameters for the connection to ZooKeeper. +- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` processes and storing tasks. Tasks are stored in `$task-path/description`. +- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` creates `clickhouse-copier_YYYYMMHHSS_` subdirectories in `$base-dir`. If this parameter is omitted, the directories are created in the directory where `clickhouse-copier` was launched. + +## Format of zookeeper.xml ```xml - + 127.0.0.1 2181 - + ``` -When you run `clickhouse-copier --config-file --task-path ` the process connects to ZooKeeper cluster, reads tasks config from `/task/path/description` and executes them. - -## Format of task config - -Here is an example of `/task/path/description` content: +## Configuration of copying tasks ```xml @@ -69,62 +83,62 @@ Here is an example of `/task/path/description` content: 0 - 3 - + 1 - - + - + source_cluster test hits - + destination_cluster test hits2 - - ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') + + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') PARTITION BY toMonday(date) ORDER BY (CounterID, EventDate) - + jumpConsistentHash(intHash64(UserID), 2) - + CounterID != 0 - '2018-02-26' @@ -133,7 +147,7 @@ Here is an example of `/task/path/description` content: - + ... @@ -142,15 +156,5 @@ Here is an example of `/task/path/description` content: ``` -cluster-copier processes watch for `/task/path/description` node update. -So, if you modify the config settings or `max_workers` params, they will be updated. +`clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change. -## Example - -```bash -clickhouse-copier copier --daemon --config /path/to/copier/zookeeper.xml --task-path /clickhouse-copier/cluster1_tables_hits --base-dir /path/to/copier_logs -``` - -`--base-dir /path/to/copier_logs` specifies where auxilary and log files of the copier process will be saved. -In this case it will create `/path/to/copier_logs/clickhouse-copier_YYYYMMHHSS_/` dir with log and status-files. -If it is not specified it will use current dir (`/clickhouse-copier_YYYYMMHHSS_/` if it is run as a `--daemon`). diff --git a/docs/en/utils/clickhouse-local.md b/docs/en/utils/clickhouse-local.md old mode 100644 new mode 100755 index d18cc200320..d5fba56271f --- a/docs/en/utils/clickhouse-local.md +++ b/docs/en/utils/clickhouse-local.md @@ -1,4 +1,6 @@ -# The clickhouse-local program + -The `clickhouse-local` program enables you to perform fast processing on local files that store tables, without having to deploy and configure clickhouse-server. +#clickhouse-local + +The `clickhouse-local` program enables you to perform fast processing on local files that store tables, without having to deploy and configure the ClickHouse server. diff --git a/docs/en/utils/index.md b/docs/en/utils/index.md old mode 100644 new mode 100755 index 7a8c5ee5138..cf541cda895 --- a/docs/en/utils/index.md +++ b/docs/en/utils/index.md @@ -1,6 +1,5 @@ -# ClickHouse utilites +# ClickHouse utility -There are several ClickHouse utilites that are separate executable files: +* [clickhouse-local](clickhouse-local.md#utils-clickhouse-local) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this. +* [clickhouse-copier](clickhouse-copier.md#utils-clickhouse-copier) — Copies (and reshards) data from one cluster to another cluster. -* `clickhouse-local` allows to execute SQL queries on a local data like `awk` -* `clickhouse-copier` copies (and reshards) immutable data from one cluster to another in a fault-tolerant manner. diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml index d5fadb3f1e1..012d498f3e2 100644 --- a/docs/mkdocs_en.yml +++ b/docs/mkdocs_en.yml @@ -86,6 +86,7 @@ pages: - 'GraphiteMergeTree': 'table_engines/graphitemergetree.md' - 'Data replication': 'table_engines/replication.md' - 'Distributed': 'table_engines/distributed.md' + - 'Dictionary': 'table_engines/dictionary.md' - 'Merge': 'table_engines/merge.md' - 'Buffer': 'table_engines/buffer.md' - 'File': 'table_engines/file.md' @@ -95,6 +96,7 @@ pages: - 'View': 'table_engines/view.md' - 'MaterializedView': 'table_engines/materializedview.md' - 'Kafka': 'table_engines/kafka.md' + - 'MySQL': 'table_engines/mysql.md' - 'External data for query processing': 'table_engines/external_data.md' - 'System tables': diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md index ff1b9c0cdd5..defb0605c0f 100644 --- a/docs/ru/dicts/external_dicts_dict_layout.md +++ b/docs/ru/dicts/external_dicts_dict_layout.md @@ -109,15 +109,15 @@ Пример: таблица содержит скидки для каждого рекламодателя в виде: ``` - +------------------+-----------------------------+------------+----------+ - | id рекламодателя | дата начала действия скидки | дата конца | величина | - +==================+=============================+============+==========+ - | 123 | 2015-01-01 | 2015-01-15 | 0.15 | - +------------------+-----------------------------+------------+----------+ - | 123 | 2015-01-16 | 2015-01-31 | 0.25 | - +------------------+-----------------------------+------------+----------+ - | 456 | 2015-01-01 | 2015-01-15 | 0.05 | - +------------------+-----------------------------+------------+----------+ ++---------------+---------------------+-------------------+--------+ +| advertiser id | discount start date | discount end date | amount | ++===============+=====================+===================+========+ +| 123 | 2015-01-01 | 2015-01-15 | 0.15 | ++---------------+---------------------+-------------------+--------+ +| 123 | 2015-01-16 | 2015-01-31 | 0.25 | ++---------------+---------------------+-------------------+--------+ +| 456 | 2015-01-01 | 2015-01-15 | 0.05 | ++---------------+---------------------+-------------------+--------+ ``` Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`. From cd59ff0d54039aac2493d819566ea1465e952d8b Mon Sep 17 00:00:00 2001 From: BayoNet Date: Sun, 25 Mar 2018 05:16:13 +0300 Subject: [PATCH 66/90] mysql.md is not translated yet --- docs/mkdocs_en.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml index 012d498f3e2..eeedc71a79b 100644 --- a/docs/mkdocs_en.yml +++ b/docs/mkdocs_en.yml @@ -96,7 +96,6 @@ pages: - 'View': 'table_engines/view.md' - 'MaterializedView': 'table_engines/materializedview.md' - 'Kafka': 'table_engines/kafka.md' - - 'MySQL': 'table_engines/mysql.md' - 'External data for query processing': 'table_engines/external_data.md' - 'System tables': From ec86ecbcdc10c7d17323cb43038bd179fa6176b4 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Sun, 25 Mar 2018 05:37:07 +0300 Subject: [PATCH 67/90] Some typos are fixed. --- docs/ru/introduction/ya_metrika_task.md | 2 +- docs/ru/roadmap.md | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md index 24e595b2c49..765c0450890 100644 --- a/docs/ru/introduction/ya_metrika_task.md +++ b/docs/ru/introduction/ya_metrika_task.md @@ -1,6 +1,6 @@ # Постановка задачи в Яндекс.Метрике -ClickHouse на данный момент обеспечивает рабту [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. +ClickHouse на данный момент обеспечивает работу [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Нужно получать произвольные отчёты на основе хитов и визитов, с произвольными сегментами, задаваемыми пользователем. Данные для отчётов обновляются в реальном времени. Запросы должны выполняться сразу (в режиме онлайн). Отчёты должно быть возможно строить за произвольный период. Требуется вычислять сложные агрегаты типа количества уникальных посетителей. На данный момент (апрель 2014), каждый день в Яндекс.Метрику поступает около 12 миллиардов событий (хитов и кликов мыши). Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать сотни миллионов строк за время не более нескольких секунд, или миллионы строк за время не более нескольких сотен миллисекунд. diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md index 13c2b60c094..bbdc740bf01 100644 --- a/docs/ru/roadmap.md +++ b/docs/ru/roadmap.md @@ -5,15 +5,15 @@ ### Новая функциональность - Поддержка `UPDATE` и `DELETE`. - Многомерные и вложенные массивы. - + Это может выглядеть например так: - + ```sql CREATE TABLE t ( - x Array(Array(String)), + x Array(Array(String)), z Nested( - x Array(String), + x Array(String), y Nested(...)) ) ENGINE = MergeTree ORDER BY x @@ -24,7 +24,7 @@ ENGINE = MergeTree ORDER BY x Внешние таблицы можно интрегрировать в ClickHouse с помощью внешних словарей. Новая функциональность станет более удобной альтернативой для подключения внешних таблиц. ```sql -SELECT ... +SELECT ... FROM mysql('host:port', 'db', 'table', 'user', 'password')` ``` @@ -40,7 +40,7 @@ INSERT INTO t SELECT * FROM remote(...) `. - O_DIRECT for merges. Улучшит производительность кэша операционной системы, а также производительность 'горячих' запросов. - + ## Q2 2018 ### Новая функциональность @@ -56,23 +56,23 @@ INSERT INTO t SELECT * FROM remote(...) `. - Настраиваемые кодировки и сжатие для каждого столбца в отдельности. Сейчас, ClickHouse поддерживает сжатие столбцов с помощью LZ4 и ZSTD, и настройки сжатия глобальные (смотрите статью [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Поколоночное сжатие и кодирование обеспечит более эффективное хранение данных, что в свою очередь ускорит выполнение запросов. - + - Хранение данных на нескольких дисках на одном сервере. Реализация это функциональности упростит расширение дискового пространства, поскольку можно будет использовать различные дисковые системы для разных баз данных или таблиц. Сейчас, пользователи вынуждены использовать символические ссылки, если базы данных и таблицы должны храниться на другом диске. - + ### Улучшения Планируется множество улучшений и исправлений в системе выполнения запросов. Например: - Использование индекса для `in (subquery)`. - Сейчас, индекс не используется, что приводит с снижению производительности. - + Сейчас, индекс не используется, что приводит к снижению производительности. + - Передача предикатов из `where` в подзапросы, а также передача предикатов в представления. Передача предикатов необходима, поскольку представление изменяется поздапросом. Сейчас производительность фильтров для представлений низкая, представления не могут использовать первичный ключ оригинальной таблицы, что делает представления для больших таблиц бесполезными. - + - Оптимизация операций с ветвлением (тернарный оператор, if, multiIf). Сейчас, ClickHouse выполняет все ветви, даже если в этом нет необходимости. @@ -88,7 +88,7 @@ INSERT INTO t SELECT * FROM remote(...) `. - Пулы ресурсов для выполнения запросов. Позволят более эффективно управлять нагрузкой. - + - Синтаксис ANSI SQL JOIN. Улучшит совместимость ClickHouse со множеством SQL-инструментов. From e07cde4363d830b69adc295f78682b049ea337e9 Mon Sep 17 00:00:00 2001 From: Jonatas Freitas Date: Sun, 25 Mar 2018 01:43:12 -0300 Subject: [PATCH 68/90] Added missing slash on graphite monitoring section --- dbms/src/Server/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Server/config.xml b/dbms/src/Server/config.xml index 0eb96cdcf6e..8cf1fd6b519 100644 --- a/dbms/src/Server/config.xml +++ b/dbms/src/Server/config.xml @@ -199,7 +199,7 @@ 0.1 60 one_min - true + true true true From 237c530b4e1564301a2bcde9e6d1ccf2abb00b62 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Mar 2018 01:42:36 +0300 Subject: [PATCH 69/90] Update StorageKafka.cpp --- dbms/src/Storages/StorageKafka.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index 4ec2307ef13..80e4942839e 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -578,11 +578,11 @@ void registerStorageKafka(StorageFactory & factory) // Parse topic list Names topics; - String topicArg = static_cast(*engine_args[1]).value.safeGet(); - boost::split(topics, topicArg , [](char c){return c == ',';}); - for(String & topic : topics) { + String topic_arg = static_cast(*engine_args[1]).value.safeGet(); + boost::split(topics, topic_arg , [](char c){ return c == ','; }); + for(String & topic : topics) boost::trim(topic); - } + // Parse consumer group String group = static_cast(*engine_args[2]).value.safeGet(); From 9ba28666e1dff9625fcf85c40fd2138059f99a25 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Mar 2018 01:44:54 +0300 Subject: [PATCH 70/90] Update Client.cpp --- dbms/src/Server/Client.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Server/Client.cpp b/dbms/src/Server/Client.cpp index 1a8d97af25a..c53a2c1b873 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/src/Server/Client.cpp @@ -337,7 +337,8 @@ private: if (is_interactive) { - query_id = ""; + if (!query_id.empty()) + throw Exception("query_id could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); From 26df2f7a858fd0b6a334490fb013125696768781 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Mar 2018 17:00:55 +0300 Subject: [PATCH 71/90] Revert "English translation is updated" --- docs/en/agg_functions/combinators.md | 0 docs/en/agg_functions/index.md | 1 - docs/en/agg_functions/parametric_functions.md | 0 docs/en/agg_functions/reference.md | 34 +++-- docs/en/data_types/array.md | 0 docs/en/data_types/boolean.md | 0 docs/en/data_types/date.md | 2 +- docs/en/data_types/datetime.md | 0 docs/en/data_types/enum.md | 0 docs/en/data_types/fixedstring.md | 0 docs/en/data_types/float.md | 10 +- docs/en/data_types/index.md | 5 +- docs/en/data_types/int_uint.md | 0 .../aggregatefunction.md | 0 .../nested_data_structures/index.md | 0 .../nested_data_structures/nested.md | 0 .../special_data_types/expression.md | 0 .../en/data_types/special_data_types/index.md | 0 docs/en/data_types/special_data_types/set.md | 0 docs/en/data_types/string.md | 0 docs/en/data_types/tuple.md | 0 docs/en/development/style.md | 65 +++++----- docs/en/dicts/external_dicts.md | 5 +- docs/en/dicts/external_dicts_dict.md | 9 +- docs/en/dicts/external_dicts_dict_layout.md | 99 ++++++++++++--- docs/en/dicts/external_dicts_dict_lifetime.md | 6 +- docs/en/dicts/external_dicts_dict_sources.md | 6 +- .../en/dicts/external_dicts_dict_structure.md | 14 +- docs/en/dicts/index.md | 0 docs/en/dicts/internal_dicts.md | 0 docs/en/formats/capnproto.md | 52 ++++---- docs/en/formats/csv.md | 0 docs/en/formats/csvwithnames.md | 0 docs/en/formats/index.md | 1 - docs/en/formats/json.md | 0 docs/en/formats/jsoncompact.md | 2 +- docs/en/formats/jsoneachrow.md | 0 docs/en/formats/native.md | 0 docs/en/formats/null.md | 0 docs/en/formats/pretty.md | 0 docs/en/formats/prettycompact.md | 0 docs/en/formats/prettycompactmonoblock.md | 0 docs/en/formats/prettynoescapes.md | 0 docs/en/formats/prettyspace.md | 0 docs/en/formats/rowbinary.md | 2 +- docs/en/formats/tabseparated.md | 0 docs/en/formats/tabseparatedraw.md | 0 docs/en/formats/tabseparatedwithnames.md | 0 .../formats/tabseparatedwithnamesandtypes.md | 0 docs/en/formats/tskv.md | 0 docs/en/formats/values.md | 2 +- docs/en/formats/vertical.md | 0 docs/en/formats/xml.md | 2 +- docs/en/functions/arithmetic_functions.md | 0 docs/en/functions/array_functions.md | 12 +- docs/en/functions/array_join.md | 1 + docs/en/functions/bit_functions.md | 1 + docs/en/functions/comparison_functions.md | 2 +- docs/en/functions/conditional_functions.md | 0 docs/en/functions/date_time_functions.md | 4 - docs/en/functions/encoding_functions.md | 0 docs/en/functions/ext_dict_functions.md | 8 +- docs/en/functions/hash_functions.md | 0 docs/en/functions/higher_order_functions.md | 3 +- docs/en/functions/in_functions.md | 0 docs/en/functions/index.md | 0 docs/en/functions/ip_address_functions.md | 0 docs/en/functions/json_functions.md | 9 +- docs/en/functions/logical_functions.md | 1 + docs/en/functions/math_functions.md | 1 + docs/en/functions/other_functions.md | 5 +- docs/en/functions/random_functions.md | 0 docs/en/functions/rounding_functions.md | 0 .../functions/splitting_merging_functions.md | 0 docs/en/functions/string_functions.md | 0 docs/en/functions/string_replace_functions.md | 1 + docs/en/functions/string_search_functions.md | 0 .../en/functions/type_conversion_functions.md | 0 docs/en/functions/url_functions.md | 0 docs/en/functions/ym_dict_functions.md | 12 +- .../example_datasets/amplab_benchmark.md | 0 .../example_datasets/criteo.md | 4 +- .../example_datasets/nyc_taxi.md | 15 ++- .../example_datasets/ontime.md | 0 .../example_datasets/star_schema.md | 3 +- .../example_datasets/wikistat.md | 2 +- docs/en/getting_started/index.md | 14 +- docs/en/index.md | 2 +- docs/en/interfaces/cli.md | 7 +- docs/en/interfaces/http_interface.md | 16 ++- docs/en/interfaces/index.md | 3 +- docs/en/interfaces/jdbc.md | 0 docs/en/interfaces/tcp.md | 0 .../third-party_client_libraries.md | 2 +- docs/en/interfaces/third-party_gui.md | 0 docs/en/introduction/distinctive_features.md | 20 +-- .../features_considered_disadvantages.md | 0 docs/en/introduction/index.md | 1 - docs/en/introduction/performance.md | 0 .../introduction/possible_silly_questions.md | 4 +- docs/en/introduction/ya_metrika_task.md | 4 +- docs/en/operations/access_rights.md | 59 +++++---- docs/en/operations/configuration_files.md | 2 +- docs/en/operations/index.md | 3 +- docs/en/operations/quotas.md | 35 ++--- docs/en/operations/server_settings/index.md | 1 - .../en/operations/server_settings/settings.md | 37 +++--- docs/en/operations/settings/index.md | 4 +- .../operations/settings/query_complexity.md | 0 docs/en/operations/settings/settings.md | 16 +-- .../operations/settings/settings_profiles.md | 2 +- docs/en/operations/tips.md | 5 +- docs/en/operators/index.md | 4 +- docs/en/query_language/index.md | 1 - docs/en/query_language/queries.md | 53 ++++---- docs/en/query_language/syntax.md | 2 +- docs/en/roadmap.md | 70 +++++----- docs/en/system_tables/index.md | 2 +- .../system.asynchronous_metrics.md | 0 docs/en/system_tables/system.clusters.md | 17 +-- docs/en/system_tables/system.columns.md | 1 + docs/en/system_tables/system.databases.md | 0 docs/en/system_tables/system.dictionaries.md | 26 ++-- docs/en/system_tables/system.events.md | 0 docs/en/system_tables/system.functions.md | 2 +- docs/en/system_tables/system.merges.md | 1 + docs/en/system_tables/system.numbers.md | 0 docs/en/system_tables/system.numbers_mt.md | 0 docs/en/system_tables/system.one.md | 0 docs/en/system_tables/system.parts.md | 0 docs/en/system_tables/system.processes.md | 6 +- docs/en/system_tables/system.replicas.md | 26 ++-- docs/en/system_tables/system.settings.md | 4 +- docs/en/system_tables/system.tables.md | 3 +- docs/en/system_tables/system.zookeeper.md | 1 + docs/en/table_engines/aggregatingmergetree.md | 31 ++--- docs/en/table_engines/buffer.md | 0 docs/en/table_engines/collapsingmergetree.md | 0 .../table_engines/custom_partitioning_key.md | 14 +- docs/en/table_engines/dictionary.md | 106 ---------------- docs/en/table_engines/distributed.md | 46 +++---- docs/en/table_engines/external_data.md | 0 docs/en/table_engines/file.md | 0 docs/en/table_engines/graphitemergetree.md | 5 +- docs/en/table_engines/index.md | 0 docs/en/table_engines/join.md | 0 docs/en/table_engines/kafka.md | 2 +- docs/en/table_engines/log.md | 0 docs/en/table_engines/materializedview.md | 2 +- docs/en/table_engines/memory.md | 0 docs/en/table_engines/merge.md | 0 docs/en/table_engines/mergetree.md | 2 +- docs/en/table_engines/null.md | 0 docs/en/table_engines/replacingmergetree.md | 0 docs/en/table_engines/replication.md | 2 +- docs/en/table_engines/set.md | 0 docs/en/table_engines/summingmergetree.md | 0 docs/en/table_engines/tinylog.md | 0 docs/en/table_engines/view.md | 0 docs/en/table_functions/index.md | 0 docs/en/table_functions/merge.md | 0 docs/en/table_functions/remote.md | 2 +- docs/en/utils/clickhouse-copier.md | 120 +++++++++--------- docs/en/utils/clickhouse-local.md | 6 +- docs/en/utils/index.md | 7 +- docs/mkdocs_en.yml | 1 - docs/ru/dicts/external_dicts_dict_layout.md | 18 +-- docs/ru/introduction/ya_metrika_task.md | 2 +- docs/ru/roadmap.md | 24 ++-- 169 files changed, 587 insertions(+), 660 deletions(-) mode change 100755 => 100644 docs/en/agg_functions/combinators.md mode change 100755 => 100644 docs/en/agg_functions/index.md mode change 100755 => 100644 docs/en/agg_functions/parametric_functions.md mode change 100755 => 100644 docs/en/agg_functions/reference.md mode change 100755 => 100644 docs/en/data_types/array.md mode change 100755 => 100644 docs/en/data_types/boolean.md mode change 100755 => 100644 docs/en/data_types/date.md mode change 100755 => 100644 docs/en/data_types/datetime.md mode change 100755 => 100644 docs/en/data_types/enum.md mode change 100755 => 100644 docs/en/data_types/fixedstring.md mode change 100755 => 100644 docs/en/data_types/float.md mode change 100755 => 100644 docs/en/data_types/index.md mode change 100755 => 100644 docs/en/data_types/int_uint.md mode change 100755 => 100644 docs/en/data_types/nested_data_structures/aggregatefunction.md mode change 100755 => 100644 docs/en/data_types/nested_data_structures/index.md mode change 100755 => 100644 docs/en/data_types/nested_data_structures/nested.md mode change 100755 => 100644 docs/en/data_types/special_data_types/expression.md mode change 100755 => 100644 docs/en/data_types/special_data_types/index.md mode change 100755 => 100644 docs/en/data_types/special_data_types/set.md mode change 100755 => 100644 docs/en/data_types/string.md mode change 100755 => 100644 docs/en/data_types/tuple.md mode change 100755 => 100644 docs/en/development/style.md mode change 100755 => 100644 docs/en/dicts/external_dicts.md mode change 100755 => 100644 docs/en/dicts/external_dicts_dict.md mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_layout.md mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_lifetime.md mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_sources.md mode change 100755 => 100644 docs/en/dicts/external_dicts_dict_structure.md mode change 100755 => 100644 docs/en/dicts/index.md mode change 100755 => 100644 docs/en/dicts/internal_dicts.md mode change 100755 => 100644 docs/en/formats/capnproto.md mode change 100755 => 100644 docs/en/formats/csv.md mode change 100755 => 100644 docs/en/formats/csvwithnames.md mode change 100755 => 100644 docs/en/formats/index.md mode change 100755 => 100644 docs/en/formats/json.md mode change 100755 => 100644 docs/en/formats/jsoncompact.md mode change 100755 => 100644 docs/en/formats/jsoneachrow.md mode change 100755 => 100644 docs/en/formats/native.md mode change 100755 => 100644 docs/en/formats/null.md mode change 100755 => 100644 docs/en/formats/pretty.md mode change 100755 => 100644 docs/en/formats/prettycompact.md mode change 100755 => 100644 docs/en/formats/prettycompactmonoblock.md mode change 100755 => 100644 docs/en/formats/prettynoescapes.md mode change 100755 => 100644 docs/en/formats/prettyspace.md mode change 100755 => 100644 docs/en/formats/rowbinary.md mode change 100755 => 100644 docs/en/formats/tabseparated.md mode change 100755 => 100644 docs/en/formats/tabseparatedraw.md mode change 100755 => 100644 docs/en/formats/tabseparatedwithnames.md mode change 100755 => 100644 docs/en/formats/tabseparatedwithnamesandtypes.md mode change 100755 => 100644 docs/en/formats/tskv.md mode change 100755 => 100644 docs/en/formats/values.md mode change 100755 => 100644 docs/en/formats/vertical.md mode change 100755 => 100644 docs/en/formats/xml.md mode change 100755 => 100644 docs/en/functions/arithmetic_functions.md mode change 100755 => 100644 docs/en/functions/array_functions.md mode change 100755 => 100644 docs/en/functions/array_join.md mode change 100755 => 100644 docs/en/functions/bit_functions.md mode change 100755 => 100644 docs/en/functions/comparison_functions.md mode change 100755 => 100644 docs/en/functions/conditional_functions.md mode change 100755 => 100644 docs/en/functions/date_time_functions.md mode change 100755 => 100644 docs/en/functions/encoding_functions.md mode change 100755 => 100644 docs/en/functions/ext_dict_functions.md mode change 100755 => 100644 docs/en/functions/hash_functions.md mode change 100755 => 100644 docs/en/functions/higher_order_functions.md mode change 100755 => 100644 docs/en/functions/in_functions.md mode change 100755 => 100644 docs/en/functions/index.md mode change 100755 => 100644 docs/en/functions/ip_address_functions.md mode change 100755 => 100644 docs/en/functions/json_functions.md mode change 100755 => 100644 docs/en/functions/logical_functions.md mode change 100755 => 100644 docs/en/functions/math_functions.md mode change 100755 => 100644 docs/en/functions/other_functions.md mode change 100755 => 100644 docs/en/functions/random_functions.md mode change 100755 => 100644 docs/en/functions/rounding_functions.md mode change 100755 => 100644 docs/en/functions/splitting_merging_functions.md mode change 100755 => 100644 docs/en/functions/string_functions.md mode change 100755 => 100644 docs/en/functions/string_replace_functions.md mode change 100755 => 100644 docs/en/functions/string_search_functions.md mode change 100755 => 100644 docs/en/functions/type_conversion_functions.md mode change 100755 => 100644 docs/en/functions/url_functions.md mode change 100755 => 100644 docs/en/functions/ym_dict_functions.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/amplab_benchmark.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/criteo.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/nyc_taxi.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/ontime.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/star_schema.md mode change 100755 => 100644 docs/en/getting_started/example_datasets/wikistat.md mode change 100755 => 100644 docs/en/getting_started/index.md mode change 100755 => 100644 docs/en/index.md mode change 100755 => 100644 docs/en/interfaces/cli.md mode change 100755 => 100644 docs/en/interfaces/http_interface.md mode change 100755 => 100644 docs/en/interfaces/index.md mode change 100755 => 100644 docs/en/interfaces/jdbc.md mode change 100755 => 100644 docs/en/interfaces/tcp.md mode change 100755 => 100644 docs/en/interfaces/third-party_client_libraries.md mode change 100755 => 100644 docs/en/interfaces/third-party_gui.md mode change 100755 => 100644 docs/en/introduction/distinctive_features.md mode change 100755 => 100644 docs/en/introduction/features_considered_disadvantages.md mode change 100755 => 100644 docs/en/introduction/index.md mode change 100755 => 100644 docs/en/introduction/performance.md mode change 100755 => 100644 docs/en/introduction/possible_silly_questions.md mode change 100755 => 100644 docs/en/introduction/ya_metrika_task.md mode change 100755 => 100644 docs/en/operations/access_rights.md mode change 100755 => 100644 docs/en/operations/configuration_files.md mode change 100755 => 100644 docs/en/operations/index.md mode change 100755 => 100644 docs/en/operations/quotas.md mode change 100755 => 100644 docs/en/operations/server_settings/index.md mode change 100755 => 100644 docs/en/operations/server_settings/settings.md mode change 100755 => 100644 docs/en/operations/settings/index.md mode change 100755 => 100644 docs/en/operations/settings/query_complexity.md mode change 100755 => 100644 docs/en/operations/settings/settings.md mode change 100755 => 100644 docs/en/operations/settings/settings_profiles.md mode change 100755 => 100644 docs/en/operations/tips.md mode change 100755 => 100644 docs/en/operators/index.md mode change 100755 => 100644 docs/en/query_language/index.md mode change 100755 => 100644 docs/en/query_language/queries.md mode change 100755 => 100644 docs/en/query_language/syntax.md mode change 100755 => 100644 docs/en/roadmap.md mode change 100755 => 100644 docs/en/system_tables/index.md mode change 100755 => 100644 docs/en/system_tables/system.asynchronous_metrics.md mode change 100755 => 100644 docs/en/system_tables/system.clusters.md mode change 100755 => 100644 docs/en/system_tables/system.columns.md mode change 100755 => 100644 docs/en/system_tables/system.databases.md mode change 100755 => 100644 docs/en/system_tables/system.dictionaries.md mode change 100755 => 100644 docs/en/system_tables/system.events.md mode change 100755 => 100644 docs/en/system_tables/system.functions.md mode change 100755 => 100644 docs/en/system_tables/system.merges.md mode change 100755 => 100644 docs/en/system_tables/system.numbers.md mode change 100755 => 100644 docs/en/system_tables/system.numbers_mt.md mode change 100755 => 100644 docs/en/system_tables/system.one.md mode change 100755 => 100644 docs/en/system_tables/system.parts.md mode change 100755 => 100644 docs/en/system_tables/system.processes.md mode change 100755 => 100644 docs/en/system_tables/system.replicas.md mode change 100755 => 100644 docs/en/system_tables/system.settings.md mode change 100755 => 100644 docs/en/system_tables/system.tables.md mode change 100755 => 100644 docs/en/system_tables/system.zookeeper.md mode change 100755 => 100644 docs/en/table_engines/aggregatingmergetree.md mode change 100755 => 100644 docs/en/table_engines/buffer.md mode change 100755 => 100644 docs/en/table_engines/collapsingmergetree.md mode change 100755 => 100644 docs/en/table_engines/custom_partitioning_key.md delete mode 100755 docs/en/table_engines/dictionary.md mode change 100755 => 100644 docs/en/table_engines/distributed.md mode change 100755 => 100644 docs/en/table_engines/external_data.md mode change 100755 => 100644 docs/en/table_engines/file.md mode change 100755 => 100644 docs/en/table_engines/graphitemergetree.md mode change 100755 => 100644 docs/en/table_engines/index.md mode change 100755 => 100644 docs/en/table_engines/join.md mode change 100755 => 100644 docs/en/table_engines/kafka.md mode change 100755 => 100644 docs/en/table_engines/log.md mode change 100755 => 100644 docs/en/table_engines/materializedview.md mode change 100755 => 100644 docs/en/table_engines/memory.md mode change 100755 => 100644 docs/en/table_engines/merge.md mode change 100755 => 100644 docs/en/table_engines/mergetree.md mode change 100755 => 100644 docs/en/table_engines/null.md mode change 100755 => 100644 docs/en/table_engines/replacingmergetree.md mode change 100755 => 100644 docs/en/table_engines/replication.md mode change 100755 => 100644 docs/en/table_engines/set.md mode change 100755 => 100644 docs/en/table_engines/summingmergetree.md mode change 100755 => 100644 docs/en/table_engines/tinylog.md mode change 100755 => 100644 docs/en/table_engines/view.md mode change 100755 => 100644 docs/en/table_functions/index.md mode change 100755 => 100644 docs/en/table_functions/merge.md mode change 100755 => 100644 docs/en/table_functions/remote.md mode change 100755 => 100644 docs/en/utils/clickhouse-copier.md mode change 100755 => 100644 docs/en/utils/clickhouse-local.md mode change 100755 => 100644 docs/en/utils/index.md diff --git a/docs/en/agg_functions/combinators.md b/docs/en/agg_functions/combinators.md old mode 100755 new mode 100644 diff --git a/docs/en/agg_functions/index.md b/docs/en/agg_functions/index.md old mode 100755 new mode 100644 index 3864f7271c4..e87bf4ff833 --- a/docs/en/agg_functions/index.md +++ b/docs/en/agg_functions/index.md @@ -8,4 +8,3 @@ ClickHouse also supports: - [Parametric aggregate functions](parametric_functions.md#aggregate_functions_parametric), which accept other parameters in addition to columns. - [Combinators](combinators.md#aggregate_functions_combinators), which change the behavior of aggregate functions. - diff --git a/docs/en/agg_functions/parametric_functions.md b/docs/en/agg_functions/parametric_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/agg_functions/reference.md b/docs/en/agg_functions/reference.md old mode 100755 new mode 100644 index 0eb896e4664..90ff9da58e7 --- a/docs/en/agg_functions/reference.md +++ b/docs/en/agg_functions/reference.md @@ -19,7 +19,7 @@ In some cases, you can rely on the order of execution. This applies to cases whe When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function. -## anyHeavy +## anyHeavy(x) Selects a frequently occurring value using the [heavy hitters](http://www.cs.umd.edu/~samir/498/karp.pdf) algorithm. If there is a value that occurs more than in half the cases in each of the query's execution threads, this value is returned. Normally, the result is nondeterministic. @@ -39,6 +39,7 @@ Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets SELECT anyHeavy(AirlineID) AS res FROM ontime ``` + ``` ┌───res─┐ │ 19690 │ @@ -124,11 +125,11 @@ The result is always Float64. Calculates the approximate number of different values of the argument. Works for numbers, strings, dates, date-with-time, and for multiple arguments and tuple arguments. Uses an adaptive sampling algorithm: for the calculation state, it uses a sample of element hash values with a size up to 65536. -This algorithm is also very accurate for data sets with low cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions). +This algorithm is also very accurate for data sets with small cardinality (up to 65536) and very efficient on CPU (when computing not too many of these functions, using `uniq` is almost as fast as using other aggregate functions). The result is determinate (it doesn't depend on the order of query processing). -This function provides excellent accuracy even for data sets with extremely high cardinality (over 10 billion elements). It is recommended for default use. +This function provides excellent accuracy even for data sets with huge cardinality (10B+ elements) and is recommended for use by default. ## uniqCombined(x) @@ -138,16 +139,16 @@ A combination of three algorithms is used: array, hash table and [HyperLogLog](h The result is determinate (it doesn't depend on the order of query processing). -The `uniqCombined` function is a good default choice for calculating the number of different values, but keep in mind that the estimation error will increase for high-cardinality data sets (200M+ elements), and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). +The `uniqCombined` function is a good default choice for calculating the number of different values, but the following should be considered: for data sets with large cardinality (200M+) error of estimate will only grow and for data sets with huge cardinality(1B+ elements) it returns result with high inaccuracy. ## uniqHLL12(x) Uses the [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm to approximate the number of different values of the argument. -212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements). +212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. Result is not very accurate (error up to ~10%) for data sets of small cardinality(<10K elements), but for data sets with large cardinality (10K - 100M) result is quite accurate (error up to ~1.6%) and after that error of estimate will only grow and for data sets with huge cardinality (1B+ elements) it returns result with high inaccuracy. The result is determinate (it doesn't depend on the order of query processing). -We don't recommend using this function. In most cases, use the `uniq` or `uniqCombined` function. +This function is not recommended for use, and in most cases, use the `uniq` or `uniqCombined` function. ## uniqExact(x) @@ -169,7 +170,7 @@ In some cases, you can still rely on the order of execution. This applies to cas -## groupArrayInsertAt +## groupArrayInsertAt(x) Inserts a value into the array in the specified position. @@ -235,8 +236,8 @@ For its purpose (calculating quantiles of page loading times), using this functi ## quantileTimingWeighted(level)(x, weight) -Differs from the `quantileTiming` function in that it has a second argument, "weights". Weight is a non-negative integer. -The result is calculated as if the `x` value were passed `weight` number of times to the `quantileTiming` function. +Differs from the 'quantileTiming' function in that it has a second argument, "weights". Weight is a non-negative integer. +The result is calculated as if the 'x' value were passed 'weight' number of times to the 'quantileTiming' function. ## quantileExact(level)(x) @@ -256,7 +257,7 @@ The performance of the function is lower than for ` quantile`, ` quantileTiming` The result depends on the order of running the query, and is nondeterministic. -## median +## median(x) All the quantile functions have corresponding median functions: `median`, `medianDeterministic`, `medianTiming`, `medianTimingWeighted`, `medianExact`, `medianExactWeighted`, `medianTDigest`. They are synonyms and their behavior is identical. @@ -274,7 +275,7 @@ Returns `Float64`. When `n <= 1`, returns `+∞`. ## varPop(x) -Calculates the amount `Σ((x - x̅)^2) / (n - 1)`, where `n` is the sample size and `x̅`is the average value of `x`. +Calculates the amount `Σ((x - x̅)^2) / n`, where `n` is the sample size and `x̅`is the average value of `x`. In other words, dispersion for a set of values. Returns `Float64`. @@ -286,33 +287,30 @@ The result is equal to the square root of `varSamp(x)`. The result is equal to the square root of `varPop(x)`. -## topK +## topK(N)(column) Returns an array of the most frequent values in the specified column. The resulting array is sorted in descending order of frequency of values (not by the values themselves). Implements the [ Filtered Space-Saving](http://www.l2f.inesc-id.pt/~fmmb/wiki/uploads/Work/misnis.ref0a.pdf) algorithm for analyzing TopK, based on the reduce-and-combine algorithm from [Parallel Space Saving](https://arxiv.org/pdf/1401.0702.pdf). -``` -topK(N)(column) -``` - This function doesn't provide a guaranteed result. In certain situations, errors might occur and it might return frequent values that aren't the most frequent values. We recommend using the `N < 10 ` value; performance is reduced with large `N` values. Maximum value of ` N = 65536`. **Arguments** -- 'N' is the number of values. +- 'N' – The number of values. - ' x ' – The column. **Example** -Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime) data set and select the three most frequently occurring values in the `AirlineID` column. +Take the [OnTime](../getting_started/example_datasets/ontime.md#example_datasets-ontime)data set and select the three most frequently occurring values in the `AirlineID` column. ```sql SELECT topK(3)(AirlineID) AS res FROM ontime ``` + ``` ┌─res─────────────────┐ │ [19393,19790,19805] │ diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md old mode 100755 new mode 100644 index cb179c0d8c4..355e5555bfa --- a/docs/en/data_types/date.md +++ b/docs/en/data_types/date.md @@ -1,6 +1,6 @@ # Date -A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105). +Date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2038, but it may be expanded to 2106). The minimum value is output as 0000-00-00. The date is stored without the time zone. diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md old mode 100755 new mode 100644 index 9d5cc2c01bb..c6d12999604 --- a/docs/en/data_types/float.md +++ b/docs/en/data_types/float.md @@ -4,8 +4,8 @@ Types are equivalent to types of C: -- `Float32` - `float` -- `Float64` - ` double` +- `Float32` - `float`; +- `Float64` - ` double`. We recommend that you store data in integer form whenever possible. For example, convert fixed precision numbers to integer values, such as monetary amounts or page load times in milliseconds. @@ -24,7 +24,9 @@ SELECT 1 - 0.9 ``` - The result of the calculation depends on the calculation method (the processor type and architecture of the computer system). + - Floating-point calculations might result in numbers such as infinity (`Inf`) and "not-a-number" (`NaN`). This should be taken into account when processing the results of calculations. + - When reading floating point numbers from rows, the result might not be the nearest machine-representable number. ## NaN and Inf @@ -42,7 +44,6 @@ SELECT 0.5 / 0 │ inf │ └────────────────┘ ``` - - `-Inf` – Negative infinity. ```sql @@ -54,7 +55,6 @@ SELECT -0.5 / 0 │ -inf │ └─────────────────┘ ``` - - `NaN` – Not a number. ``` @@ -67,5 +67,5 @@ SELECT 0 / 0 └──────────────┘ ``` -See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by). + See the rules for ` NaN` sorting in the section [ORDER BY clause](../query_language/queries.md#query_language-queries-order_by). diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md old mode 100755 new mode 100644 index 4008a872161..c17b51c08a2 --- a/docs/en/data_types/index.md +++ b/docs/en/data_types/index.md @@ -2,7 +2,6 @@ # Data types -ClickHouse can store various types of data in table cells. - -This section describes the supported data types and special considerations when using and/or implementing them, if any. +ClickHouse table fields can contain data of different types. +The topic contains descriptions of data types supported and specificity of their usage of implementation if exists. \ No newline at end of file diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.md b/docs/en/data_types/nested_data_structures/aggregatefunction.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/special_data_types/expression.md b/docs/en/data_types/special_data_types/expression.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/special_data_types/index.md b/docs/en/data_types/special_data_types/index.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/special_data_types/set.md b/docs/en/data_types/special_data_types/set.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/string.md b/docs/en/data_types/string.md old mode 100755 new mode 100644 diff --git a/docs/en/data_types/tuple.md b/docs/en/data_types/tuple.md old mode 100755 new mode 100644 diff --git a/docs/en/development/style.md b/docs/en/development/style.md old mode 100755 new mode 100644 index 700fede5373..ef6490187c8 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -4,7 +4,7 @@ 1. The following are recommendations, not requirements. 2. If you are editing code, it makes sense to follow the formatting of the existing code. -3. Code style is needed for consistency. Consistency makes it easier to read the code, and it also makes it easier to search the code. +3. Code style is needed for consistency. Consistency makes it easier to read the code. and it also makes it easier to search the code. 4. Many of the rules do not have logical reasons; they are dictated by established practices. ## Formatting @@ -93,25 +93,25 @@ 14. In classes and structures, public, private, and protected are written on the same level as the class/struct, but all other internal elements should be deeper. ```cpp - template -class MultiVersion -{ -public: - /// Version of object for usage. shared_ptr manage lifetime of version. - using Version = std::shared_ptr; - ... -} + template > + class MultiVersion + { + public: + /// The specific version of the object to use. + using Version = Ptr; + ... + } ``` 15. If the same namespace is used for the entire file, and there isn't anything else significant, an offset is not necessary inside namespace. 16. If the block for if, for, while... expressions consists of a single statement, you don't need to use curly brackets. Place the statement on a separate line, instead. The same is true for a nested if, for, while... statement. But if the inner statement contains curly brackets or else, the external block should be written in curly brackets. - ```cpp - /// Finish write. -for (auto & stream : streams) - stream.second->finalize(); - ``` + ```cpp + /// Finish write. + for (auto & stream : streams) + stream.second->finalize(); + ``` 17. There should be any spaces at the ends of lines. @@ -218,11 +218,11 @@ for (auto & stream : streams) */ void executeQuery( ReadBuffer & istr, /// Where to read the query from (and data for INSERT, if applicable) - WriteBuffer & ostr, /// Where to write the result - Context & context, /// DB, tables, data types, engines, functions, aggregate functions... - BlockInputStreamPtr & query_plan, /// A description of query processing can be included here - QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to - ) + WriteBuffer & ostr, /// Where to write the result + Context & context, /// DB, tables, data types, engines, functions, aggregate functions... + BlockInputStreamPtr & query_plan, /// A description of query processing can be included here + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// The last stage to process the SELECT query to + ) ``` 4. Comments should be written in English only. @@ -252,7 +252,7 @@ for (auto & stream : streams) */ ``` - (the example is borrowed from the resource [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) + (Example taken from: [http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/)](http://home.tamk.fi/~jaalto/course/coding-style/doc/unmaintainable-code/) 7. Do not write garbage comments (author, creation date ..) at the beginning of each file. @@ -497,15 +497,7 @@ This is not recommended, but it is allowed. You can create a separate code block inside a single function in order to make certain variables local, so that the destructors are called when exiting the block. ```cpp - Block block = data.in->read(); - - { - std::lock_guard lock(mutex); - data.ready = true; - data.block = block; - } - - ready_any.set(); + Block block = data.in->read();{ std::lock_guard lock(mutex); data.ready = true; data.block = block;}ready_any.set(); ``` 7. Multithreading. @@ -568,12 +560,13 @@ This is not recommended, but it is allowed. ```cpp using AggregateFunctionPtr = std::shared_ptr; - - /** Creates an aggregate function by name. */ + + /** Creates an aggregate function by name. + */ class AggregateFunctionFactory { public: - AggregateFunctionFactory(); + AggregateFunctionFactory(); AggregateFunctionPtr get(const String & name, const DataTypes & argument_types) const; ``` @@ -598,10 +591,10 @@ This is not recommended, but it is allowed. If later you’ll need to delay initialization, you can add a default constructor that will create an invalid object. Or, for a small number of objects, you can use shared_ptr/unique_ptr. ```cpp - Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); - - /// For delayed initialization - Loader() {} + Loader(DB::Connection * connection_, const std::string & query, size_t max_block_size_); + + /// For delayed initialization + Loader() {} ``` 17. Virtual functions. diff --git a/docs/en/dicts/external_dicts.md b/docs/en/dicts/external_dicts.md old mode 100755 new mode 100644 index a6af84a313f..b99b02bbf57 --- a/docs/en/dicts/external_dicts.md +++ b/docs/en/dicts/external_dicts.md @@ -21,11 +21,12 @@ The dictionary config file has the following format: /etc/metrika.xml + - + - + ... diff --git a/docs/en/dicts/external_dicts_dict.md b/docs/en/dicts/external_dicts_dict.md old mode 100755 new mode 100644 index 6d2f4128704..4133b036e1f --- a/docs/en/dicts/external_dicts_dict.md +++ b/docs/en/dicts/external_dicts_dict.md @@ -27,8 +27,7 @@ The dictionary configuration has the following structure: ``` - name – The identifier that can be used to access the dictionary. Use the characters `[a-zA-Z0-9_\-]`. -- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Source of the dictionary . -- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) — Dictionary layout in memory. -- [source](external_dicts_dict_sources.html/#dicts-external_dicts_dict_sources) — Structure of the dictionary . A key and attributes that can be retrieved by this key. -- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) — Frequency of dictionary updates. - +- [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) – Source of the dictionary. +- [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) – Location of the dictionary in memory. +- [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure) – Structure of the dictionary. A key and attributes that can be retrieved by this key. +- [lifetime](external_dicts_dict_lifetime.md#dicts-external_dicts_dict_lifetime) – How frequently to update dictionaries. diff --git a/docs/en/dicts/external_dicts_dict_layout.md b/docs/en/dicts/external_dicts_dict_layout.md old mode 100755 new mode 100644 index 8b7cad24b65..ad635db94f5 --- a/docs/en/dicts/external_dicts_dict_layout.md +++ b/docs/en/dicts/external_dicts_dict_layout.md @@ -2,11 +2,11 @@ # Storing dictionaries in memory -There are a [variety of ways](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory. +There are [many different ways](external_dicts_dict_layout#dicts-external_dicts_dict_layout-manner) to store dictionaries in memory. -We recommend [flat](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-hashed)and[complex_key_hashed](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed. +We recommend [flat](external_dicts_dict_layout#dicts-external_dicts_dict_layout-flat), [hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-hashed), and [complex_key_hashed](external_dicts_dict_layout#dicts-external_dicts_dict_layout-complex_key_hashed). which provide optimal processing speed. -Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more in the section " [cache](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout-cache)". +Caching is not recommended because of potentially poor performance and difficulties in selecting optimal parameters. Read more about this in the "[cache](external_dicts_dict_layout#dicts-external_dicts_dict_layout-cache)" section. There are several ways to improve dictionary performance: @@ -46,6 +46,7 @@ The configuration looks like this: - [range_hashed](#dicts-external_dicts_dict_layout-range_hashed) - [complex_key_hashed](#dicts-external_dicts_dict_layout-complex_key_hashed) - [complex_key_cache](#dicts-external_dicts_dict_layout-complex_key_cache) +- [ip_trie](#dicts-external_dicts_dict_layout-ip_trie) @@ -87,7 +88,7 @@ Configuration example: ### complex_key_hashed -This type is for use with composite [keys](external_dicts_dict_structure.md/#dicts-external_dicts_dict_structure). Similar to `hashed`. +This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). It is similar to hashed. Configuration example: @@ -108,18 +109,18 @@ This storage method works the same way as hashed and allows using date/time rang Example: The table contains discounts for each advertiser in the format: ``` -+---------------+---------------------+-------------------+--------+ -| advertiser id | discount start date | discount end date | amount | -+===============+=====================+===================+========+ -| 123 | 2015-01-01 | 2015-01-15 | 0.15 | -+---------------+---------------------+-------------------+--------+ -| 123 | 2015-01-16 | 2015-01-31 | 0.25 | -+---------------+---------------------+-------------------+--------+ -| 456 | 2015-01-01 | 2015-01-15 | 0.05 | -+---------------+---------------------+-------------------+--------+ + +---------------+---------------------+-------------------+--------+ + | advertiser id | discount start date | discount end date | amount | + +===============+=====================+===================+========+ + | 123 | 2015-01-01 | 2015-01-15 | 0.15 | + +---------------+---------------------+-------------------+--------+ + | 123 | 2015-01-16 | 2015-01-31 | 0.25 | + +---------------+---------------------+-------------------+--------+ + | 456 | 2015-01-01 | 2015-01-15 | 0.05 | + +---------------+---------------------+-------------------+--------+ ``` -To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). +To use a sample for date ranges, define `range_min` and `range_max` in [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Example: @@ -196,15 +197,15 @@ This is the least effective of all the ways to store dictionaries. The speed of To improve cache performance, use a subquery with ` LIMIT`, and call the function with the dictionary externally. -Supported [sources](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP. +Supported [sources](external_dicts_dict_sources#dicts-external_dicts_dict_sources): MySQL, ClickHouse, executable, HTTP. Example of settings: ```xml - - 1000000000 + + 1000000000 ``` @@ -226,4 +227,66 @@ Do not use ClickHouse as a source, because it is slow to process queries with ra ### complex_key_cache -This type of storage is for use with composite [keys](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure). Similar to `cache`. +This type of storage is designed for use with compound [keys](external_dicts_dict_structure#dicts-external_dicts_dict_structure). Similar to `cache`. + + + +### ip_trie + + +The table stores IP prefixes for each key (IP address), which makes it possible to map IP addresses to metadata such as ASN or threat score. + +Example: in the table there are prefixes matches to AS number and country: + +``` + +-----------------+-------+--------+ + | prefix | asn | cca2 | + +=================+=======+========+ + | 202.79.32.0/20 | 17501 | NP | + +-----------------+-------+--------+ + | 2620:0:870::/48 | 3856 | US | + +-----------------+-------+--------+ + | 2a02:6b8:1::/48 | 13238 | RU | + +-----------------+-------+--------+ + | 2001:db8::/32 | 65536 | ZZ | + +-----------------+-------+--------+ +``` + +When using such a layout, the structure should have the "key" element. + +Example: + +```xml + + + + prefix + String + + + + asn + UInt32 + + + + cca2 + String + ?? + + ... +``` + +These key must have only one attribute of type String, containing a valid IP prefix. Other types are not yet supported. + +For querying, same functions (dictGetT with tuple) as for complex key dictionaries have to be used: + + dictGetT('dict_name', 'attr_name', tuple(ip)) + +The function accepts either UInt32 for IPv4 address or FixedString(16) for IPv6 address in wire format: + + dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) + +No other type is supported. The function returns attribute for a prefix matching the given IP address. If there are overlapping prefixes, the most specific one is returned. + +The data is stored currently in a bitwise trie, it has to fit in memory. diff --git a/docs/en/dicts/external_dicts_dict_lifetime.md b/docs/en/dicts/external_dicts_dict_lifetime.md old mode 100755 new mode 100644 index 52ee7a4aa78..6431fb3de48 --- a/docs/en/dicts/external_dicts_dict_lifetime.md +++ b/docs/en/dicts/external_dicts_dict_lifetime.md @@ -36,13 +36,13 @@ Example of settings: When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources): > - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. -- For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. -- Dictionaries from other sources are updated every time by default. +> - For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. +> - Dictionaries from other sources are updated every time by default. For MySQL (InnoDB) and ODBC sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: > - The dictionary table must have a field that always changes when the source data is updated. -- The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `` field in the settings for the [source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources). +> - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. The query must be specified in the `` field in the [ source](external_dicts_dict_sources.md#dicts-external_dicts_dict_sources) settings. Example of settings: diff --git a/docs/en/dicts/external_dicts_dict_sources.md b/docs/en/dicts/external_dicts_dict_sources.md old mode 100755 new mode 100644 index 6cb4e0ea44d..721302cd556 --- a/docs/en/dicts/external_dicts_dict_sources.md +++ b/docs/en/dicts/external_dicts_dict_sources.md @@ -80,7 +80,7 @@ Setting fields: ## HTTP(s) -Working with an HTTP(s) server depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method. +Working with executable files depends on [how the dictionary is stored in memory](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request via the `POST` method. Example of settings: @@ -135,9 +135,9 @@ Installing unixODBC and the ODBC driver for PostgreSQL: Configuring `/etc/odbc.ini` (or `~/.odbc.ini`): ``` -[DEFAULT] + [DEFAULT] Driver = myconnection - + [myconnection] Description = PostgreSQL connection to my_db Driver = PostgreSQL Unicode diff --git a/docs/en/dicts/external_dicts_dict_structure.md b/docs/en/dicts/external_dicts_dict_structure.md old mode 100755 new mode 100644 index 2542af00ec6..5a6d349b350 --- a/docs/en/dicts/external_dicts_dict_structure.md +++ b/docs/en/dicts/external_dicts_dict_structure.md @@ -25,8 +25,8 @@ Overall structure: Columns are described in the structure: -- `` - [key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key). -- `` - [data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns. +- `` – [Key column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-key). +- `` – [Data column](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). There can be a large number of columns. @@ -63,10 +63,12 @@ Configuration fields: ### Composite key -The key can be a `tuple` from any types of fields. The [layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`. +The key can be a `tuple` from any types of fields. The [ layout](external_dicts_dict_layout.md#dicts-external_dicts_dict_layout) in this case must be `complex_key_hashed` or `complex_key_cache`.
-A composite key can consist of a single element. This makes it possible to use a string as the key, for instance. + +A composite key can also consist of a single element, which makes it possible to use a string as the key, for instance. +
The key structure is set in the element ``. Key fields are specified in the same format as the dictionary [attributes](external_dicts_dict_structure.md#dicts-external_dicts_dict_structure-attributes). Example: @@ -117,6 +119,6 @@ Configuration fields: - `null_value` – The default value for a non-existing element. In the example, it is an empty string. - `expression` – The attribute can be an expression. The tag is not required. - `hierarchical` – Hierarchical support. Mirrored to the parent identifier. By default, ` false`. -- `injective` – Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`. -- `is_object_id` – Whether the query is executed for a MongoDB document by `ObjectID`. +- `injective` Whether the `id -> attribute` image is injective. If ` true`, then you can optimize the ` GROUP BY` clause. By default, `false`. +- `is_object_id` - Used for query mongo documents by ObjectId diff --git a/docs/en/dicts/index.md b/docs/en/dicts/index.md old mode 100755 new mode 100644 diff --git a/docs/en/dicts/internal_dicts.md b/docs/en/dicts/internal_dicts.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/capnproto.md b/docs/en/formats/capnproto.md old mode 100755 new mode 100644 index 918197b2bd9..0d482e20887 --- a/docs/en/formats/capnproto.md +++ b/docs/en/formats/capnproto.md @@ -1,26 +1,26 @@ - - -# CapnProto - -Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack. - -Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query. - -```sql -SELECT SearchPhrase, count() AS c FROM test.hits - GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' -``` - -Where `schema.capnp` looks like this: - -``` -struct Message { - SearchPhrase @0 :Text; - c @1 :Uint64; -} -``` - -Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration. - -Deserialization is effective and usually doesn't increase the system load. - + + +# CapnProto + +Cap'n Proto is a binary message format similar to Protocol Buffers and Thrift, but not like JSON or MessagePack. + +Cap'n Proto messages are strictly typed and not self-describing, meaning they need an external schema description. The schema is applied on the fly and cached for each query. + +```sql +SELECT SearchPhrase, count() AS c FROM test.hits + GROUP BY SearchPhrase FORMAT CapnProto SETTINGS schema = 'schema:Message' +``` + +Where `schema.capnp` looks like this: + +``` +struct Message { + SearchPhrase @0 :Text; + c @1 :Uint64; +} +``` + +Schema files are in the file that is located in the directory specified in [ format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) in the server configuration. + +Deserialization is effective and usually doesn't increase the system load. + diff --git a/docs/en/formats/csv.md b/docs/en/formats/csv.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/csvwithnames.md b/docs/en/formats/csvwithnames.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/index.md b/docs/en/formats/index.md old mode 100755 new mode 100644 index 112a13ff5e5..815a2d060cb --- a/docs/en/formats/index.md +++ b/docs/en/formats/index.md @@ -3,4 +3,3 @@ # Formats The format determines how data is returned to you after SELECTs (how it is written and formatted by the server), and how it is accepted for INSERTs (how it is read and parsed by the server). - diff --git a/docs/en/formats/json.md b/docs/en/formats/json.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/jsoncompact.md b/docs/en/formats/jsoncompact.md old mode 100755 new mode 100644 index d870b6dff08..e4ce0867bc2 --- a/docs/en/formats/jsoncompact.md +++ b/docs/en/formats/jsoncompact.md @@ -24,7 +24,7 @@ Example: ["bathroom interior design", "2166"], ["yandex", "1655"], ["spring 2014 fashion", "1549"], - ["freeform photo", "1480"] + ["freeform photos", "1480"] ], "totals": ["","8873898"], diff --git a/docs/en/formats/jsoneachrow.md b/docs/en/formats/jsoneachrow.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/native.md b/docs/en/formats/native.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/null.md b/docs/en/formats/null.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/pretty.md b/docs/en/formats/pretty.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/prettycompact.md b/docs/en/formats/prettycompact.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/prettycompactmonoblock.md b/docs/en/formats/prettycompactmonoblock.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/prettynoescapes.md b/docs/en/formats/prettynoescapes.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/prettyspace.md b/docs/en/formats/prettyspace.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/rowbinary.md b/docs/en/formats/rowbinary.md old mode 100755 new mode 100644 index aeb3df4c8a8..bc8479332ba --- a/docs/en/formats/rowbinary.md +++ b/docs/en/formats/rowbinary.md @@ -9,5 +9,5 @@ Date is represented as a UInt16 object that contains the number of days since 19 String is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the bytes of the string. FixedString is represented simply as a sequence of bytes. -Array is represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by successive elements of the array. +Arrays are represented as a varint length (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), followed by the array elements in order. diff --git a/docs/en/formats/tabseparated.md b/docs/en/formats/tabseparated.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/tabseparatedraw.md b/docs/en/formats/tabseparatedraw.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/tabseparatedwithnames.md b/docs/en/formats/tabseparatedwithnames.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/tabseparatedwithnamesandtypes.md b/docs/en/formats/tabseparatedwithnamesandtypes.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/tskv.md b/docs/en/formats/tskv.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/values.md b/docs/en/formats/values.md old mode 100755 new mode 100644 index a672723f33d..2e929369848 --- a/docs/en/formats/values.md +++ b/docs/en/formats/values.md @@ -4,5 +4,5 @@ Prints every row in brackets. Rows are separated by commas. There is no comma af The minimum set of characters that you need to escape when passing data in Values ​​format: single quotes and backslashes. -This is the format that is used in `INSERT INTO t VALUES ...`, but you can also use it for formatting query results. +This is the format that is used in `INSERT INTO t VALUES ...` but you can also use it for query result. diff --git a/docs/en/formats/vertical.md b/docs/en/formats/vertical.md old mode 100755 new mode 100644 diff --git a/docs/en/formats/xml.md b/docs/en/formats/xml.md old mode 100755 new mode 100644 index 0da55875cc3..f91adec9356 --- a/docs/en/formats/xml.md +++ b/docs/en/formats/xml.md @@ -35,7 +35,7 @@ XML format is suitable only for output, not for parsing. Example: 1549 - freeform photo + freeform photos 1480 diff --git a/docs/en/functions/arithmetic_functions.md b/docs/en/functions/arithmetic_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/array_functions.md b/docs/en/functions/array_functions.md old mode 100755 new mode 100644 index 232f6a20427..493a465ac82 --- a/docs/en/functions/array_functions.md +++ b/docs/en/functions/array_functions.md @@ -39,7 +39,7 @@ Accepts an empty array and returns a one-element array that is equal to the defa Returns an array of numbers from 0 to N-1. Just in case, an exception is thrown if arrays with a total length of more than 100,000,000 elements are created in a data block. -## array(x1, ...), оператор \[x1, ...\] +## array(x1, ...), operator \[x1, ...\] Creates an array from the function arguments. The arguments must be constants and have types that have the smallest common type. At least one argument must be passed, because otherwise it isn't clear which type of array to create. That is, you can't use this function to create an empty array (to do that, use the 'emptyArray\*' function described above). @@ -62,6 +62,7 @@ arrayConcat(arrays) ```sql SELECT arrayConcat([1, 2], [3, 4], [5, 6]) AS res ``` + ``` ┌─res───────────┐ │ [1,2,3,4,5,6] │ @@ -202,6 +203,7 @@ arrayPopBack(array) ```sql SELECT arrayPopBack([1, 2, 3]) AS res ``` + ``` ┌─res───┐ │ [1,2] │ @@ -243,13 +245,14 @@ arrayPushBack(array, single_value) **Arguments** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)". +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". **Example** ```sql SELECT arrayPushBack(['a'], 'b') AS res ``` + ``` ┌─res───────┐ │ ['a','b'] │ @@ -267,7 +270,7 @@ arrayPushFront(array, single_value) **Arguments** - `array` – Array. -- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about ClickHouse data types, read the section "[Data types](../data_types/index.md#data_types)". +- `single_value` – A single value. Only numbers can be added to an array with numbers, and only strings can be added to an array of strings. When adding numbers, ClickHouse automatically sets the `single_value` type for the data type of the array. For more information about the types of data in ClickHouse, see "[Data types](../data_types/index.md#data_types)". **Example** @@ -292,7 +295,7 @@ arraySlice(array, offset[, length]) **Arguments** - `array` – Array of data. -- `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. +- `offset` – Offset from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. - `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`. **Example** @@ -300,6 +303,7 @@ arraySlice(array, offset[, length]) ```sql SELECT arraySlice([1, 2, 3, 4, 5], 2, 3) AS res ``` + ``` ┌─res─────┐ │ [2,3,4] │ diff --git a/docs/en/functions/array_join.md b/docs/en/functions/array_join.md old mode 100755 new mode 100644 index f94b2707f52..6e18f8203c0 --- a/docs/en/functions/array_join.md +++ b/docs/en/functions/array_join.md @@ -28,3 +28,4 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src │ 3 │ Hello │ [1,2,3] │ └─────┴───────────┴─────────┘ ``` + diff --git a/docs/en/functions/bit_functions.md b/docs/en/functions/bit_functions.md old mode 100755 new mode 100644 index c5a032aa5d6..523413f200a --- a/docs/en/functions/bit_functions.md +++ b/docs/en/functions/bit_functions.md @@ -15,3 +15,4 @@ The result type is an integer with bits equal to the maximum bits of its argumen ## bitShiftLeft(a, b) ## bitShiftRight(a, b) + diff --git a/docs/en/functions/comparison_functions.md b/docs/en/functions/comparison_functions.md old mode 100755 new mode 100644 index 9b95966ba84..e37642d42ed --- a/docs/en/functions/comparison_functions.md +++ b/docs/en/functions/comparison_functions.md @@ -15,7 +15,7 @@ For example, you can't compare a date with a string. You have to use a function Strings are compared by bytes. A shorter string is smaller than all strings that start with it and that contain at least one more character. -Note. Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 > -1. This behavior changed in version 1.1.54134 and is now mathematically correct. +Note: Up until version 1.1.54134, signed and unsigned numbers were compared the same way as in C++. In other words, you could get an incorrect result in cases like SELECT 9223372036854775807 > -1. This behavior changed in version 1.1.54134 and is now mathematically correct. ## equals, a = b and a == b operator diff --git a/docs/en/functions/conditional_functions.md b/docs/en/functions/conditional_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/date_time_functions.md b/docs/en/functions/date_time_functions.md old mode 100755 new mode 100644 index a7529e5f0e1..da6ad9b4a7c --- a/docs/en/functions/date_time_functions.md +++ b/docs/en/functions/date_time_functions.md @@ -79,10 +79,6 @@ Rounds down a date with time to the start of the minute. Rounds down a date with time to the start of the hour. -## toStartOfFifteenMinutes - -Rounds down the date with time to the start of the fifteen-minute interval. - Note: If you need to round a date with time to any other number of seconds, minutes, or hours, you can convert it into a number by using the toUInt32 function, then round the number using intDiv and multiplication, and convert it back using the toDateTime function. ## toStartOfHour diff --git a/docs/en/functions/encoding_functions.md b/docs/en/functions/encoding_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/ext_dict_functions.md b/docs/en/functions/ext_dict_functions.md old mode 100755 new mode 100644 index 002e2f55845..949e805d9ab --- a/docs/en/functions/ext_dict_functions.md +++ b/docs/en/functions/ext_dict_functions.md @@ -18,20 +18,18 @@ For information on connecting and configuring external dictionaries, see "[Exter `dictGetT('dict_name', 'attr_name', id)` -- Get the value of the attr_name attribute from the dict_name dictionary using the 'id' key. -`dict_name` and `attr_name` are constant strings. -`id`must be UInt64. +- Get the value of the attr_name attribute from the dict_name dictionary using the 'id' key.`dict_name` and `attr_name` are constant strings.`id`must be UInt64. If there is no `id` key in the dictionary, it returns the default value specified in the dictionary description. ## dictGetTOrDefault `dictGetT('dict_name', 'attr_name', id, default)` -The same as the `dictGetT` functions, but the default value is taken from the function's last argument. +Similar to the functions dictGetT, but the default value is taken from the last argument of the function. ## dictIsIn -`dictIsIn('dict_name', child_id, ancestor_id)` +`dictIsIn ('dict_name', child_id, ancestor_id)` - For the 'dict_name' hierarchical dictionary, finds out whether the 'child_id' key is located inside 'ancestor_id' (or matches 'ancestor_id'). Returns UInt8. diff --git a/docs/en/functions/hash_functions.md b/docs/en/functions/hash_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/higher_order_functions.md b/docs/en/functions/higher_order_functions.md old mode 100755 new mode 100644 index 15cc40dead1..ab9bdc50786 --- a/docs/en/functions/higher_order_functions.md +++ b/docs/en/functions/higher_order_functions.md @@ -73,7 +73,7 @@ Returns the index of the first element in the 'arr1' array for which 'func' retu ### arrayCumSum(\[func,\] arr1, ...) -Returns an array of partial sums of elements in the source array (a running sum). If the `func` function is specified, then the values of the array elements are converted by this function before summing. +Returns the cumulative sum of the array obtained from the original application of the 'func' function to each element in the 'arr' array. Example: @@ -86,3 +86,4 @@ SELECT arrayCumSum([1, 1, 1, 1]) AS res │ [1, 2, 3, 4] │ └──────────────┘ ``` + diff --git a/docs/en/functions/in_functions.md b/docs/en/functions/in_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/index.md b/docs/en/functions/index.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/ip_address_functions.md b/docs/en/functions/ip_address_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/json_functions.md b/docs/en/functions/json_functions.md old mode 100755 new mode 100644 index 90a2ddc47dd..1bf10e9cf0c --- a/docs/en/functions/json_functions.md +++ b/docs/en/functions/json_functions.md @@ -1,11 +1,11 @@ -# Functions for working with JSON +# Functions for working with JSON. In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done. The following assumptions are made: 1. The field name (function argument) must be a constant. -2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, но `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` +2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` 3. Fields are searched for on any nesting level, indiscriminately. If there are multiple matching fields, the first occurrence is used. 4. The JSON doesn't have space characters outside of string literals. @@ -47,10 +47,7 @@ Parses the string in double quotes. The value is unescaped. If unescaping failed Examples: ```text -visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0' -visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺' -visitParamExtractString('{"abc":"\\u263"}', 'abc') = '' -visitParamExtractString('{"abc":"hello}', 'abc') = '' +visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''visitParamExtractString('{"abc":"hello}', 'abc') = '' ``` There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8). diff --git a/docs/en/functions/logical_functions.md b/docs/en/functions/logical_functions.md old mode 100755 new mode 100644 index d396640a49d..4ef0fe5fd32 --- a/docs/en/functions/logical_functions.md +++ b/docs/en/functions/logical_functions.md @@ -11,3 +11,4 @@ Zero as an argument is considered "false," while any non-zero value is considere ## not, NOT operator ## xor + diff --git a/docs/en/functions/math_functions.md b/docs/en/functions/math_functions.md old mode 100755 new mode 100644 index 42e3f3e8018..d606c87a509 --- a/docs/en/functions/math_functions.md +++ b/docs/en/functions/math_functions.md @@ -97,3 +97,4 @@ The arc tangent. ## pow(x, y) xy. + diff --git a/docs/en/functions/other_functions.md b/docs/en/functions/other_functions.md old mode 100755 new mode 100644 index 8a0063750fe..befd94ecd4e --- a/docs/en/functions/other_functions.md +++ b/docs/en/functions/other_functions.md @@ -59,8 +59,7 @@ For elements in a nested data structure, the function checks for the existence o Allows building a unicode-art diagram. -`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max. -`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction. +`bar (x, min, max, width)` – Draws a band with a width proportional to (x - min) and equal to 'width' characters when x == max.`min, max` – Integer constants. The value must fit in Int64.`width` – Constant, positive number, may be a fraction. The band is drawn with accuracy to one eighth of a symbol. @@ -138,7 +137,7 @@ Example: ```sql SELECT - transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other') AS title, + transform(SearchEngineID, [2, 3], ['Yandex', 'Google'], 'Other' AS title, count() AS c FROM test.hits WHERE SearchEngineID != 0 diff --git a/docs/en/functions/random_functions.md b/docs/en/functions/random_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/rounding_functions.md b/docs/en/functions/rounding_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/splitting_merging_functions.md b/docs/en/functions/splitting_merging_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/string_functions.md b/docs/en/functions/string_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/string_replace_functions.md b/docs/en/functions/string_replace_functions.md old mode 100755 new mode 100644 index d70d8f404de..d3773504278 --- a/docs/en/functions/string_replace_functions.md +++ b/docs/en/functions/string_replace_functions.md @@ -76,3 +76,4 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res │ here: Hello, World! │ └─────────────────────┘ ``` + diff --git a/docs/en/functions/string_search_functions.md b/docs/en/functions/string_search_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/type_conversion_functions.md b/docs/en/functions/type_conversion_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/url_functions.md b/docs/en/functions/url_functions.md old mode 100755 new mode 100644 diff --git a/docs/en/functions/ym_dict_functions.md b/docs/en/functions/ym_dict_functions.md old mode 100755 new mode 100644 index 7ba7e7012cf..540b5dd601a --- a/docs/en/functions/ym_dict_functions.md +++ b/docs/en/functions/ym_dict_functions.md @@ -21,9 +21,7 @@ All functions for working with regions have an optional argument at the end – Example: ```text -regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt -regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt -regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt +regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txtregionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt ``` ### regionToCity(id[, geobase]) @@ -35,9 +33,7 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. If this regio Converts a region to an area (type 5 in the geobase). In every other way, this function is the same as 'regionToCity'. ```sql -SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua')) -FROM system.numbers -LIMIT 15 +SELECT DISTINCT regionToName(regionToArea(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 ``` ```text @@ -65,9 +61,7 @@ LIMIT 15 Converts a region to a federal district (type 4 in the geobase). In every other way, this function is the same as 'regionToCity'. ```sql -SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua')) -FROM system.numbers -LIMIT 15 +SELECT DISTINCT regionToName(regionToDistrict(toUInt32(number), 'ua'))FROM system.numbersLIMIT 15 ``` ```text diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md old mode 100755 new mode 100644 diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md old mode 100755 new mode 100644 index 3c60a68f430..9b59d6e5f3d --- a/docs/en/getting_started/example_datasets/criteo.md +++ b/docs/en/getting_started/example_datasets/criteo.md @@ -66,8 +66,6 @@ CREATE TABLE criteo Transform data from the raw log and put it in the second table: ```sql -INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log; - -DROP TABLE criteo_log; +INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int7, int8, int9, int10, int11, int12, int13, reinterpretAsUInt32(unhex(cat1)) AS icat1, reinterpretAsUInt32(unhex(cat2)) AS icat2, reinterpretAsUInt32(unhex(cat3)) AS icat3, reinterpretAsUInt32(unhex(cat4)) AS icat4, reinterpretAsUInt32(unhex(cat5)) AS icat5, reinterpretAsUInt32(unhex(cat6)) AS icat6, reinterpretAsUInt32(unhex(cat7)) AS icat7, reinterpretAsUInt32(unhex(cat8)) AS icat8, reinterpretAsUInt32(unhex(cat9)) AS icat9, reinterpretAsUInt32(unhex(cat10)) AS icat10, reinterpretAsUInt32(unhex(cat11)) AS icat11, reinterpretAsUInt32(unhex(cat12)) AS icat12, reinterpretAsUInt32(unhex(cat13)) AS icat13, reinterpretAsUInt32(unhex(cat14)) AS icat14, reinterpretAsUInt32(unhex(cat15)) AS icat15, reinterpretAsUInt32(unhex(cat16)) AS icat16, reinterpretAsUInt32(unhex(cat17)) AS icat17, reinterpretAsUInt32(unhex(cat18)) AS icat18, reinterpretAsUInt32(unhex(cat19)) AS icat19, reinterpretAsUInt32(unhex(cat20)) AS icat20, reinterpretAsUInt32(unhex(cat21)) AS icat21, reinterpretAsUInt32(unhex(cat22)) AS icat22, reinterpretAsUInt32(unhex(cat23)) AS icat23, reinterpretAsUInt32(unhex(cat24)) AS icat24, reinterpretAsUInt32(unhex(cat25)) AS icat25, reinterpretAsUInt32(unhex(cat26)) AS icat26 FROM criteo_log;DROP TABLE criteo_log; ``` diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md old mode 100755 new mode 100644 index a9f04f595d1..11ed81d1a43 --- a/docs/en/getting_started/example_datasets/nyc_taxi.md +++ b/docs/en/getting_started/example_datasets/nyc_taxi.md @@ -1,8 +1,8 @@ -# New York Taxi data +# Data about New York taxis -## How to import the raw data +## How to import raw data -See and for the description of the dataset and instructions for downloading. +See and for description of the dataset and loading instructions. Downloading will result in about 227 GB of uncompressed data in CSV files. The download takes about an hour over a 1 Gbit connection (parallel downloading from s3.amazonaws.com recovers at least half of a 1 Gbit channel). Some of the files might not download fully. Check the file sizes and re-download any that seem doubtful. @@ -301,14 +301,19 @@ SELECT passenger_count, toYear(pickup_date) AS year, count(*) FROM trips_mergetr Q4: ```sql -SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*)FROM trips_mergetreeGROUP BY passenger_count, year, distanceORDER BY year, count(*) DESC +SELECT passenger_count, toYear(pickup_date) AS year, round(trip_distance) AS distance, count(*) +FROM trips_mergetree +GROUP BY passenger_count, year, distance +ORDER BY year, count(*) DESC ``` 3.593 seconds. The following server was used: -Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total,128 GiB RAM,8x6 TB HD on hardware RAID-5 +Two Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz, 16 physical kernels total, +128 GiB RAM, +8x6 TB HD on hardware RAID-5 Execution time is the best of three runsBut starting from the second run, queries read data from the file system cache. No further caching occurs: the data is read out and processed in each run. diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md old mode 100755 new mode 100644 diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md old mode 100755 new mode 100644 index 664ba59f48c..8807de3e670 --- a/docs/en/getting_started/example_datasets/star_schema.md +++ b/docs/en/getting_started/example_datasets/star_schema.md @@ -1,4 +1,4 @@ -# Star Schema Benchmark +# Star scheme Compiling dbgen: @@ -82,4 +82,3 @@ Downloading data (change 'customer' to 'customerd' in the distributed version): cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INTO customer FORMAT CSV" cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" ``` - diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md old mode 100755 new mode 100644 index fee0a56b52c..6cbc3b15561 --- a/docs/en/getting_started/example_datasets/wikistat.md +++ b/docs/en/getting_started/example_datasets/wikistat.md @@ -20,7 +20,7 @@ CREATE TABLE wikistat Loading data: ```bash -for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sSL "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt +for i in {2007..2016}; do for j in {01..12}; do echo $i-$j >&2; curl -sS "http://dumps.wikimedia.org/other/pagecounts-raw/$i/$i-$j/" | grep -oE 'pagecounts-[0-9]+-[0-9]+\.gz'; done; done | sort | uniq | tee links.txt cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pagecounts-raw/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1/')/$(echo $link | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})[0-9]{2}-[0-9]+\.gz/\1-\2/')/$link; done ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done ``` diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md old mode 100755 new mode 100644 index 07d0d91a224..42fc1c75551 --- a/docs/en/getting_started/index.md +++ b/docs/en/getting_started/index.md @@ -16,15 +16,14 @@ The terminal must use UTF-8 encoding (the default in Ubuntu). For testing and development, the system can be installed on a single server or on a desktop computer. -### Installing from packages +### Installing from packages Debian/Ubuntu In `/etc/apt/sources.list` (or in a separate `/etc/apt/sources.list.d/clickhouse.list` file), add the repository: ```text -deb http://repo.yandex.ru/clickhouse/trusty stable main +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ ``` -On other versions of Ubuntu, replace `trusty` with `xenial` or `precise`. If you want to use the most recent test version, replace 'stable' with 'testing'. Then run: @@ -35,10 +34,7 @@ sudo apt-get update sudo apt-get install clickhouse-client clickhouse-server-common ``` -You can also download and install packages manually from here: - - - +You can also download and install packages manually from here: ClickHouse contains access restriction settings. They are located in the 'users.xml' file (next to 'config.xml'). By default, access is allowed from anywhere for the 'default' user, without a password. See 'user/default/networks'. @@ -104,8 +100,7 @@ clickhouse-client ``` The default parameters indicate connecting with localhost:9000 on behalf of the user 'default' without a password. -The client can be used for connecting to a remote server. -Example: +The client can be used for connecting to a remote server. Example: ```bash clickhouse-client --host=example.com @@ -137,4 +132,3 @@ SELECT 1 **Congratulations, the system works!** To continue experimenting, you can try to download from the test data sets. - diff --git a/docs/en/index.md b/docs/en/index.md old mode 100755 new mode 100644 index 72efa70802b..586c18297a8 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -39,7 +39,7 @@ We'll say that the following is true for the OLAP (online analytical processing) - Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. - Data is added to the DB but is not modified. - For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. -- Tables are "wide," meaning they contain a large number of columns. +- Tables are "wide", meaning they contain a large number of columns. - Queries are relatively rare (usually hundreds of queries per server or less per second). - For simple queries, latencies around 50 ms are allowed. - Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md old mode 100755 new mode 100644 index 76549b46b36..4fd998fed66 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -6,9 +6,7 @@ To work from the command line, you can use ` clickhouse-client`: $ clickhouse-client ClickHouse client version 0.0.26176. Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.26176. - -:) +Connected to ClickHouse server version 0.0.26176.:) ``` The client supports command-line options and configuration files. For more information, see "[Configuring](#interfaces_cli_configuration)". @@ -31,6 +29,7 @@ _EOF cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV"; ``` +In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query. By default, you can only process a single query in batch mode. To make multiple queries from a "script," use the --multiquery parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators. Similarly, to process a large number of queries, you can run 'clickhouse-client' for each query. Note that it may take tens of milliseconds to launch the 'clickhouse-client' program. @@ -65,7 +64,7 @@ The command-line client allows passing external data (external temporary tables) -## Configuring +## Configure You can pass parameters to `clickhouse-client` (all parameters have a default value) using: diff --git a/docs/en/interfaces/http_interface.md b/docs/en/interfaces/http_interface.md old mode 100755 new mode 100644 index 38a70feef46..5c989a59d65 --- a/docs/en/interfaces/http_interface.md +++ b/docs/en/interfaces/http_interface.md @@ -37,8 +37,7 @@ Date: Fri, 16 Nov 2012 19:21:50 GMT 1 ``` -As you can see, curl is somewhat inconvenient in that spaces must be URL escaped. -Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. +As you can see, curl is somewhat inconvenient in that spaces must be URL escaped.Although wget escapes everything itself, we don't recommend using it because it doesn't work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. ```bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -131,11 +130,15 @@ POST 'http://localhost:8123/?query=DROP TABLE t' For successful requests that don't return a data table, an empty response body is returned. -You can use compression when transmitting data. The compressed data has a non-standard format, and you will need to use the special compressor program to work with it (sudo apt-get install compressor-metrika-yandex). +You can use compression when transmitting data. +For using ClickHouse internal compression format, and you will need to use the special clickhouse-compressor program to work with it (installed as a part of clickhouse-client package). If you specified 'compress=1' in the URL, the server will compress the data it sends you. If you specified 'decompress=1' in the URL, the server will decompress the same data that you pass in the POST method. +Also standard gzip-based HTTP compression can be used. To send gzip compressed POST data just add `Content-Encoding: gzip` to request headers, and gzip POST body. +To get response compressed, you need to add `Accept-Encoding: gzip` to request headers, and turn on ClickHouse setting called `enable_http_compression`. + You can use this to reduce network traffic when transmitting a large amount of data, or for creating dumps that are immediately compressed. You can use the 'database' URL parameter to specify the default database. @@ -191,7 +194,11 @@ $ echo 'SELECT number FROM system.numbers LIMIT 10' | curl 'http://localhost:812 For information about other parameters, see the section "SET". -In contrast to the native interface, the HTTP interface does not support the concept of sessions or session settings, does not allow aborting a query (to be exact, it allows this in only a few cases), and does not show the progress of query processing. Parsing and data formatting are performed on the server side, and using the network might be ineffective. +You can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` GET parameter in HTTP request. You can use any alphanumeric string as a session_id. By default session will be timed out after 60 seconds of inactivity. You can change that by setting `default_session_timeout` in server config file, or by adding GET parameter `session_timeout`. You can also check the status of the session by using GET parameter `session_check=1`. When using sessions you can't run 2 queries with the same session_id simultaneously. + +You can get the progress of query execution in X-ClickHouse-Progress headers, by enabling setting send_progress_in_http_headers. + +Running query are not aborted automatically after closing HTTP connection. Parsing and data formatting are performed on the server side, and using the network might be ineffective. The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query". The optional 'quota_key' parameter can be passed as the quota key (any string). For more information, see the section "Quotas". @@ -213,4 +220,3 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa ``` Use buffering to avoid situations where a query processing error occurred after the response code and HTTP headers were sent to the client. In this situation, an error message is written at the end of the response body, and on the client side, the error can only be detected at the parsing stage. - diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md old mode 100755 new mode 100644 index e43f4474271..3e3e3df4853 --- a/docs/en/interfaces/index.md +++ b/docs/en/interfaces/index.md @@ -2,5 +2,4 @@ # Interfaces -To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program. - +To explore the system's capabilities, download data to tables, or make manual queries, use the clickhouse-client program. \ No newline at end of file diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md old mode 100755 new mode 100644 diff --git a/docs/en/interfaces/tcp.md b/docs/en/interfaces/tcp.md old mode 100755 new mode 100644 diff --git a/docs/en/interfaces/third-party_client_libraries.md b/docs/en/interfaces/third-party_client_libraries.md old mode 100755 new mode 100644 index 8437be23b99..cc8ff1f4307 --- a/docs/en/interfaces/third-party_client_libraries.md +++ b/docs/en/interfaces/third-party_client_libraries.md @@ -2,7 +2,7 @@ There are libraries for working with ClickHouse for: -- Python +- Python: - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) - [sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse) - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) diff --git a/docs/en/interfaces/third-party_gui.md b/docs/en/interfaces/third-party_gui.md old mode 100755 new mode 100644 diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md old mode 100755 new mode 100644 index 59853b8e202..3927405579f --- a/docs/en/introduction/distinctive_features.md +++ b/docs/en/introduction/distinctive_features.md @@ -1,10 +1,10 @@ # Distinctive features of ClickHouse -## True column-oriented DBMS +## True column-oriented DBMS. -In a true column-oriented DBMS, there isn't any "garbage" stored with the values. Among other things, this means that constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data. +In a true column-oriented DBMS, there isn't any "garbage" stored with the values. For example, constant-length values must be supported, to avoid storing their length "number" next to the values. As an example, a billion UInt8-type values should actually consume around 1 GB uncompressed, or this will strongly affect the CPU use. It is very important to store data compactly (without any "garbage") even when uncompressed, since the speed of decompression (CPU usage) depends mainly on the volume of uncompressed data. -This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Examples are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second. +This is worth noting because there are systems that can store values of separate columns separately, but that can't effectively process analytical queries due to their optimization for other scenarios. Example are HBase, BigTable, Cassandra, and HyperTable. In these systems, you will get throughput around a hundred thousand rows per second, but not hundreds of millions of rows per second. Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows creating tables and databases in runtime, loading data, and running queries without reconfiguring and restarting the server. @@ -12,15 +12,15 @@ Also note that ClickHouse is a DBMS, not a single database. ClickHouse allows cr Some column-oriented DBMSs (InfiniDB CE and MonetDB) do not use data compression. However, data compression really improves performance. -## Disk storage of data +## Disk storage of data. Many column-oriented DBMSs (such as SAP HANA and Google PowerDrill) can only work in RAM. But even on thousands of servers, the RAM is too small for storing all the pageviews and sessions in Yandex.Metrica. -## Parallel processing on multiple cores +## Parallel processing on multiple cores. Large queries are parallelized in a natural way. -## Distributed processing on multiple servers +## Distributed processing on multiple servers. Almost none of the columnar DBMSs listed above have support for distributed processing. In ClickHouse, data can reside on different shards. Each shard can be a group of replicas that are used for fault tolerance. The query is processed on all the shards in parallel. This is transparent for the user. @@ -30,12 +30,12 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of If you are familiar with standard SQL, we can't really talk about SQL support. All the functions have different names. However, this is a declarative query language based on SQL that can't be differentiated from SQL in many instances. -JOINs are supported. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries. +Support for JOINs. Subqueries are supported in FROM, IN, and JOIN clauses, as well as scalar subqueries. Dependent subqueries are not supported. ## Vector engine -Data is not only stored by columns, but is processed by vectors (parts of columns). This allows us to achieve high CPU performance. +Data is not only stored by columns, but is processed by vectors – parts of columns. This allows us to achieve high CPU performance. ## Real-time data updates @@ -43,13 +43,13 @@ ClickHouse supports primary key tables. In order to quickly perform queries on t ## Indexes -Having a primary key makes it possible to extract data for specific clients (for instance, Yandex.Metrica tracking tags) for a specific time range, with low latency less than several dozen milliseconds. +Having a primary key allows, for example, extracting data for specific clients (Metrica counters) for a specific time range, with low latency less than several dozen milliseconds. ## Suitable for online queries This lets us use the system as the back-end for a web interface. Low latency means queries can be processed without delay, while the Yandex.Metrica interface page is loading. In other words, in online mode. -## Support for approximated calculations +## Support for approximated calculations. 1. The system contains aggregate functions for approximated calculation of the number of various values, medians, and quantiles. 2. Supports running a query based on a part (sample) of data and getting an approximated result. In this case, proportionally less data is retrieved from the disk. diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md old mode 100755 new mode 100644 diff --git a/docs/en/introduction/index.md b/docs/en/introduction/index.md old mode 100755 new mode 100644 index 3d07efe555d..e10b99d0138 --- a/docs/en/introduction/index.md +++ b/docs/en/introduction/index.md @@ -1,2 +1 @@ # Introduction - diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md old mode 100755 new mode 100644 diff --git a/docs/en/introduction/possible_silly_questions.md b/docs/en/introduction/possible_silly_questions.md old mode 100755 new mode 100644 index cf7b2c48032..36363ebe247 --- a/docs/en/introduction/possible_silly_questions.md +++ b/docs/en/introduction/possible_silly_questions.md @@ -1,8 +1,8 @@ -# Questions you were afraid to ask +# Everything you were afraid to ask ## Why not use something like MapReduce? -We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop, and YT (YT is developed at Yandex for internal use). +We can refer to systems like map-reduce as distributed computing systems in which the reduce operation is based on distributed sorting. In this sense, they include Hadoop and YT (Yandex proprietary technology). These systems aren't appropriate for online queries due to their high latency. In other words, they can't be used as the back-end for a web interface. These types of systems aren't useful for real-time data updates. diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md old mode 100755 new mode 100644 index 10f45f061d6..6a488be9b5f --- a/docs/en/introduction/ya_metrika_task.md +++ b/docs/en/introduction/ya_metrika_task.md @@ -1,6 +1,6 @@ -# Yandex.Metrica use case +# The Yandex.Metrica task -ClickHouse currently powers [Yandex.Metrica](https://metrika.yandex.ru/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all). With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data. +ClickHouse currently powers [ Yandex.Metrica](https://metrika.yandex.ru/), [ the second largest platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all) for Web Analytics. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows you generating custom reports on the fly directly from non-aggregated data. We need to get custom reports based on hits and sessions, with custom segments set by the user. Data for the reports is updated in real-time. Queries must be run immediately (in online mode). We must be able to build reports for any time period. Complex aggregates must be calculated, such as the number of unique visitors. At this time (April 2014), Yandex.Metrica receives approximately 12 billion events (pageviews and mouse clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning hundreds of millions of rows over a few seconds, or millions of rows in no more than a few hundred milliseconds. diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md old mode 100755 new mode 100644 index 1c72bf13b3e..9879dab9a99 --- a/docs/en/operations/access_rights.md +++ b/docs/en/operations/access_rights.md @@ -2,14 +2,14 @@ Users and access rights are set up in the user config. This is usually `users.xml`. -Users are recorded in the 'users' section. Here is a fragment of the `users.xml` file: +Users are recorded in the 'users' section. We'll look at a fragment of the `users.xml` file: ```xml - - - - + How to generate a decent password: + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + The first line has the password and the second line has the corresponding SHA256. + --> + - - - - default - + + ::/0 + + --> + + + + default + default - - - - - - - web - default - - test + + + + + + + web + default + + test ``` You can see a declaration from two users: `default`and`web`. We added the `web` user separately. -The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify the `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#table_engines-distributed) engine). +The `default` user is chosen in cases when the username is not passed. The `default` user is also used for distributed query processing, if the configuration of the server or cluster doesn't specify `user` and `password` (see the section on the [Distributed](../table_engines/distributed.md#distributed_distributed) engine). The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail. diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md old mode 100755 new mode 100644 index 52e9e10ffea..c3122617bf1 --- a/docs/en/operations/configuration_files.md +++ b/docs/en/operations/configuration_files.md @@ -14,7 +14,7 @@ If `replace` is specified, it replaces the entire element with the specified one If ` remove` is specified, it deletes the element. -The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](server_settings/settings.md#server_settings-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros]()server_settings/settings.md#server_settings-macros)). +The config can also define "substitutions". If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the config in the [include_from](server_settings/settings.md#server_settings-include_from) element. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in ` incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional = "true"` attribute (for example, settings for [ macros](server_settings/settings.md#server_settings-macros)). Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at ` /path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element. diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md old mode 100755 new mode 100644 index 0ff38af8086..eb90f937cff --- a/docs/en/operations/index.md +++ b/docs/en/operations/index.md @@ -1,2 +1 @@ -# Usage - +# Operation diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md old mode 100755 new mode 100644 index fb1238b257d..d7b1a61ce7f --- a/docs/en/operations/quotas.md +++ b/docs/en/operations/quotas.md @@ -18,10 +18,10 @@ Let's look at the section of the 'users.xml' file that defines quotas. - + - 3600 - + 3600 + 0 0 @@ -39,19 +39,21 @@ The resource consumption calculated for each interval is output to the server lo - - 3600 - 1000 - 100 - 1000000000 - 100000000000 - 900 - + + 3600 - - 86400 - 10000 - 1000 + 1000 + 100 + 1000000000 + 100000000000 + 900 + + + + 86400 + + 10000 + 1000 5000000000 500000000000 7200 @@ -87,7 +89,7 @@ Quotas can use the "quota key" feature in order to report on resources for multi Using keys makes sense only if quota_key is transmitted by the program, not by a user. You can also write so the IP address is used as the quota key.(But keep in mind that users can change the IPv6 address fairly easily.) - --> + --> ``` @@ -96,3 +98,4 @@ The quota is assigned to users in the 'users' section of the config. See the sec For distributed query processing, the accumulated amounts are stored on the requestor server. So if the user goes to another server, the quota there will "start over". When the server is restarted, quotas are reset. + diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md old mode 100755 new mode 100644 index 208deec710c..2293e86f5c7 --- a/docs/en/operations/server_settings/index.md +++ b/docs/en/operations/server_settings/index.md @@ -9,4 +9,3 @@ These settings are stored in the ` config.xml` file on the ClickHouse server. Other settings are described in the "[Settings](../settings/index.md#settings)" section. Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes). - diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md old mode 100755 new mode 100644 index e1575df2f88..8818f8ec932 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -67,7 +67,7 @@ ClickHouse checks ` min_part_size` and ` min_part_size_ratio` and processes th The default database. -To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/queries.md#query_language_queries_show_databases). +Use a [ SHOW DATABASES](../../query_language/queries.md#query_language_queries_show_databases) query to get a list of databases. **Example** @@ -81,7 +81,7 @@ To get a list of databases, use the [ SHOW DATABASES]( query./../query_language/ Default settings profile. -Settings profiles are located in the file specified in the parameter [user_config](#server_settings-users_config). +Settings profiles are located in the file specified in the [user_config](#server_settings-users_config) parameter. **Example** @@ -100,7 +100,7 @@ Path: - Specify the absolute path or the path relative to the server config file. - The path can contain wildcards \* and ?. -See also "[External dictionaries]("./../dicts/external_dicts.md#dicts-external_dicts)". +See also "[External dictionaries](../../dicts/external_dicts.md#dicts-external_dicts)". **Example** @@ -130,12 +130,12 @@ The default is ` true`. ## format_schema_path -The path to the directory with the schemes for the input data, such as schemas for the [CapnProto](../../formats/capnproto.md#format_capnproto) format. +The path to the directory with the schemas for the input data, such as schemas for the [ CapnProto](../../formats/capnproto.md#format_capnproto) format. **Example** ```xml - + format_schemas/ ``` @@ -179,7 +179,7 @@ You can configure multiple `` clauses. For instance, you can use this Settings for thinning data for Graphite. -For more information, see [GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree). +For more details, see [ GraphiteMergeTree](../../table_engines/graphitemergetree.md#table_engines-graphitemergetree). **Example** @@ -241,7 +241,7 @@ Opens `https://tabix.io/` when accessing ` http://localhost: http_port`. The path to the file with substitutions. -For more information, see the section "[Configuration files](../configuration_files.md#configuration_files)". +For details, see the section "[Configuration files](../configuration_files.md#configuration_files)". **Example** @@ -298,8 +298,7 @@ Restriction on hosts that requests can come from. If you want the server to answ Examples: ```xml -::1 -127.0.0.1 +::1127.0.0.1 ``` @@ -348,7 +347,7 @@ For more information, see the section "[Creating replicated tables](../../table_ ## mark_cache_size -Approximate size (in bytes) of the cache of "marks" used by [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. +Approximate size (in bytes) of the cache of "marks" used by [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) engines. The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120. @@ -404,7 +403,7 @@ We recommend using this option in Mac OS X, since the ` getrlimit()` function re Restriction on deleting tables. -If the size of a [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. +If the size of a [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) type table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. If you still need to delete the table without restarting the ClickHouse server, create the ` /flags/force_drop_table` file and run the DROP query. @@ -440,17 +439,17 @@ For more information, see the MergeTreeSettings.h header file. SSL client/server configuration. -Support for SSL is provided by the `` libpoco`` library. The interface is described in the file [SSLManager.h](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) +Support for SSL is provided by the `` libpoco`` library. The description of the interface is in the [ SSLManager.h file.](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) Keys for server/client settings: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `` privateKeyFile`` contains the certificate. - caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/yandex/ClickHouse/blob/master/contrib/libpoco/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``none``, ``relaxed``, ``strict``, ``once``. +- verificationMode – The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Acceptable values: ``none``, ``relaxed``, ``strict``, ``once``. - verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. - loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `` true``, `` false``. | -- cipherList - Поддерживаемые OpenSSL-шифры. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``. +- cipherList - Supported OpenSSL-ciphers. For example: `` ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH``. - cacheSessions – Enables or disables caching sessions. Must be used in combination with ``sessionIdContext``. Acceptable values: `` true``, `` false``. - sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed ``SSL_MAX_SSL_SESSION_ID_LENGTH``. This parameter is always recommended, since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: ``${application.name}``. - sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. @@ -499,7 +498,7 @@ Keys for server/client settings: ## part_log -Logging events that are associated with [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. +Logging events that are associated with the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) data type. For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. Queries are logged in the ClickHouse table, not in a separate file. @@ -519,7 +518,7 @@ Use the following parameters to configure logging: - database – Name of the database. - table – Name of the table. -- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). +- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). - flush_interval_milliseconds – Interval for flushing data from memory to the disk. **Example** @@ -563,7 +562,7 @@ Use the following parameters to configure logging: - database – Name of the database. - table – Name of the table. -- partition_by – Sets a [custom partitioning key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). +- partition_by - Sets the [custom partition key](../../table_engines/custom_partitioning_key.md#custom-partitioning-key). - flush_interval_milliseconds – Interval for flushing data from memory to the disk. If the table doesn't exist, ClickHouse will create it. If the structure of the query log changed when the ClickHouse server was updated, the table with the old structure is renamed, and a new table is created automatically. @@ -585,7 +584,7 @@ If the table doesn't exist, ClickHouse will create it. If the structure of the q Configuration of clusters used by the Distributed table engine. -For more information, see the section "[Table engines/Distributed](../../table_engines/distributed.md#table_engines-distributed)". +For more information, see the section "[Duplicated table engine](../../table_engines/distributed.md#table_engines-distributed)". **Example** @@ -645,7 +644,7 @@ The end slash is mandatory. ## uncompressed_cache_size -Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. +Cache size (in bytes) for uncompressed data used by table engines from the [ MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) family. There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#settings-use_uncompressed_cache) is enabled. diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md old mode 100755 new mode 100644 index 0c5ca5d5171..0e967a4c081 --- a/docs/en/operations/settings/index.md +++ b/docs/en/operations/settings/index.md @@ -9,9 +9,9 @@ Ways to configure settings, in order of priority: - Settings in the server config file. - Settings from user profiles. + Set via user profiles. -- Session settings. +- For the session. Send ` SET setting=value` from the ClickHouse console client in interactive mode. Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to specify the `session_id` HTTP parameter. diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md old mode 100755 new mode 100644 diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md old mode 100755 new mode 100644 index 25c804b0035..e006f302c68 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4,7 +4,7 @@ ## distributed_product_mode -Changes the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables. +Alters the behavior of [distributed subqueries](../../query_language/queries.md#queries-distributed-subrequests), i.e. in cases when the query contains the product of distributed tables. ClickHouse applies the configuration if the subqueries on any level have a distributed table that exists on the local server and has more than one shard. @@ -12,9 +12,9 @@ Restrictions: - Only applied for IN and JOIN subqueries. - Used only if a distributed table is used in the FROM clause. -- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote) function. +- Not used for a table-valued [ remote](../../table_functions/remote.md#table_functions-remote)function. -The possible values ​​are: +Possible values: @@ -36,7 +36,7 @@ Disables query execution if the index can't be used by date. Works with tables in the MergeTree family. -If `force_index_by_date=1`, ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". +If `force_index_by_date=1`, ClickHouse checks whether the query has a date key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For example, the condition `Date != ' 2000-01-01 '` is acceptable even when it matches all the data in the table (i.e., running the query requires a full scan). For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". @@ -46,7 +46,7 @@ Disables query execution if indexing by the primary key is not possible. Works with tables in the MergeTree family. -If `force_primary_key=1`, ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about data ranges in MergeTree tables, see "[MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". +If `force_primary_key=1`, ClickHouse checks to see if the query has a primary key condition that can be used for restricting data ranges. If there is no suitable condition, it throws an exception. However, it does not check whether the condition actually reduces the amount of data to read. For more information about ranges of data in MergeTree tables, see [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)". @@ -158,7 +158,7 @@ Don't confuse blocks for compression (a chunk of memory consisting of bytes) and ## min_compress_block_size -For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree)" tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536. +For [MergeTree](../../table_engines/mergetree.md#table_engines-mergetree) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least 'min_compress_block_size'. By default, 65,536. The actual size of the block, if the uncompressed data is less than 'max_compress_block_size', is no less than this value and no less than the volume of data for one mark. @@ -253,13 +253,13 @@ Yandex.Metrica uses this parameter set to 1 for implementing suggestions for seg ## schema -This parameter is useful when you are using formats that require a schema definition, such as [Cap'n Proto](https://capnproto.org/). The value depends on the format. +This parameter is useful when you are using formats that require a schema definition, such as [ Cap'n Proto](https://capnproto.org/). The value depends on the format. ## stream_flush_interval_ms -Works for tables with streaming in the case of a timeout, or when a thread generates[max_insert_block_size](#settings-settings-max_insert_block_size) rows. +Works for tables with streaming in the case of a timeout, or when a thread generates [ max_insert_block_size](#settings-settings-max_insert_block_size) rows. The default value is 7500. diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md old mode 100755 new mode 100644 index f1fce41ba75..c978c599bd5 --- a/docs/en/operations/settings/settings_profiles.md +++ b/docs/en/operations/settings/settings_profiles.md @@ -17,7 +17,7 @@ Example: - + 8 diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md old mode 100755 new mode 100644 index 652698fe24c..11fc8f6da11 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -105,7 +105,7 @@ Use at least a 10 GB network, if possible. 1 Gb will also work, but it will be m You are probably already using ZooKeeper for other purposes. You can use the same installation of ZooKeeper, if it isn't already overloaded. -It's best to use a fresh version of ZooKeeper – 3.4.9 or later. The version in stable Linux distributions may be outdated. +It's best to use a fresh version of ZooKeeper – 3.5 or later. The version in stable Linux distributions may be outdated. With the default settings, ZooKeeper is a time bomb: @@ -174,7 +174,8 @@ dynamicConfigFile=/etc/zookeeper-{{ cluster['name'] }}/conf/zoo.cfg.dynamic Java version: ```text -Java(TM) SE Runtime Environment (build 1.8.0_25-b17)Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) +Java(TM) SE Runtime Environment (build 1.8.0_25-b17) +Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode) ``` JVM parameters: diff --git a/docs/en/operators/index.md b/docs/en/operators/index.md old mode 100755 new mode 100644 index 411cde34b50..779f3bf3843 --- a/docs/en/operators/index.md +++ b/docs/en/operators/index.md @@ -67,11 +67,11 @@ Groups of operators are listed in order of priority (the higher it is in the lis `NOT a` The `not(a) function.` -## Logical AND operator +## Logical 'AND' operator `a AND b` – The`and(a, b) function.` -## Logical OR operator +## Logical 'OR' operator `a OR b` – The `or(a, b) function.` diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md old mode 100755 new mode 100644 index 769d94eb4fd..247d76fc6ed --- a/docs/en/query_language/index.md +++ b/docs/en/query_language/index.md @@ -1,2 +1 @@ # Query language - diff --git a/docs/en/query_language/queries.md b/docs/en/query_language/queries.md old mode 100755 new mode 100644 index d235945a646..743706a551f --- a/docs/en/query_language/queries.md +++ b/docs/en/query_language/queries.md @@ -11,6 +11,7 @@ CREATE DATABASE [IF NOT EXISTS] db_name `A database` is just a directory for tables. If `IF NOT EXISTS` is included, the query won't return an error if the database already exists. + ## CREATE TABLE @@ -183,7 +184,7 @@ Deletes all tables inside the 'db' database, then deletes the 'db' database itse If `IF EXISTS` is specified, it doesn't return an error if the database doesn't exist. ```sql -DROP TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` Deletes the table. @@ -307,7 +308,8 @@ SELECT * FROM system.parts WHERE active `active` – Only count active parts. Inactive parts are, for example, source parts remaining after merging to a larger part – these parts are deleted approximately 10 minutes after merging. Another way to view a set of parts and partitions is to go into the directory with table data. -Data directory: `/var/lib/clickhouse/data/database/table/`,where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: +Data directory: `/var/lib/clickhouse/data/database/table/`, +where `/var/lib/clickhouse/` is the path to the ClickHouse data, 'database' is the database name, and 'table' is the table name. Example: ```bash $ ls -l /var/lib/clickhouse/data/test/visits/ @@ -323,7 +325,7 @@ Here, `20140317_20140323_2_2_0` and ` 20140317_20140323_4_4_0` are the directori Let's break down the name of the first part: `20140317_20140323_2_2_0`. - `20140317` is the minimum date of the data in the chunk. -- `20140323` is the maximum data of the data in the chunk. +- `20140323` is the maximum date of the data in the chunk. - `2` is the minimum number of the data block. - `2` is the maximum number of the data block. - `0` is the chunk level (the depth of the merge tree it is formed from). @@ -450,7 +452,7 @@ See also the section "Formats". ## SHOW TABLES ```sql -SHOW TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] +SHOW [TEMPORARY] TABLES [FROM db] [LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format] ``` Displays a list of tables @@ -460,7 +462,7 @@ Displays a list of tables This query is identical to: `SELECT name FROM system.tables WHERE database = 'db' [AND name LIKE 'pattern'] [INTO OUTFILE filename] [FORMAT format]`. -See also the section "LIKE operator". +See the section "LIKE operator" also. ## SHOW PROCESSLIST @@ -484,7 +486,7 @@ Prints a table containing the columns: **query** – The query itself. In INSERT queries, the data for insertion is not output. -**query_id** – The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. +**query_id** - The query identifier. Non-empty only if it was explicitly defined by the user. For distributed processing, the query ID is not passed to remote servers. This query is identical to: `SELECT * FROM system.processes [INTO OUTFILE filename] [FORMAT format]`. @@ -497,7 +499,7 @@ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" ## SHOW CREATE TABLE ```sql -SHOW CREATE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] +SHOW CREATE [TEMPORARY] TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] ``` Returns a single `String`-type 'statement' column, which contains a single value – the `CREATE` query used for creating the specified table. @@ -515,7 +517,7 @@ Nested data structures are output in "expanded" format. Each column is shown sep ## EXISTS ```sql -EXISTS TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] +EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] ``` Returns a single `UInt8`-type column, which contains the single value `0` if the table or database doesn't exist, or `1` if the table exists in the specified database. @@ -571,9 +573,9 @@ The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case - The values calculated from the `DEFAULT` expressions specified in the table definition. - Zeros and empty strings, if `DEFAULT` expressions are not defined. -If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have `DEFAULT` defined must be listed in the query. +If [strict_insert_defaults=1](../operations/settings/settings.md#settings-strict_insert_defaults), columns that do not have ` DEFAULT` defined must be listed in the query. -Data can be passed to the INSERT in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: +The INSERT can pass data in any [format](../formats/index.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: ```sql INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set @@ -972,7 +974,8 @@ All columns that are not needed for the JOIN are deleted from the subquery. There are several types of JOINs: -`INNER` or `LEFT` type:If INNER is specified, the result will contain only those rows that have a matching row in the right table. +`INNER` or `LEFT` type: +If INNER is specified, the result will contain only those rows that have a matching row in the right table. If LEFT is specified, any rows in the left table that don't have matching rows in the right table will be assigned the default value - zeros or empty rows. LEFT OUTER may be written instead of LEFT; the word OUTER does not affect anything. `ANY` or `ALL` stringency:If `ANY` is specified and the right table has several matching rows, only the first one found is joined. @@ -1103,7 +1106,7 @@ Example: SELECT domainWithoutWWW(URL) AS domain, count(), - any(Title) AS title -- getting the first occurring page header for each domain. + any(Title) AS title -- getting the first occurred page header for each domain. FROM hits GROUP BY domain ``` @@ -1348,7 +1351,7 @@ There are two options for IN-s with subqueries (similar to JOINs): normal `IN`
-Remember that the algorithms described below may work differently depending on the [settings](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting. +Remember that the algorithms described below may work differently depending on the [](../operations/settings/settings.md#settings-distributed_product_mode) `distributed_product_mode` setting.
@@ -1476,34 +1479,34 @@ In all other cases, we don't recommend using the asterisk, since it only gives y ## KILL QUERY ```sql -KILL QUERY WHERE [SYNC|ASYNC|TEST] [FORMAT format] +KILL QUERY + WHERE + [SYNC|ASYNC|TEST] + [FORMAT format] ``` Attempts to terminate queries currently running. -The queries to terminate are selected from the system.processes table for which expression_for_system.processes is true. +The queries to terminate are selected from the system.processes table for which `WHERE` expression is true. Examples: ```sql +-- Terminates all queries with the specified query_id. KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90' -``` -Terminates all queries with the specified query_id. - -```sql +-- Synchronously terminates all queries run by `username`. KILL QUERY WHERE user='username' SYNC ``` -Synchronously terminates all queries run by `username`. - Readonly-users can only terminate their own requests. -By default, the asynchronous version of queries is used (`ASYNC`), which terminates without waiting for queries to complete. -The synchronous version (`SYNC`) waits for all queries to be completed and displays information about each process as it terminates. + +By default, the asynchronous version of queries is used (`ASYNC`), which doesn't wait for query termination. + +The synchronous version (`SYNC`) waits for all queries to be killed and displays information about each process as it terminates. The response contains the `kill_status` column, which can take the following values: -1. 'finished' – The query completed successfully. +1. 'finished' – The query terminated successfully. 2. 'waiting' – Waiting for the query to finish after sending it a signal to terminate. 3. The other values ​​explain why the query can't be terminated. A test query (`TEST`) only checks the user's rights and displays a list of queries to terminate. - diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md old mode 100755 new mode 100644 index 4928f2d4a12..e151d2ee3d9 --- a/docs/en/query_language/syntax.md +++ b/docs/en/query_language/syntax.md @@ -46,7 +46,7 @@ There are numeric literals, string literals, and compound literals. A numeric literal tries to be parsed: -- First as a 64-bit signed number, using the 'strtoull' function. +- first as a 64-bit signed number, using the 'strtoull' function. - If unsuccessful, as a 64-bit unsigned number, using the 'strtoll' function. - If unsuccessful, as a floating-point number using the 'strtod' function. - Otherwise, an error is returned. diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md old mode 100755 new mode 100644 index 8241b0a65ae..46b08a89607 --- a/docs/en/roadmap.md +++ b/docs/en/roadmap.md @@ -3,13 +3,11 @@ ## Q1 2018 ### New functionality - -- Support for `UPDATE` and `DELETE`. - -- Multidimensional and nested arrays. - - It can look something like this: - +- Initial support for `UPDATE` and `DELETE`. +- Multi-dimensional and nested arrays. + + It may look like this: + ```sql CREATE TABLE t ( @@ -23,7 +21,7 @@ ENGINE = MergeTree ORDER BY x - External MySQL and ODBC tables. - External tables can be integrated into ClickHouse using external dictionaries. This new functionality is a convenient alternative to connecting external tables. + External tables can be integrated to ClickHouse using external dictionaries. This will be an alternative and a more convenient way to do so. ```sql SELECT ... @@ -32,68 +30,66 @@ FROM mysql('host:port', 'db', 'table', 'user', 'password')` ### Improvements -- Effective data copying between ClickHouse clusters. +- Efficient data copy between ClickHouse clusters. - Now you can copy data with the remote() function. For example: ` + Currently, it is possible to copy data using remote() function, e.g.: ` INSERT INTO t SELECT * FROM remote(...) `. - This operation will have improved performance. + The performance of this will be improved by proper distributed execution. - O_DIRECT for merges. - This will improve the performance of the OS cache and "hot" queries. + Should improve OS cache performance and correspondingly query performance for 'hot' queries. + ## Q2 2018 ### New functionality -- UPDATE/DELETE conform to the EU GDPR. -- Protobuf and Parquet input and output formats. -- Creating dictionaries using DDL queries. +- UPDATE/DELETE in order to comply with Europe GDPR. +- Protobuf and Parquet input/output formats. +- Create dictionaries by DDL queries. - Currently, dictionaries that are part of the database schema are defined in external XML files. This is inconvenient and counter-intuitive. The new approach should fix it. - -- Integration with LDAP. + Currently, it is inconvenient and confusing that dictionaries are defined in external XML files while being a part of DB schema. The new approach will fix that. +- LDAP integration. - WITH ROLLUP and WITH CUBE for GROUP BY. +- Custom encoding/compression for columns. -- Custom encoding and compression for each column individually. + Currently, ClickHouse support LZ4 and ZSTD compressions for columns, and compressions settings are global (see our article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)) for more details). Column level encoding (e.g. delta encoding) and compression will allow more efficient data storage and therefore faster queries. - As of now, ClickHouse supports LZ4 and ZSTD compression of columns, and compression settings are global (see the article [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Per-column compression and encoding will provide more efficient data storage, which in turn will speed up queries. +- Store data at multiple disk volumes of a single server. -- Storing data on multiple disks on the same server. - - This functionality will make it easier to extend the disk space, since different disk systems can be used for different databases or tables. Currently, users are forced to use symbolic links if the databases and tables must be stored on a different disk. + That will make it easier to extend disk system as well as use different disk systems for different DBs or tables. Currently, users have to use symlinks if DB/table needs to be stored in another volume. ### Improvements -Many improvements and fixes are planned for the query execution system. For example: +A lot of enhancements and fixes are planned for query execution. In particular: -- Using an index for `in (subquery)`. +- Using index for ‘in (subquery)’. - The index is not used right now, which reduces performance. + Currently, index is not used for such queries resulting in lower performance. -- Passing predicates from `where` to subqueries, and passing predicates to views. +- Predicate pushdown from ‘where’ into subqueries and Predicate pushdown for views. - The predicates must be passed, since the view is changed by the subquery. Performance is still low for view filters, and views can't use the primary key of the original table, which makes views useless for large tables. + These two are related since view is replaced by subquery. Currently, performance of filter conditions for views is significantly degraded, views can not use primary key of the underlying table, that makes views on big tables pretty much useless. -- Optimizing branching operations (ternary operator, if, multiIf). +- Short-circuit expressions evaluation (ternary operator, if, multiIf). - ClickHouse currently performs all branches, even if they aren't necessary. + Currently, ClickHouse evaluates all branches even if the first one needs to be returned due to logical condition result. -- Using a primary key for GROUP BY and ORDER BY. +- Using primary key for GROUP BY and ORDER BY. - This will speed up certain types of queries with partially sorted data. + This may speed up certain types of queries since data is already partially pre-sorted. ## Q3-Q4 2018 -We don't have any set plans yet, but the main projects will be: +Longer term plans are not yet finalized. There are two major projects on the list so far. -- Resource pools for executing queries. +- Resource pools for query execution. - This will make load management more efficient. + That will allow managing workloads more efficiently. - ANSI SQL JOIN syntax. - Improve ClickHouse compatibility with many SQL tools. - + That will make ClickHouse more friendly for numerous SQL tools. diff --git a/docs/en/system_tables/index.md b/docs/en/system_tables/index.md old mode 100755 new mode 100644 index 240105a684b..614ce4020ec --- a/docs/en/system_tables/index.md +++ b/docs/en/system_tables/index.md @@ -4,5 +4,5 @@ System tables are used for implementing part of the system's functionality, and You can't delete a system table (but you can perform DETACH). System tables don't have files with data on the disk or files with metadata. The server creates all the system tables when it starts. System tables are read-only. -They are located in the 'system' database. +System tables are located in the 'system' database. diff --git a/docs/en/system_tables/system.asynchronous_metrics.md b/docs/en/system_tables/system.asynchronous_metrics.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.clusters.md b/docs/en/system_tables/system.clusters.md old mode 100755 new mode 100644 index bc8dab86b3c..c0bc3dd13fa --- a/docs/en/system_tables/system.clusters.md +++ b/docs/en/system_tables/system.clusters.md @@ -4,12 +4,13 @@ Contains information about clusters available in the config file and the servers Columns: ```text -cluster String - Cluster name. -shard_num UInt32 - Number of a shard in the cluster, starting from 1. -shard_weight UInt32 - Relative weight of a shard when writing data. -replica_num UInt32 - Number of a replica in the shard, starting from 1. -host_name String - Host name as specified in the config. -host_address String - Host's IP address obtained from DNS. -port UInt16 - The port used to access the server. -user String - The username to use for connecting to the server. +cluster String – Cluster name. +shard_num UInt32 – Number of a shard in the cluster, starting from 1. +shard_weight UInt32 – Relative weight of a shard when writing data. +replica_num UInt32 – Number of a replica in the shard, starting from 1. +host_name String – Host name as specified in the config. +host_address String – Host's IP address obtained from DNS. +port UInt16 – The port used to access the server. +user String – The username to use for connecting to the server. ``` + diff --git a/docs/en/system_tables/system.columns.md b/docs/en/system_tables/system.columns.md old mode 100755 new mode 100644 index bf05616fbef..975b84fe9d4 --- a/docs/en/system_tables/system.columns.md +++ b/docs/en/system_tables/system.columns.md @@ -11,3 +11,4 @@ type String - Column type. default_type String - Expression type (DEFAULT, MATERIALIZED, ALIAS) for the default value, or an empty string if it is not defined. default_expression String - Expression for the default value, or an empty string if it is not defined. ``` + diff --git a/docs/en/system_tables/system.databases.md b/docs/en/system_tables/system.databases.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.dictionaries.md b/docs/en/system_tables/system.dictionaries.md old mode 100755 new mode 100644 index 4ef0d7707b8..f3c9929d38e --- a/docs/en/system_tables/system.dictionaries.md +++ b/docs/en/system_tables/system.dictionaries.md @@ -5,19 +5,19 @@ Contains information about external dictionaries. Columns: ```text -name String - Dictionary name. -type String - Dictionary type: Flat, Hashed, Cache. -origin String - Path to the config file where the dictionary is described. -attribute.names Array(String) - Array of attribute names provided by the dictionary. -attribute.types Array(String) - Corresponding array of attribute types provided by the dictionary. -has_hierarchy UInt8 - Whether the dictionary is hierarchical. -bytes_allocated UInt64 - The amount of RAM used by the dictionary. -hit_rate Float64 - For cache dictionaries, the percent of usage for which the value was in the cache. -element_count UInt64 - The number of items stored in the dictionary. -load_factor Float64 - The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). -creation_time DateTime - Time spent for the creation or last successful reload of the dictionary. -last_exception String - Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. -source String - Text describing the data source for the dictionary. +name String – Dictionary name. +type String – Dictionary type: Flat, Hashed, Cache. +origin String – Path to the config file where the dictionary is described.attribute. +names Array(String) – Array of attribute names provided by the dictionary. +attribute.types Array(String) – Corresponding array of attribute types provided by the dictionary. +has_hierarchy UInt8 – Whether the dictionary is hierarchical. +bytes_allocated UInt64 – The amount of RAM used by the dictionary. +hit_rate Float64 – For cache dictionaries, the percent of usage for which the value was in the cache. +element_count UInt64 – The number of items stored in the dictionary. +load_factor Float64 – The filled percentage of the dictionary (for a hashed dictionary, it is the filled percentage of the hash table). +creation_time DateTime – Time spent for the creation or last successful reload of the dictionary. +last_exception String – Text of an error that occurred when creating or reloading the dictionary, if the dictionary couldn't be created. +source String – Text describing the data source for the dictionary. ``` Note that the amount of memory used by the dictionary is not proportional to the number of items stored in it. So for flat and cached dictionaries, all the memory cells are pre-assigned, regardless of how full the dictionary actually is. diff --git a/docs/en/system_tables/system.events.md b/docs/en/system_tables/system.events.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.functions.md b/docs/en/system_tables/system.functions.md old mode 100755 new mode 100644 index a1022a5e557..ac550acc14b --- a/docs/en/system_tables/system.functions.md +++ b/docs/en/system_tables/system.functions.md @@ -6,6 +6,6 @@ Columns: ```text name String – Function name. -is_aggregate UInt8 – Whether it is an aggregate function. +is_aggregate UInt8 – Whether it is an aggregate function. ``` diff --git a/docs/en/system_tables/system.merges.md b/docs/en/system_tables/system.merges.md old mode 100755 new mode 100644 index 59870922ea5..0a10e4a5a8c --- a/docs/en/system_tables/system.merges.md +++ b/docs/en/system_tables/system.merges.md @@ -18,3 +18,4 @@ rows_read UInt64 - Number of rows read. bytes_written_uncompressed UInt64 - Amount of bytes written, uncompressed. rows_written UInt64 - Number of rows written. ``` + diff --git a/docs/en/system_tables/system.numbers.md b/docs/en/system_tables/system.numbers.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.numbers_mt.md b/docs/en/system_tables/system.numbers_mt.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.one.md b/docs/en/system_tables/system.one.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.parts.md b/docs/en/system_tables/system.parts.md old mode 100755 new mode 100644 diff --git a/docs/en/system_tables/system.processes.md b/docs/en/system_tables/system.processes.md old mode 100755 new mode 100644 index 0802e555648..b9ad6a44e81 --- a/docs/en/system_tables/system.processes.md +++ b/docs/en/system_tables/system.processes.md @@ -14,12 +14,12 @@ rows_read UInt64 – The number of rows read from the table. For distribu bytes_read UInt64 – The number of uncompressed bytes read from the table. For distributed processing, on the requestor server, this is the total for all remote servers. -UInt64 total_rows_approx – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. +total_rows_approx UInt64 – The approximate total number of rows that must be read. For distributed processing, on the requestor server, this is the total for all remote servers. It can be updated during request processing, when new sources to process become known. memory_usage UInt64 – Memory consumption by the query. It might not include some types of dedicated memory. -Query String – The query text. For INSERT, it doesn't include the data to insert. +query String – The query text. For INSERT, it doesn't include the data to insert. -query_id – Query ID, if defined. +query_id String - The query ID, if defined. ``` diff --git a/docs/en/system_tables/system.replicas.md b/docs/en/system_tables/system.replicas.md old mode 100755 new mode 100644 index 75cd8e34340..ec1341198dc --- a/docs/en/system_tables/system.replicas.md +++ b/docs/en/system_tables/system.replicas.md @@ -42,41 +42,37 @@ Columns: database: database name table: table name engine: table engine name - is_leader: whether the replica is the leader Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform. Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. -is_readonly: Whether the replica is in read-only mode. -This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. +is_readonly: Whether the replica is in read-only mode.This mode is turned on if the config doesn't have sections with ZK, if an unknown error occurred when reinitializing sessions in ZK, and during session reinitialization in ZK. is_session_expired: Whether the ZK session expired. Basically, the same thing as is_readonly. -future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. +future_parts: The number of data parts that will appear as the result of INSERTs or merges that haven't been done yet. -parts_to_check: The number of data parts in the queue for verification. -A part is put in the verification queue if there is suspicion that it might be damaged. +parts_to_check: The number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged. -zookeeper_path: The path to the table data in ZK. +zookeeper_path: The path to the table data in ZK. replica_name: Name of the replica in ZK. Different replicas of the same table have different names. -replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path. +replica_path: The path to the replica data in ZK. The same as concatenating zookeeper_path/replicas/replica_path. -columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. +columns_version: Version number of the table structure. Indicates how many times ALTER was performed. If replicas have different versions, it means some replicas haven't made all of the ALTERs yet. -queue_size: Size of the queue for operations waiting to be performed. -Operations include inserting blocks of data, merges, and certain other actions. +queue_size: Size of the queue for operations waiting to be performed. Operations include inserting blocks of data, merges, and certain other actions. Normally coincides with future_parts. -inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong. +inserts_in_queue: Number of inserts of blocks of data that need to be made. Insertions are usually replicated fairly quickly. If the number is high, something is wrong. -merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time. +merges_in_queue: The number of merges waiting to be made. Sometimes merges are lengthy, so this value may be greater than zero for a long time. The next 4 columns have a non-null value only if the ZK session is active. -log_max_index: Maximum entry number in the log of general activity. log_pointer: Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. -If log_pointer is much smaller than log_max_index, something is wrong. +log_max_index: Maximum entry number in the log of general activity. +log_pointer: Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. If log_pointer is much smaller than log_max_index, something is wrong. total_replicas: Total number of known replicas of this table. active_replicas: Number of replicas of this table that have a ZK session (the number of active replicas). diff --git a/docs/en/system_tables/system.settings.md b/docs/en/system_tables/system.settings.md old mode 100755 new mode 100644 index 90a392bcc24..a055135ebcf --- a/docs/en/system_tables/system.settings.md +++ b/docs/en/system_tables/system.settings.md @@ -6,9 +6,9 @@ I.e. used for executing the query you are using to read from the system.settings Columns: ```text -name String – Setting name. +name String – Setting name. value String – Setting value. -changed UInt8 - Whether the setting was explicitly defined in the config or explicitly changed. +changed UInt8 -–Whether the setting was explicitly defined in the config or explicitly changed. ``` Example: diff --git a/docs/en/system_tables/system.tables.md b/docs/en/system_tables/system.tables.md old mode 100755 new mode 100644 index 5757a8ac3da..fabddf4dbb1 --- a/docs/en/system_tables/system.tables.md +++ b/docs/en/system_tables/system.tables.md @@ -1,7 +1,6 @@ # system.tables This table contains the String columns 'database', 'name', and 'engine'. -The table also contains three virtual columns: metadata_modification_time (DateTime type), create_table_query, and engine_full (String type). +Also, the table has three virtual columns: metadata_modification_time of type DateTime, create_table_query and engine_full of type String. Each table that the server knows about is entered in the 'system.tables' table. This system table is used for implementing SHOW TABLES queries. - diff --git a/docs/en/system_tables/system.zookeeper.md b/docs/en/system_tables/system.zookeeper.md old mode 100755 new mode 100644 index 46b40e7a08f..c456e7d9207 --- a/docs/en/system_tables/system.zookeeper.md +++ b/docs/en/system_tables/system.zookeeper.md @@ -70,3 +70,4 @@ numChildren: 7 pzxid: 987021252247 path: /clickhouse/tables/01-08/visits/replicas ``` + diff --git a/docs/en/table_engines/aggregatingmergetree.md b/docs/en/table_engines/aggregatingmergetree.md old mode 100755 new mode 100644 index 987c102508b..d75d8353e6d --- a/docs/en/table_engines/aggregatingmergetree.md +++ b/docs/en/table_engines/aggregatingmergetree.md @@ -1,11 +1,10 @@ # AggregatingMergeTree -This engine differs from `MergeTree` in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. +This engine differs from MergeTree in that the merge combines the states of aggregate functions stored in the table for rows with the same primary key value. -For this to work, it uses the `AggregateFunction` data type, as well as `-State` and `-Merge` modifiers for aggregate functions. Let's examine it more closely. - -There is an `AggregateFunction` data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. +In order for this to work, it uses the AggregateFunction data type and the -State and -Merge modifiers for aggregate functions. Let's examine it more closely. +There is an AggregateFunction data type. It is a parametric data type. As parameters, the name of the aggregate function is passed, then the types of its arguments. Examples: ```sql @@ -20,16 +19,12 @@ CREATE TABLE t This type of column stores the state of an aggregate function. To get this type of value, use aggregate functions with the `State` suffix. +Example: `uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` – in contrast to the corresponding 'uniq' and 'quantiles' functions, these functions return the state, rather than the prepared value. In other words, they return an AggregateFunction type value. -Example: -`uniqState(UserID), quantilesState(0.5, 0.9)(SendTiming)` +An AggregateFunction type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The AggregateFunction type values are not intended for output or saving in a dump. -In contrast to the corresponding `uniq` and `quantiles` functions, these functions return the state, rather than the prepared value. In other words, they return an `AggregateFunction` type value. - -An `AggregateFunction` type value can't be output in Pretty formats. In other formats, these types of values are output as implementation-specific binary data. The `AggregateFunction` type values are not intended for output or saving in a dump. - -The only useful thing you can do with `AggregateFunction` type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. -Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction` type. +The only useful thing you can do with AggregateFunction type values is combine the states and get a result, which essentially means to finish aggregation. Aggregate functions with the 'Merge' suffix are used for this purpose. +Example: `uniqMerge(UserIDState), where UserIDState has the AggregateFunction type`. In other words, an aggregate function with the 'Merge' suffix takes a set of states, combines them, and returns the result. As an example, these two queries return the same result: @@ -42,15 +37,15 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP There is an ` AggregatingMergeTree` engine. Its job during a merge is to combine the states of aggregate functions from different table rows with the same primary key value. -You can't use a normal INSERT to insert a row in a table containing `AggregateFunction` columns, because you can't explicitly define the `AggregateFunction` value. Instead, use `INSERT SELECT` with `-State` aggregate functions for inserting data. +You can't use a normal INSERT to insert a row in a table containing AggregateFunction columns, because you can't explicitly define the AggregateFunction value. Instead, use INSERT SELECT with '-State' aggregate functions for inserting data. -With SELECT from an `AggregatingMergeTree` table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. +With SELECT from an AggregatingMergeTree table, use GROUP BY and aggregate functions with the '-Merge' modifier in order to complete data aggregation. -You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views. +You can use AggregatingMergeTree tables for incremental data aggregation, including for aggregated materialized views. Example: -Create an `AggregatingMergeTree` materialized view that watches the `test.visits` table: +Creating a materialized AggregatingMergeTree view that tracks the 'test.visits' table: ```sql CREATE MATERIALIZED VIEW test.basic @@ -64,13 +59,13 @@ FROM test.visits GROUP BY CounterID, StartDate; ``` -Insert data in the `test.visits` table. Data will also be inserted in the view, where it will be aggregated: +Inserting data in the 'test.visits' table. Data will also be inserted in the view, where it will be aggregated: ```sql INSERT INTO test.visits ... ``` -Perform `SELECT` from the view using `GROUP BY` in order to complete data aggregation: +Performing SELECT from the view using GROUP BY to finish data aggregation: ```sql SELECT diff --git a/docs/en/table_engines/buffer.md b/docs/en/table_engines/buffer.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/collapsingmergetree.md b/docs/en/table_engines/collapsingmergetree.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/custom_partitioning_key.md b/docs/en/table_engines/custom_partitioning_key.md old mode 100755 new mode 100644 index bfcb3c2c545..6a468a1137d --- a/docs/en/table_engines/custom_partitioning_key.md +++ b/docs/en/table_engines/custom_partitioning_key.md @@ -2,7 +2,7 @@ # Custom partitioning key -Starting with version 1.1.54310, you can create tables in the MergeTree family with any partitioning expression (not only partitioning by month). +Starting with version 1.1.54310, you can create tables in the MergeTree family with any partition expression (not only partitioning by month). The partition key can be an expression from the table columns, or a tuple of such expressions (similar to the primary key). The partition key can be omitted. When creating a table, specify the partition key in the ENGINE description with the new syntax: @@ -10,7 +10,7 @@ The partition key can be an expression from the table columns, or a tuple of suc ENGINE [=] Name(...) [PARTITION BY expr] [ORDER BY expr] [SAMPLE BY expr] [SETTINGS name=value, ...] ``` -For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). The other engine parameters are specified in parentheses after the engine name, as previously. Example: +For MergeTree tables, the partition expression is specified after `PARTITION BY`, the primary key after `ORDER BY`, the sampling key after `SAMPLE BY`, and `SETTINGS` can specify `index_granularity` (optional; the default value is 8192), as well as other settings from [MergeTreeSettings.h](https://github.com/yandex/ClickHouse/blob/master/dbms/src/Storages/MergeTree/MergeTreeSettings.h). Example: ```sql ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/name', 'replica1', Sign) @@ -23,7 +23,7 @@ The traditional partitioning by month is expressed as `toYYYYMM(date_column)`. You can't convert an old-style table to a table with custom partitions (only via INSERT SELECT). -After this table is created, merge will only work for data parts that have the same value for the partitioning expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly. +After this table is created, merge will only work for data parts that have the same value for the partition expression. Note: This means that you shouldn't make overly granular partitions (more than about a thousand partitions), or SELECT will perform poorly. To specify a partition in ALTER PARTITION commands, specify the value of the partition expression (or a tuple). Constants and constant expressions are supported. Example: @@ -35,13 +35,13 @@ Deletes the partition for the current week with event type 1. The same is true f Note: For old-style tables, the partition can be specified either as a number `201710` or a string `'201710'`. The syntax for the new style of tables is stricter with types (similar to the parser for the VALUES input format). In addition, ALTER TABLE FREEZE PARTITION uses exact match for new-style tables (not prefix match). -In the `system.parts` table, the `partition` column specifies the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format. +In the `system.parts` table, the `partition` column should specify the value of the partition expression to use in ALTER queries (if quotas are removed). The `name` column should specify the name of the data part that has a new format. -Was: `20140317_20140323_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level). +Before: `20140317_20140323_2_2_0` (minimal data - maximal data - number of minimal block - number of maximal block - level). -Now: `201403_2_2_0` (partition ID - minimum block number - maximum block number - level). +After: `201403_2_2_0` (partition ID - number of minimal block - number of maximal block - level). The partition ID is its string identifier (human-readable, if possible) that is used for the names of data parts in the file system and in ZooKeeper. You can specify it in ALTER queries in place of the partition key. Example: Partition key `toYYYYMM(EventDate)`; ALTER can specify either `PARTITION 201710` or `PARTITION ID '201710'`. -For more examples, see the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). +There are more examples in the tests [`00502_custom_partitioning_local`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/yandex/ClickHouse/blob/master/dbms/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). diff --git a/docs/en/table_engines/dictionary.md b/docs/en/table_engines/dictionary.md deleted file mode 100755 index ae8cca90d7c..00000000000 --- a/docs/en/table_engines/dictionary.md +++ /dev/null @@ -1,106 +0,0 @@ - - -# Dictionary - -The `Dictionary` engine displays the dictionary data as a ClickHouse table. - -As an example, consider a dictionary of `products` with the following configuration: - -```xml - - - products - - - products
- DSN=some-db-server - - - - 300 - 360 - - - - - - - product_id - - - title - String - - - - - -``` - -Query the dictionary data: - -```sql -select name, type, key, attribute.names, attribute.types, bytes_allocated, element_count,source from system.dictionaries where name = 'products'; - -SELECT - name, - type, - key, - attribute.names, - attribute.types, - bytes_allocated, - element_count, - source -FROM system.dictionaries -WHERE name = 'products' -``` -``` -┌─name─────┬─type─┬─key────┬─attribute.names─┬─attribute.types─┬─bytes_allocated─┬─element_count─┬─source──────────┐ -│ products │ Flat │ UInt64 │ ['title'] │ ['String'] │ 23065376 │ 175032 │ ODBC: .products │ -└──────────┴──────┴────────┴─────────────────┴─────────────────┴─────────────────┴───────────────┴─────────────────┘ -``` - -You can use the [dictGet*](../functions/ext_dict_functions.md#ext_dict_functions) function to get the dictionary data in this format. - -This view isn't helpful when you need to get raw data, or when performing a `JOIN` operation. For these cases, you can use the `Dictionary` engine, which displays the dictionary data in a table. - -Syntax: - -``` -CREATE TABLE %table_name% (%fields%) engine = Dictionary(%dictionary_name%)` -``` - -Usage example: - -```sql -create table products (product_id UInt64, title String) Engine = Dictionary(products); - -CREATE TABLE products -( - product_id UInt64, - title String, -) -ENGINE = Dictionary(products) -``` -``` -Ok. - -0 rows in set. Elapsed: 0.004 sec. -``` - -Take a look at what's in the table. - -```sql -select * from products limit 1; - -SELECT * -FROM products -LIMIT 1 -``` -``` -┌────product_id─┬─title───────────┐ -│ 152689 │ Некоторый товар │ -└───────────────┴─────────────────┘ - -1 rows in set. Elapsed: 0.006 sec. -``` diff --git a/docs/en/table_engines/distributed.md b/docs/en/table_engines/distributed.md old mode 100755 new mode 100644 index dd2ffe27fe5..b8643461fbb --- a/docs/en/table_engines/distributed.md +++ b/docs/en/table_engines/distributed.md @@ -25,29 +25,29 @@ Clusters are set like this: - - 1 - - false - - example01-01-1 - 9000 - - - example01-01-2 - 9000 - - - - 2 - false - - example01-02-1 - 9000 - - - example01-02-2 - 9000 + + 1 + + false + + example01-01-1 + 9000 + + + example01-01-2 + 9000 + + + + 2 + false + + example01-02-1 + 9000 + + + example01-02-2 + 9000 diff --git a/docs/en/table_engines/external_data.md b/docs/en/table_engines/external_data.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/file.md b/docs/en/table_engines/file.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/graphitemergetree.md b/docs/en/table_engines/graphitemergetree.md old mode 100755 new mode 100644 index a4b62424954..6452377ac15 --- a/docs/en/table_engines/graphitemergetree.md +++ b/docs/en/table_engines/graphitemergetree.md @@ -2,13 +2,13 @@ # GraphiteMergeTree -This engine is designed for rollup (thinning and aggregating/averaging) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. +This engine is designed for rollup (thinning and aggregating/averaging) [ Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite. Graphite stores full data in ClickHouse, and data can be retrieved in the following ways: - Without thinning. - Uses the [MergeTree](mergetree.md#table_engines-mergetree) engine. + Using the [MergeTree](mergetree.md#table_engines-mergetree) engine. - With thinning. @@ -83,3 +83,4 @@ Example of settings: ``` + diff --git a/docs/en/table_engines/index.md b/docs/en/table_engines/index.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/join.md b/docs/en/table_engines/join.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/kafka.md b/docs/en/table_engines/kafka.md old mode 100755 new mode 100644 index 4f10e55d029..85943e44fc5 --- a/docs/en/table_engines/kafka.md +++ b/docs/en/table_engines/kafka.md @@ -59,7 +59,7 @@ Example: level String, total UInt64 ) ENGINE = SummingMergeTree(day, (day, level), 8192); - + CREATE MATERIALIZED VIEW consumer TO daily AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total FROM queue GROUP BY day, level; diff --git a/docs/en/table_engines/log.md b/docs/en/table_engines/log.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/materializedview.md b/docs/en/table_engines/materializedview.md old mode 100755 new mode 100644 index 5e2741c6aa1..00f70bd72bd --- a/docs/en/table_engines/materializedview.md +++ b/docs/en/table_engines/materializedview.md @@ -1,4 +1,4 @@ # MaterializedView -Used for implementing materialized views (for more information, see the [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)) query. For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. +Used for implementing materialized views (for more information, see [CREATE TABLE](../query_language/queries.md#query_language-queries-create_table)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine. diff --git a/docs/en/table_engines/memory.md b/docs/en/table_engines/memory.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/merge.md b/docs/en/table_engines/merge.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/mergetree.md b/docs/en/table_engines/mergetree.md old mode 100755 new mode 100644 index fea02e01d72..71197f21b34 --- a/docs/en/table_engines/mergetree.md +++ b/docs/en/table_engines/mergetree.md @@ -56,7 +56,7 @@ In this example, the index can't be used: SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' ``` -To check whether ClickHouse can use the index when executing the query, use the settings [force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date)and[force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key). +To check whether ClickHouse can use the index when executing the query, use the settings [ force_index_by_date](../operations/settings/settings.md#settings-settings-force_index_by_date) and [ force_primary_key](../operations/settings/settings.md#settings-settings-force_primary_key). The index by date only allows reading those parts that contain dates from the desired range. However, a data part may contain data for many dates (up to an entire month), while within a single part the data is ordered by the primary key, which might not contain the date as the first column. Because of this, using a query with only a date condition that does not specify the primary key prefix will cause more data to be read than for a single date. diff --git a/docs/en/table_engines/null.md b/docs/en/table_engines/null.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/replacingmergetree.md b/docs/en/table_engines/replacingmergetree.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/replication.md b/docs/en/table_engines/replication.md old mode 100755 new mode 100644 index 20dd17e444f..1e58878c34e --- a/docs/en/table_engines/replication.md +++ b/docs/en/table_engines/replication.md @@ -46,7 +46,7 @@ You can specify any existing ZooKeeper cluster and the system will use a directo If ZooKeeper isn't set in the config file, you can't create replicated tables, and any existing replicated tables will be read-only. -ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries). +ZooKeeper isn't used for SELECT queries. In other words, replication doesn't affect the productivity of SELECT queries – they work just as fast as for non-replicated tables. When querying distributed replicated tables, ClickHouse behavior is controlled by the settings [max_replica_delay_for_distributed_queries](../operations/settings/settings.md#settings_settings_max_replica_delay_for_distributed_queries) and [fallback_to_stale_replicas_for_distributed_queries](../operations/settings/settings.md#settings-settings-fallback_to_stale_replicas_for_distributed_queries). For each INSERT query (more precisely, for each inserted block of data; the INSERT query contains a single block, or per block for every max_insert_block_size = 1048576 rows), approximately ten entries are made in ZooKeeper in several transactions. This leads to slightly longer latencies for INSERT compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one INSERT per second, it doesn't create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred INSERTs per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. diff --git a/docs/en/table_engines/set.md b/docs/en/table_engines/set.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/summingmergetree.md b/docs/en/table_engines/summingmergetree.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/tinylog.md b/docs/en/table_engines/tinylog.md old mode 100755 new mode 100644 diff --git a/docs/en/table_engines/view.md b/docs/en/table_engines/view.md old mode 100755 new mode 100644 diff --git a/docs/en/table_functions/index.md b/docs/en/table_functions/index.md old mode 100755 new mode 100644 diff --git a/docs/en/table_functions/merge.md b/docs/en/table_functions/merge.md old mode 100755 new mode 100644 diff --git a/docs/en/table_functions/remote.md b/docs/en/table_functions/remote.md old mode 100755 new mode 100644 index e26e245207b..99b0c7bb116 --- a/docs/en/table_functions/remote.md +++ b/docs/en/table_functions/remote.md @@ -52,7 +52,7 @@ example01-{01..02}-1 If you have multiple pairs of curly brackets, it generates the direct product of the corresponding sets. -Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. However, the replicas are iterated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting. +Addresses and parts of addresses in curly brackets can be separated by the pipe symbol (|). In this case, the corresponding sets of addresses are interpreted as replicas, and the query will be sent to the first healthy replica. The replicas are evaluated in the order currently set in the [load_balancing](../operations/settings/settings.md#settings-load_balancing) setting. Example: diff --git a/docs/en/utils/clickhouse-copier.md b/docs/en/utils/clickhouse-copier.md old mode 100755 new mode 100644 index 9d15053fe06..25d22f19222 --- a/docs/en/utils/clickhouse-copier.md +++ b/docs/en/utils/clickhouse-copier.md @@ -1,54 +1,40 @@ - +# clickhouse-copier util -# clickhouse-copier +The util copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed and fault-tolerant manner. -Copies data from the tables in one cluster to tables in another (or the same) cluster. +Configuration of copying tasks is set in special ZooKeeper node (called the `/description` node). +A ZooKeeper path to the description node is specified via `--task-path ` parameter. +So, node `/task/path/description` should contain special XML content describing copying tasks. -You can run multiple `clickhouse-copier` instances on different servers to perform the same job. ZooKeeper is used for syncing the processes. +Simultaneously many `clickhouse-copier` processes located on any servers could execute the same task. +ZooKeeper node `/task/path/` is used by the processes to coordinate their work. +You must not add additional child nodes to `/task/path/`. -After starting, `clickhouse-copier`: +Currently you are responsible for manual launching of all `cluster-copier` processes. +You can launch as many processes as you want, whenever and wherever you want. +Each process try to select the nearest available shard of source cluster and copy some part of data (partition) from it to the whole +destination cluster (with resharding). +Therefore it makes sense to launch cluster-copier processes on the source cluster nodes to reduce the network usage. -- Connects to ZooKeeper and receives: - - Copying jobs. - - The state of the copying jobs. - -- It performs the jobs. - - Each running process chooses the "closest" shard of the source cluster and copies the data into the destination cluster, resharding the data if necessary. - -`clickhouse-copier` tracks the changes in ZooKeeper and applies them on the fly. - -To reduce network traffic, we recommend running `clickhouse-copier` on the same server where the source data is located. - -## Running clickhouse-copier - -The utility should be run manually: - -```bash -clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/path --base-dir /path/to/dir -``` - -Parameters: - -- `daemon` — Starts `clickhouse-copier` in daemon mode. -- `config` — The path to the `zookeeper.xml` file with the parameters for the connection to ZooKeeper. -- `task-path` — The path to the ZooKeeper node. This node is used for syncing `clickhouse-copier` processes and storing tasks. Tasks are stored in `$task-path/description`. -- `base-dir` — The path to logs and auxiliary files. When it starts, `clickhouse-copier` creates `clickhouse-copier_YYYYMMHHSS_` subdirectories in `$base-dir`. If this parameter is omitted, the directories are created in the directory where `clickhouse-copier` was launched. - -## Format of zookeeper.xml +Since the workers coordinate their work via ZooKeeper, in addition to `--task-path ` you have to specify ZooKeeper +cluster configuration via `--config-file ` parameter. Example of `zookeeper.xml`: ```xml - + 127.0.0.1 2181 - + ``` -## Configuration of copying tasks +When you run `clickhouse-copier --config-file --task-path ` the process connects to ZooKeeper cluster, reads tasks config from `/task/path/description` and executes them. + +## Format of task config + +Here is an example of `/task/path/description` content: ```xml @@ -83,62 +69,62 @@ Parameters: 0 - 3 - + 1 - - + - + source_cluster test hits - + destination_cluster test hits2 - - - ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hits2', '{replica}') PARTITION BY toMonday(date) ORDER BY (CounterID, EventDate) - + jumpConsistentHash(intHash64(UserID), 2) - + CounterID != 0 - '2018-02-26' @@ -147,7 +133,7 @@ Parameters: - + ... @@ -156,5 +142,15 @@ Parameters: ``` -`clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change. +cluster-copier processes watch for `/task/path/description` node update. +So, if you modify the config settings or `max_workers` params, they will be updated. +## Example + +```bash +clickhouse-copier copier --daemon --config /path/to/copier/zookeeper.xml --task-path /clickhouse-copier/cluster1_tables_hits --base-dir /path/to/copier_logs +``` + +`--base-dir /path/to/copier_logs` specifies where auxilary and log files of the copier process will be saved. +In this case it will create `/path/to/copier_logs/clickhouse-copier_YYYYMMHHSS_/` dir with log and status-files. +If it is not specified it will use current dir (`/clickhouse-copier_YYYYMMHHSS_/` if it is run as a `--daemon`). diff --git a/docs/en/utils/clickhouse-local.md b/docs/en/utils/clickhouse-local.md old mode 100755 new mode 100644 index d5fba56271f..d18cc200320 --- a/docs/en/utils/clickhouse-local.md +++ b/docs/en/utils/clickhouse-local.md @@ -1,6 +1,4 @@ - +# The clickhouse-local program -#clickhouse-local - -The `clickhouse-local` program enables you to perform fast processing on local files that store tables, without having to deploy and configure the ClickHouse server. +The `clickhouse-local` program enables you to perform fast processing on local files that store tables, without having to deploy and configure clickhouse-server. diff --git a/docs/en/utils/index.md b/docs/en/utils/index.md old mode 100755 new mode 100644 index cf541cda895..7a8c5ee5138 --- a/docs/en/utils/index.md +++ b/docs/en/utils/index.md @@ -1,5 +1,6 @@ -# ClickHouse utility +# ClickHouse utilites -* [clickhouse-local](clickhouse-local.md#utils-clickhouse-local) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this. -* [clickhouse-copier](clickhouse-copier.md#utils-clickhouse-copier) — Copies (and reshards) data from one cluster to another cluster. +There are several ClickHouse utilites that are separate executable files: +* `clickhouse-local` allows to execute SQL queries on a local data like `awk` +* `clickhouse-copier` copies (and reshards) immutable data from one cluster to another in a fault-tolerant manner. diff --git a/docs/mkdocs_en.yml b/docs/mkdocs_en.yml index eeedc71a79b..d5fadb3f1e1 100644 --- a/docs/mkdocs_en.yml +++ b/docs/mkdocs_en.yml @@ -86,7 +86,6 @@ pages: - 'GraphiteMergeTree': 'table_engines/graphitemergetree.md' - 'Data replication': 'table_engines/replication.md' - 'Distributed': 'table_engines/distributed.md' - - 'Dictionary': 'table_engines/dictionary.md' - 'Merge': 'table_engines/merge.md' - 'Buffer': 'table_engines/buffer.md' - 'File': 'table_engines/file.md' diff --git a/docs/ru/dicts/external_dicts_dict_layout.md b/docs/ru/dicts/external_dicts_dict_layout.md index defb0605c0f..ff1b9c0cdd5 100644 --- a/docs/ru/dicts/external_dicts_dict_layout.md +++ b/docs/ru/dicts/external_dicts_dict_layout.md @@ -109,15 +109,15 @@ Пример: таблица содержит скидки для каждого рекламодателя в виде: ``` -+---------------+---------------------+-------------------+--------+ -| advertiser id | discount start date | discount end date | amount | -+===============+=====================+===================+========+ -| 123 | 2015-01-01 | 2015-01-15 | 0.15 | -+---------------+---------------------+-------------------+--------+ -| 123 | 2015-01-16 | 2015-01-31 | 0.25 | -+---------------+---------------------+-------------------+--------+ -| 456 | 2015-01-01 | 2015-01-15 | 0.05 | -+---------------+---------------------+-------------------+--------+ + +------------------+-----------------------------+------------+----------+ + | id рекламодателя | дата начала действия скидки | дата конца | величина | + +==================+=============================+============+==========+ + | 123 | 2015-01-01 | 2015-01-15 | 0.15 | + +------------------+-----------------------------+------------+----------+ + | 123 | 2015-01-16 | 2015-01-31 | 0.25 | + +------------------+-----------------------------+------------+----------+ + | 456 | 2015-01-01 | 2015-01-15 | 0.05 | + +------------------+-----------------------------+------------+----------+ ``` Чтобы использовать выборку по диапазонам дат, необходимо в [structure](external_dicts_dict_structure#dicts-external_dicts_dict_structure) определить элементы `range_min`, `range_max`. diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md index 765c0450890..24e595b2c49 100644 --- a/docs/ru/introduction/ya_metrika_task.md +++ b/docs/ru/introduction/ya_metrika_task.md @@ -1,6 +1,6 @@ # Постановка задачи в Яндекс.Метрике -ClickHouse на данный момент обеспечивает работу [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. +ClickHouse на данный момент обеспечивает рабту [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Нужно получать произвольные отчёты на основе хитов и визитов, с произвольными сегментами, задаваемыми пользователем. Данные для отчётов обновляются в реальном времени. Запросы должны выполняться сразу (в режиме онлайн). Отчёты должно быть возможно строить за произвольный период. Требуется вычислять сложные агрегаты типа количества уникальных посетителей. На данный момент (апрель 2014), каждый день в Яндекс.Метрику поступает около 12 миллиардов событий (хитов и кликов мыши). Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать сотни миллионов строк за время не более нескольких секунд, или миллионы строк за время не более нескольких сотен миллисекунд. diff --git a/docs/ru/roadmap.md b/docs/ru/roadmap.md index bbdc740bf01..13c2b60c094 100644 --- a/docs/ru/roadmap.md +++ b/docs/ru/roadmap.md @@ -5,15 +5,15 @@ ### Новая функциональность - Поддержка `UPDATE` и `DELETE`. - Многомерные и вложенные массивы. - + Это может выглядеть например так: - + ```sql CREATE TABLE t ( - x Array(Array(String)), + x Array(Array(String)), z Nested( - x Array(String), + x Array(String), y Nested(...)) ) ENGINE = MergeTree ORDER BY x @@ -24,7 +24,7 @@ ENGINE = MergeTree ORDER BY x Внешние таблицы можно интрегрировать в ClickHouse с помощью внешних словарей. Новая функциональность станет более удобной альтернативой для подключения внешних таблиц. ```sql -SELECT ... +SELECT ... FROM mysql('host:port', 'db', 'table', 'user', 'password')` ``` @@ -40,7 +40,7 @@ INSERT INTO t SELECT * FROM remote(...) `. - O_DIRECT for merges. Улучшит производительность кэша операционной системы, а также производительность 'горячих' запросов. - + ## Q2 2018 ### Новая функциональность @@ -56,23 +56,23 @@ INSERT INTO t SELECT * FROM remote(...) `. - Настраиваемые кодировки и сжатие для каждого столбца в отдельности. Сейчас, ClickHouse поддерживает сжатие столбцов с помощью LZ4 и ZSTD, и настройки сжатия глобальные (смотрите статью [Compression in ClickHouse](https://www.altinity.com/blog/2017/11/21/compression-in-clickhouse)). Поколоночное сжатие и кодирование обеспечит более эффективное хранение данных, что в свою очередь ускорит выполнение запросов. - + - Хранение данных на нескольких дисках на одном сервере. Реализация это функциональности упростит расширение дискового пространства, поскольку можно будет использовать различные дисковые системы для разных баз данных или таблиц. Сейчас, пользователи вынуждены использовать символические ссылки, если базы данных и таблицы должны храниться на другом диске. - + ### Улучшения Планируется множество улучшений и исправлений в системе выполнения запросов. Например: - Использование индекса для `in (subquery)`. - Сейчас, индекс не используется, что приводит к снижению производительности. - + Сейчас, индекс не используется, что приводит с снижению производительности. + - Передача предикатов из `where` в подзапросы, а также передача предикатов в представления. Передача предикатов необходима, поскольку представление изменяется поздапросом. Сейчас производительность фильтров для представлений низкая, представления не могут использовать первичный ключ оригинальной таблицы, что делает представления для больших таблиц бесполезными. - + - Оптимизация операций с ветвлением (тернарный оператор, if, multiIf). Сейчас, ClickHouse выполняет все ветви, даже если в этом нет необходимости. @@ -88,7 +88,7 @@ INSERT INTO t SELECT * FROM remote(...) `. - Пулы ресурсов для выполнения запросов. Позволят более эффективно управлять нагрузкой. - + - Синтаксис ANSI SQL JOIN. Улучшит совместимость ClickHouse со множеством SQL-инструментов. From c5b04f7f17e40402d70cabc27328a945a3656757 Mon Sep 17 00:00:00 2001 From: Alex Zatelepin Date: Mon, 26 Mar 2018 17:18:04 +0300 Subject: [PATCH 72/90] Add info about total uncompressed part size to system tables (#2052) * column size calculating routines now take into account multiple streams [#CLICKHOUSE-2831] * rename for clarity * use more appropriate method for calculating table size * add info about total uncompressed part size to system.parts and system.parts_columns [#CLICKHOUSE-2831] * rename columns for clarity [#CLICKHOUSE-2831] * count shared substreams only once [#CLICKHOUSE-2831] --- dbms/src/Interpreters/PartLog.cpp | 2 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 9 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 70 ++++-------- dbms/src/Storages/MergeTree/MergeTreeData.h | 30 +---- .../MergeTree/MergeTreeDataMerger.cpp | 8 +- .../Storages/MergeTree/MergeTreeDataPart.cpp | 107 ++++++++---------- .../Storages/MergeTree/MergeTreeDataPart.h | 43 ++++--- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 4 +- .../Storages/StorageReplicatedMergeTree.cpp | 12 +- .../Storages/System/StorageSystemColumns.cpp | 2 +- .../Storages/System/StorageSystemParts.cpp | 51 +++++---- .../System/StorageSystemPartsBase.cpp | 29 ++++- .../System/StorageSystemPartsColumns.cpp | 84 +++++++------- 16 files changed, 225 insertions(+), 232 deletions(-) diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp index 8043f451402..646aae527b1 100644 --- a/dbms/src/Interpreters/PartLog.cpp +++ b/dbms/src/Interpreters/PartLog.cpp @@ -108,7 +108,7 @@ bool PartLog::addNewPartToTheLog(Context & context, const MergeTreeDataPart & pa elem.table_name = part.storage.getTableName(); elem.part_name = part.name; - elem.bytes_compressed_on_disk = part.size_in_bytes; + elem.bytes_compressed_on_disk = part.bytes_on_disk; elem.rows = part.rows_count; elem.error = static_cast(execution_status.code); diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 5d52eedfda1..a2bfffabeb2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -102,8 +102,6 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N const String & column_name = column_with_type_and_name.name; const ColumnPtr & column_data = column_with_type_and_name.column; - const auto column_checksum = data_part->tryGetBinChecksum(column_name); - if (!from_update && !names_set.count(column_name)) continue; @@ -122,8 +120,11 @@ void MergeTreeBlockSizePredictor::initialize(const Block & sample_block, const N ColumnInfo info; info.name = column_name; /// If column isn't fixed and doesn't have checksum, than take first - info.bytes_per_row_global = column_checksum - ? column_checksum->uncompressed_size / number_of_rows_in_part + MergeTreeDataPart::ColumnSize column_size = data_part->getColumnSize( + column_name, *column_with_type_and_name.type); + + info.bytes_per_row_global = column_size.data_uncompressed + ? column_size.data_uncompressed / number_of_rows_in_part : column_data->byteSize() / std::max(1, column_data->size()); dynamic_columns_infos.emplace_back(info); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ef4b8022323..5b5a6b87c8e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -747,7 +747,7 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa for (auto & part : parts) { part_log_elem.part_name = part->name; - part_log_elem.bytes_compressed_on_disk = part->size_in_bytes; + part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk; part_log_elem.rows = part->rows_count; part_log->add(part_log_elem); @@ -1211,8 +1211,8 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart( false, nullptr, "", false, 0, DBMS_DEFAULT_BUFFER_SIZE, false); auto compression_settings = this->context.chooseCompressionSettings( - part->size_in_bytes, - static_cast(part->size_in_bytes) / this->getTotalActiveSizeInBytes()); + part->bytes_on_disk, + static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); ExpressionBlockInputStream in(part_in, expression); /** Don't write offsets for arrays, because ALTER never change them @@ -1314,7 +1314,7 @@ void MergeTreeData::AlterDataPartTransaction::commit() file.remove(); } - mutable_part.size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(path); + mutable_part.bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(path); /// TODO: we can skip resetting caches when the column is added. data_part->storage.context.dropCaches(); @@ -1674,7 +1674,7 @@ size_t MergeTreeData::getTotalActiveSizeInBytes() const std::lock_guard lock(data_parts_mutex); for (auto & part : getDataPartsStateRange(DataPartState::Committed)) - res += part->size_in_bytes; + res += part->bytes_on_disk; } return res; @@ -1832,59 +1832,37 @@ void MergeTreeData::calculateColumnSizesImpl() void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) { std::shared_lock lock(part->columns_lock); - const auto & files = part->checksums.files; - /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : getColumns().getAllPhysical()) + for (const auto & column : part->columns) { - const auto escaped_name = escapeForFileName(column.name); - const auto bin_file_name = escaped_name + ".bin"; - const auto mrk_file_name = escaped_name + ".mrk"; - - ColumnSize & column_size = column_sizes[column.name]; - - if (files.count(bin_file_name)) - { - const auto & bin_file_checksums = files.at(bin_file_name); - column_size.data_compressed += bin_file_checksums.file_size; - column_size.data_uncompressed += bin_file_checksums.uncompressed_size; - } - - if (files.count(mrk_file_name)) - column_size.marks += files.at(mrk_file_name).file_size; + DataPart::ColumnSize & total_column_size = column_sizes[column.name]; + DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); + total_column_size.add(part_column_size); } } -static inline void logSubtract(size_t & from, size_t value, Logger * log, const String & variable) -{ - if (value > from) - LOG_ERROR(log, "Possibly incorrect subtraction: " << from << " - " << value << " = " << from - value << ", variable " << variable); - - from -= value; -} - void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part) { - const auto & files = part->checksums.files; + std::shared_lock lock(part->columns_lock); - /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : getColumns().getAllPhysical()) + for (const auto & column : part->columns) { - const auto escaped_name = escapeForFileName(column.name); - const auto bin_file_name = escaped_name + ".bin"; - const auto mrk_file_name = escaped_name + ".mrk"; + DataPart::ColumnSize & total_column_size = column_sizes[column.name]; + DataPart::ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); - auto & column_size = column_sizes[column.name]; - - if (files.count(bin_file_name)) + auto log_subtract = [&](size_t & from, size_t value, const char * field) { - const auto & bin_file_checksums = files.at(bin_file_name); - logSubtract(column_size.data_compressed, bin_file_checksums.file_size, log, bin_file_name + ".file_size"); - logSubtract(column_size.data_uncompressed, bin_file_checksums.uncompressed_size, log, bin_file_name + ".uncompressed_size"); - } + if (value > from) + LOG_ERROR(log, "Possibly incorrect column size subtraction: " + << from << " - " << value << " = " << from - value + << ", column: " << column.name << ", field: " << field); - if (files.count(mrk_file_name)) - logSubtract(column_size.marks, files.at(mrk_file_name).file_size, log, mrk_file_name + ".file_size"); + from -= value; + }; + + log_subtract(total_column_size.data_compressed, part_column_size.data_compressed, ".data_compressed"); + log_subtract(total_column_size.data_uncompressed, part_column_size.data_uncompressed, ".data_uncompressed"); + log_subtract(total_column_size.marks, part_column_size.marks, ".marks"); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 47abd959645..f412419459c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -452,18 +452,6 @@ public: /// Returns the size of partition in bytes. size_t getPartitionSize(const std::string & partition_id) const; - struct ColumnSize - { - size_t marks = 0; - size_t data_compressed = 0; - size_t data_uncompressed = 0; - - size_t getTotalCompressedSize() const - { - return marks + data_compressed; - } - }; - size_t getColumnCompressedSize(const std::string & name) const { std::lock_guard lock{data_parts_mutex}; @@ -472,25 +460,13 @@ public: return it == std::end(column_sizes) ? 0 : it->second.data_compressed; } - using ColumnSizes = std::unordered_map; - ColumnSizes getColumnSizes() const + using ColumnSizeByName = std::unordered_map; + ColumnSizeByName getColumnSizes() const { std::lock_guard lock{data_parts_mutex}; return column_sizes; } - /// NOTE Could be off after DROPped and MODIFYed columns in ALTER. Doesn't include primary.idx. - size_t getTotalCompressedSize() const - { - std::lock_guard lock{data_parts_mutex}; - size_t total_size = 0; - - for (const auto & col : column_sizes) - total_size += col.second.getTotalCompressedSize(); - - return total_size; - } - /// Calculates column sizes in compressed form for the current state of data_parts. void recalculateColumnSizes() { @@ -555,7 +531,7 @@ private: String full_path; /// Current column sizes in compressed and uncompressed form. - ColumnSizes column_sizes; + ColumnSizeByName column_sizes; /// Engine-specific methods BrokenPartCallback broken_part_callback; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index ae8677c57be..f2d6e773be1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -180,7 +180,7 @@ bool MergeTreeDataMerger::selectPartsToMerge( } IMergeSelector::Part part_info; - part_info.size = part->size_in_bytes; + part_info.size = part->bytes_on_disk; part_info.age = current_time - part->modification_time; part_info.level = part->info.level; part_info.data = ∂ @@ -266,7 +266,7 @@ bool MergeTreeDataMerger::selectAllPartsToMergeWithinPartition( return false; } - sum_bytes += (*it)->size_in_bytes; + sum_bytes += (*it)->bytes_on_disk; prev_it = it; ++it; @@ -532,7 +532,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart { std::shared_lock part_lock(part->columns_lock); - merge_entry->total_size_bytes_compressed += part->size_in_bytes; + merge_entry->total_size_bytes_compressed += part->bytes_on_disk; merge_entry->total_size_marks += part->marks_count; } @@ -883,7 +883,7 @@ size_t MergeTreeDataMerger::estimateDiskSpaceForMerge(const MergeTreeData::DataP { size_t res = 0; for (const MergeTreeData::DataPartPtr & part : parts) - res += part->size_in_bytes; + res += part->bytes_on_disk; return static_cast(res * DISK_USAGE_COEFFICIENT_TO_RESERVE); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 79f3ddb523f..26f7c2c4493 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -134,26 +134,51 @@ MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const String & na { } -/// Returns the size of .bin file for column `name` if found, zero otherwise. -UInt64 MergeTreeDataPart::getColumnCompressedSize(const String & name) const +/// Takes into account the fact that several columns can e.g. share their .size substreams. +/// When calculating totals these should be counted only once. +MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const { - const Checksum * checksum = tryGetBinChecksum(name); + ColumnSize size; + if (checksums.empty()) + return size; - /// Probably a logic error, not sure if this can ever happen if checksums are not empty - return checksum ? checksum->file_size : 0; + type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(name, substream_path); + + if (processed_substreams && !processed_substreams->insert(file_name).second) + return; + + auto bin_checksum = checksums.files.find(file_name + ".bin"); + if (bin_checksum != checksums.files.end()) + { + size.data_compressed += bin_checksum->second.file_size; + size.data_uncompressed += bin_checksum->second.uncompressed_size; + } + + auto mrk_checksum = checksums.files.find(file_name + ".mrk"); + if (mrk_checksum != checksums.files.end()) + size.marks += mrk_checksum->second.file_size; + }, {}); + + return size; } -UInt64 MergeTreeDataPart::getColumnUncompressedSize(const String & name) const +MergeTreeDataPart::ColumnSize MergeTreeDataPart::getColumnSize(const String & name, const IDataType & type) const { - const Checksum * checksum = tryGetBinChecksum(name); - return checksum ? checksum->uncompressed_size : 0; + return getColumnSizeImpl(name, type, nullptr); } - -UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const +MergeTreeDataPart::ColumnSize MergeTreeDataPart::getTotalColumnsSize() const { - const Checksum * checksum = tryGetMrkChecksum(name); - return checksum ? checksum->file_size : 0; + ColumnSize totals; + std::unordered_set processed_substreams; + for (const NameAndTypePair & column : columns) + { + ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams); + totals.add(size); + } + return totals; } @@ -171,7 +196,7 @@ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const if (!hasColumnFiles(column.name)) continue; - const auto size = getColumnCompressedSize(column.name); + const auto size = getColumnSize(column.name, *column.type).data_compressed; if (size < minimum_size) { minimum_size = size; @@ -251,7 +276,7 @@ MergeTreeDataPart::~MergeTreeDataPart() } } -UInt64 MergeTreeDataPart::calculateTotalSize(const String & from) +UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) { Poco::File cur(from); if (cur.isFile()) @@ -260,7 +285,7 @@ UInt64 MergeTreeDataPart::calculateTotalSize(const String & from) cur.list(files); UInt64 res = 0; for (const auto & file : files) - res += calculateTotalSize(from + file); + res += calculateTotalSizeOnDisk(from + file); return res; } @@ -420,7 +445,7 @@ void MergeTreeDataPart::loadIndex() index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); } - size_in_bytes = calculateTotalSize(getFullPath()); + bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); } void MergeTreeDataPart::loadPartitionAndMinMaxIndex() @@ -484,20 +509,21 @@ void MergeTreeDataPart::loadRowsCount() for (const NameAndTypePair & column : columns) { ColumnPtr column_col = column.type->createColumn(); - const auto checksum = tryGetBinChecksum(column.name); + if (!column_col->isFixedAndContiguous()) + continue; - /// Should be fixed non-nullable column - if (!checksum || !column_col->isFixedAndContiguous()) + size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed; + if (!column_size) continue; size_t sizeof_field = column_col->sizeOfValueIfFixed(); - rows_count = checksum->uncompressed_size / sizeof_field; + rows_count = column_size / sizeof_field; - if (checksum->uncompressed_size % sizeof_field != 0) + if (column_size % sizeof_field != 0) { throw Exception( - "Column " + column.name + " has indivisible uncompressed size " + toString(checksum->uncompressed_size) - + ", sizeof " + toString(sizeof_field), + "Uncompressed size of column " + column.name + "(" + toString(column_size) + + ") is not divisible by the size of value (" + toString(sizeof_field) + ")", ErrorCodes::LOGICAL_ERROR); } @@ -670,29 +696,6 @@ bool MergeTreeDataPart::hasColumnFiles(const String & column) const } -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetChecksum(const String & name, const String & ext) const -{ - if (checksums.empty()) - return nullptr; - - const auto & files = checksums.files; - const auto file_name = escapeForFileName(name) + ext; - auto it = files.find(file_name); - - return (it == files.end()) ? nullptr : &it->second; -} - -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetBinChecksum(const String & name) const -{ - return tryGetChecksum(name, ".bin"); -} - -const MergeTreeDataPartChecksums::Checksum * MergeTreeDataPart::tryGetMrkChecksum(const String & name) const -{ - return tryGetChecksum(name, ".mrk"); -} - - UInt64 MergeTreeDataPart::getIndexSizeInBytes() const { UInt64 res = 0; @@ -709,18 +712,6 @@ UInt64 MergeTreeDataPart::getIndexSizeInAllocatedBytes() const return res; } -UInt64 MergeTreeDataPart::getTotalMrkSizeInBytes() const -{ - UInt64 res = 0; - for (const NameAndTypePair & it : columns) - { - const Checksum * checksum = tryGetMrkChecksum(it.name); - if (checksum) - res += checksum->file_size; - } - return res; -} - String MergeTreeDataPart::stateToString(MergeTreeDataPart::State state) { switch (state) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h index a617dfedacd..c1c1274f7b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.h @@ -29,22 +29,31 @@ struct MergeTreeDataPart MergeTreeDataPart(MergeTreeData & storage_, const String & name_); - const Checksum * tryGetChecksum(const String & name, const String & ext) const; - /// Returns checksum of column's binary file. - const Checksum * tryGetBinChecksum(const String & name) const; - /// Returns checksum of column's mrk file. - const Checksum * tryGetMrkChecksum(const String & name) const; - - /// Returns the size of .bin file for column `name` if found, zero otherwise - UInt64 getColumnCompressedSize(const String & name) const; - UInt64 getColumnUncompressedSize(const String & name) const; - /// Returns the size of .mrk file for column `name` if found, zero otherwise - UInt64 getColumnMrkSize(const String & name) const; - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const; + struct ColumnSize + { + size_t marks = 0; + size_t data_compressed = 0; + size_t data_uncompressed = 0; + + void add(const ColumnSize & other) + { + marks += other.marks; + data_compressed += other.data_compressed; + data_uncompressed += other.data_uncompressed; + } + }; + + /// NOTE: Returns zeros if column files are not found in checksums. + /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. + /// (either by locking columns_lock, or by locking table structure). + ColumnSize getColumnSize(const String & name, const IDataType & type) const; + + ColumnSize getTotalColumnsSize() const; + /// Returns full path to part dir String getFullPath() const; @@ -68,8 +77,8 @@ struct MergeTreeDataPart size_t rows_count = 0; size_t marks_count = 0; - std::atomic size_in_bytes {0}; /// size in bytes, 0 - if not counted; - /// is used from several threads without locks (it is changed with ALTER). + std::atomic bytes_on_disk {0}; /// 0 - if not counted; + /// Is used from several threads without locks (it is changed with ALTER). time_t modification_time = 0; /// When the part is removed from the working set. Changes once. mutable std::atomic remove_time { std::numeric_limits::max() }; @@ -208,7 +217,7 @@ struct MergeTreeDataPart ~MergeTreeDataPart(); /// Calculate the total size of the entire directory with all the files - static UInt64 calculateTotalSize(const String & from); + static UInt64 calculateTotalSizeOnDisk(const String & from); void remove() const; @@ -232,8 +241,6 @@ struct MergeTreeDataPart /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; - /// Total size of *.mrk files - UInt64 getTotalMrkSizeInBytes() const; private: /// Reads columns names and types from columns.txt @@ -252,6 +259,8 @@ private: void loadPartitionAndMinMaxIndex(); void checkConsistency(bool require_part_metadata); + + ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index e85efa435fd..9d6d41fa9d5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -214,7 +214,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterUncompressedBytes, block.bytes()); - ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->size_in_bytes); + ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->bytes_on_disk); return new_data_part; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index d3162810a1c..6c9409be833 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -347,7 +347,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->columns = *total_column_list; new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); new_part->checksums = checksums; - new_part->size_in_bytes = MergeTreeData::DataPart::calculateTotalSize(new_part->getFullPath()); + new_part->bytes_on_disk = MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_part->getFullPath()); } void MergedBlockOutputStream::init() diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index c5d4a41d496..32d7b263f80 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -634,7 +634,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( auto part = data.getPartIfExists(name, {MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); if (part) - sum_parts_size_in_bytes += part->size_in_bytes; + sum_parts_size_in_bytes += part->bytes_on_disk; } if (merger.merges_blocker.isCancelled()) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 293c3254a8f..da205e9a293 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -116,7 +116,7 @@ BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Set bool StorageMergeTree::checkTableCanBeDropped() const { const_cast(getData()).recalculateColumnSizes(); - context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize()); + context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); return true; } @@ -334,7 +334,7 @@ bool StorageMergeTree::merge( part_log_elem.part_name = future_part.name; if (new_part) - part_log_elem.bytes_compressed_on_disk = new_part->size_in_bytes; + part_log_elem.bytes_compressed_on_disk = new_part->bytes_on_disk; part_log_elem.source_part_names.reserve(future_part.parts.size()); for (const auto & source_part : future_part.parts) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index f11d9fe28b8..13d50500d09 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1142,11 +1142,11 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre /// If entry is old enough, and have enough size, and part are exists in any replica, /// then prefer fetching of merged part from replica. - size_t sum_parts_size_in_bytes = 0; + size_t sum_parts_bytes_on_disk = 0; for (const auto & part : parts) - sum_parts_size_in_bytes += part->size_in_bytes; + sum_parts_bytes_on_disk += part->bytes_on_disk; - if (sum_parts_size_in_bytes >= data.settings.prefer_fetch_merged_part_size_threshold) + if (sum_parts_bytes_on_disk >= data.settings.prefer_fetch_merged_part_size_threshold) { String replica = findReplicaHavingPart(entry.new_part_name, true); /// NOTE excessive ZK requests for same data later, may remove. if (!replica.empty()) @@ -1205,7 +1205,7 @@ void StorageReplicatedMergeTree::tryExecuteMerge(const StorageReplicatedMergeTre part_log_elem.part_name = entry.new_part_name; if (part) - part_log_elem.bytes_compressed_on_disk = part->size_in_bytes; + part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk; part_log_elem.source_part_names.reserve(parts.size()); for (const auto & source_part : parts) @@ -2293,7 +2293,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin if (part) { - part_log_elem.bytes_compressed_on_disk = part->size_in_bytes; + part_log_elem.bytes_compressed_on_disk = part->bytes_on_disk; part_log_elem.rows = part->rows_count; /// Could be approximate (?) } @@ -2949,7 +2949,7 @@ bool StorageReplicatedMergeTree::checkTableCanBeDropped() const { /// Consider only synchronized data const_cast(getData()).recalculateColumnSizes(); - context.checkTableCanBeDropped(database_name, table_name, getData().getTotalCompressedSize()); + context.checkTableCanBeDropped(database_name, table_name, getData().getTotalActiveSizeInBytes()); return true; } diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 18598b1c1dc..d42d8a80394 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -114,7 +114,7 @@ BlockInputStreams StorageSystemColumns::read( NamesAndTypesList columns; ColumnDefaults column_defaults; - MergeTreeData::ColumnSizes column_sizes; + MergeTreeData::ColumnSizeByName column_sizes; { StoragePtr storage = storages.at(std::make_pair(database_name, table_name)); diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index a07e8bc26dd..b2f3d2208c7 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -17,28 +17,30 @@ namespace DB StorageSystemParts::StorageSystemParts(const std::string & name) : StorageSystemPartsBase(name, { - {"partition", std::make_shared()}, - {"name", std::make_shared()}, - {"active", std::make_shared()}, - {"marks", std::make_shared()}, - {"marks_size", std::make_shared()}, - {"rows", std::make_shared()}, - {"bytes", std::make_shared()}, - {"modification_time", std::make_shared()}, - {"remove_time", std::make_shared()}, - {"refcount", std::make_shared()}, - {"min_date", std::make_shared()}, - {"max_date", std::make_shared()}, - {"min_block_number", std::make_shared()}, - {"max_block_number", std::make_shared()}, - {"level", std::make_shared()}, - {"primary_key_bytes_in_memory", std::make_shared()}, - {"primary_key_bytes_in_memory_allocated", std::make_shared()}, + {"partition", std::make_shared()}, + {"name", std::make_shared()}, + {"active", std::make_shared()}, + {"marks", std::make_shared()}, + {"rows", std::make_shared()}, + {"bytes_on_disk", std::make_shared()}, + {"data_compressed_bytes", std::make_shared()}, + {"data_uncompressed_bytes", std::make_shared()}, + {"marks_bytes", std::make_shared()}, + {"modification_time", std::make_shared()}, + {"remove_time", std::make_shared()}, + {"refcount", std::make_shared()}, + {"min_date", std::make_shared()}, + {"max_date", std::make_shared()}, + {"min_block_number", std::make_shared()}, + {"max_block_number", std::make_shared()}, + {"level", std::make_shared()}, + {"primary_key_bytes_in_memory", std::make_shared()}, + {"primary_key_bytes_in_memory_allocated", std::make_shared()}, - {"database", std::make_shared()}, - {"table", std::make_shared()}, - {"engine", std::make_shared()}, - {"path", std::make_shared()} + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"engine", std::make_shared()}, + {"path", std::make_shared()}, } ) { @@ -52,6 +54,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor { const auto & part = info.all_parts[part_number]; auto part_state = info.all_parts_state[part_number]; + MergeTreeDataPart::ColumnSize columns_size = part->getTotalColumnsSize(); size_t i = 0; { @@ -62,9 +65,11 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor columns[i++]->insert(part->name); columns[i++]->insert(static_cast(part_state == State::Committed)); columns[i++]->insert(static_cast(part->marks_count)); - columns[i++]->insert(static_cast(part->getTotalMrkSizeInBytes())); columns[i++]->insert(static_cast(part->rows_count)); - columns[i++]->insert(static_cast(part->size_in_bytes)); + columns[i++]->insert(static_cast(part->bytes_on_disk)); + columns[i++]->insert(static_cast(columns_size.data_compressed)); + columns[i++]->insert(static_cast(columns_size.data_uncompressed)); + columns[i++]->insert(static_cast(columns_size.marks)); columns[i++]->insert(static_cast(part->modification_time)); time_t remove_time = part->remove_time.load(std::memory_order_relaxed); diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 9d94655fc18..9a2538cbe6f 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB @@ -160,7 +161,7 @@ public: try { - /// For table not to be dropped. + /// For table not to be dropped and set of columns to remain constant. info.table_lock = info.storage->lockStructure(false, __PRETTY_FUNCTION__); } catch (const Exception & e) @@ -280,7 +281,31 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) : name(std::move(name_)) { - setColumns(ColumnsDescription(std::move(columns_))); + NamesAndTypesList aliases; + ColumnDefaults defaults; + auto add_alias = [&](const String & alias_name, const String & column_name) + { + DataTypePtr type; + for (const NameAndTypePair & col : columns_) + { + if (col.name == column_name) + { + type = col.type; + break; + } + } + if (!type) + throw Exception("No column " + column_name + " in table system." + name, ErrorCodes::LOGICAL_ERROR); + + aliases.push_back({alias_name, type}); + defaults[alias_name] = ColumnDefault{ColumnDefaultKind::Alias, std::make_shared(column_name)}; + }; + + /// Add aliases for old column names for backwards compatibility. + add_alias("bytes", "bytes_on_disk"); + add_alias("marks_size", "marks_bytes"); + + setColumns(ColumnsDescription(std::move(columns_), {}, std::move(aliases), std::move(defaults))); } } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 8e55625c5cd..921b32b6c0a 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -16,39 +16,43 @@ namespace DB StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name) - : StorageSystemPartsBase(name, - { - {"partition", std::make_shared()}, - {"name", std::make_shared()}, - {"active", std::make_shared()}, - {"marks", std::make_shared()}, - {"marks_bytes_in_block", std::make_shared()}, - {"rows", std::make_shared()}, - {"bytes", std::make_shared()}, - {"modification_time", std::make_shared()}, - {"remove_time", std::make_shared()}, - {"refcount", std::make_shared()}, - {"min_date", std::make_shared()}, - {"max_date", std::make_shared()}, - {"min_block_number", std::make_shared()}, - {"max_block_number", std::make_shared()}, - {"level", std::make_shared()}, - {"primary_key_bytes_in_memory", std::make_shared()}, - {"primary_key_bytes_in_memory_allocated", std::make_shared()}, + : StorageSystemPartsBase(name, + { + {"partition", std::make_shared()}, + {"name", std::make_shared()}, + {"active", std::make_shared()}, + {"marks", std::make_shared()}, + {"rows", std::make_shared()}, + {"bytes_on_disk", std::make_shared()}, + {"data_compressed_bytes", std::make_shared()}, + {"data_uncompressed_bytes", std::make_shared()}, + {"marks_bytes", std::make_shared()}, + {"modification_time", std::make_shared()}, + {"remove_time", std::make_shared()}, + {"refcount", std::make_shared()}, + {"min_date", std::make_shared()}, + {"max_date", std::make_shared()}, + {"min_block_number", std::make_shared()}, + {"max_block_number", std::make_shared()}, + {"level", std::make_shared()}, + {"primary_key_bytes_in_memory", std::make_shared()}, + {"primary_key_bytes_in_memory_allocated", std::make_shared()}, - {"database", std::make_shared()}, - {"table", std::make_shared()}, - {"engine", std::make_shared()}, - {"path", std::make_shared()}, - {"column", std::make_shared()}, - {"type", std::make_shared() }, - {"default_kind", std::make_shared() }, - {"default_expression", std::make_shared() }, - {"data_compressed_bytes", std::make_shared() }, - {"data_uncompressed_bytes", std::make_shared() }, - {"marks_bytes_in_column", std::make_shared() }, - } - ) + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"engine", std::make_shared()}, + {"path", std::make_shared()}, + + {"column", std::make_shared()}, + {"type", std::make_shared() }, + {"default_kind", std::make_shared() }, + {"default_expression", std::make_shared() }, + {"column_bytes_on_disk", std::make_shared() }, + {"column_data_compressed_bytes", std::make_shared() }, + {"column_data_uncompressed_bytes", std::make_shared() }, + {"column_marks_bytes", std::make_shared() }, + } + ) { } @@ -84,8 +88,8 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con { const auto & part = info.all_parts[part_number]; auto part_state = info.all_parts_state[part_number]; + auto columns_size = part->getTotalColumnsSize(); - auto total_mrk_size_in_bytes = part->getTotalMrkSizeInBytes(); /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts. auto use_count = part.use_count() - 1; auto min_date = part->getMinDate(); @@ -106,10 +110,12 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con columns[j++]->insert(part->name); columns[j++]->insert(static_cast(part_state == State::Committed)); columns[j++]->insert(static_cast(part->marks_count)); - columns[j++]->insert(static_cast(total_mrk_size_in_bytes)); columns[j++]->insert(static_cast(part->rows_count)); - columns[j++]->insert(static_cast(part->size_in_bytes)); + columns[j++]->insert(static_cast(part->bytes_on_disk)); + columns[j++]->insert(static_cast(columns_size.data_compressed)); + columns[j++]->insert(static_cast(columns_size.data_uncompressed)); + columns[j++]->insert(static_cast(columns_size.marks)); columns[j++]->insert(static_cast(part->modification_time)); columns[j++]->insert(static_cast(part->remove_time.load(std::memory_order_relaxed))); @@ -142,9 +148,11 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con columns[j++]->insertDefault(); } - columns[j++]->insert(static_cast(part->getColumnCompressedSize(column.name))); - columns[j++]->insert(static_cast(part->getColumnUncompressedSize(column.name))); - columns[j++]->insert(static_cast(part->getColumnMrkSize(column.name))); + MergeTreeDataPart::ColumnSize column_size = part->getColumnSize(column.name, *column.type); + columns[j++]->insert(static_cast(column_size.data_compressed + column_size.marks)); + columns[j++]->insert(static_cast(column_size.data_compressed)); + columns[j++]->insert(static_cast(column_size.data_uncompressed)); + columns[j++]->insert(static_cast(column_size.marks)); if (has_state_column) columns[j++]->insert(part->stateString()); From d63d9a4ffbfebb7132e4ea5894b6af27e4b4a221 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 23 Mar 2018 22:56:24 +0300 Subject: [PATCH 73/90] fix test [#CLICKHOUSE-3590] --- dbms/src/Databases/DatabaseDictionary.cpp | 36 ++++++++++++++------ dbms/src/Databases/DatabaseDictionary.h | 26 +++++++------- dbms/src/Databases/DatabaseMemory.cpp | 7 ---- dbms/src/Databases/DatabaseMemory.h | 9 ++--- dbms/src/Databases/DatabaseOrdinary.cpp | 34 ++++++++++-------- dbms/src/Databases/DatabaseOrdinary.h | 7 ++-- dbms/src/Databases/IDatabase.h | 12 +++---- dbms/src/Interpreters/Context.cpp | 1 + dbms/src/Interpreters/ExternalDictionaries.h | 5 +++ dbms/src/Interpreters/ExternalLoader.cpp | 24 ++++++++++--- dbms/src/Interpreters/ExternalLoader.h | 3 ++ 11 files changed, 99 insertions(+), 65 deletions(-) diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index c68c801e4cb..3979f71f602 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -13,10 +13,11 @@ namespace DB { namespace ErrorCodes { -extern const int TABLE_ALREADY_EXISTS; -extern const int UNKNOWN_TABLE; -extern const int LOGICAL_ERROR; -extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int TABLE_ALREADY_EXISTS; + extern const int UNKNOWN_TABLE; + extern const int LOGICAL_ERROR; + extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int SYNTAX_ERROR; } DatabaseDictionary::DatabaseDictionary(const String & name_, const Context & context) @@ -163,15 +164,17 @@ time_t DatabaseDictionary::getTableMetadataModificationTime( return static_cast(0); } -ASTPtr DatabaseDictionary::getCreateTableQuery( - const Context & context, - const String & table_name) const +ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context, + const String & table_name, bool throw_on_error) const { String query; { WriteBufferFromString buffer(query); - auto dictionary = context.getExternalDictionaries().getDictionary(table_name); + const auto & dictionaries = context.getExternalDictionaries(); + auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name) + : dictionaries.tryGetDictionary(table_name); + auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure()); buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " ("; buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end()); @@ -179,12 +182,25 @@ ASTPtr DatabaseDictionary::getCreateTableQuery( } ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), ""); + const char * pos = query.data(); + std::string error_message; + auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, + /* hilite = */ false, "", /* allow_multi_statements = */ false); + + if (!ast && throw_on_error) + throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); + + return ast; +} + +ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const +{ + return getCreateTableQueryImpl(context, table_name, true); } ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const { - return getCreateTableQuery(context, table_name); + return getCreateTableQueryImpl(context, table_name, false); } ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index e2a036362b0..a234694b768 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -22,16 +22,6 @@ class ExternalDictionaries; */ class DatabaseDictionary : public IDatabase { -private: - const String name; - mutable std::mutex mutex; - const ExternalDictionaries & external_dictionaries; - std::unordered_set deleted_tables; - - Poco::Logger * log; - - Tables loadTables(); - public: DatabaseDictionary(const String & name_, const Context & context); @@ -94,13 +84,25 @@ public: const String & table_name) const override; ASTPtr tryGetCreateTableQuery( - const Context & context, - const String & table_name) const override; + const Context & context, + const String & table_name) const override; ASTPtr getCreateDatabaseQuery(const Context & context) const override; void shutdown() override; void drop() override; + +private: + const String name; + mutable std::mutex mutex; + const ExternalDictionaries & external_dictionaries; + std::unordered_set deleted_tables; + + Poco::Logger * log; + + Tables loadTables(); + + ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const; }; } diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 8eacc7f856f..b50fb78468e 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -128,13 +128,6 @@ ASTPtr DatabaseMemory::getCreateTableQuery( throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); } -ASTPtr DatabaseMemory::tryGetCreateTableQuery( - const Context &, - const String &) const -{ - throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); -} - ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context &) const { throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 1b5f0a20c62..2c68e0a093b 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -80,13 +80,8 @@ public: const Context & context, const String & table_name) override; - ASTPtr getCreateTableQuery( - const Context & context, - const String & table_name) const override; - - ASTPtr tryGetCreateTableQuery( - const Context & context, - const String & table_name) const override; + ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override; + ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; } ASTPtr getCreateDatabaseQuery(const Context & context) const override; diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 4f63834d396..027ce07148d 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -31,6 +31,7 @@ namespace ErrorCodes extern const int FILE_DOESNT_EXIST; extern const int LOGICAL_ERROR; extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int SYNTAX_ERROR; } @@ -335,7 +336,7 @@ void DatabaseOrdinary::removeTable( } } -static ASTPtr getQueryFromMetadata(const String & metadata_path) +static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true) { if (!Poco::File(metadata_path).exists()) return nullptr; @@ -348,12 +349,20 @@ static ASTPtr getQueryFromMetadata(const String & metadata_path) } ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), "in file " + metadata_path); + const char * pos = query.data(); + std::string error_message; + auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, + "in file " + metadata_path, /* allow_multi_statements = */ false); + + if (!ast && throw_on_error) + throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); + + return ast; } -static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database) +static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error) { - ASTPtr ast = getQueryFromMetadata(metadata_path); + ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error); if (ast) { @@ -428,21 +437,18 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime( } } - -ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const +ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, + const String & table_name, bool throw_on_error) const { ASTPtr ast; auto table_metadata_path = detail::getTableMetadataPath(metadata_path, table_name); - ast = getCreateQueryFromMetadata(table_metadata_path, name); - if (!ast) + ast = getCreateQueryFromMetadata(table_metadata_path, name, throw_on_error); + if (!ast && throw_on_error) { /// Handle system.* tables for which there are no table.sql files. bool has_table = tryGetTable(context, table_name) != nullptr; - if (has_table && try_get) - return nullptr; - auto msg = has_table ? "There is no CREATE TABLE query for table " : "There is no metadata file for table "; @@ -455,12 +461,12 @@ ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, const ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const { - return getCreateTableQueryImpl(context, table_name, false); + return getCreateTableQueryImpl(context, table_name, true); } ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const { - return getCreateTableQueryImpl(context, table_name, true); + return getCreateTableQueryImpl(context, table_name, false); } ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const @@ -468,7 +474,7 @@ ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) con ASTPtr ast; auto database_metadata_path = detail::getDatabaseMetadataPath(metadata_path); - ast = getCreateQueryFromMetadata(database_metadata_path, name); + ast = getCreateQueryFromMetadata(database_metadata_path, name, true); if (!ast) { /// Handle databases (such as default) for which there are no database.sql files. diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index cae6686d5ba..86279eff4c7 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -60,8 +60,8 @@ public: const String & table_name) const override; ASTPtr tryGetCreateTableQuery( - const Context & context, - const String & table_name) const override; + const Context & context, + const String & table_name) const override; ASTPtr getCreateDatabaseQuery(const Context & context) const override; @@ -74,7 +74,8 @@ public: private: void startupTables(ThreadPool * thread_pool); - ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool try_get) const; + + ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const; }; } diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index 6b84f3fa165..7416fe70fb1 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -125,14 +125,12 @@ public: const String & name) = 0; /// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata. - virtual ASTPtr getCreateTableQuery( - const Context & context, - const String & name) const = 0; + virtual ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const = 0; - /// Like getCreateTableQuery, but don't throw for existing tables without metadata. - virtual ASTPtr tryGetCreateTableQuery( - const Context & context, - const String & name) const = 0; + virtual ASTPtr getCreateTableQuery(const Context & context, const String & name) const + { + return tryGetCreateTableQuery(context, name); + } /// Get the CREATE DATABASE query for current database. virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index e3e0e268f98..b011a1a3d3c 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -82,6 +82,7 @@ namespace ErrorCodes extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; + extern const int CANNOT_GET_CREATE_TABLE_QUERY; } diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h index cc03c9b9389..0b00ab9f632 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ b/dbms/src/Interpreters/ExternalDictionaries.h @@ -31,6 +31,11 @@ public: return std::static_pointer_cast(getLoadable(name)); } + DictPtr tryGetDictionary(const std::string & name) const + { + return std::static_pointer_cast(tryGetLoadable(name)); + } + protected: std::unique_ptr create(const std::string & name, const Configuration & config, diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 527c3d495ea..fc25327558e 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -385,21 +385,35 @@ void ExternalLoader::reload(const std::string & name) throw Exception("Failed to load " + object_name + " '" + name + "' during the reload process", ErrorCodes::BAD_ARGUMENTS); } -ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const +ExternalLoader::LoadablePtr ExternalLoader::getLoadableImpl(const std::string & name, bool throw_on_error) const { const std::lock_guard lock{map_mutex}; const auto it = loadable_objects.find(name); if (it == std::end(loadable_objects)) - throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS); + { + if (throw_on_error) + throw Exception("No such " + object_name + ": " + name, ErrorCodes::BAD_ARGUMENTS); + return nullptr; + } - if (!it->second.loadable) - it->second.exception ? std::rethrow_exception(it->second.exception) : - throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR}; + if (!it->second.loadable && throw_on_error) + it->second.exception ? std::rethrow_exception(it->second.exception) + : throw Exception{object_name + " '" + name + "' is not loaded", ErrorCodes::LOGICAL_ERROR}; return it->second.loadable; } +ExternalLoader::LoadablePtr ExternalLoader::getLoadable(const std::string & name) const +{ + return getLoadableImpl(name, true); +} + +ExternalLoader::LoadablePtr ExternalLoader::tryGetLoadable(const std::string & name) const +{ + return getLoadableImpl(name, false); +} + ExternalLoader::LockedObjectsMap ExternalLoader::getObjectsMap() const { return LockedObjectsMap(map_mutex, loadable_objects); diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index a2f64b61dc7..2cf42942bc9 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -104,6 +104,7 @@ public: void reload(const std::string & name); LoadablePtr getLoadable(const std::string & name) const; + LoadablePtr tryGetLoadable(const std::string & name) const; protected: virtual std::unique_ptr create(const std::string & name, const Configuration & config, @@ -172,6 +173,8 @@ private: void reloadAndUpdate(bool throw_on_error = false); void reloadPeriodically(); + + LoadablePtr getLoadableImpl(const std::string & name, bool throw_on_error) const; }; } From 190964f98e5f6db860bc2c185654cdfca8d718bd Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Mon, 26 Mar 2018 21:39:28 +0300 Subject: [PATCH 74/90] Avoid extra waits in copier. [#CLICKHOUSE-2] --- dbms/src/Server/ClusterCopier.cpp | 36 +++++++++++-------- .../DistributedBlockOutputStream.cpp | 9 ++--- .../DistributedBlockOutputStream.h | 1 + 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp index 3bcbad95d76..74d5eb5753d 100644 --- a/dbms/src/Server/ClusterCopier.cpp +++ b/dbms/src/Server/ClusterCopier.cpp @@ -1128,7 +1128,7 @@ protected: catch (const zkutil::KeeperException & e) { LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name - << ". Will recheck the partition. Error: " << e.what()); + << ". Will recheck the partition. Error: " << e.displayText()); return false; } @@ -1285,6 +1285,7 @@ protected: Stopwatch watch; TasksShard expected_shards; size_t num_failed_shards = 0; + bool previous_shard_is_instantly_finished = false; ++cluster_partition.total_tries; @@ -1330,15 +1331,20 @@ protected: expected_shards.emplace_back(shard); + /// Do not sleep if there is a sequence of already processed shards to increase startup + bool sleep_before_execution = !previous_shard_is_instantly_finished && shard->priority.is_remote; PartitionTaskStatus task_status = PartitionTaskStatus::Error; + bool was_error = false; for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - task_status = tryProcessPartitionTask(partition); + task_status = tryProcessPartitionTask(partition, sleep_before_execution); /// Exit if success if (task_status == PartitionTaskStatus::Finished) break; + was_error = true; + /// Skip if the task is being processed by someone if (task_status == PartitionTaskStatus::Active) break; @@ -1349,6 +1355,8 @@ protected: if (task_status == PartitionTaskStatus::Error) ++num_failed_shards; + + previous_shard_is_instantly_finished = !was_error; } cluster_partition.elapsed_time_seconds += watch.elapsedSeconds(); @@ -1415,13 +1423,13 @@ protected: Error, }; - PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition) + PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition, bool sleep_before_execution) { PartitionTaskStatus res; try { - res = processPartitionTaskImpl(task_partition); + res = processPartitionTaskImpl(task_partition, sleep_before_execution); } catch (...) { @@ -1442,7 +1450,7 @@ protected: return res; } - PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition) + PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition, bool sleep_before_execution) { TaskShard & task_shard = task_partition.task_shard; TaskTable & task_table = task_shard.task_table; @@ -1482,7 +1490,7 @@ protected: }; /// Load balancing - auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, task_shard.priority.is_remote); + auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, sleep_before_execution); LOG_DEBUG(log, "Processing " << current_task_status_path); @@ -1656,7 +1664,7 @@ protected: } using ExistsFuture = zkutil::ZooKeeper::ExistsFuture; - auto future_is_dirty_checker = std::make_unique(zookeeper->asyncExists(is_dirty_flag_path)); + std::unique_ptr future_is_dirty_checker; Stopwatch watch(CLOCK_MONOTONIC_COARSE); constexpr size_t check_period_milliseconds = 500; @@ -1667,9 +1675,15 @@ protected: if (zookeeper->expired()) throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED); - if (future_is_dirty_checker != nullptr) + if (!future_is_dirty_checker) + future_is_dirty_checker = std::make_unique(zookeeper->asyncExists(is_dirty_flag_path)); + + /// check_period_milliseconds should less than average insert time of single block + /// Otherwise, the insertion will slow a little bit + if (watch.elapsedMilliseconds() >= check_period_milliseconds) { zkutil::ZooKeeper::StatAndExists status; + try { status = future_is_dirty_checker->get(); @@ -1689,12 +1703,6 @@ protected: throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); } - if (watch.elapsedMilliseconds() >= check_period_milliseconds) - { - watch.restart(); - future_is_dirty_checker = std::make_unique(zookeeper->asyncExists(is_dirty_flag_path)); - } - return false; }; diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 98416437497..d739badba88 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -193,13 +193,12 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp auto memory_tracker = current_memory_tracker; return [this, memory_tracker, &job, ¤t_block]() { - SCOPE_EXIT({++finished_jobs_count;}); - - Stopwatch watch; ++job.blocks_started; SCOPE_EXIT({ - UInt64 elapsed_time_for_block_ms = watch.elapsedMilliseconds(); + ++finished_jobs_count; + + UInt64 elapsed_time_for_block_ms = watch_current_block.elapsedMilliseconds(); job.elapsed_time_ms += elapsed_time_for_block_ms; job.max_elapsed_time_for_block_ms = std::max(job.max_elapsed_time_for_block_ms, elapsed_time_for_block_ms); }); @@ -322,6 +321,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block) watch.restart(); } + watch_current_block.restart(); + if (num_shards > 1) { auto current_selector = createSelector(block); diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h index 0b37d51c856..fe9e765021b 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -88,6 +88,7 @@ private: /// Sync-related stuff UInt64 insert_timeout; // in seconds Stopwatch watch; + Stopwatch watch_current_block; std::optional pool; ThrottlerPtr throttler; String query_string; From 348b7c3bda816630a17004fe4bbb0e1e0dd3e2fb Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 27 Mar 2018 18:08:01 +0300 Subject: [PATCH 75/90] clickhouse-test: Fix run without server config --- dbms/tests/clickhouse-test | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 4e9886f8b70..b88eab93c7c 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -81,9 +81,12 @@ def main(args): # TODO ! use clickhouse-extract-from-config here: if args.zookeeper is None: code, out = commands.getstatusoutput(args.binary + "-extract-from-config --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l') - if int(out) > 0: - args.zookeeper = True - else: + try: + if int(out) > 0: + args.zookeeper = True + else: + args.zookeeper = False + except ValueError: args.zookeeper = False if args.shard is None: From d58483b08415037baf8e7e7bd4aea193b8a95fa7 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 27 Mar 2018 20:08:41 +0300 Subject: [PATCH 76/90] squash blocks when inserting into materialized views #2084 [#CLICKHOUSE-3540] --- dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index b1f0dc8a9cd..815b8ebf303 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -35,8 +36,14 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( auto & materialized_view = dynamic_cast(*dependent_table); auto query = materialized_view.getInnerQuery(); - auto out = std::make_shared( + BlockOutputStreamPtr out; + out = std::make_shared( database_table.first, database_table.second, dependent_table, *views_context, ASTPtr()); + /// Squashing is needed here because the materialized view query can generate a lot of blocks + /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY + /// and two-level aggregation is triggered). + out = std::make_shared( + out, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)}); } } From 9744ddf70f7c29e87cb4bc3664bf0189533fc979 Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 27 Mar 2018 21:36:50 +0300 Subject: [PATCH 77/90] Use ninja for package build --- debian/control | 1 + debian/rules | 2 ++ 2 files changed, 3 insertions(+) diff --git a/debian/control b/debian/control index 79666ace2e5..5edac5f66a8 100644 --- a/debian/control +++ b/debian/control @@ -4,6 +4,7 @@ Priority: optional Maintainer: Alexey Milovidov Build-Depends: debhelper (>= 9), cmake3 | cmake, + ninja-build, gcc-7, g++-7, libc6-dev, libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev, diff --git a/debian/rules b/debian/rules index 7990db33b1c..4cef8702e05 100755 --- a/debian/rules +++ b/debian/rules @@ -41,6 +41,8 @@ endif endif CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)` +CMAKE_FLAGS += -GNinja +export MAKE=ninja ifndef DH_VERBOSE CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0 From 88a9b7de91e2356a40130c88ba6b1d5466c0903e Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 27 Mar 2018 23:04:40 +0300 Subject: [PATCH 78/90] fix calls to writeSuffix() in PushingToViewsBlockOutputStream #2084 [#CLICKHOUSE-3540] don't use copyData because it calls writeSuffix() prematurely. --- dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 3 ++- dbms/src/DataStreams/PushingToViewsBlockOutputStream.h | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 815b8ebf303..743c24cc6ca 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -74,7 +74,8 @@ void PushingToViewsBlockOutputStream::write(const Block & block) BlockInputStreamPtr from = std::make_shared(block); InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from); BlockInputStreamPtr data = std::make_shared(select.execute().in); - copyData(*data, *view.out); + while (Block result_block = data->read()) + view.out->write(result_block); } catch (Exception & ex) { diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h index 7703b81dc4e..4ff953fd265 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -29,18 +29,27 @@ public: { if (output) output->flush(); + + for (auto & view : views) + view.out->flush(); } void writePrefix() override { if (output) output->writePrefix(); + + for (auto & view : views) + view.out->writePrefix(); } void writeSuffix() override { if (output) output->writeSuffix(); + + for (auto & view : views) + view.out->writeSuffix(); } private: From 1be7c0970273ece874f3df5186c13675f78d0ad1 Mon Sep 17 00:00:00 2001 From: Alex Zatelepin Date: Tue, 27 Mar 2018 22:25:50 +0000 Subject: [PATCH 79/90] add readPrefix()/readSuffix() calls for completeness #2084 [#CLICKHOUSE-3540] --- dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 743c24cc6ca..dc6e2ef2ea7 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -74,8 +74,13 @@ void PushingToViewsBlockOutputStream::write(const Block & block) BlockInputStreamPtr from = std::make_shared(block); InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from); BlockInputStreamPtr data = std::make_shared(select.execute().in); + + data->readPrefix(); + while (Block result_block = data->read()) view.out->write(result_block); + + data->readSuffix(); } catch (Exception & ex) { From f48caff3606c28dd5d9f4891d8b30dfe1c4b96be Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 28 Mar 2018 05:27:25 +0300 Subject: [PATCH 80/90] Update COWPtr.h --- dbms/src/Common/COWPtr.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbms/src/Common/COWPtr.h b/dbms/src/Common/COWPtr.h index 39c912f8abc..525f2372c6c 100644 --- a/dbms/src/Common/COWPtr.h +++ b/dbms/src/Common/COWPtr.h @@ -68,10 +68,9 @@ * of this shared state. * * Caveats: - * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere - * and it can still become shared. Also it would be better to make 'mutate' method rvalue-qualified. + * - after a call to 'mutate' method, you can still have a reference to immutable ptr somewhere. * - as 'mutable_ptr' should be unique, it's refcount is redundant - probably it would be better - * to use std::unique_ptr for it, but see above. + * to use std::unique_ptr for it somehow. */ template class COWPtr : public boost::intrusive_ref_counter From 5877498c4ae94457c7ae49205737c4b39c7d968b Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 27 Mar 2018 19:32:54 -0700 Subject: [PATCH 81/90] Add clickhouse-report script to server package (#2123) * Add clickhouse-report script to server package * fix * fix * fix * Revert "fix" This reverts commit ef3ded092028ce3666e099c844e30fe686a7bb33. --- debian/clickhouse-server-base.install | 1 + utils/CMakeLists.txt | 1 + utils/report/CMakeLists.txt | 1 + utils/report/clickhouse-report | 30 +++++++++++++++++++++++++++ utils/report/clickhouse_report.sh | 23 -------------------- 5 files changed, 33 insertions(+), 23 deletions(-) create mode 100644 utils/report/CMakeLists.txt create mode 100755 utils/report/clickhouse-report delete mode 100755 utils/report/clickhouse_report.sh diff --git a/debian/clickhouse-server-base.install b/debian/clickhouse-server-base.install index 9d3d470b040..573f34042bb 100644 --- a/debian/clickhouse-server-base.install +++ b/debian/clickhouse-server-base.install @@ -3,6 +3,7 @@ /usr/bin/clickhouse-clang /usr/bin/clickhouse-lld /usr/bin/clickhouse-copier +/usr/bin/clickhouse-report /etc/systemd/system/clickhouse-server.service /etc/init.d/clickhouse-server /etc/cron.d/clickhouse-server diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index bd055efc791..010e38ebfe0 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -15,3 +15,4 @@ add_subdirectory (wikistat-loader) add_subdirectory (fill-factor) add_subdirectory (check-marks) add_subdirectory (test-data-generator) +add_subdirectory (report) diff --git a/utils/report/CMakeLists.txt b/utils/report/CMakeLists.txt new file mode 100644 index 00000000000..e39dd155b15 --- /dev/null +++ b/utils/report/CMakeLists.txt @@ -0,0 +1 @@ +install (PROGRAMS clickhouse-report DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) diff --git a/utils/report/clickhouse-report b/utils/report/clickhouse-report new file mode 100755 index 00000000000..1cd212febcb --- /dev/null +++ b/utils/report/clickhouse-report @@ -0,0 +1,30 @@ +#!/bin/sh -x +# Usages: +# sh -x clickhouse-report > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 +# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 + +# Also dump some system info (can contain some private data) +# sh -x clickhouse-report system > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 +# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse-report system | sh -x > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 + +clickhouse-client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes' +clickhouse-client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes' +tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log +tail -n200 /var/log/clickhouse-server/clickhouse-server.log +tail -n100 /var/log/clickhouse-server/stderr + +if [ "$1" = "system" ]; then + uname -a + cat /etc/lsb-release + df -h + top -bn1 + ps auxw +fi diff --git a/utils/report/clickhouse_report.sh b/utils/report/clickhouse_report.sh deleted file mode 100755 index ab1ec396e80..00000000000 --- a/utils/report/clickhouse_report.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -x -# Usages: -# sh clickhouse_report.sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 -# curl https://raw.githubusercontent.com/yandex/ClickHouse/master/utils/report/clickhouse_report.sh | sh > ch.`hostname`.`date '+%Y%M%''d%H%M%''S'`.dmp 2>&1 - -clickhouse --client -q 'SELECT * FROM system.events FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.metrics FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.asynchronous_metrics FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.build_options FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.processes FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.merges FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.parts FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.replication_queue FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.replicas FORMAT PrettyCompactNoEscapes' -clickhouse --client -q 'SELECT * FROM system.dictionaries FORMAT PrettyCompactNoEscapes' -ps auxw -df -h -top -bn1 -tail -n200 /var/log/clickhouse-server/clickhouse-server.err.log -tail -n200 /var/log/clickhouse-server/clickhouse-server.log -tail -n100 /var/log/clickhouse-server/stderr -cat /etc/lsb-release -uname -a From c0aa85436cacf316d9669b20a1c36ad9b1410d3f Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 27 Mar 2018 14:29:11 +0300 Subject: [PATCH 82/90] METRIQA-2349 centos package: fix adduser: unrecognized option '--disabled-login' --- debian/clickhouse-server-base.postinst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst index c7aa47b5ef9..3bcfa9e22bc 100644 --- a/debian/clickhouse-server-base.postinst +++ b/debian/clickhouse-server-base.postinst @@ -27,8 +27,13 @@ if [ "$1" = configure ]; then # Make sure the administrative user exists if ! getent passwd ${CLICKHOUSE_USER} > /dev/null; then - adduser --system --disabled-login --no-create-home --home /nonexistent \ - --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null + if [ "$OS" == "rhel" ] || [ "$OS" == "centos" ] || [ "$OS" == "fedora" ]; then + adduser --system --no-create-home --home /nonexistent \ + --shell /bin/false clickhouse > /dev/null + else + adduser --system --disabled-login --no-create-home --home /nonexistent \ + --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null + fi fi # if the user was created manually, make sure the group is there as well From 0f9501b21f5e73780ec498b0027c81456b5b67af Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 28 Mar 2018 05:38:07 +0300 Subject: [PATCH 83/90] Update clickhouse-server-base.postinst --- debian/clickhouse-server-base.postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/clickhouse-server-base.postinst b/debian/clickhouse-server-base.postinst index 3bcfa9e22bc..6946f8b2728 100644 --- a/debian/clickhouse-server-base.postinst +++ b/debian/clickhouse-server-base.postinst @@ -32,7 +32,7 @@ if [ "$1" = configure ]; then --shell /bin/false clickhouse > /dev/null else adduser --system --disabled-login --no-create-home --home /nonexistent \ - --shell /bin/false --group --gecos "Clickhouse server" clickhouse > /dev/null + --shell /bin/false --group --gecos "ClickHouse server" clickhouse > /dev/null fi fi From f2db72a5e5b0846de3bc3fa412dd050406453a2e Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Wed, 28 Mar 2018 14:38:06 +0300 Subject: [PATCH 84/90] move squashing to read side #2084 [#CLICKHOUSE-3540] This way there is a clear correspondence between blocks inserted into the parent table and blocks inserted into the materialized view. --- .../PushingToViewsBlockOutputStream.cpp | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp index dc6e2ef2ea7..b3dd05aee9c 100644 --- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -36,14 +36,8 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( auto & materialized_view = dynamic_cast(*dependent_table); auto query = materialized_view.getInnerQuery(); - BlockOutputStreamPtr out; - out = std::make_shared( + BlockOutputStreamPtr out = std::make_shared( database_table.first, database_table.second, dependent_table, *views_context, ASTPtr()); - /// Squashing is needed here because the materialized view query can generate a lot of blocks - /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY - /// and two-level aggregation is triggered). - out = std::make_shared( - out, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); views.emplace_back(ViewInfo{std::move(query), database_table.first, database_table.second, std::move(out)}); } } @@ -73,14 +67,19 @@ void PushingToViewsBlockOutputStream::write(const Block & block) { BlockInputStreamPtr from = std::make_shared(block); InterpreterSelectQuery select(view.query, *views_context, {}, QueryProcessingStage::Complete, 0, from); - BlockInputStreamPtr data = std::make_shared(select.execute().in); + BlockInputStreamPtr in = std::make_shared(select.execute().in); + /// Squashing is needed here because the materialized view query can generate a lot of blocks + /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY + /// and two-level aggregation is triggered). + in = std::make_shared( + in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); - data->readPrefix(); + in->readPrefix(); - while (Block result_block = data->read()) + while (Block result_block = in->read()) view.out->write(result_block); - data->readSuffix(); + in->readSuffix(); } catch (Exception & ex) { From 0d70314eadad1e28d7dc88c6de11c0557a35b4cb Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 28 Mar 2018 05:17:23 -0700 Subject: [PATCH 85/90] Fix package make with ninja (#2129) * Fix package make with ninja --- debian/rules | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/debian/rules b/debian/rules index 4cef8702e05..1cb7de10224 100755 --- a/debian/rules +++ b/debian/rules @@ -9,7 +9,8 @@ export DEB_BUILD_MAINT_OPTIONS=hardening=+all,-pie # because copy_headers.sh have hardcoded path to build/include_directories.txt BUILDDIR = build -DESTDIR = debian/tmp +CURDIR = $(shell pwd) +DESTDIR = $(CURDIR)/debian/tmp DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) @@ -94,12 +95,15 @@ override_dh_install: CLANG=$(DESTDIR)/usr/bin/clickhouse-clang ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers # fake metrika files when private dir is empty - mkdir -p debian/tmp/etc/clickhouse-server/metrika - touch debian/tmp/etc/clickhouse-server/metrika/config.xml - touch debian/tmp/etc/clickhouse-server/metrika/users.xml + mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika + touch $(DESTDIR)/etc/clickhouse-server/metrika/config.xml + touch $(DESTDIR)/etc/clickhouse-server/metrika/users.xml dh_install --list-missing --sourcedir=$(DESTDIR) +override_dh_auto_install: + env DESTDIR=$(DESTDIR) ninja -C $(BUILDDIR) install + override_dh_shlibdeps: true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency. From 7f60bb0a9b62c991510457efac3386dc573baafb Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Wed, 28 Mar 2018 16:45:48 +0300 Subject: [PATCH 86/90] SystemLog: execute prepareTable() on each flush --- dbms/src/Interpreters/SystemLog.h | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 5c6071e647d..64796bc13db 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -127,11 +127,6 @@ protected: void threadFunction(); void flush(); - /** Creates new table if it does not exist. - * Renames old table if its structure is not suitable. - * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. - */ - bool is_prepared = false; void prepareTable(); }; @@ -233,8 +228,7 @@ void SystemLog::flush() { LOG_TRACE(log, "Flushing system log"); - if (!is_prepared) /// BTW, flush method is called from single thread. - prepareTable(); + prepareTable(); Block block = LogElement::createBlock(); for (const LogElement & elem : data) @@ -267,7 +261,10 @@ void SystemLog::flush() } } - +/** Creates new table if it does not exist. + * Renames old table if its structure is not suitable. + * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. + */ template void SystemLog::prepareTable() { @@ -341,7 +338,6 @@ void SystemLog::prepareTable() table = context.getTable(database_name, table_name); } - is_prepared = true; } From d78a46d958cffeabf9543cb0e35e744c7e141bd6 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 29 Mar 2018 08:16:58 +0300 Subject: [PATCH 87/90] #2131 deleted excessive logging --- dbms/src/Interpreters/SystemLog.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 64796bc13db..f99b2f684cb 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -308,8 +308,6 @@ void SystemLog::prepareTable() /// The required table will be created. table = nullptr; } - else - LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name()); } if (!table) From 042a197e91dfda1197dad69b6d5ac1e80952a803 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 29 Mar 2018 16:43:24 +0300 Subject: [PATCH 88/90] Revert "#2131 deleted excessive logging" This reverts commit d78a46d958cffeabf9543cb0e35e744c7e141bd6. --- dbms/src/Interpreters/SystemLog.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index f99b2f684cb..64796bc13db 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -308,6 +308,8 @@ void SystemLog::prepareTable() /// The required table will be created. table = nullptr; } + else + LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name()); } if (!table) From d095f69ae6c8ed596e5e68ec21397dde89d67a67 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 29 Mar 2018 16:46:42 +0300 Subject: [PATCH 89/90] Revert "SystemLog: execute prepareTable() on each flush" This reverts commit 7f60bb0a9b62c991510457efac3386dc573baafb. --- dbms/src/Interpreters/SystemLog.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 64796bc13db..5c6071e647d 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -127,6 +127,11 @@ protected: void threadFunction(); void flush(); + /** Creates new table if it does not exist. + * Renames old table if its structure is not suitable. + * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. + */ + bool is_prepared = false; void prepareTable(); }; @@ -228,7 +233,8 @@ void SystemLog::flush() { LOG_TRACE(log, "Flushing system log"); - prepareTable(); + if (!is_prepared) /// BTW, flush method is called from single thread. + prepareTable(); Block block = LogElement::createBlock(); for (const LogElement & elem : data) @@ -261,10 +267,7 @@ void SystemLog::flush() } } -/** Creates new table if it does not exist. - * Renames old table if its structure is not suitable. - * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. - */ + template void SystemLog::prepareTable() { @@ -338,6 +341,7 @@ void SystemLog::prepareTable() table = context.getTable(database_name, table_name); } + is_prepared = true; } From 44fb6eb66d0f3e91f0c5a6e5ce87f1e33c5cf799 Mon Sep 17 00:00:00 2001 From: Kirill Shvakov Date: Thu, 29 Mar 2018 16:54:50 +0300 Subject: [PATCH 90/90] SystemLog: execute prepareTable() on each flush --- dbms/src/Interpreters/SystemLog.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 5c6071e647d..8e04076dc14 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -233,8 +233,7 @@ void SystemLog::flush() { LOG_TRACE(log, "Flushing system log"); - if (!is_prepared) /// BTW, flush method is called from single thread. - prepareTable(); + prepareTable(); Block block = LogElement::createBlock(); for (const LogElement & elem : data) @@ -311,7 +310,7 @@ void SystemLog::prepareTable() /// The required table will be created. table = nullptr; } - else + else if (!is_prepared) LOG_DEBUG(log, "Will use existing table " << description << " for " + LogElement::name()); }