From 6277747e8aa22b75a788b3e97372678b5c5df756 Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 20 Sep 2021 05:37:18 +0000 Subject: [PATCH 001/126] First draft --- .../functions/other-functions.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 11cd522c622..af17954fec7 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2354,3 +2354,39 @@ Result: │ 1 │ └─────────┘ ``` + +## shardNum {#shard-num} + +Returns the number of a shard which executes the query for a distributed query. +If query is not distributed then *constant value* is returned. + +**Syntax** + +``` sql +shardNum() +``` + +**Returned value** + +- Shard number. + +Type: [UInt32](../../sql-reference/data-types/int-uint.md). + +## shardCount {#shard-count} + +Returns the total number of shards which execute a distributed query. +If query is not distributed then *constant value* is returned. + +**Syntax** + +``` sql +shardCount() +``` + +**Returned value** + +- Total number of shards. + +Type: [UInt32](../../sql-reference/data-types/int-uint.md). + + From b3325772f78e5b91ca6aaaee9cc1dd9beafda089 Mon Sep 17 00:00:00 2001 From: Viachaslau Boben Date: Mon, 6 Sep 2021 02:25:22 +0300 Subject: [PATCH 002/126] Add normalizeUTF8 function with NFC normalization --- src/Common/ErrorCodes.cpp | 1 + src/Functions/normalizeString.cpp | 126 ++++++++++++++++++ src/Functions/registerFunctionsString.cpp | 8 ++ .../02011_normalize_utf8.reference | 3 + .../0_stateless/02011_normalize_utf8.sql | 19 +++ 5 files changed, 157 insertions(+) create mode 100644 src/Functions/normalizeString.cpp create mode 100644 tests/queries/0_stateless/02011_normalize_utf8.reference create mode 100644 tests/queries/0_stateless/02011_normalize_utf8.sql diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 53276f5b196..b6d9b65c28b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -588,6 +588,7 @@ M(618, LZ4_DECODER_FAILED) \ M(619, POSTGRESQL_REPLICATION_INTERNAL_ERROR) \ M(620, QUERY_NOT_ALLOWED) \ + M(621, CANNOT_NORMALIZE_STRING) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp new file mode 100644 index 00000000000..178c2dc2cf1 --- /dev/null +++ b/src/Functions/normalizeString.cpp @@ -0,0 +1,126 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/logger_useful.h" +#include "Columns/ColumnString.h" +#include "Parsers/IAST_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_NORMALIZE_STRING; +} + +namespace +{ + +struct NormalizeUTF8Impl +{ + + static void vector(const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + UErrorCode err = U_ZERO_ERROR; + + const UNormalizer2 *normalizer = unorm2_getNFCInstance(&err); + if (U_FAILURE(err)) { + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); + } + + size_t size = offsets.size(); + res_offsets.resize(size); + + ColumnString::Offset current_from_offset = 0; + ColumnString::Offset current_to_offset = 0; + + icu::UnicodeString to_string; + + PODArray from_uchars; + PODArray to_uchars; + + for (size_t i = 0; i < size; ++i) + { + size_t from_size = offsets[i] - current_from_offset - 1; + + from_uchars.resize(from_size + 1); + int32_t from_code_points; + u_strFromUTF8( + from_uchars.data(), + from_uchars.size(), + &from_code_points, + reinterpret_cast(&data[current_from_offset]), + from_size, + &err); + if (U_FAILURE(err)) { + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); + } + + // NFC should produce no more than 3x code points + // https://unicode.org/faq/normalization.html#12 + to_uchars.resize(from_code_points * 3 + 1); + + int32_t to_code_points = unorm2_normalize( + normalizer, + from_uchars.data(), + from_code_points, + to_uchars.data(), + to_uchars.size(), + &err); + if (U_FAILURE(err)) { + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); + } + + size_t max_to_size = current_to_offset + 2 * to_code_points + 1; + if (res_data.size() < max_to_size) { + res_data.resize(max_to_size); + } + + int32_t to_size; + u_strToUTF8( + reinterpret_cast(&res_data[current_to_offset]), + res_data.size() - current_to_offset, + &to_size, + to_uchars.data(), + to_code_points, + &err); + if (U_FAILURE(err)) { + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); + } + + current_to_offset += to_size; + res_data[current_to_offset] = 0; + ++current_to_offset; + res_offsets[i] = current_to_offset; + + current_from_offset = offsets[i]; + } + } + + [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &) + { + throw Exception("Cannot apply function normalizeUTF8 to fixed string.", ErrorCodes::ILLEGAL_COLUMN); + } +}; + +struct NameNormalizeUTF8 +{ + static constexpr auto name = "normalizeUTF8"; +}; + +using FunctionNormalizeUTF8 = FunctionStringToString; +} + +void registerFunctionNormalizeUTF8(FunctionFactory & factory) { + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index ba6a294abba..f2439a3373b 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -52,6 +52,10 @@ void registerFunctionSynonyms(FunctionFactory &); void registerFunctionLemmatize(FunctionFactory &); #endif +#if USE_ICU +void registerFunctionNormalizeUTF8(FunctionFactory &); +#endif + void registerFunctionsString(FunctionFactory & factory) { registerFunctionRepeat(factory); @@ -97,6 +101,10 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionSynonyms(factory); registerFunctionLemmatize(factory); #endif + +#if USE_ICU + registerFunctionNormalizeUTF8(factory); +#endif } } diff --git a/tests/queries/0_stateless/02011_normalize_utf8.reference b/tests/queries/0_stateless/02011_normalize_utf8.reference new file mode 100644 index 00000000000..6878a38ca0d --- /dev/null +++ b/tests/queries/0_stateless/02011_normalize_utf8.reference @@ -0,0 +1,3 @@ +ё ё 2 4 ё ё 2 2 +ё 4 ё 2 +ё 2 ё 2 diff --git a/tests/queries/0_stateless/02011_normalize_utf8.sql b/tests/queries/0_stateless/02011_normalize_utf8.sql new file mode 100644 index 00000000000..c28a0c0a794 --- /dev/null +++ b/tests/queries/0_stateless/02011_normalize_utf8.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS normalize_test; +CREATE TABLE normalize_test (value String) ENGINE = MergeTree ORDER BY value; + +SELECT + 'ё' AS norm, + 'ё' AS denorm, + length(norm), + length(denorm), + normalizeUTF8(norm), + normalizeUTF8(denorm), + length(normalizeUTF8(norm)), + length(normalizeUTF8(denorm)); + +INSERT INTO normalize_test (value) VALUES ('ё'); +INSERT INTO normalize_test (value) VALUES ('ё'); + +SELECT value, length(value), normalizeUTF8(value) AS normalized, length(normalized) FROM normalize_test; + +SELECT char(228) AS value, normalizeUTF8(value); -- { serverError 619 } From 762904adbda95dc24b771250b1f32ccd404db739 Mon Sep 17 00:00:00 2001 From: Viachaslau Boben Date: Mon, 27 Sep 2021 18:45:04 +0300 Subject: [PATCH 003/126] Add nfd and perf test --- src/Functions/normalizeString.cpp | 114 +++++++++++++----- tests/performance/normalize_utf8.xml | 15 +++ .../02011_normalize_utf8.reference | 12 +- .../0_stateless/02011_normalize_utf8.sql | 51 ++++++-- 4 files changed, 148 insertions(+), 44 deletions(-) create mode 100644 tests/performance/normalize_utf8.xml diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 178c2dc2cf1..5beca566cd1 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -1,6 +1,10 @@ +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_ICU #include #include -#include #include #include #include @@ -15,12 +19,67 @@ namespace DB namespace ErrorCodes { + extern const int ILLEGAL_COLUMN; extern const int CANNOT_NORMALIZE_STRING; } namespace { +// Expansion factors are specified for UTF-32, since icu uses UTF-32 for normalization +// Maximum expansion factors for different normalization forms +// https://unicode.org/faq/normalization.html#12 + +struct NormalizeNFCImpl +{ + static constexpr auto name = "normalizeUTF8NFC"; + + static constexpr auto expansionFactor = 3; + + static const UNormalizer2 *getNormalizer(UErrorCode *err) + { + return unorm2_getNFCInstance(err); + } +}; + +struct NormalizeNFDImpl +{ + static constexpr auto name = "normalizeUTF8NFD"; + + static constexpr auto expansionFactor = 4; + + static const UNormalizer2 *getNormalizer(UErrorCode *err) + { + return unorm2_getNFDInstance(err); + } +}; + +struct NormalizeNFKCImpl +{ + static constexpr auto name = "normalizeUTF8NFKC"; + + static constexpr auto expansionFactor = 18; + + static const UNormalizer2 *getNormalizer(UErrorCode *err) + { + return unorm2_getNFKCInstance(err); + } +}; + + +struct NormalizeNFKDImpl +{ + static constexpr auto name = "normalizeUTF8NFKD"; + + static constexpr auto expansionFactor = 18; + + static const UNormalizer2 *getNormalizer(UErrorCode *err) + { + return unorm2_getNFKDInstance(err); + } +}; + +template struct NormalizeUTF8Impl { @@ -31,10 +90,9 @@ struct NormalizeUTF8Impl { UErrorCode err = U_ZERO_ERROR; - const UNormalizer2 *normalizer = unorm2_getNFCInstance(&err); - if (U_FAILURE(err)) { - throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); - } + const UNormalizer2 *normalizer = NormalizeImpl::getNormalizer(&err); + if (U_FAILURE(err)) + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed (getNormalizer): {}", u_errorName(err)); size_t size = offsets.size(); res_offsets.resize(size); @@ -60,13 +118,10 @@ struct NormalizeUTF8Impl reinterpret_cast(&data[current_from_offset]), from_size, &err); - if (U_FAILURE(err)) { - throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); - } + if (U_FAILURE(err)) + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed (strFromUTF8): {}", u_errorName(err)); - // NFC should produce no more than 3x code points - // https://unicode.org/faq/normalization.html#12 - to_uchars.resize(from_code_points * 3 + 1); + to_uchars.resize(from_code_points * NormalizeImpl::expansionFactor + 1); int32_t to_code_points = unorm2_normalize( normalizer, @@ -75,14 +130,12 @@ struct NormalizeUTF8Impl to_uchars.data(), to_uchars.size(), &err); - if (U_FAILURE(err)) { - throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); - } + if (U_FAILURE(err)) + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed (normalize): {}", u_errorName(err)); - size_t max_to_size = current_to_offset + 2 * to_code_points + 1; - if (res_data.size() < max_to_size) { + size_t max_to_size = current_to_offset + 4 * to_code_points + 1; + if (res_data.size() < max_to_size) res_data.resize(max_to_size); - } int32_t to_size; u_strToUTF8( @@ -92,9 +145,8 @@ struct NormalizeUTF8Impl to_uchars.data(), to_code_points, &err); - if (U_FAILURE(err)) { - throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed: {}", u_errorName(err)); - } + if (U_FAILURE(err)) + throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed (strToUTF8): {}", u_errorName(err)); current_to_offset += to_size; res_data[current_to_offset] = 0; @@ -111,16 +163,20 @@ struct NormalizeUTF8Impl } }; -struct NameNormalizeUTF8 +using FunctionNormalizeUTF8NFC = FunctionStringToString, NormalizeNFCImpl>; +using FunctionNormalizeUTF8NFD = FunctionStringToString, NormalizeNFDImpl>; +using FunctionNormalizeUTF8NFKC = FunctionStringToString, NormalizeNFKCImpl>; +using FunctionNormalizeUTF8NFKD = FunctionStringToString, NormalizeNFKDImpl>; +} + +void registerFunctionNormalizeUTF8(FunctionFactory & factory) { - static constexpr auto name = "normalizeUTF8"; -}; - -using FunctionNormalizeUTF8 = FunctionStringToString; -} - -void registerFunctionNormalizeUTF8(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } + +#endif diff --git a/tests/performance/normalize_utf8.xml b/tests/performance/normalize_utf8.xml new file mode 100644 index 00000000000..de9bd87fdf8 --- /dev/null +++ b/tests/performance/normalize_utf8.xml @@ -0,0 +1,15 @@ + + + hits_10m_single + + + CREATE TABLE strings (words String) ENGINE Memory + INSERT INTO strings SELECT SearchPhrase FROM hits_10m_single WHERE length(SearchPhrase) > 0 + + SELECT normalizeUTF8NFC(words) FROM strings FORMAT Null + SELECT normalizeUTF8NFD(words) FROM strings FORMAT Null + SELECT normalizeUTF8NFKC(words) FROM strings FORMAT Null + SELECT normalizeUTF8NFKD(words) FROM strings FORMAT Null + + DROP TABLE IF EXISTS strings + diff --git a/tests/queries/0_stateless/02011_normalize_utf8.reference b/tests/queries/0_stateless/02011_normalize_utf8.reference index 6878a38ca0d..b97f0ee5a01 100644 --- a/tests/queries/0_stateless/02011_normalize_utf8.reference +++ b/tests/queries/0_stateless/02011_normalize_utf8.reference @@ -1,3 +1,11 @@ ё ё 2 4 ё ё 2 2 -ё 4 ё 2 -ё 2 ё 2 +1 ё 4 ё 2 ё 4 ё 2 ё 4 +2 ё 2 ё 2 ё 4 ё 2 ё 4 +3 జ్ఞ‌ా 15 జ్ఞ‌ా 15 జ్ఞ‌ా 15 జ్ఞ‌ా 15 జ్ఞ‌ా 15 +4 本気ですか 15 本気ですか 15 本気ですか 18 本気ですか 15 本気ですか 18 +5 ﷺ 3 ﷺ 3 ﷺ 3 صلى الله عليه وسلم 33 صلى الله عليه وسلم 33 +6 ᾂ 3 ᾂ 3 ᾂ 8 ᾂ 3 ᾂ 8 +7 ΐ 2 ΐ 2 ΐ 6 ΐ 2 ΐ 6 +8 שּׁ 6 שּׁ 6 שּׁ 6 שּׁ 6 שּׁ 6 +9 𝅘𝅥𝅮 12 𝅘𝅥𝅮 12 𝅘𝅥𝅮 12 𝅘𝅥𝅮 12 𝅘𝅥𝅮 12 +10 Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒ 281 Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒ 281 Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒ 282 Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒ 281 Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒ 282 diff --git a/tests/queries/0_stateless/02011_normalize_utf8.sql b/tests/queries/0_stateless/02011_normalize_utf8.sql index c28a0c0a794..5abb6b4d8fb 100644 --- a/tests/queries/0_stateless/02011_normalize_utf8.sql +++ b/tests/queries/0_stateless/02011_normalize_utf8.sql @@ -1,19 +1,44 @@ +-- Tags: no-fasttest + DROP TABLE IF EXISTS normalize_test; -CREATE TABLE normalize_test (value String) ENGINE = MergeTree ORDER BY value; +CREATE TABLE normalize_test (id int, value String) ENGINE = MergeTree ORDER BY value; + SELECT - 'ё' AS norm, - 'ё' AS denorm, - length(norm), - length(denorm), - normalizeUTF8(norm), - normalizeUTF8(denorm), - length(normalizeUTF8(norm)), - length(normalizeUTF8(denorm)); + 'ё' AS norm, 'ё' AS denorm, + length(norm), length(denorm), + normalizeUTF8NFC(norm) AS norm_nfc, + normalizeUTF8NFC(denorm) AS denorm_nfc, + length(norm_nfc), + length(denorm_nfc); -INSERT INTO normalize_test (value) VALUES ('ё'); -INSERT INTO normalize_test (value) VALUES ('ё'); -SELECT value, length(value), normalizeUTF8(value) AS normalized, length(normalized) FROM normalize_test; +INSERT INTO normalize_test (id, value) VALUES (1, 'ё'); +INSERT INTO normalize_test (id, value) VALUES (2, 'ё'); +INSERT INTO normalize_test (id, value) VALUES (3, 'జ్ఞ‌ా'); +INSERT INTO normalize_test (id, value) VALUES (4, '本気ですか'); +INSERT INTO normalize_test (id, value) VALUES (5, 'ﷺ'); +INSERT INTO normalize_test (id, value) VALUES (6, 'ᾂ'); +INSERT INTO normalize_test (id, value) VALUES (7, 'ΐ'); +INSERT INTO normalize_test (id, value) VALUES (8, 'שּׁ'); +INSERT INTO normalize_test (id, value) VALUES (9, '𝅘𝅥𝅮'); -SELECT char(228) AS value, normalizeUTF8(value); -- { serverError 619 } + +INSERT INTO normalize_test (id, value) VALUES (10, 'Q̹̣̩̭̰̰̹̄ͬ̿͋̃ṷ̬̰ͥe̘͚͈̰̺̍͐s͎̜̖t͔̣̯̲̜̠ͣ̑ͨ̉̈̈o̲͙̺͊ͯͣ̐̋̂̔ ̳͉͍̒̂è̗ͥͯͨ̍ͮ͛ ̦̹̣̰̐̅̑͑̅̂t͙̭̻̖͛̾e̺͙ͣ͒̚ṣ̠͉͓͔̲̦̎t̖͖̝͓̣ͭ͑̈́̂ỏ̥͕͈͛̓ ̀ͦ̽ͅZͯ̑̎a͆l̻ͨ̋ͧͣͨͬg͉̙̟̾̅̾ͬo̠ͮ͒'); + + + +SELECT + id, value, length(value), + normalizeUTF8NFC(value) AS nfc, length(nfc) AS nfc_len, + normalizeUTF8NFD(value) AS nfd, length(nfd) AS nfd_len, + normalizeUTF8NFKC(value) AS nfkc, length(nfkc) AS nfkc_len, + normalizeUTF8NFKD(value) AS nfkd, length(nfkd) AS nfkd_len +FROM normalize_test +ORDER BY id; + + +SELECT char(228) AS value, normalizeUTF8NFC(value); -- { serverError 621 } +SELECT char(228) AS value, normalizeUTF8NFD(value); -- { serverError 621 } +SELECT char(228) AS value, normalizeUTF8NFKC(value); -- { serverError 621 } +SELECT char(228) AS value, normalizeUTF8NFKD(value); -- { serverError 621 } From d75136c3b1b3040b87dde90463e9a0e8a087b16b Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 29 Sep 2021 16:59:56 +0300 Subject: [PATCH 004/126] Update hash functions (SHA) en --- .../sql-reference/functions/hash-functions.md | 145 +++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 227e2885417..a3154e5c200 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -139,17 +139,160 @@ It works faster than intHash32. Average quality. ## SHA1 {#sha1} +Calculates SHA-1 hash from a string and returns the resulting set of bytes as [FixedString(20)](../data-types/fixedstring.md). + +**Syntax** + +``` sql +SHA1('s') +``` + +**Arguments** + +- `s` — Input string for SHA-1 hash calculation. [String](..data-types/string.md). + +**Returned value** + +- SHA-1 hash as a hex-unencoded FixedString(10). + +Type: [FixedString](../data-types/fixedstring.md). + +**Example** + +Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. + +Query: + +``` sql +SELECT hex(SHA1('abc')); +``` + +Result: + +``` text +┌─hex(SHA1('abc'))─────────────────────────┐ +│ A9993E364706816ABA3E25717850C26C9CD0D89D │ +└──────────────────────────────────────────┘ +``` + ## SHA224 {#sha224} +Calculates SHA-224 hash from a string and returns the resulting set of bytes as [FixedString(28)](../data-types/fixedstring.md). + +**Syntax** + +``` sql +SHA224('s') +``` + +**Arguments** + +- `s` — Input string for SHA-224 hash calculation. [String](..data-types/string.md). + +**Returned value** + +- SHA-224 hash as a hex-unencoded FixedString(28). + +Type: [FixedString](../data-types/fixedstring.md). + +**Example** + +Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. + +Query: + +``` sql +SELECT hex(SHA224('abc')); +``` + +Result: + +``` text +┌─hex(SHA224('abc'))───────────────────────────────────────┐ +│ 23097D223405D8228642A477BDA255B32AADBCE4BDA0B3F7E36C9DA7 │ +└──────────────────────────────────────────────────────────┘ +``` + ## SHA256 {#sha256} +Calculates SHA-256 hash from a string and returns the resulting set of bytes as [FixedString(32)](../data-types/fixedstring.md). + +**Syntax** + +``` sql +SHA256('s') +``` + +**Arguments** + +- `s` — Input string for SHA-256 hash calculation. [String](..data-types/string.md). + +**Returned value** + +- SHA-256 hash as a hex-unencoded FixedString(32). + +Type: [FixedString](../data-types/fixedstring.md). + +**Example** + +Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. + +Query: + +``` sql +SELECT hex(SHA256('abc')); +``` + +Result: + +``` text +┌─hex(SHA256('abc'))───────────────────────────────────────────────┐ +│ BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD │ +└──────────────────────────────────────────────────────────────────┘ +``` + ## SHA512 {#sha512} -Calculates SHA-1, SHA-224, SHA-256 or SHA-512 from a string and returns the resulting set of bytes as FixedString(20), FixedString(28), FixedString(32), or FixedString(64). +Calculates SHA-512 hash from a string and returns the resulting set of bytes as [FixedString(64)](../data-types/fixedstring.md). + +**Syntax** + +``` sql +SHA512('s') +``` + The function works fairly slowly (SHA-1 processes about 5 million short strings per second per processor core, while SHA-224 and SHA-256 process about 2.2 million). We recommend using this function only in cases when you need a specific hash function and you can’t select it. Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in SELECTS. +**Arguments** + +- `s` — Input string for SHA-512 hash calculation. [String](..data-types/string.md). + +**Returned value** + +- SHA-512 hash as a hex-unencoded FixedString(64). + +Type: [FixedString](../data-types/fixedstring.md). + +**Example** + +Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. + +Query: + +``` sql +SELECT hex(SHA512('abc')); +``` + +Result: + +``` text +┌─hex(SHA512('abc'))───────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ DDAF35A193617ABACC417349AE20413112E6FA4E89A97EA20A9EEEE64B55D39A2192992A274FC1A836BA3C23A3FEEBBD454D4423643CE80E2A9AC94FA54CA49F │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## URLHash(url\[, N\]) {#urlhashurl-n} A fast, decent-quality non-cryptographic hash function for a string obtained from a URL using some type of normalization. From e312156b1c2a6a8a79177fb543c5d110ea47a058 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 29 Sep 2021 17:52:39 +0300 Subject: [PATCH 005/126] Add note about Materialized views --- docs/en/sql-reference/functions/hash-functions.md | 2 +- docs/en/sql-reference/statements/create/view.md | 7 +++---- docs/ru/sql-reference/statements/create/view.md | 5 ++++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index a3154e5c200..dc4c749865a 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -263,7 +263,7 @@ SHA512('s') The function works fairly slowly (SHA-1 processes about 5 million short strings per second per processor core, while SHA-224 and SHA-256 process about 2.2 million). We recommend using this function only in cases when you need a specific hash function and you can’t select it. -Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in SELECTS. +Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in `SELECT` queries. **Arguments** diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index b6a09e25f95..84213020925 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -50,14 +50,13 @@ When creating a materialized view with `TO [db].[table]`, you must not use `POPU A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view. !!! important "Important" - Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not `Nullable`. A safe practice would be to add aliases for every column when using Materialized views. + Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not [Nullable](../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. -!!! important "Important" Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. -If you specify `POPULATE`, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using POPULATE, since data inserted in the table during the view creation will not be inserted in it. +If you specify `POPULATE`, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using `POPULATE`, since data inserted in the table during the view creation will not be inserted in it. -A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`. +A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`. Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`. The execution of [ALTER](../../../sql-reference/statements/alter/view.md) queries on materialized views has limitations, so they might be inconvenient. If the materialized view uses the construction `TO [db.]name`, you can `DETACH` the view, run `ALTER` for the target table, and then `ATTACH` the previously detached (`DETACH`) view. diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index ccbf79baa73..53d75b78dd1 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -48,9 +48,12 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na Материализованное представление устроено следующим образом: при вставке данных в таблицу, указанную в SELECT-е, кусок вставляемых данных преобразуется этим запросом SELECT, и полученный результат вставляется в представление. !!! important "Важно" + + Материализованные представления в ClickHouse используют **имена столбцов** вместо порядка следования столбцов при вставке в целевую таблицу. Если в результатах запроса `SELECT` некоторые имена столбцов отсутствуют, то ClickHouse будет использовать значение по умолчанию, даже если столбец не является [Nullable](../data-types/nullable.md). Безопасной практикой, при использовании материализованных представлений, считается добавление псевдонимов для каждого столбца. + Материализованные представления в ClickHouse больше похожи на `after insert` триггеры. Если в запросе материализованного представления есть агрегирование, оно применяется только к вставляемому блоку записей. Любые изменения существующих данных исходной таблицы (например обновление, удаление, удаление раздела и т.д.) не изменяют материализованное представление. -Если указано `POPULATE`, то при создании представления, в него будут вставлены имеющиеся данные таблицы, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Иначе, представление будет содержать только данные, вставляемые в таблицу после создания представления. Не рекомендуется использовать POPULATE, так как вставляемые в таблицу данные во время создания представления, не попадут в него. +Если указано `POPULATE`, то при создании представления, в него будут вставлены имеющиеся данные таблицы, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Иначе, представление будет содержать только данные, вставляемые в таблицу после создания представления. Не рекомендуется использовать `POPULATE`, так как вставляемые в таблицу данные во время создания представления, не попадут в него. Запрос `SELECT` может содержать `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Следует иметь ввиду, что соответствующие преобразования будут выполняться независимо, на каждый блок вставляемых данных. Например, при наличии `GROUP BY`, данные будут агрегироваться при вставке, но только в рамках одной пачки вставляемых данных. Далее, данные не будут доагрегированы. Исключение - использование ENGINE, производящего агрегацию данных самостоятельно, например, `SummingMergeTree`. From b226429435eeda0cae88b7553f471b6e413cff3d Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 29 Sep 2021 18:41:15 +0300 Subject: [PATCH 006/126] Fix links, add 512 translation. --- .../sql-reference/functions/hash-functions.md | 8 ++-- .../sql-reference/functions/hash-functions.md | 45 ++++++++++++++++++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index dc4c749865a..e28594540be 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -149,7 +149,7 @@ SHA1('s') **Arguments** -- `s` — Input string for SHA-1 hash calculation. [String](..data-types/string.md). +- `s` — Input string for SHA-1 hash calculation. [String](../data-types/string.md). **Returned value** @@ -187,7 +187,7 @@ SHA224('s') **Arguments** -- `s` — Input string for SHA-224 hash calculation. [String](..data-types/string.md). +- `s` — Input string for SHA-224 hash calculation. [String](../data-types/string.md). **Returned value** @@ -225,7 +225,7 @@ SHA256('s') **Arguments** -- `s` — Input string for SHA-256 hash calculation. [String](..data-types/string.md). +- `s` — Input string for SHA-256 hash calculation. [String](../data-types/string.md). **Returned value** @@ -267,7 +267,7 @@ Even in these cases, we recommend applying the function offline and pre-calculat **Arguments** -- `s` — Input string for SHA-512 hash calculation. [String](..data-types/string.md). +- `s` — Input string for SHA-512 hash calculation. [String](../data-types/string.md). **Returned value** diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 07c741e0588..d7e86d5a540 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -143,10 +143,51 @@ SELECT groupBitXor(cityHash64(*)) FROM table ## SHA256 {#sha256} -Вычисляет SHA-1, SHA-224, SHA-256 от строки и возвращает полученный набор байт в виде FixedString(20), FixedString(28), FixedString(32). + + +## SHA512 {#sha512} + +Вычисляет SHA-1, SHA-224, SHA-256 хеш строки и возвращает полученный набор байт в виде FixedString(20), FixedString(28), FixedString(32), [FixedString(64)](../data-types/fixedstring.md) + +Вычисляет SHA-512 хеш строки и возвращает полученный набор байт в виде [FixedString(64)](../data-types/fixedstring.md) + +**Синтаксис** + +``` sql +SHA512('s') +``` + Функция работает достаточно медленно (SHA-1 - примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 - примерно 2.2 миллионов). Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хэш-функция и вы не можете её выбрать. -Даже в этих случаях, рекомендуется применять функцию оффлайн - заранее вычисляя значения при вставке в таблицу, вместо того, чтобы применять её при SELECT-ах. +Даже в этих случаях, рекомендуется применять функцию оффлайн - заранее вычисляя значения при вставке в таблицу, вместо того, чтобы применять её при выполнении `SELECT`. + +**Параметры** + +- `s` — входная строка для вычисления хеша SHA-512. [String](../data-types/string.md). + +**Возвращаемое значение** + +- Хеш SHA-512 в виде шестнадцатеричной некодированной строки FixedString(64). + +Тип: [FixedString](../data-types/fixedstring.md). + +**Пример** + +Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. + +Запрос: + +``` sql +SELECT hex(SHA512('abc')); +``` + +Результат: + +``` text +┌─hex(SHA512('abc'))───────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ DDAF35A193617ABACC417349AE20413112E6FA4E89A97EA20A9EEEE64B55D39A2192992A274FC1A836BA3C23A3FEEBBD454D4423643CE80E2A9AC94FA54CA49F │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` ## URLHash(url\[, N\]) {#urlhashurl-n} From 02205492e5cef5119455bbed48d339349cb4575e Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 29 Sep 2021 19:55:11 +0300 Subject: [PATCH 007/126] Update hash-functions.md Add ru translation. --- .../sql-reference/functions/hash-functions.md | 110 +++++++++++++++++- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index d7e86d5a540..98b5ed6df27 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -139,16 +139,120 @@ SELECT groupBitXor(cityHash64(*)) FROM table ## SHA1 {#sha1} +Вычисляет SHA-1 хеш строки и возвращает полученный набор байт в виде [FixedString(20)](../data-types/fixedstring.md). + +**Синтаксис** + +``` sql +SHA1('s') +``` + +**Параметры** + +- `s` — входная строка для вычисления хеша SHA-1. [String](../data-types/string.md). + +**Возвращаемое значение** + +- Хеш SHA-1 в виде шестнадцатеричной некодированной строки FixedString(20). + +Тип: [FixedString](../data-types/fixedstring.md). + +**Пример** + +Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. + +Запрос: + +``` sql +SELECT hex(SHA1('abc')); +``` + +Результат: + +``` text +┌─hex(SHA1('abc'))─────────────────────────┐ +│ A9993E364706816ABA3E25717850C26C9CD0D89D │ +└──────────────────────────────────────────┘ +``` + ## SHA224 {#sha224} +Вычисляет SHA-224 хеш строки и возвращает полученный набор байт в виде [FixedString(28)](../data-types/fixedstring.md). + +**Синтаксис** + +``` sql +SHA224('s') +``` + +**Параметры** + +- `s` — входная строка для вычисления хеша SHA-224. [String](../data-types/string.md). + +**Возвращаемое значение** + +- Хеш SHA-224 в виде шестнадцатеричной некодированной строки FixedString(28). + +Тип: [FixedString](../data-types/fixedstring.md). + +**Пример** + +Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. + +Запрос: + +``` sql +SELECT hex(SHA224('abc')); +``` + +Результат: + +``` text +┌─hex(SHA224('abc'))───────────────────────────────────────┐ +│ 23097D223405D8228642A477BDA255B32AADBCE4BDA0B3F7E36C9DA7 │ +└──────────────────────────────────────────────────────────┘ +``` + ## SHA256 {#sha256} +Вычисляет SHA-256 хеш строки и возвращает полученный набор байт в виде [FixedString(32)](../data-types/fixedstring.md). +**Синтаксис** + +``` sql +SHA256('s') +``` + +**Параметры** + +- `s` — входная строка для вычисления хеша SHA-256. [String](../data-types/string.md). + +**Возвращаемое значение** + +- Хеш SHA-256 в виде шестнадцатеричной некодированной строки FixedString(32). + +Тип: [FixedString](../data-types/fixedstring.md). + +**Пример** + +Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. + +Запрос: + +``` sql +SELECT hex(SHA256('abc')); +``` + +Результат: + +``` text +┌─hex(SHA256('abc'))───────────────────────────────────────────────┐ +│ BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD │ +└──────────────────────────────────────────────────────────────────┘ +``` ## SHA512 {#sha512} -Вычисляет SHA-1, SHA-224, SHA-256 хеш строки и возвращает полученный набор байт в виде FixedString(20), FixedString(28), FixedString(32), [FixedString(64)](../data-types/fixedstring.md) - Вычисляет SHA-512 хеш строки и возвращает полученный набор байт в виде [FixedString(64)](../data-types/fixedstring.md) **Синтаксис** @@ -157,7 +261,7 @@ SELECT groupBitXor(cityHash64(*)) FROM table SHA512('s') ``` -Функция работает достаточно медленно (SHA-1 - примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 - примерно 2.2 миллионов). +Функция работает достаточно медленно (SHA-1 — примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 — примерно 2.2 миллионов). Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хэш-функция и вы не можете её выбрать. Даже в этих случаях, рекомендуется применять функцию оффлайн - заранее вычисляя значения при вставке в таблицу, вместо того, чтобы применять её при выполнении `SELECT`. From 66bb857a1a7988ec0f94bc9c83668ff662b651b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 29 Sep 2021 19:11:38 +0200 Subject: [PATCH 008/126] Add test for JOIN engine deadlock --- .../02033_join_engine_deadlock.reference | 0 .../0_stateless/02033_join_engine_deadlock.sh | 71 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 tests/queries/0_stateless/02033_join_engine_deadlock.reference create mode 100755 tests/queries/0_stateless/02033_join_engine_deadlock.sh diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock.reference b/tests/queries/0_stateless/02033_join_engine_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock.sh b/tests/queries/0_stateless/02033_join_engine_deadlock.sh new file mode 100755 index 00000000000..7a4ca1c8bb1 --- /dev/null +++ b/tests/queries/0_stateless/02033_join_engine_deadlock.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Tags: deadlock + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +create_table () { + $CLICKHOUSE_CLIENT --query " + CREATE TABLE join_block_test + ( + id String, + num Int64 + ) + ENGINE = Join(ANY, LEFT, id) + " +} + +drop_table () { + # Force a sync drop to free the memory before ending the test + # Otherwise things get interesting if you run the test many times before the database is finally dropped + $CLICKHOUSE_CLIENT --query " + DROP TABLE join_block_test SYNC + " +} + +populate_table_bg () { + ( + $CLICKHOUSE_CLIENT --query " + INSERT INTO join_block_test + SELECT toString(number) as id, number * number as num + FROM system.numbers LIMIT 3000000 + " --lock_acquire_timeout=20 >/dev/null + ) & +} + +read_table_bg () { + ( + $CLICKHOUSE_CLIENT --query " + SELECT * + FROM + ( + SELECT toString(number) AS user_id + FROM system.numbers LIMIT 10000 OFFSET 20000 + ) AS t1 + LEFT JOIN + ( + SELECT + * + FROM join_block_test AS i1 + ANY LEFT JOIN + ( + SELECT * + FROM join_block_test + ) AS i2 ON i1.id = toString(i2.num) + ) AS t2 ON t1.user_id = t2.id + " --lock_acquire_timeout=20 >/dev/null + ) & +} + +create_table +for _ in {1..5}; +do + populate_table_bg + sleep 0.05 + read_table_bg + sleep 0.05 +done + +wait +drop_table From 0ee5c0bff570ec676a394e2e60dc934b1e640b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 29 Sep 2021 19:30:07 +0200 Subject: [PATCH 009/126] Use RWLock in StorageJoin to avoid deadlocks --- src/Functions/FunctionJoinGet.cpp | 6 +-- src/Functions/FunctionJoinGet.h | 16 +++++--- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/HashJoin.cpp | 2 +- src/Interpreters/HashJoin.h | 7 ++-- src/Storages/IStorage.h | 1 + src/Storages/StorageJoin.cpp | 53 +++++++++++++++---------- src/Storages/StorageJoin.h | 13 +++--- src/Storages/StorageSet.cpp | 21 +++++----- src/Storages/StorageSet.h | 8 ++-- 10 files changed, 76 insertions(+), 53 deletions(-) diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index f0dff0ac7e4..df131538275 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -25,14 +25,14 @@ ColumnPtr ExecutableFunctionJoinGet::executeImpl(const ColumnsWithTypeA auto key = arguments[i]; keys.emplace_back(std::move(key)); } - return storage_join->joinGet(keys, result_columns).column; + return storage_join->joinGet(keys, result_columns, getContext()).column; } template ExecutableFunctionPtr FunctionJoinGet::prepare(const ColumnsWithTypeAndName &) const { Block result_columns {{return_type->createColumn(), return_type, attr_name}}; - return std::make_unique>(table_lock, storage_join, result_columns); + return std::make_unique>(getContext(), table_lock, storage_join, result_columns); } static std::pair, String> @@ -89,7 +89,7 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null); auto table_lock = storage_join->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - return std::make_unique>(table_lock, storage_join, attr_name, argument_types, return_type); + return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); } void registerFunctionJoinGet(FunctionFactory & factory) diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 3ddab51e2d9..2dd0cb9fdea 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -14,13 +14,15 @@ class StorageJoin; using StorageJoinPtr = std::shared_ptr; template -class ExecutableFunctionJoinGet final : public IExecutableFunction +class ExecutableFunctionJoinGet final : public IExecutableFunction, WithContext { public: - ExecutableFunctionJoinGet(TableLockHolder table_lock_, + ExecutableFunctionJoinGet(ContextPtr context_, + TableLockHolder table_lock_, StorageJoinPtr storage_join_, const DB::Block & result_columns_) - : table_lock(std::move(table_lock_)) + : WithContext(context_) + , table_lock(std::move(table_lock_)) , storage_join(std::move(storage_join_)) , result_columns(result_columns_) {} @@ -42,15 +44,17 @@ private: }; template -class FunctionJoinGet final : public IFunctionBase +class FunctionJoinGet final : public IFunctionBase, WithContext { public: static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; - FunctionJoinGet(TableLockHolder table_lock_, + FunctionJoinGet(ContextPtr context_, + TableLockHolder table_lock_, StorageJoinPtr storage_join_, String attr_name_, DataTypes argument_types_, DataTypePtr return_type_) - : table_lock(std::move(table_lock_)) + : WithContext(context_) + , table_lock(std::move(table_lock_)) , storage_join(storage_join_) , attr_name(std::move(attr_name_)) , argument_types(std::move(argument_types_)) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 566ee60a3e6..89d7624f203 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -938,7 +938,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( if (auto storage = analyzed_join->getStorageJoin()) { std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, {}); - return storage->getJoinLocked(analyzed_join); + return storage->getJoinLocked(analyzed_join, getContext()); } joined_plan = buildJoinedPlan(getContext(), join_element, *analyzed_join, query_options); diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 07872df8ce5..d88df9d3e30 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -744,7 +744,7 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) size_t total_rows = 0; size_t total_bytes = 0; { - if (storage_join_lock.mutex()) + if (storage_join_lock) throw DB::Exception("addJoinedBlock called when HashJoin locked to prevent updates", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 07fd6d5b89f..f1f1198e7d9 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -334,9 +335,9 @@ public: /// We keep correspondence between used_flags and hash table internal buffer. /// Hash table cannot be modified during HashJoin lifetime and must be protected with lock. - void setLock(std::shared_mutex & rwlock) + void setLock(RWLockImpl::LockHolder rwlock_holder) { - storage_join_lock = std::shared_lock(rwlock); + storage_join_lock = rwlock_holder; } void reuseJoinedData(const HashJoin & join); @@ -391,7 +392,7 @@ private: /// Should be set via setLock to protect hash table from modification from StorageJoin /// If set HashJoin instance is not available for modification (addJoinedBlock) - std::shared_lock storage_join_lock; + RWLockImpl::LockHolder storage_join_lock = nullptr; void dataMapInit(MapsVariant &); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6ce17552ba1..2013cc5ecb6 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -219,6 +219,7 @@ private: /// without locks. MultiVersionStorageMetadataPtr metadata; +protected: RWLockImpl::LockHolder tryLockTimed( const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & acquire_timeout) const; diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index e45183591f2..e5574708de0 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -1,13 +1,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -67,6 +67,14 @@ StorageJoin::StorageJoin( restore(); } +RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr ctx) const +{ + const String query_id = ctx ? ctx->getInitialQueryId() : RWLockImpl::NO_QUERY; + const std::chrono::milliseconds acquire_timeout + = ctx ? ctx->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + return tryLockTimed(lock, type, query_id, acquire_timeout); +} + SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { std::lock_guard mutate_lock(mutate_mutex); @@ -74,10 +82,10 @@ SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataP } void StorageJoin::truncate( - const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder&) + const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr ctx, TableExclusiveLockHolder&) { std::lock_guard mutate_lock(mutate_mutex); - std::unique_lock lock(rwlock); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, ctx); disk->removeRecursive(path); disk->createDirectories(path); @@ -128,7 +136,7 @@ void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) } /// Now acquire exclusive lock and modify storage. - std::unique_lock lock(rwlock); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); join = std::move(new_data); increment = 1; @@ -152,7 +160,7 @@ void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) } } -HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join) const +HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, ContextPtr ctx) const { auto metadata_snapshot = getInMemoryMetadataPtr(); if (!analyzed_join->sameStrictnessAndKind(strictness, kind)) @@ -171,34 +179,36 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join) analyzed_join->setRightKeys(key_names); HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); - join_clone->setLock(rwlock); + + RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); + join_clone->setLock(holder); join_clone->reuseJoinedData(*join); return join_clone; } -void StorageJoin::insertBlock(const Block & block) +void StorageJoin::insertBlock(const Block & block, ContextPtr ctx) { - std::unique_lock lock(rwlock); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, ctx); join->addJoinedBlock(block, true); } -size_t StorageJoin::getSize() const +size_t StorageJoin::getSize(ContextPtr ctx) const { - std::shared_lock lock(rwlock); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); return join->getTotalRowCount(); } -std::optional StorageJoin::totalRows(const Settings &) const +std::optional StorageJoin::totalRows(const Settings &settings) const { - std::shared_lock lock(rwlock); + TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings.lock_acquire_timeout); return join->getTotalRowCount(); } -std::optional StorageJoin::totalBytes(const Settings &) const +std::optional StorageJoin::totalBytes(const Settings &settings) const { - std::shared_lock lock(rwlock); + TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings.lock_acquire_timeout); return join->getTotalByteCount(); } @@ -207,9 +217,9 @@ DataTypePtr StorageJoin::joinGetCheckAndGetReturnType(const DataTypes & data_typ return join->joinGetCheckAndGetReturnType(data_types, column_name, or_null); } -ColumnWithTypeAndName StorageJoin::joinGet(const Block & block, const Block & block_with_columns_to_add) const +ColumnWithTypeAndName StorageJoin::joinGet(const Block & block, const Block & block_with_columns_to_add, ContextPtr ctx) const { - std::shared_lock lock(rwlock); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); return join->joinGet(block, block_with_columns_to_add); } @@ -370,10 +380,10 @@ size_t rawSize(const StringRef & t) class JoinSource : public SourceWithProgress { public: - JoinSource(HashJoinPtr join_, std::shared_mutex & rwlock, UInt64 max_block_size_, Block sample_block_) + JoinSource(HashJoinPtr join_, TableLockHolder lock_holder_, UInt64 max_block_size_, Block sample_block_) : SourceWithProgress(sample_block_) , join(join_) - , lock(rwlock) + , lock_holder(lock_holder_) , max_block_size(max_block_size_) , sample_block(std::move(sample_block_)) { @@ -421,7 +431,7 @@ protected: private: HashJoinPtr join; - std::shared_lock lock; + TableLockHolder lock_holder; UInt64 max_block_size; Block sample_block; @@ -571,7 +581,7 @@ Pipe StorageJoin::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & /*query_info*/, - ContextPtr /*context*/, + ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned /*num_streams*/) @@ -579,7 +589,8 @@ Pipe StorageJoin::read( metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); Block source_sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); - return Pipe(std::make_shared(join, rwlock, max_block_size, source_sample_block)); + RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); + return Pipe(std::make_shared(join, std::move(holder), max_block_size, source_sample_block)); } } diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index 6a08773ecc8..4926194433c 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -2,7 +2,9 @@ #include +#include #include +#include #include #include @@ -35,7 +37,7 @@ public: /// Return instance of HashJoin holding lock that protects from insertions to StorageJoin. /// HashJoin relies on structure of hash table that's why we need to return it with locked mutex. - HashJoinPtr getJoinLocked(std::shared_ptr analyzed_join) const; + HashJoinPtr getJoinLocked(std::shared_ptr analyzed_join, ContextPtr ctx) const; /// Get result type for function "joinGet(OrNull)" DataTypePtr joinGetCheckAndGetReturnType(const DataTypes & data_types, const String & column_name, bool or_null) const; @@ -43,7 +45,7 @@ public: /// Execute function "joinGet(OrNull)" on data block. /// Takes rwlock for read to prevent parallel StorageJoin updates during processing data block /// (but not during processing whole query, it's safe for joinGet that doesn't involve `used_flags` from HashJoin) - ColumnWithTypeAndName joinGet(const Block & block, const Block & block_with_columns_to_add) const; + ColumnWithTypeAndName joinGet(const Block & block, const Block & block_with_columns_to_add, ContextPtr context) const; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; @@ -73,12 +75,13 @@ private: /// Protect state for concurrent use in insertFromBlock and joinBlock. /// Lock is stored in HashJoin instance during query and blocks concurrent insertions. - mutable std::shared_mutex rwlock; + mutable RWLock rwlock = RWLockImpl::create(); mutable std::mutex mutate_mutex; - void insertBlock(const Block & block) override; + void insertBlock(const Block & block, ContextPtr ctx) override; void finishInsert() override {} - size_t getSize() const override; + size_t getSize(ContextPtr context) const override; + RWLockImpl::LockHolder tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr ctx) const; protected: StorageJoin( diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index fe55123335a..c57dadf6d52 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -34,11 +34,11 @@ namespace ErrorCodes } -class SetOrJoinSink : public SinkToStorage +class SetOrJoinSink : public SinkToStorage, WithContext { public: SetOrJoinSink( - StorageSetOrJoinBase & table_, const StorageMetadataPtr & metadata_snapshot_, + ContextPtr ctx, StorageSetOrJoinBase & table_, const StorageMetadataPtr & metadata_snapshot_, const String & backup_path_, const String & backup_tmp_path_, const String & backup_file_name_, bool persistent_); @@ -60,6 +60,7 @@ private: SetOrJoinSink::SetOrJoinSink( + ContextPtr ctx, StorageSetOrJoinBase & table_, const StorageMetadataPtr & metadata_snapshot_, const String & backup_path_, @@ -67,6 +68,7 @@ SetOrJoinSink::SetOrJoinSink( const String & backup_file_name_, bool persistent_) : SinkToStorage(metadata_snapshot_->getSampleBlock()) + , WithContext(ctx) , table(table_) , metadata_snapshot(metadata_snapshot_) , backup_path(backup_path_) @@ -84,7 +86,7 @@ void SetOrJoinSink::consume(Chunk chunk) /// Sort columns in the block. This is necessary, since Set and Join count on the same column order in different blocks. Block sorted_block = getHeader().cloneWithColumns(chunk.detachColumns()).sortColumns(); - table.insertBlock(sorted_block); + table.insertBlock(sorted_block, getContext()); if (persistent) backup_stream.write(sorted_block); } @@ -104,10 +106,10 @@ void SetOrJoinSink::onFinish() } -SinkToStoragePtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr /*context*/) +SinkToStoragePtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr ctx) { UInt64 id = ++increment; - return std::make_shared(*this, metadata_snapshot, path, fs::path(path) / "tmp/", toString(id) + ".bin", persistent); + return std::make_shared(ctx, *this, metadata_snapshot, path, fs::path(path) / "tmp/", toString(id) + ".bin", persistent); } @@ -155,10 +157,10 @@ StorageSet::StorageSet( } -void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block.getColumnsWithTypeAndName()); } +void StorageSet::insertBlock(const Block & block, ContextPtr) { set->insertFromBlock(block.getColumnsWithTypeAndName()); } void StorageSet::finishInsert() { set->finishInsert(); } -size_t StorageSet::getSize() const { return set->getTotalRowCount(); } +size_t StorageSet::getSize(ContextPtr) const { return set->getTotalRowCount(); } std::optional StorageSet::totalRows(const Settings &) const { return set->getTotalRowCount(); } std::optional StorageSet::totalBytes(const Settings &) const { return set->getTotalByteCount(); } @@ -210,6 +212,7 @@ void StorageSetOrJoinBase::restore() void StorageSetOrJoinBase::restoreFromFile(const String & file_path) { + ContextPtr ctx = nullptr; auto backup_buf = disk->readFile(file_path); CompressedReadBuffer compressed_backup_buf(*backup_buf); NativeBlockInputStream backup_stream(compressed_backup_buf, 0); @@ -217,14 +220,14 @@ void StorageSetOrJoinBase::restoreFromFile(const String & file_path) backup_stream.readPrefix(); while (Block block = backup_stream.read()) - insertBlock(block); + insertBlock(block, ctx); finishInsert(); backup_stream.readSuffix(); /// TODO Add speed, compressed bytes, data volume in memory, compression ratio ... Generalize all statistics logging in project. LOG_INFO(&Poco::Logger::get("StorageSetOrJoinBase"), "Loaded from backup file {}. {} rows, {}. State has {} unique rows.", - file_path, backup_stream.getProfileInfo().rows, ReadableSize(backup_stream.getProfileInfo().bytes), getSize()); + file_path, backup_stream.getProfileInfo().rows, ReadableSize(backup_stream.getProfileInfo().bytes), getSize(ctx)); } diff --git a/src/Storages/StorageSet.h b/src/Storages/StorageSet.h index 1166557ec8e..1b78676b6c5 100644 --- a/src/Storages/StorageSet.h +++ b/src/Storages/StorageSet.h @@ -51,10 +51,10 @@ private: void restoreFromFile(const String & file_path); /// Insert the block into the state. - virtual void insertBlock(const Block & block) = 0; + virtual void insertBlock(const Block & block, ContextPtr context) = 0; /// Call after all blocks were inserted. virtual void finishInsert() = 0; - virtual size_t getSize() const = 0; + virtual size_t getSize(ContextPtr context) const = 0; }; @@ -81,9 +81,9 @@ public: private: SetPtr set; - void insertBlock(const Block & block) override; + void insertBlock(const Block & block, ContextPtr) override; void finishInsert() override; - size_t getSize() const override; + size_t getSize(ContextPtr) const override; protected: StorageSet( From c0ba8d1a043c962c365c142e64a75a8e5db993b0 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 29 Sep 2021 21:31:18 +0300 Subject: [PATCH 010/126] Fix crosslink. --- docs/en/sql-reference/statements/create/view.md | 2 +- docs/ru/sql-reference/statements/create/view.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 84213020925..39c5760ecf3 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -50,7 +50,7 @@ When creating a materialized view with `TO [db].[table]`, you must not use `POPU A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view. !!! important "Important" - Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not [Nullable](../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. + Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index 53d75b78dd1..9eb0baf5a98 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -49,7 +49,7 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na !!! important "Важно" - Материализованные представления в ClickHouse используют **имена столбцов** вместо порядка следования столбцов при вставке в целевую таблицу. Если в результатах запроса `SELECT` некоторые имена столбцов отсутствуют, то ClickHouse будет использовать значение по умолчанию, даже если столбец не является [Nullable](../data-types/nullable.md). Безопасной практикой, при использовании материализованных представлений, считается добавление псевдонимов для каждого столбца. + Материализованные представления в ClickHouse используют **имена столбцов** вместо порядка следования столбцов при вставке в целевую таблицу. Если в результатах запроса `SELECT` некоторые имена столбцов отсутствуют, то ClickHouse будет использовать значение по умолчанию, даже если столбец не является [Nullable](../../data-types/nullable.md). Безопасной практикой, при использовании материализованных представлений, считается добавление псевдонимов для каждого столбца. Материализованные представления в ClickHouse больше похожи на `after insert` триггеры. Если в запросе материализованного представления есть агрегирование, оно применяется только к вставляемому блоку записей. Любые изменения существующих данных исходной таблицы (например обновление, удаление, удаление раздела и т.д.) не изменяют материализованное представление. From e53a48fb3089ab114b8fa907fc2efa3e27a1c960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 30 Sep 2021 10:15:44 +0200 Subject: [PATCH 011/126] Raise lock acquire timeout for the test Needed for check test under ASAN --- tests/queries/0_stateless/02033_join_engine_deadlock.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock.sh b/tests/queries/0_stateless/02033_join_engine_deadlock.sh index 7a4ca1c8bb1..f4ae564e2a7 100755 --- a/tests/queries/0_stateless/02033_join_engine_deadlock.sh +++ b/tests/queries/0_stateless/02033_join_engine_deadlock.sh @@ -30,7 +30,7 @@ populate_table_bg () { INSERT INTO join_block_test SELECT toString(number) as id, number * number as num FROM system.numbers LIMIT 3000000 - " --lock_acquire_timeout=20 >/dev/null + " >/dev/null ) & } @@ -54,7 +54,7 @@ read_table_bg () { FROM join_block_test ) AS i2 ON i1.id = toString(i2.num) ) AS t2 ON t1.user_id = t2.id - " --lock_acquire_timeout=20 >/dev/null + " >/dev/null ) & } From f58742014c0a72e9d02d9c02d4172b3d36989461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 30 Sep 2021 10:47:15 +0200 Subject: [PATCH 012/126] Consistent naming --- src/Storages/StorageJoin.cpp | 27 +++++++++++++-------------- src/Storages/StorageJoin.h | 6 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index e5574708de0..b17315106ce 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -67,11 +67,11 @@ StorageJoin::StorageJoin( restore(); } -RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr ctx) const +RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) const { - const String query_id = ctx ? ctx->getInitialQueryId() : RWLockImpl::NO_QUERY; + const String query_id = context ? context->getInitialQueryId() : RWLockImpl::NO_QUERY; const std::chrono::milliseconds acquire_timeout - = ctx ? ctx->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + = context ? context->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); return tryLockTimed(lock, type, query_id, acquire_timeout); } @@ -81,11 +81,10 @@ SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataP return StorageSetOrJoinBase::write(query, metadata_snapshot, context); } -void StorageJoin::truncate( - const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr ctx, TableExclusiveLockHolder&) +void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, TableExclusiveLockHolder &) { std::lock_guard mutate_lock(mutate_mutex); - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, ctx); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); disk->removeRecursive(path); disk->createDirectories(path); @@ -160,7 +159,7 @@ void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) } } -HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, ContextPtr ctx) const +HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, ContextPtr context) const { auto metadata_snapshot = getInMemoryMetadataPtr(); if (!analyzed_join->sameStrictnessAndKind(strictness, kind)) @@ -180,7 +179,7 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); - RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); + RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); join_clone->setLock(holder); join_clone->reuseJoinedData(*join); @@ -188,15 +187,15 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, } -void StorageJoin::insertBlock(const Block & block, ContextPtr ctx) +void StorageJoin::insertBlock(const Block & block, ContextPtr context) { - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, ctx); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); join->addJoinedBlock(block, true); } -size_t StorageJoin::getSize(ContextPtr ctx) const +size_t StorageJoin::getSize(ContextPtr context) const { - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); return join->getTotalRowCount(); } @@ -217,9 +216,9 @@ DataTypePtr StorageJoin::joinGetCheckAndGetReturnType(const DataTypes & data_typ return join->joinGetCheckAndGetReturnType(data_types, column_name, or_null); } -ColumnWithTypeAndName StorageJoin::joinGet(const Block & block, const Block & block_with_columns_to_add, ContextPtr ctx) const +ColumnWithTypeAndName StorageJoin::joinGet(const Block & block, const Block & block_with_columns_to_add, ContextPtr context) const { - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, ctx); + TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); return join->joinGet(block, block_with_columns_to_add); } diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index 4926194433c..cdc47531999 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -37,7 +37,7 @@ public: /// Return instance of HashJoin holding lock that protects from insertions to StorageJoin. /// HashJoin relies on structure of hash table that's why we need to return it with locked mutex. - HashJoinPtr getJoinLocked(std::shared_ptr analyzed_join, ContextPtr ctx) const; + HashJoinPtr getJoinLocked(std::shared_ptr analyzed_join, ContextPtr context) const; /// Get result type for function "joinGet(OrNull)" DataTypePtr joinGetCheckAndGetReturnType(const DataTypes & data_types, const String & column_name, bool or_null) const; @@ -78,10 +78,10 @@ private: mutable RWLock rwlock = RWLockImpl::create(); mutable std::mutex mutate_mutex; - void insertBlock(const Block & block, ContextPtr ctx) override; + void insertBlock(const Block & block, ContextPtr context) override; void finishInsert() override {} size_t getSize(ContextPtr context) const override; - RWLockImpl::LockHolder tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr ctx) const; + RWLockImpl::LockHolder tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) const; protected: StorageJoin( From 6f2447c027526d06e6f0125ea496815f01d40052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 30 Sep 2021 15:48:54 +0200 Subject: [PATCH 013/126] clang-tidy fix --- src/Storages/StorageSet.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index c57dadf6d52..fd06c2975b6 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -106,10 +106,11 @@ void SetOrJoinSink::onFinish() } -SinkToStoragePtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr ctx) +SinkToStoragePtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { UInt64 id = ++increment; - return std::make_shared(ctx, *this, metadata_snapshot, path, fs::path(path) / "tmp/", toString(id) + ".bin", persistent); + return std::make_shared( + context, *this, metadata_snapshot, path, fs::path(path) / "tmp/", toString(id) + ".bin", persistent); } From 50ef202b12aba727011ea2fe14f588f56e66a2d5 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:25:57 +0300 Subject: [PATCH 014/126] Update docs/en/sql-reference/statements/create/view.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/create/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 39c5760ecf3..f174d561cc6 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -50,7 +50,7 @@ When creating a materialized view with `TO [db].[table]`, you must not use `POPU A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view. !!! important "Important" - Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in `SELECT`'s result ClickHouse will use a default value, even if column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. + Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in the `SELECT` query result, ClickHouse uses a default value, even if the column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. From bb5c92276d0d9b5d838624b5cc345f33bbb41fdf Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:26:09 +0300 Subject: [PATCH 015/126] Update docs/en/sql-reference/statements/create/view.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/create/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index f174d561cc6..ec34c57a4cd 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -54,7 +54,7 @@ A materialized view is implemented as follows: when inserting data to the table Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. -If you specify `POPULATE`, the existing table data is inserted in the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using `POPULATE`, since data inserted in the table during the view creation will not be inserted in it. +If you specify `POPULATE`, the existing table data is inserted into the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using `POPULATE`, since data inserted in the table during the view creation will not be inserted in it. A `SELECT` query can contain `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`. Note that the corresponding conversions are performed independently on each block of inserted data. For example, if `GROUP BY` is set, data is aggregated during insertion, but only within a single packet of inserted data. The data won’t be further aggregated. The exception is when using an `ENGINE` that independently performs data aggregation, such as `SummingMergeTree`. From fbe95f9c9d1834e09edd6c04b58ce58d09732f4e Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:26:17 +0300 Subject: [PATCH 016/126] Update docs/ru/sql-reference/functions/hash-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/hash-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 98b5ed6df27..975efdae71c 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -263,7 +263,7 @@ SHA512('s') Функция работает достаточно медленно (SHA-1 — примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 — примерно 2.2 миллионов). Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хэш-функция и вы не можете её выбрать. -Даже в этих случаях, рекомендуется применять функцию оффлайн - заранее вычисляя значения при вставке в таблицу, вместо того, чтобы применять её при выполнении `SELECT`. +Даже в этих случаях рекомендуется применять функцию офлайн — заранее вычисляя значения при вставке в таблицу, вместо того чтобы применять её при выполнении `SELECT`. **Параметры** From 7d5ea307f1d51b434ca44072eaf51aba6e57e992 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:26:25 +0300 Subject: [PATCH 017/126] Update docs/ru/sql-reference/functions/hash-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/hash-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 975efdae71c..18197f88ce3 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -262,7 +262,7 @@ SHA512('s') ``` Функция работает достаточно медленно (SHA-1 — примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 — примерно 2.2 миллионов). -Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хэш-функция и вы не можете её выбрать. +Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хеш-функция и вы не можете её выбрать. Даже в этих случаях рекомендуется применять функцию офлайн — заранее вычисляя значения при вставке в таблицу, вместо того чтобы применять её при выполнении `SELECT`. **Параметры** From 89f4830180ae0120797d75cc81e46ab5abd2ff2e Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:26:33 +0300 Subject: [PATCH 018/126] Update docs/ru/sql-reference/statements/create/view.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/statements/create/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index 9eb0baf5a98..77bdc7249c7 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -49,7 +49,7 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na !!! important "Важно" - Материализованные представления в ClickHouse используют **имена столбцов** вместо порядка следования столбцов при вставке в целевую таблицу. Если в результатах запроса `SELECT` некоторые имена столбцов отсутствуют, то ClickHouse будет использовать значение по умолчанию, даже если столбец не является [Nullable](../../data-types/nullable.md). Безопасной практикой, при использовании материализованных представлений, считается добавление псевдонимов для каждого столбца. + Материализованные представления в ClickHouse используют **имена столбцов** вместо порядка следования столбцов при вставке в целевую таблицу. Если в результатах запроса `SELECT` некоторые имена столбцов отсутствуют, то ClickHouse использует значение по умолчанию, даже если столбец не является [Nullable](../../data-types/nullable.md). Безопасной практикой при использовании материализованных представлений считается добавление псевдонимов для каждого столбца. Материализованные представления в ClickHouse больше похожи на `after insert` триггеры. Если в запросе материализованного представления есть агрегирование, оно применяется только к вставляемому блоку записей. Любые изменения существующих данных исходной таблицы (например обновление, удаление, удаление раздела и т.д.) не изменяют материализованное представление. From 61a7db9612ed67ce4320bda3c193ec07669f4242 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 1 Oct 2021 18:26:55 +0300 Subject: [PATCH 019/126] Update docs/ru/sql-reference/statements/create/view.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/statements/create/view.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index 77bdc7249c7..7ebb154d6b6 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -53,7 +53,7 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na Материализованные представления в ClickHouse больше похожи на `after insert` триггеры. Если в запросе материализованного представления есть агрегирование, оно применяется только к вставляемому блоку записей. Любые изменения существующих данных исходной таблицы (например обновление, удаление, удаление раздела и т.д.) не изменяют материализованное представление. -Если указано `POPULATE`, то при создании представления, в него будут вставлены имеющиеся данные таблицы, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Иначе, представление будет содержать только данные, вставляемые в таблицу после создания представления. Не рекомендуется использовать `POPULATE`, так как вставляемые в таблицу данные во время создания представления, не попадут в него. +Если указано `POPULATE`, то при создании представления в него будут добавлены данные, уже содержащиеся в исходной таблице, как если бы был сделан запрос `CREATE TABLE ... AS SELECT ...` . Если `POPULATE` не указано, представление будет содержать только данные, добавленные в таблицу после создания представления. Использовать `POPULATE` не рекомендуется, так как в представление не попадут данные, добавляемые в таблицу во время создания представления. Запрос `SELECT` может содержать `DISTINCT`, `GROUP BY`, `ORDER BY`, `LIMIT`… Следует иметь ввиду, что соответствующие преобразования будут выполняться независимо, на каждый блок вставляемых данных. Например, при наличии `GROUP BY`, данные будут агрегироваться при вставке, но только в рамках одной пачки вставляемых данных. Далее, данные не будут доагрегированы. Исключение - использование ENGINE, производящего агрегацию данных самостоятельно, например, `SummingMergeTree`. From 271f7995c03f5e24a047e692fad06f81181c0d93 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Mon, 4 Oct 2021 00:19:37 +0300 Subject: [PATCH 020/126] Fix PR comments. --- .../sql-reference/functions/hash-functions.md | 130 ++--------------- .../sql-reference/functions/hash-functions.md | 132 ++---------------- 2 files changed, 19 insertions(+), 243 deletions(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index e28594540be..20fe6d14e86 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -137,23 +137,29 @@ This is a relatively fast non-cryptographic hash function of average quality for Calculates a 64-bit hash code from any type of integer. It works faster than intHash32. Average quality. -## SHA1 {#sha1} +## SHA1, SHA224, SHA256, SHA512 {#sha} -Calculates SHA-1 hash from a string and returns the resulting set of bytes as [FixedString(20)](../data-types/fixedstring.md). +Calculates SHA-1, SHA-224, SHA-256, SHA-512 hash from a string and returns the resulting set of bytes as [FixedString](../data-types/fixedstring.md). **Syntax** ``` sql SHA1('s') +... +SHA512('s') ``` +The function works fairly slowly (SHA-1 processes about 5 million short strings per second per processor core, while SHA-224 and SHA-256 process about 2.2 million). +We recommend using this function only in cases when you need a specific hash function and you can’t select it. +Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in `SELECT` queries. + **Arguments** - `s` — Input string for SHA-1 hash calculation. [String](../data-types/string.md). **Returned value** -- SHA-1 hash as a hex-unencoded FixedString(10). +- SHA hash as a hex-unencoded FixedString. SHA-1 returns as FixedString(20), SHA-224 as FixedString(28), SHA-256 — FixedString(32), SHA-512 — FixedString(64). Type: [FixedString](../data-types/fixedstring.md). @@ -175,124 +181,6 @@ Result: └──────────────────────────────────────────┘ ``` -## SHA224 {#sha224} - -Calculates SHA-224 hash from a string and returns the resulting set of bytes as [FixedString(28)](../data-types/fixedstring.md). - -**Syntax** - -``` sql -SHA224('s') -``` - -**Arguments** - -- `s` — Input string for SHA-224 hash calculation. [String](../data-types/string.md). - -**Returned value** - -- SHA-224 hash as a hex-unencoded FixedString(28). - -Type: [FixedString](../data-types/fixedstring.md). - -**Example** - -Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. - -Query: - -``` sql -SELECT hex(SHA224('abc')); -``` - -Result: - -``` text -┌─hex(SHA224('abc'))───────────────────────────────────────┐ -│ 23097D223405D8228642A477BDA255B32AADBCE4BDA0B3F7E36C9DA7 │ -└──────────────────────────────────────────────────────────┘ -``` - -## SHA256 {#sha256} - -Calculates SHA-256 hash from a string and returns the resulting set of bytes as [FixedString(32)](../data-types/fixedstring.md). - -**Syntax** - -``` sql -SHA256('s') -``` - -**Arguments** - -- `s` — Input string for SHA-256 hash calculation. [String](../data-types/string.md). - -**Returned value** - -- SHA-256 hash as a hex-unencoded FixedString(32). - -Type: [FixedString](../data-types/fixedstring.md). - -**Example** - -Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. - -Query: - -``` sql -SELECT hex(SHA256('abc')); -``` - -Result: - -``` text -┌─hex(SHA256('abc'))───────────────────────────────────────────────┐ -│ BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD │ -└──────────────────────────────────────────────────────────────────┘ -``` - -## SHA512 {#sha512} - -Calculates SHA-512 hash from a string and returns the resulting set of bytes as [FixedString(64)](../data-types/fixedstring.md). - -**Syntax** - -``` sql -SHA512('s') -``` - -The function works fairly slowly (SHA-1 processes about 5 million short strings per second per processor core, while SHA-224 and SHA-256 process about 2.2 million). -We recommend using this function only in cases when you need a specific hash function and you can’t select it. -Even in these cases, we recommend applying the function offline and pre-calculating values when inserting them into the table, instead of applying it in `SELECT` queries. - -**Arguments** - -- `s` — Input string for SHA-512 hash calculation. [String](../data-types/string.md). - -**Returned value** - -- SHA-512 hash as a hex-unencoded FixedString(64). - -Type: [FixedString](../data-types/fixedstring.md). - -**Example** - -Use the [hex](../functions/encoding-functions.md#hex) function to represent the result as a hex-encoded string. - -Query: - -``` sql -SELECT hex(SHA512('abc')); -``` - -Result: - -``` text -┌─hex(SHA512('abc'))───────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ DDAF35A193617ABACC417349AE20413112E6FA4E89A97EA20A9EEEE64B55D39A2192992A274FC1A836BA3C23A3FEEBBD454D4423643CE80E2A9AC94FA54CA49F │ -└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - ## URLHash(url\[, N\]) {#urlhashurl-n} A fast, decent-quality non-cryptographic hash function for a string obtained from a URL using some type of normalization. diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 18197f88ce3..f3b14625a8c 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -137,23 +137,29 @@ SELECT groupBitXor(cityHash64(*)) FROM table Вычисляет 64-битный хэш-код от целого числа любого типа. Работает быстрее, чем intHash32. Качество среднее. -## SHA1 {#sha1} +## SHA1, SHA224, SHA256, SHA512 {#sha} -Вычисляет SHA-1 хеш строки и возвращает полученный набор байт в виде [FixedString(20)](../data-types/fixedstring.md). +Вычисляет SHA-1, SHA-224, SHA-256, SHA-512 хеш строки и возвращает полученный набор байт в виде [FixedString](../data-types/fixedstring.md). **Синтаксис** ``` sql SHA1('s') +... +SHA512('s') ``` +Функция работает достаточно медленно (SHA-1 — примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 — примерно 2.2 миллионов). +Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хеш-функция и вы не можете её выбрать. +Даже в этих случаях рекомендуется применять функцию офлайн — заранее вычисляя значения при вставке в таблицу, вместо того чтобы применять её при выполнении `SELECT`. + **Параметры** -- `s` — входная строка для вычисления хеша SHA-1. [String](../data-types/string.md). +- `s` — входная строка для вычисления хеша SHA. [String](../data-types/string.md). **Возвращаемое значение** -- Хеш SHA-1 в виде шестнадцатеричной некодированной строки FixedString(20). +- Хеш SHA в виде шестнадцатеричной некодированной строки FixedString. SHA-1 хеш как FixedString(20), SHA-224 как FixedString(28), SHA-256 — FixedString(32), SHA-512 — FixedString(64). Тип: [FixedString](../data-types/fixedstring.md). @@ -175,124 +181,6 @@ SELECT hex(SHA1('abc')); └──────────────────────────────────────────┘ ``` -## SHA224 {#sha224} - -Вычисляет SHA-224 хеш строки и возвращает полученный набор байт в виде [FixedString(28)](../data-types/fixedstring.md). - -**Синтаксис** - -``` sql -SHA224('s') -``` - -**Параметры** - -- `s` — входная строка для вычисления хеша SHA-224. [String](../data-types/string.md). - -**Возвращаемое значение** - -- Хеш SHA-224 в виде шестнадцатеричной некодированной строки FixedString(28). - -Тип: [FixedString](../data-types/fixedstring.md). - -**Пример** - -Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. - -Запрос: - -``` sql -SELECT hex(SHA224('abc')); -``` - -Результат: - -``` text -┌─hex(SHA224('abc'))───────────────────────────────────────┐ -│ 23097D223405D8228642A477BDA255B32AADBCE4BDA0B3F7E36C9DA7 │ -└──────────────────────────────────────────────────────────┘ -``` - -## SHA256 {#sha256} - -Вычисляет SHA-256 хеш строки и возвращает полученный набор байт в виде [FixedString(32)](../data-types/fixedstring.md). - -**Синтаксис** - -``` sql -SHA256('s') -``` - -**Параметры** - -- `s` — входная строка для вычисления хеша SHA-256. [String](../data-types/string.md). - -**Возвращаемое значение** - -- Хеш SHA-256 в виде шестнадцатеричной некодированной строки FixedString(32). - -Тип: [FixedString](../data-types/fixedstring.md). - -**Пример** - -Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. - -Запрос: - -``` sql -SELECT hex(SHA256('abc')); -``` - -Результат: - -``` text -┌─hex(SHA256('abc'))───────────────────────────────────────────────┐ -│ BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD │ -└──────────────────────────────────────────────────────────────────┘ -``` - -## SHA512 {#sha512} - -Вычисляет SHA-512 хеш строки и возвращает полученный набор байт в виде [FixedString(64)](../data-types/fixedstring.md) - -**Синтаксис** - -``` sql -SHA512('s') -``` - -Функция работает достаточно медленно (SHA-1 — примерно 5 миллионов коротких строк в секунду на одном процессорном ядре, SHA-224 и SHA-256 — примерно 2.2 миллионов). -Рекомендуется использовать эти функции лишь в тех случаях, когда вам нужна конкретная хеш-функция и вы не можете её выбрать. -Даже в этих случаях рекомендуется применять функцию офлайн — заранее вычисляя значения при вставке в таблицу, вместо того чтобы применять её при выполнении `SELECT`. - -**Параметры** - -- `s` — входная строка для вычисления хеша SHA-512. [String](../data-types/string.md). - -**Возвращаемое значение** - -- Хеш SHA-512 в виде шестнадцатеричной некодированной строки FixedString(64). - -Тип: [FixedString](../data-types/fixedstring.md). - -**Пример** - -Используйте функцию [hex](../functions/encoding-functions.md#hex) для представления результата в виде строки с шестнадцатеричной кодировкой. - -Запрос: - -``` sql -SELECT hex(SHA512('abc')); -``` - -Результат: - -``` text -┌─hex(SHA512('abc'))───────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ DDAF35A193617ABACC417349AE20413112E6FA4E89A97EA20A9EEEE64B55D39A2192992A274FC1A836BA3C23A3FEEBBD454D4423643CE80E2A9AC94FA54CA49F │ -└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -``` - ## URLHash(url\[, N\]) {#urlhashurl-n} Быстрая не криптографическая хэш-функция неплохого качества для строки, полученной из URL путём некоторой нормализации. From 57c5d9d3828b69935a71d6f472762bdbce93bb46 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Mon, 4 Oct 2021 00:29:46 +0300 Subject: [PATCH 021/126] Update hash-functions.md minor fix --- docs/en/sql-reference/functions/hash-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index d8659b406df..21ed8d33098 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -159,7 +159,7 @@ Even in these cases, we recommend applying the function offline and pre-calculat **Arguments** -- `s` — Input string for SHA-1 hash calculation. [String](../data-types/string.md). +- `s` — Input string for SHA hash calculation. [String](../data-types/string.md). **Returned value** From 4497f5094e518f0a9d16068c152c88b6bc4c5c98 Mon Sep 17 00:00:00 2001 From: Mikhail <71978106+michon470@users.noreply.github.com> Date: Mon, 4 Oct 2021 16:19:51 +0300 Subject: [PATCH 022/126] Moved changes to this new branch --- .../sql-reference/statements/alter/column.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 801690afbb2..8f9273c81ba 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -10,7 +10,7 @@ A set of queries that allow changing the table structure. Syntax: ``` sql -ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... +ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|MODIFY|MATERIALIZE COLUMN ... ``` In the query, specify a list of one or more comma-separated actions. @@ -25,6 +25,7 @@ The following actions are supported: - [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. - [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. - [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties. +- [MATERIALIZE COLUMN](#materialize-column) — Materializes the column in the parts where the column is missing. These actions are described in detail below. @@ -193,6 +194,39 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; - [REMOVE TTL](ttl.md). +## MATERIALIZE COLUMN {#materialize-column} + +Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier as shown below. + +Syntax: + +```sql +ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +``` + +**Example:** + +```sql +DROP TABLE IF EXISTS tmp; +SET mutations_sync = 2; +CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 20; +ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); +SELECT groupArray(x), groupArray(s) FROM tmp; +``` + +**Result:** + +```sql +┌─groupArray(x)───────────────────────────────────────┬─groupArray(s)───────────────────────────────────────────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] │ ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'] │ +└─────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +**See Also** + +- [MATERIALIZED](../../statements/create/table.md#materialized). + ## Limitations {#alter-query-limitations} The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot. From 52c5f2da7203eaaae8ea819bc8ef405dafacb1c2 Mon Sep 17 00:00:00 2001 From: Mikhail <71978106+michon470@users.noreply.github.com> Date: Mon, 4 Oct 2021 16:41:50 +0300 Subject: [PATCH 023/126] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=B2=D0=BE?= =?UTF-8?q?=D0=B4=20+=20=D0=B4=D0=BE=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B0=D0=BD=D0=B3=D0=BB=20=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D1=81=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sql-reference/statements/alter/column.md | 2 +- .../sql-reference/statements/alter/column.md | 38 +++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 8f9273c81ba..31874ef208d 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -204,7 +204,7 @@ Syntax: ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; ``` -**Example:** +**Example with the creation of new column:** ```sql DROP TABLE IF EXISTS tmp; diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 9f59c79bfdd..366caf6a2a0 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -10,7 +10,7 @@ toc_title: "Манипуляции со столбцами" Синтаксис: ``` sql -ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... +ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|MODIFY|MATERIALIZE COLUMN ... ``` В запросе можно указать сразу несколько действий над одной таблицей через запятую. @@ -20,11 +20,12 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN - [ADD COLUMN](#alter_add-column) — добавляет столбец в таблицу; - [DROP COLUMN](#alter_drop-column) — удаляет столбец; -- [RENAME COLUMN](#alter_rename-column) — переименовывает существующий столбец. +- [RENAME COLUMN](#alter_rename-column) — переименовывает существующий столбец; - [CLEAR COLUMN](#alter_clear-column) — сбрасывает все значения в столбце для заданной партиции; - [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; -- [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL. -- [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца. +- [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL; +- [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца; +- [MATERIALIZE COLUMN](#materialize-column) — делает столбец материализованным (MATERIALIZED) в его частях, у которых отсутствуют значения. Подробное описание для каждого действия приведено ниже. @@ -193,6 +194,35 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; - [REMOVE TTL](ttl.md). +## MATERIALIZE COLUMN {#materialize-column} + +С помощью этого запроса можно сделать столбец таблицы материализованным (`MATERIALIZED`) в его частях, у которых отсутствуют значения. Это полезно, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`). Если вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо большим, для него может оказаться целесообразным использовать `MATERIALIZE COLUMN`. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`, как показано ниже. + +Синтаксис: + +```sql +ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +``` + +**Пример использования при создании нового столбца:** + +```sql +DROP TABLE IF EXISTS tmp; +SET mutations_sync = 2; +CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 20; +ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); +SELECT groupArray(x), groupArray(s) FROM tmp; +``` + +**Результат:** + +```sql +┌─groupArray(x)───────────────────────────────────────┬─groupArray(s)───────────────────────────────────────────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] │ ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'] │ +└─────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки. From fbfdd605eea214466bbd8d32a58f214aa5e5ca8e Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Tue, 5 Oct 2021 21:58:49 +0300 Subject: [PATCH 024/126] Update metrica.md Update ru with for hits_100m_obfuscated --- docs/ru/getting-started/example-datasets/metrica.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index c82048a445e..27105ca8488 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -38,6 +38,9 @@ $ curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads= $ # теперь создадим таблицу $ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" $ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +# for hits_100m_obfuscated +clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" + $ # импортируем данные $ cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 $ # опционально можно оптимизировать таблицу From af94e30a955dbfe271f412ee5ebe384994448f8e Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Tue, 5 Oct 2021 22:12:44 +0300 Subject: [PATCH 025/126] Update H3 functions Update en and add ru draft. --- docs/en/sql-reference/functions/geo/h3.md | 14 +-- docs/ru/sql-reference/functions/geo/h3.md | 112 +++++++++++++++++++++- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 3c3ed7b8932..9cdd3bcf947 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -481,7 +481,7 @@ Type: [UInt64](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3ToParent(599405990164561919, 3) as parent; +SELECT h3ToParent(599405990164561919, 3) AS parent; ``` Result: @@ -515,7 +515,7 @@ Type: [String](../../../sql-reference/data-types/string.md). Query: ``` sql -SELECT h3ToString(617420388352917503) as h3_string; +SELECT h3ToString(617420388352917503) AS h3_string; ``` Result: @@ -549,7 +549,7 @@ stringToH3(index_str) Query: ``` sql -SELECT stringToH3('89184926cc3ffff') as index; +SELECT stringToH3('89184926cc3ffff') AS index; ``` Result: @@ -583,7 +583,7 @@ h3GetResolution(index) Query: ``` sql -SELECT h3GetResolution(617420388352917503) as res; +SELECT h3GetResolution(617420388352917503) AS res; ``` Result: @@ -620,7 +620,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3IsResClassIII(617420388352917503) as res; +SELECT h3IsResClassIII(617420388352917503) AS res; ``` Result: @@ -657,7 +657,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT SELECT h3IsPentagon(644721767722457330) as pentagon; +SELECT h3IsPentagon(644721767722457330) AS pentagon; ``` Result: @@ -693,7 +693,7 @@ Type: [Array](../../../sql-reference/data-types/array.md)([UInt64](../../../sql- Query: ``` sql -SELECT SELECT h3GetFaces(599686042433355775) as faces; +SELECT h3GetFaces(599686042433355775) AS faces; ``` Result: diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index bc47ca72a39..e8871d856c4 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -548,7 +548,7 @@ h3GetResolution(index) Запрос: ``` sql -SELECT h3GetResolution(617420388352917503) as res; +SELECT h3GetResolution(617420388352917503) AS res; ``` Результат: @@ -559,3 +559,113 @@ SELECT h3GetResolution(617420388352917503) as res; └─────┘ ``` +## h3IsResClassIII {#h3isresclassIII} + +Returns whether [H3](#h3index) index has a resolution with Class III orientation. + +**Синтаксис** + +``` sql +h3IsResClassIII(index) +``` + +**Параметр** + +- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- `1` — Index has a resolution with Class III orientation. +- `0` — Index doesn't have a resolution with Class III orientation. + +Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). + +**Пример** + +Запрос: + +``` sql +SELECT h3IsResClassIII(617420388352917503) AS res; +``` + +Результат: + +``` text +┌─res─┐ +│ 1 │ +└─────┘ +``` + +## h3IsPentagon {#h3ispentagon } + +Returns whether this [H3](#h3index) index represents a pentagonal cell. + +**Синтаксис** + +``` sql +h3IsPentagon(index) +``` + +**Параметр** + +- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- `1` — Index represents a pentagonal cell. +- `0` — Index doesn't represent a pentagonal cell. + +Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). + +**Пример** + +Запрос: + +``` sql +SELECT h3IsPentagon(644721767722457330) AS pentagon; +``` + +Результат: + +``` text +┌─pentagon─┐ +│ 0 │ +└──────────┘ +``` + +## h3GetFaces {#h3getfaces} + +Returns icosahedron faces intersected by a given [H3](#h3index) index. + +**Синтаксис** + +``` sql +h3GetFaces(index) +``` + +**Параметр** + +- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Возвращаемое значение** + +- Array containing icosahedron faces intersected by a given H3 index. + +Тип: [Array](../../../sql-reference/data-types/array.md)([UInt64](../../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Запрос: + +``` sql +SELECT h3GetFaces(599686042433355775) AS faces; +``` + +Результат: + +``` text +┌─faces─┐ +│ [7] │ +└───────┘ + +[Оригинальная статья](https://clickhouse.com/docs/ru/sql-reference/functions/geo/h3) From 1550c167bb8b725376968d2b1f2779c669f59a3a Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 6 Oct 2021 18:14:51 +0300 Subject: [PATCH 026/126] Update ru translation. --- docs/en/sql-reference/functions/geo/h3.md | 2 +- docs/ru/sql-reference/functions/geo/h3.md | 26 +++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 9cdd3bcf947..410cb9d3cc2 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -631,7 +631,7 @@ Result: └─────┘ ``` -## h3IsPentagon {#h3ispentagon } +## h3IsPentagon {#h3ispentagon} Returns whether this [H3](#h3index) index represents a pentagonal cell. diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index e8871d856c4..cd807ade04a 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -6,7 +6,7 @@ toc_title: "Функции для работы с индексами H3" [H3](https://eng.uber.com/h3/) — это система геокодирования, которая делит поверхность Земли на равные шестигранные ячейки. Система поддерживает иерархию (вложенность) ячеек, т.е. каждый "родительский" шестигранник может быть поделен на семь одинаковых вложенных "дочерних" шестигранников, и так далее. -Уровень вложенности назвается `разрешением` и может принимать значение от `0` до `15`, где `0` соответствует `базовым` ячейкам самого верхнего уровня (наиболее крупным). +Уровень вложенности называется `разрешением` и может принимать значение от `0` до `15`, где `0` соответствует `базовым` ячейкам самого верхнего уровня (наиболее крупным). Для каждой точки, имеющей широту и долготу, можно получить 64-битный индекс H3, соответствующий номеру шестигранной ячейки, где эта точка находится. @@ -561,7 +561,7 @@ SELECT h3GetResolution(617420388352917503) AS res; ## h3IsResClassIII {#h3isresclassIII} -Returns whether [H3](#h3index) index has a resolution with Class III orientation. +Проверяет, имеет ли индекс [H3](#h3index) разрешение с ориентацией Class III. **Синтаксис** @@ -571,12 +571,12 @@ h3IsResClassIII(index) **Параметр** -- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `index` — порядковый номер шестигранника. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** -- `1` — Index has a resolution with Class III orientation. -- `0` — Index doesn't have a resolution with Class III orientation. +- `1` — индекс имеет разрешение с ориентацией Class III. +- `0` — индекс не имеет разрешения с ориентацией Class III. Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). @@ -596,9 +596,9 @@ SELECT h3IsResClassIII(617420388352917503) AS res; └─────┘ ``` -## h3IsPentagon {#h3ispentagon } +## h3IsPentagon {#h3ispentagon} -Returns whether this [H3](#h3index) index represents a pentagonal cell. +Проверяет, является ли указанный индекс [H3](#h3index) пятиугольной ячейкой. **Синтаксис** @@ -608,12 +608,12 @@ h3IsPentagon(index) **Параметр** -- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `index` — порядковый номер шестигранника. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** -- `1` — Index represents a pentagonal cell. -- `0` — Index doesn't represent a pentagonal cell. +- `1` — индекс представляет собой пятиугольную ячейку. +- `0` — индекс не является пятиугольной ячейкой. Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). @@ -635,7 +635,7 @@ SELECT h3IsPentagon(644721767722457330) AS pentagon; ## h3GetFaces {#h3getfaces} -Returns icosahedron faces intersected by a given [H3](#h3index) index. +Возвращает все грани многоугольника (икосаэдра), пересекаемые заданным [H3](#h3index) индексом. **Синтаксис** @@ -645,11 +645,11 @@ h3GetFaces(index) **Параметр** -- `index` — Hexagon index number. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `index` — индекс шестиугольной ячейки. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). **Возвращаемое значение** -- Array containing icosahedron faces intersected by a given H3 index. +- Массив, содержащий грани многоугольника (икосаэдра), пересекаемые заданным H3 индексом. Тип: [Array](../../../sql-reference/data-types/array.md)([UInt64](../../../sql-reference/data-types/int-uint.md)). From 259da1ccf07e8ed788ed7a418884c91801bff1fa Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 6 Oct 2021 18:32:55 +0300 Subject: [PATCH 027/126] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index cd807ade04a..6bc6943ec93 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -668,4 +668,4 @@ SELECT h3GetFaces(599686042433355775) AS faces; │ [7] │ └───────┘ -[Оригинальная статья](https://clickhouse.com/docs/ru/sql-reference/functions/geo/h3) +[Оригинальная статья](https://clickhouse.com/docs/ru/sql-reference/functions/geo/h3/) From 4894588f2751189a55b0dce9ca218e4b0040ec7b Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 6 Oct 2021 19:50:05 +0000 Subject: [PATCH 028/126] description improved new example --- .../functions/other-functions.md | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 44702f4097f..2bb38684eb3 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -2357,8 +2357,8 @@ Result: ## shardNum {#shard-num} -Returns the number of a shard which executes the query for a distributed query. -If query is not distributed then *constant value* is returned. +Returns the index of a shard which processes a part of data for a distributed query. Indices are started from `1`. +If a query is not distributed then constant value `0` is returned. **Syntax** @@ -2368,14 +2368,39 @@ shardNum() **Returned value** -- Shard number. +- Shard index or constant `0`. Type: [UInt32](../../sql-reference/data-types/int-uint.md). +**Example** + +In the following example a configuration with two shards is used. The query is executed on the [system.one](../../operations/system-tables/one.md) table on every shard. + +Query: + +``` sql +CREATE TABLE shard_num_example (dummy UInt8) + ENGINE=Distributed(test_cluster_two_shards_localhost, system, one, dummy); +SELECT dummy, shardNum(), shardCount() FROM shard_num_example; +``` + +Result: + +``` text +┌─dummy─┬─shardNum()─┬─shardCount()─┐ +│ 0 │ 2 │ 2 │ +│ 0 │ 1 │ 2 │ +└───────┴────────────┴──────────────┘ +``` + +**See Also** + +- [Distributed Table Engine](../../engines/table-engines/special/distributed.md) + ## shardCount {#shard-count} -Returns the total number of shards which execute a distributed query. -If query is not distributed then *constant value* is returned. +Returns the total number of shards for a distributed query. +If a query is not distributed then constant value `0` is returned. **Syntax** @@ -2385,8 +2410,10 @@ shardCount() **Returned value** -- Total number of shards. +- Total number of shards or `0`. Type: [UInt32](../../sql-reference/data-types/int-uint.md). +**See Also** +- [shardNum()](#shard-num) function example also contains `shardCount()` function call. From c2533b974394c2f1b4c356fa8408ad9dc0526d0f Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 6 Oct 2021 20:13:46 +0000 Subject: [PATCH 029/126] constant or column note added for other functions --- docs/en/sql-reference/functions/date-time-functions.md | 1 + docs/en/sql-reference/functions/other-functions.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index f54ef635e0c..b85f105758b 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -26,6 +26,7 @@ SELECT ## timeZone {#timezone} Returns the timezone of the server. +If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. **Syntax** diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 2bb38684eb3..afcc9563b58 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -8,6 +8,7 @@ toc_title: Other ## hostName() {#hostname} Returns a string with the name of the host that this function was performed on. For distributed processing, this is the name of the remote server host, if the function is performed on a remote server. +If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. ## getMacro {#getmacro} @@ -691,10 +692,12 @@ Returns the largest value of a and b. ## uptime() {#uptime} Returns the server’s uptime in seconds. +If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. ## version() {#version} Returns the version of the server as a string. +If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. ## blockNumber {#blocknumber} @@ -2101,6 +2104,7 @@ UNSUPPORTED_METHOD ## tcpPort {#tcpPort} Returns [native interface](../../interfaces/tcp.md) TCP port number listened by this server. +If it is executed in the context of a distributed table, then it generates a normal column, otherwise it produces a constant value. **Syntax** From c41923c5958067f487b31a27f860cc1e775accdc Mon Sep 17 00:00:00 2001 From: Haavard Kvaalen Date: Thu, 7 Oct 2021 13:29:38 +0200 Subject: [PATCH 030/126] MaterializedMySQL: Update GTID set at end of transaction We would update the set of seen GTIDs as soon as we saw a GTID_EVENT, which arrives before a transaction. This would mostly work fine, but if we lost the connection to MySQL in the middle of a large transaction we would persist that the transaction had been processed as soon as the transaction had started. When the connection was reestablished, we would not process the transaction again, which meant that we only applied parts of it. Fix this by updating the seen GTIDs at the end of the transaction instead. --- src/Core/MySQL/MySQLReplication.cpp | 29 ++++++++++++++++-- src/Core/MySQL/MySQLReplication.h | 4 +++ .../materialize_with_ddl.py | 30 +++++++++++++++++++ .../test_materialized_mysql_database/test.py | 5 ++++ 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 9c90b2ff220..b5468d15edc 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -105,12 +105,16 @@ namespace MySQLReplication if (query.starts_with("BEGIN") || query.starts_with("COMMIT")) { typ = QUERY_EVENT_MULTI_TXN_FLAG; + if (!query.starts_with("COMMIT")) + transaction_complete = false; } else if (query.starts_with("XA")) { if (query.starts_with("XA ROLLBACK")) throw ReplicationError("ParseQueryEvent: Unsupported query event:" + query, ErrorCodes::LOGICAL_ERROR); typ = QUERY_EVENT_XA; + if (!query.starts_with("XA COMMIT")) + transaction_complete = false; } else if (query.starts_with("SAVEPOINT")) { @@ -711,9 +715,26 @@ namespace MySQLReplication { switch (event->header.type) { - case FORMAT_DESCRIPTION_EVENT: - case QUERY_EVENT: + case FORMAT_DESCRIPTION_EVENT: { + binlog_pos = event->header.log_pos; + break; + } + case QUERY_EVENT: { + auto query = std::static_pointer_cast(event); + if (query->transaction_complete && pending_gtid) + { + gtid_sets.update(*pending_gtid); + pending_gtid.reset(); + } + binlog_pos = event->header.log_pos; + break; + } case XID_EVENT: { + if (pending_gtid) + { + gtid_sets.update(*pending_gtid); + pending_gtid.reset(); + } binlog_pos = event->header.log_pos; break; } @@ -724,9 +745,11 @@ namespace MySQLReplication break; } case GTID_EVENT: { + if (pending_gtid) + gtid_sets.update(*pending_gtid); auto gtid_event = std::static_pointer_cast(event); binlog_pos = event->header.log_pos; - gtid_sets.update(gtid_event->gtid); + pending_gtid = gtid_event->gtid; break; } default: diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h index a57cc246eaa..cb67ce73de9 100644 --- a/src/Core/MySQL/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -383,6 +383,7 @@ namespace MySQLReplication String schema; String query; QueryType typ = QUERY_EVENT_DDL; + bool transaction_complete = true; QueryEvent(EventHeader && header_) : EventBase(std::move(header_)), thread_id(0), exec_time(0), schema_len(0), error_code(0), status_len(0) @@ -536,6 +537,9 @@ namespace MySQLReplication void update(BinlogEventPtr event); void update(UInt64 binlog_pos_, const String & binlog_name_, const String & gtid_sets_); void dump(WriteBuffer & out) const; + + private: + std::optional pending_gtid; }; class IFlavor : public MySQLProtocol::IMySQLReadPacket diff --git a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py index 23fa9894a84..5f6daea24ac 100644 --- a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py @@ -980,3 +980,33 @@ def mysql_settings_test(clickhouse_node, mysql_node, service_name): clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") +def materialized_mysql_large_transaction(clickhouse_node, mysql_node, service_name): + mysql_node.query("DROP DATABASE IF EXISTS largetransaction") + clickhouse_node.query("DROP DATABASE IF EXISTS largetransaction") + mysql_node.query("CREATE DATABASE largetransaction") + + mysql_node.query("CREATE TABLE largetransaction.test_table (" + "`key` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " + "`value` INT NOT NULL) ENGINE = InnoDB;") + num_rows = 200000 + rows_per_insert = 5000 + values = ",".join(["(1)" for _ in range(rows_per_insert)]) + for i in range(num_rows//rows_per_insert): + mysql_node.query(f"INSERT INTO largetransaction.test_table (`value`) VALUES {values};") + + + clickhouse_node.query("CREATE DATABASE largetransaction ENGINE = MaterializedMySQL('{}:3306', 'largetransaction', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "SELECT COUNT() FROM largetransaction.test_table", f"{num_rows}\n") + + mysql_node.query("UPDATE largetransaction.test_table SET value = 2;") + + # Attempt to restart clickhouse after it has started processing + # the transaction, but before it has completed it. + while int(clickhouse_node.query("SELECT COUNT() FROM largetransaction.test_table WHERE value = 2")) == 0: + time.sleep(0.2) + clickhouse_node.restart_clickhouse() + + check_query(clickhouse_node, "SELECT COUNT() FROM largetransaction.test_table WHERE value = 2", f"{num_rows}\n") + + clickhouse_node.query("DROP DATABASE largetransaction") + mysql_node.query("DROP DATABASE largetransaction") diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 18cb5b3b87c..feade1b60a0 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -237,3 +237,8 @@ def test_materialize_with_enum(started_cluster, started_mysql_8_0, started_mysql def test_mysql_settings(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): materialize_with_ddl.mysql_settings_test(clickhouse_node, started_mysql_5_7, "mysql57") materialize_with_ddl.mysql_settings_test(clickhouse_node, started_mysql_8_0, "mysql80") + +@pytest.mark.parametrize(('clickhouse_node'), [pytest.param(node_db_ordinary, id="ordinary"), pytest.param(node_db_atomic, id="atomic")]) +def test_large_transaction(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): + materialize_with_ddl.materialized_mysql_large_transaction(clickhouse_node, started_mysql_8_0, "mysql80") + materialize_with_ddl.materialized_mysql_large_transaction(clickhouse_node, started_mysql_5_7, "mysql57") From 00fbf48a683ec009b26f0c47f931c7441013dbbe Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Thu, 7 Oct 2021 19:09:40 +0300 Subject: [PATCH 031/126] Minor fixes. --- docs/en/sql-reference/functions/geo/h3.md | 14 +++++++------- docs/ru/sql-reference/functions/geo/h3.md | 23 ++++++++++++----------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 410cb9d3cc2..048834806d1 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -40,7 +40,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3IsValid(630814730351855103) as h3IsValid; +SELECT h3IsValid(630814730351855103) AS h3IsValid; ``` Result: @@ -77,7 +77,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3GetResolution(639821929606596015) as resolution; +SELECT h3GetResolution(639821929606596015) AS resolution; ``` Result: @@ -111,7 +111,7 @@ h3EdgeAngle(resolution) Query: ``` sql -SELECT h3EdgeAngle(10) as edgeAngle; +SELECT h3EdgeAngle(10) AS edgeAngle; ``` Result: @@ -145,7 +145,7 @@ h3EdgeLengthM(resolution) Query: ``` sql -SELECT h3EdgeLengthM(15) as edgeLengthM; +SELECT h3EdgeLengthM(15) AS edgeLengthM; ``` Result: @@ -184,7 +184,7 @@ Type: [UInt64](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; +SELECT geoToH3(37.79506683, 55.71290588, 15) AS h3Index; ``` Result: @@ -333,7 +333,7 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3GetBaseCell(612916788725809151) as basecell; +SELECT h3GetBaseCell(612916788725809151) AS basecell; ``` Result: @@ -369,7 +369,7 @@ Type: [Float64](../../../sql-reference/data-types/float.md). Query: ``` sql -SELECT h3HexAreaM2(13) as area; +SELECT h3HexAreaM2(13) AS area; ``` Result: diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 6bc6943ec93..e85236848f6 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -38,7 +38,7 @@ h3IsValid(h3index) Запрос: ``` sql -SELECT h3IsValid(630814730351855103) as h3IsValid; +SELECT h3IsValid(630814730351855103) AS h3IsValid; ``` Результат: @@ -75,7 +75,7 @@ h3GetResolution(h3index) Запрос: ``` sql -SELECT h3GetResolution(639821929606596015) as resolution; +SELECT h3GetResolution(639821929606596015) AS resolution; ``` Результат: @@ -109,7 +109,7 @@ h3EdgeAngle(resolution) Запрос: ``` sql -SELECT h3EdgeAngle(10) as edgeAngle; +SELECT h3EdgeAngle(10) AS edgeAngle; ``` Результат: @@ -143,7 +143,7 @@ h3EdgeLengthM(resolution) Запрос: ``` sql -SELECT h3EdgeLengthM(15) as edgeLengthM; +SELECT h3EdgeLengthM(15) AS edgeLengthM; ``` Результат: @@ -182,7 +182,7 @@ geoToH3(lon, lat, resolution) Запрос: ``` sql -SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; +SELECT geoToH3(37.79506683, 55.71290588, 15) AS h3Index; ``` Результат: @@ -295,7 +295,7 @@ h3GetBaseCell(index) Запрос: ``` sql -SELECT h3GetBaseCell(612916788725809151) as basecell; +SELECT h3GetBaseCell(612916788725809151) AS basecell; ``` Результат: @@ -329,7 +329,7 @@ h3HexAreaM2(resolution) Запрос: ``` sql -SELECT h3HexAreaM2(13) as area; +SELECT h3HexAreaM2(13) AS area; ``` Результат: @@ -441,7 +441,7 @@ h3ToParent(index, resolution) Запрос: ``` sql -SELECT h3ToParent(599405990164561919, 3) as parent; +SELECT h3ToParent(599405990164561919, 3) AS parent; ``` Результат: @@ -475,7 +475,7 @@ h3ToString(index) Запрос: ``` sql -SELECT h3ToString(617420388352917503) as h3_string; +SELECT h3ToString(617420388352917503) AS h3_string; ``` Результат: @@ -512,7 +512,7 @@ stringToH3(index_str) Запрос: ``` sql -SELECT stringToH3('89184926cc3ffff') as index; +SELECT stringToH3('89184926cc3ffff') AS index; ``` Результат: @@ -667,5 +667,6 @@ SELECT h3GetFaces(599686042433355775) AS faces; ┌─faces─┐ │ [7] │ └───────┘ +``` -[Оригинальная статья](https://clickhouse.com/docs/ru/sql-reference/functions/geo/h3/) +[Оригинальная статья](https://clickhouse.com/docs/ru/sql-reference/functions/geo/h3) From 1df9afb47cf5204be24edbe0e8c8c631ea1e759f Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:27:22 +0300 Subject: [PATCH 032/126] Update docs/en/sql-reference/statements/alter/column.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/statements/alter/column.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 31874ef208d..ef4b88af6ba 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -204,7 +204,7 @@ Syntax: ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; ``` -**Example with the creation of new column:** +**Example** ```sql DROP TABLE IF EXISTS tmp; From fc0bccb0c6f6fec55800235fde76ef6669c5b5f9 Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:27:39 +0300 Subject: [PATCH 033/126] Update docs/ru/sql-reference/statements/alter/column.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/statements/alter/column.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 366caf6a2a0..5ab7207c580 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -196,7 +196,7 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -С помощью этого запроса можно сделать столбец таблицы материализованным (`MATERIALIZED`) в его частях, у которых отсутствуют значения. Это полезно, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`). Если вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо большим, для него может оказаться целесообразным использовать `MATERIALIZE COLUMN`. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`, как показано ниже. +Материализует столбец таблицы в кусках, в которых отсутствуют значения. Используется, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`), потому как вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`. Синтаксис: From 0eaf2f12a31e1ed0f9dff5bfcd2059123541603f Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:27:46 +0300 Subject: [PATCH 034/126] Update docs/ru/sql-reference/statements/alter/column.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/statements/alter/column.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 5ab7207c580..c6269f0eb62 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -204,7 +204,7 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; ``` -**Пример использования при создании нового столбца:** +**Пример** ```sql DROP TABLE IF EXISTS tmp; From c030756c38f75deaf0c3fd9e00c762e376d515c3 Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:28:02 +0300 Subject: [PATCH 035/126] Update docs/en/sql-reference/statements/alter/column.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/statements/alter/column.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index ef4b88af6ba..aee3823bc05 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -196,7 +196,7 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier as shown below. +Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier. Syntax: From 76e3ef686a244d13cbc37249ae260873ed36fae2 Mon Sep 17 00:00:00 2001 From: michon470 <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:28:12 +0300 Subject: [PATCH 036/126] Update docs/ru/sql-reference/statements/alter/column.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/statements/alter/column.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index c6269f0eb62..ef3d98fc10e 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -25,7 +25,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|RENAME|CLEAR|COMMENT|MODIFY| - [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; - [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL; - [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца; -- [MATERIALIZE COLUMN](#materialize-column) — делает столбец материализованным (MATERIALIZED) в его частях, у которых отсутствуют значения. +- [MATERIALIZE COLUMN](#materialize-column) — делает столбец материализованным (`MATERIALIZED`) в кусках, в которых отсутствуют значения. Подробное описание для каждого действия приведено ниже. From 9389cb7c7702574dcf6224ef0e7c4d83e7a30896 Mon Sep 17 00:00:00 2001 From: Mikhail <71978106+michon470@users.noreply.github.com> Date: Thu, 7 Oct 2021 22:31:44 +0300 Subject: [PATCH 037/126] Example corrected --- docs/en/sql-reference/statements/alter/column.md | 8 ++++---- docs/ru/sql-reference/statements/alter/column.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index aee3823bc05..4eb251b88cd 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -210,7 +210,7 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 20; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); SELECT groupArray(x), groupArray(s) FROM tmp; ``` @@ -218,9 +218,9 @@ SELECT groupArray(x), groupArray(s) FROM tmp; **Result:** ```sql -┌─groupArray(x)───────────────────────────────────────┬─groupArray(s)───────────────────────────────────────────────────────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] │ ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'] │ -└─────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────┘ +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ +└───────────────────────┴───────────────────────────────────────────┘ ``` **See Also** diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index ef3d98fc10e..bfd52801210 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -210,7 +210,7 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 20; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); SELECT groupArray(x), groupArray(s) FROM tmp; ``` @@ -218,9 +218,9 @@ SELECT groupArray(x), groupArray(s) FROM tmp; **Результат:** ```sql -┌─groupArray(x)───────────────────────────────────────┬─groupArray(s)───────────────────────────────────────────────────────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] │ ['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19'] │ -└─────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────┘ +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ +└───────────────────────┴───────────────────────────────────────────┘ ``` ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} From f854065744f57607e23a0de3edcca1b06f06c11a Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 8 Oct 2021 05:05:12 +0000 Subject: [PATCH 038/126] buildID() description --- docs/en/sql-reference/functions/other-functions.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index afcc9563b58..9828c91909b 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -699,6 +699,12 @@ If it is executed in the context of a distributed table, then it generates a nor Returns the version of the server as a string. If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. +## buildId() {#buildid} + +Returns the compiler build id of the running binary. +If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. + + ## blockNumber {#blocknumber} Returns the sequence number of the data block where the row is located. From d454a9affe73ee5844f19f8e85a4143c89c1d016 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 8 Oct 2021 10:07:55 +0300 Subject: [PATCH 039/126] Update docs/ru/getting-started/example-datasets/metrica.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/getting-started/example-datasets/metrica.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 27105ca8488..4d862bae423 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -35,7 +35,7 @@ $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` bash $ curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -$ # теперь создадим таблицу +# создадим таблицу hits_v1 $ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" $ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" # for hits_100m_obfuscated From 9d97a1263f4bd31c8520d9a54a35c8aa8877b982 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 8 Oct 2021 10:08:02 +0300 Subject: [PATCH 040/126] Update docs/ru/getting-started/example-datasets/metrica.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/getting-started/example-datasets/metrica.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 4d862bae423..765642ce3ae 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -38,7 +38,7 @@ $ curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads= # создадим таблицу hits_v1 $ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" $ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -# for hits_100m_obfuscated +# создадим таблицу hits_100m_obfuscated clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" $ # импортируем данные From 2c5341df33a3410db2aed4f57d4429d681064186 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 8 Oct 2021 10:08:25 +0300 Subject: [PATCH 041/126] Update docs/ru/sql-reference/functions/geo/h3.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index e85236848f6..db96f0caa1d 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -6,7 +6,7 @@ toc_title: "Функции для работы с индексами H3" [H3](https://eng.uber.com/h3/) — это система геокодирования, которая делит поверхность Земли на равные шестигранные ячейки. Система поддерживает иерархию (вложенность) ячеек, т.е. каждый "родительский" шестигранник может быть поделен на семь одинаковых вложенных "дочерних" шестигранников, и так далее. -Уровень вложенности называется `разрешением` и может принимать значение от `0` до `15`, где `0` соответствует `базовым` ячейкам самого верхнего уровня (наиболее крупным). +Уровень вложенности называется "разрешением" и может принимать значение от `0` до `15`, где `0` соответствует "базовым" ячейкам самого верхнего уровня (наиболее крупным). Для каждой точки, имеющей широту и долготу, можно получить 64-битный индекс H3, соответствующий номеру шестигранной ячейки, где эта точка находится. From 0fdbf867a45689308f0072a005a70b83757d8cc0 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 8 Oct 2021 10:08:31 +0300 Subject: [PATCH 042/126] Update docs/ru/getting-started/example-datasets/metrica.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/getting-started/example-datasets/metrica.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 765642ce3ae..761e298fc54 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -41,9 +41,9 @@ $ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, Ja # создадим таблицу hits_100m_obfuscated clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -$ # импортируем данные +# импортируем данные $ cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 -$ # опционально можно оптимизировать таблицу +# опционально можно оптимизировать таблицу $ clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` From d11cae26178fd114f69c763de4dbd18190175486 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Fri, 8 Oct 2021 10:26:02 +0300 Subject: [PATCH 043/126] Remove `$ #` from metrica.md --- docs/ru/getting-started/example-datasets/metrica.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 761e298fc54..0f5e7197fe5 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -14,7 +14,7 @@ toc_title: "Анонимизированные данные Яндекс.Мет ``` bash $ curl -O https://datasets.clickhouse.com/hits/partitions/hits_v1.tar $ tar xvf hits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse -$ # убедитесь, что установлены корректные права доступа на файлы +# убедитесь, что установлены корректные права доступа на файлы $ sudo service clickhouse-server restart $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` @@ -24,7 +24,7 @@ $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` bash $ curl -O https://datasets.clickhouse.com/visits/partitions/visits_v1.tar $ tar xvf visits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse -$ # убедитесь, что установлены корректные права доступа на файлы +# убедитесь, что установлены корректные права доступа на файлы $ sudo service clickhouse-server restart $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` @@ -52,12 +52,12 @@ $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` bash $ curl https://datasets.clickhouse.com/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -$ # теперь создадим таблицу +# теперь создадим таблицу $ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" $ clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" -$ # импортируем данные +# импортируем данные $ cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 -$ # опционально можно оптимизировать таблицу +# опционально можно оптимизировать таблицу $ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` From a4d1ad61d0e220b6c215eaffd4cf3531e4c01423 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Fri, 8 Oct 2021 12:09:12 +0300 Subject: [PATCH 044/126] Remove $ symbols in bash commands. --- .../example-datasets/metrica.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 0f5e7197fe5..15c0cc14517 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -12,21 +12,21 @@ toc_title: "Анонимизированные данные Яндекс.Мет **Скачивание и импортирование партиций hits:** ``` bash -$ curl -O https://datasets.clickhouse.com/hits/partitions/hits_v1.tar -$ tar xvf hits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse +curl -O https://datasets.clickhouse.com/hits/partitions/hits_v1.tar +tar xvf hits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse # убедитесь, что установлены корректные права доступа на файлы -$ sudo service clickhouse-server restart -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` **Скачивание и импортирование партиций visits:** ``` bash -$ curl -O https://datasets.clickhouse.com/visits/partitions/visits_v1.tar -$ tar xvf visits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse +curl -O https://datasets.clickhouse.com/visits/partitions/visits_v1.tar +tar xvf visits_v1.tar -C /var/lib/clickhouse # путь к папке с данными ClickHouse # убедитесь, что установлены корректные права доступа на файлы -$ sudo service clickhouse-server restart -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +sudo service clickhouse-server restart +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` ## Получение таблиц из сжатых tsv-файлов {#poluchenie-tablits-iz-szhatykh-tsv-failov} @@ -34,32 +34,32 @@ $ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" **Скачивание и импортирование hits из сжатого tsv-файла** ``` bash -$ curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv # создадим таблицу hits_v1 -$ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -$ clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" # создадим таблицу hits_100m_obfuscated clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" # импортируем данные -$ cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 +cat hits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.hits_v1 FORMAT TSV" --max_insert_block_size=100000 # опционально можно оптимизировать таблицу -$ clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" +clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` **Скачивание и импортирование visits из сжатого tsv-файла** ``` bash -$ curl https://datasets.clickhouse.com/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +curl https://datasets.clickhouse.com/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv # теперь создадим таблицу -$ clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -$ clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" +clickhouse-client --query "CREATE TABLE datasets.visits_v1 ( CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" # импортируем данные -$ cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 +cat visits_v1.tsv | clickhouse-client --query "INSERT INTO datasets.visits_v1 FORMAT TSV" --max_insert_block_size=100000 # опционально можно оптимизировать таблицу -$ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" -$ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" +clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" +clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` ## Запросы {#zaprosy} From 5cc379392579978f5f398497c8a494508c2e40b3 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 9 Oct 2021 11:50:06 +0800 Subject: [PATCH 045/126] Add shutdown_wait_unfinished_queries setting --- programs/server/Server.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index cd5d72cfba4..79a41078a77 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1550,7 +1550,8 @@ if (ThreadFuzzer::instance().isEffective()) LOG_INFO(log, "Closed all listening sockets."); /// Killing remaining queries. - global_context->getProcessList().killAllQueries(); + if (!config().getBool("shutdown_wait_unfinished_queries", false)) + global_context->getProcessList().killAllQueries(); if (current_connections) current_connections = waitServersToFinish(*servers, config().getInt("shutdown_wait_unfinished", 5)); From fecfcf9b66afa96a3107c5b2debe420c5c894161 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Sat, 9 Oct 2021 13:17:47 +0300 Subject: [PATCH 046/126] Update metrica.md --- docs/ru/getting-started/example-datasets/metrica.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 15c0cc14517..ee764ff4879 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -37,7 +37,7 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" curl https://datasets.clickhouse.com/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv # создадим таблицу hits_v1 clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" -clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" +clickhouse-client --query "CREATE TABLE datasets.hits_v1 ( WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" # создадим таблицу hits_100m_obfuscated clickhouse-client --query="CREATE TABLE hits_100m_obfuscated (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, Refresh UInt8, RefererCategoryID UInt16, RefererRegionID UInt32, URLCategoryID UInt16, URLRegionID UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, OriginalURL String, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), LocalEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, RemoteIP UInt32, WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming UInt32, DNSTiming UInt32, ConnectTiming UInt32, ResponseStartTiming UInt32, ResponseEndTiming UInt32, FetchTiming UInt32, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192" From 7f5852a7114e5e4da08f363ec81dcbfe4079d2f0 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 9 Oct 2021 18:37:28 +0000 Subject: [PATCH 047/126] New buildId variant Links from Distributed --- docs/en/engines/table-engines/special/distributed.md | 3 ++- docs/en/sql-reference/functions/other-functions.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index 368849359ef..9503944a7a8 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -197,5 +197,6 @@ When the `max_parallel_replicas` option is enabled, query processing is parallel - [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns) - [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) +- [shardNum()](../../../sql-reference/functions/other-functions.md#shard-num) and [shardCount()](../../../sql-reference/functions/other-functions.md#shard-count) functions + -[Original article](https://clickhouse.com/docs/en/operations/table_engines/distributed/) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 9828c91909b..6864ba7705b 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -701,7 +701,7 @@ If it is executed in the context of a distributed table, then it generates a nor ## buildId() {#buildid} -Returns the compiler build id of the running binary. +Returns the build ID generated by a compiler for the running ClickHouse server binary. If it is executed in the context of a distributed table, then it generates a normal column with values relevant to each shard. Otherwise it produces a constant value. From 2b272f5781ac22659dc8da6e4f657a359a74dac6 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 9 Oct 2021 19:17:02 +0000 Subject: [PATCH 048/126] Virtual column in Distributed updated, link fixed, links added Translated that part --- docs/en/engines/table-engines/special/distributed.md | 8 ++++---- docs/ru/engines/table-engines/special/distributed.md | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index 9503944a7a8..6593a5dc17f 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -188,15 +188,15 @@ When the `max_parallel_replicas` option is enabled, query processing is parallel ## Virtual Columns {#virtual-columns} -- `_shard_num` — Contains the `shard_num` (from `system.clusters`). Type: [UInt32](../../../sql-reference/data-types/int-uint.md). +- `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). !!! note "Note" - Since [`remote`](../../../sql-reference/table-functions/remote.md)/`cluster` table functions internally create temporary instance of the same Distributed engine, `_shard_num` is available there too. + Since [remote](../../../sql-reference/table-functions/remote.md) and [cluster](../../../sql-reference/table-functions/cluster.md) table functions internally create temporary Distributed table, `_shard_num` is available there too. **See Also** -- [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns) -- [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) +- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) description +- [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting - [shardNum()](../../../sql-reference/functions/other-functions.md#shard-num) and [shardCount()](../../../sql-reference/functions/other-functions.md#shard-count) functions diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index b1f6f56623d..ff1dc7c4057 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -136,3 +136,15 @@ logs - имя кластера в конфигурационном файле с При выставлении опции max_parallel_replicas выполнение запроса распараллеливается по всем репликам внутри одного шарда. Подробнее смотрите раздел [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +## Виртуальные столбцы {#virtual-columns} + +- `_shard_num` — содержит значение `shard_num` из таблицы `system.clusters`. Тип: [UInt32](../../../sql-reference/data-types/int-uint.md). + +!!! note "Примечание" + Так как табличные функции [remote](../../../sql-reference/table-functions/remote.md) и [cluster](../../../sql-reference/table-functions/cluster.md) создают временную таблицу на движке `Distributed`, то в ней также доступен столбец `_shard_num`. + +**Смотрите также** + +- общее описание [виртуальных столбцов](../../../engines/table-engines/index.md#table_engines-virtual_columns) +- настройка [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) +- функции [shardNum()](../../../sql-reference/functions/other-functions.md#shard-num) и [shardCount()](../../../sql-reference/functions/other-functions.md#shard-count) From 17552931af3806c528c34a3788b53d572f6726e0 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 9 Oct 2021 19:27:24 +0000 Subject: [PATCH 049/126] ru other functuins addons --- docs/ru/sql-reference/functions/date-time-functions.md | 1 + docs/ru/sql-reference/functions/other-functions.md | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 282962b9e3f..d4777faf354 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -26,6 +26,7 @@ SELECT ## timeZone {#timezone} Возвращает часовой пояс сервера. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. **Синтаксис** diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index d3e0f8b946e..31e81b04330 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -8,6 +8,7 @@ toc_title: "Прочие функции" ## hostName() {#hostname} Возвращает строку - имя хоста, на котором эта функция была выполнена. При распределённой обработке запроса, это будет имя хоста удалённого сервера, если функция выполняется на удалённом сервере. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. ## getMacro {#getmacro} @@ -643,10 +644,17 @@ SELECT ## uptime() {#uptime} Возвращает аптайм сервера в секундах. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. ## version() {#version} Возвращает версию сервера в виде строки. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. + +## buildId() {#buildid} + +Возвращает ID сборки, сгенерированный компилятором для запущенного сервера ClickHouse. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. ## rowNumberInBlock {#function-rownumberinblock} @@ -2304,3 +2312,4 @@ SELECT count(DISTINCT t) FROM (SELECT initialQueryID() AS t FROM remote('127.0.0 │ 1 │ └─────────┘ ``` + From 2bab572caf91d2fc8962f409fea34149ecec8c95 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 10 Oct 2021 00:46:28 +0300 Subject: [PATCH 050/126] Fix lock-order-inversion between periodic dictionary reload and config reload Integration tests found [1], TSan report: WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=11) Cycle in lock order graph: M3152 (0x7b9000000058) => M3153 (0x7b9000000438) => M3152 Mutex M3153 acquired here while holding mutex M3152 in main thread: 3 std::__1::lock_guard::lock_guard(std::__1::recursive_mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse+0x15716b81) 4 DB::ExternalLoader::addConfigRepository() const obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:1283:21 (clickhouse+0x15716b81) 5 DB::Context::loadOrReloadDictionaries() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1453:62 (clickhouse+0x155d7bad) 6 DB::Server::main()::$_1::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:852:29 (clickhouse+0x9b1944c) 13 DB::ConfigReloader::reloadIfNewer() obj-x86_64-linux-gnu/../src/Common/Config/ConfigReloader.cpp:137:13 (clickhouse+0x17045e2e) 14 DB::ConfigReloader::ConfigReloader() obj-x86_64-linux-gnu/../src/Common/Config/ConfigReloader.cpp:33:9 (clickhouse+0x17044e51) 16 DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:803:33 (clickhouse+0x9b0c41d) 17 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse+0x19ffc08b) 18 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:405:25 (clickhouse+0x9b03ebe) 19 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse+0x1a01c246) 20 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:183:20 (clickhouse+0x9b02576) 21 main obj-x86_64-linux-gnu/../programs/main.cpp:372:12 (clickhouse+0x9b00a3a) Mutex M3152 acquired here while holding mutex M3153 in thread T2: 3 std::__1::lock_guard::lock_guard() obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse+0x155d63b8) 4 DB::Context::getExternalDictionariesLoader() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1337:21 (clickhouse+0x155d63b8) 5 DB::Context::getExternalDictionariesLoader() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1332:41 (clickhouse+0x155d6359) 6 DB::DatabaseDictionary::tryGetTable() const obj-x86_64-inux-gnu/../src/Databases/DatabaseDictionary.cpp:76:38 (clickhouse+0x157819ad) 7 DB::DatabaseCatalog::getTableImpl() const obj-x86_64-linux-gnu/../src/Interpreters/DatabaseCatalog.cpp:285:28 (clickhouse+0x1564a1fa) 8 DB::DatabaseCatalog::getTable() const obj-x86_64-linux-gnu/../src/Interpreters/DatabaseCatalog.cpp:656:16 (clickhouse+0x1564fa2a) 9 DB::JoinedTables::getLeftTableStorage() obj-x86_64-linux-gnu/../src/Interpreters/JoinedTables.cpp:219:40 (clickhouse+0x15eeef45) 10 DB::InterpreterSelectQuery::InterpreterSelectQuery() obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:321:33 (clickhouse+0x15b792be) 19 DB::ClickHouseDictionarySource::doInvalidateQuery() const obj-x86_64-linux-gnu/../src/Dictionaries/ClickHouseDictionarySource.cpp:207:36 (clickhouse+0x12872d2d) 20 DB::ClickHouseDictionarySource::isModified() const obj-x86_64-linux-gnu/../src/Dictionaries/ClickHouseDictionarySource.cpp:144:25 (clickhouse+0x12872534) 21 DB::IDictionary::isModified() const (clickhouse+0x128ce39b) 22 DB::ExternalLoader::LoadingDispatcher::reloadOutdated() obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:660:50 (clickhouse+0x157305f7) 23 DB::ExternalLoader::PeriodicUpdater::doPeriodicUpdates() obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:1248:36 (clickhouse+0x1572fff7) [1]: https://clickhouse-test-reports.s3.yandex.net/29856/42ca2b4bb241827edf69bbd6938d6b19c31935f1/integration_tests_(thread).html#fail1 --- src/Interpreters/ExternalLoader.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index e4c8e46980d..dc8466f3c26 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1243,8 +1243,10 @@ private: { lock.unlock(); { - std::lock_guard config_lock{config_mutex}; - loading_dispatcher.setConfiguration(config_files_reader.read()); + { + std::lock_guard config_lock{config_mutex}; + loading_dispatcher.setConfiguration(config_files_reader.read()); + } loading_dispatcher.reloadOutdated(); } lock.lock(); From 4e6ed5c45c85166dcf7c98d4adaf02873b695183 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Oct 2021 04:10:52 +0300 Subject: [PATCH 051/126] Remove trash from SentryWriter --- base/daemon/SentryWriter.cpp | 40 ++---------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index ad914ff8cf4..efd915b1e5b 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -64,41 +64,6 @@ void setExtras() sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(fs::space(server_data_path).free).c_str())); } -void sentry_logger(sentry_level_e level, const char * message, va_list args, void *) -{ - auto * logger = &Poco::Logger::get("SentryWriter"); - size_t size = 1024; - char buffer[size]; -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" -#endif - if (vsnprintf(buffer, size, message, args) >= 0) - { -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - switch (level) - { - case SENTRY_LEVEL_DEBUG: - logger->debug(buffer); - break; - case SENTRY_LEVEL_INFO: - logger->information(buffer); - break; - case SENTRY_LEVEL_WARNING: - logger->warning(buffer); - break; - case SENTRY_LEVEL_ERROR: - logger->error(buffer); - break; - case SENTRY_LEVEL_FATAL: - logger->fatal(buffer); - break; - } - } -} - } @@ -107,13 +72,13 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) bool enabled = false; bool debug = config.getBool("send_crash_reports.debug", false); auto * logger = &Poco::Logger::get("SentryWriter"); + if (config.getBool("send_crash_reports.enabled", false)) { if (debug || (strlen(VERSION_OFFICIAL) > 0)) //-V560 - { enabled = true; - } } + if (enabled) { server_data_path = config.getString("path", ""); @@ -126,7 +91,6 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config) sentry_options_t * options = sentry_options_new(); /// will be freed by sentry_init or sentry_shutdown sentry_options_set_release(options, VERSION_STRING_SHORT); - sentry_options_set_logger(options, &sentry_logger, nullptr); if (debug) { sentry_options_set_debug(options, 1); From 2a20bf4909e38fae7acc1fd97646af10ba150696 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Oct 2021 04:16:32 +0300 Subject: [PATCH 052/126] Remove function bayesAB #26233 --- src/Functions/abtesting.cpp | 315 ------------------ src/Functions/abtesting.h | 35 -- src/Functions/registerFunctions.cpp | 7 - .../01411_bayesian_ab_testing.reference | 4 - .../0_stateless/01411_bayesian_ab_testing.sql | 6 - 5 files changed, 367 deletions(-) delete mode 100644 src/Functions/abtesting.cpp delete mode 100644 src/Functions/abtesting.h delete mode 100644 tests/queries/0_stateless/01411_bayesian_ab_testing.reference delete mode 100644 tests/queries/0_stateless/01411_bayesian_ab_testing.sql diff --git a/src/Functions/abtesting.cpp b/src/Functions/abtesting.cpp deleted file mode 100644 index 312fdf6fb48..00000000000 --- a/src/Functions/abtesting.cpp +++ /dev/null @@ -1,315 +0,0 @@ -#include - -#if !defined(ARCADIA_BUILD) && USE_STATS - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#define STATS_ENABLE_STDVEC_WRAPPERS -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int BAD_ARGUMENTS; -} - -static const String BETA = "beta"; -static const String GAMMA = "gamma"; - -template -Variants bayesian_ab_test(String distribution, PODArray & xs, PODArray & ys) -{ - const size_t r = 1000, c = 100; - - Variants variants(xs.size(), {0.0, 0.0, 0.0, 0.0}); - std::vector> samples_matrix; - - for (size_t i = 0; i < xs.size(); ++i) - { - variants[i].x = xs[i]; - variants[i].y = ys[i]; - } - - if (distribution == BETA) - { - Float64 alpha, beta; - - for (size_t i = 0; i < xs.size(); ++i) - if (xs[i] < ys[i]) - throw Exception("Conversions cannot be larger than trials", ErrorCodes::BAD_ARGUMENTS); - - for (size_t i = 0; i < xs.size(); ++i) - { - alpha = 1.0 + ys[i]; - beta = 1.0 + xs[i] - ys[i]; - - samples_matrix.emplace_back(stats::rbeta>(r, c, alpha, beta)); - } - } - else if (distribution == GAMMA) - { - Float64 shape, scale; - - for (size_t i = 0; i < xs.size(); ++i) - { - shape = 1.0 + xs[i]; - scale = 250.0 / (1 + 250.0 * ys[i]); - - std::vector samples = stats::rgamma>(r, c, shape, scale); - for (auto & sample : samples) - sample = 1 / sample; - samples_matrix.emplace_back(std::move(samples)); - } - } - - PODArray means; - for (auto & samples : samples_matrix) - { - Float64 total = 0.0; - for (auto sample : samples) - total += sample; - means.push_back(total / samples.size()); - } - - // Beats control - for (size_t i = 1; i < xs.size(); ++i) - { - for (size_t n = 0; n < r * c; ++n) - { - if (higher_is_better) - { - if (samples_matrix[i][n] > samples_matrix[0][n]) - ++variants[i].beats_control; - } - else - { - if (samples_matrix[i][n] < samples_matrix[0][n]) - ++variants[i].beats_control; - } - } - } - - for (auto & variant : variants) - variant.beats_control = static_cast(variant.beats_control) / r / c; - - // To be best - PODArray count_m(xs.size(), 0); - PODArray row(xs.size(), 0); - - for (size_t n = 0; n < r * c; ++n) - { - for (size_t i = 0; i < xs.size(); ++i) - row[i] = samples_matrix[i][n]; - - Float64 m; - if (higher_is_better) - m = *std::max_element(row.begin(), row.end()); - else - m = *std::min_element(row.begin(), row.end()); - - for (size_t i = 0; i < xs.size(); ++i) - { - if (m == samples_matrix[i][n]) - { - ++variants[i].best; - break; - } - } - } - - for (auto & variant : variants) - variant.best = static_cast(variant.best) / r / c; - - return variants; -} - -String convertToJson(const PODArray & variant_names, const Variants & variants) -{ - FormatSettings settings; - - WriteBufferFromOwnString buf; - - writeCString("{\"data\":[", buf); - for (size_t i = 0; i < variants.size(); ++i) - { - writeCString("{\"variant_name\":", buf); - writeJSONString(variant_names[i], buf, settings); - writeCString(",\"x\":", buf); - writeText(variants[i].x, buf); - writeCString(",\"y\":", buf); - writeText(variants[i].y, buf); - writeCString(",\"beats_control\":", buf); - writeText(variants[i].beats_control, buf); - writeCString(",\"to_be_best\":", buf); - writeText(variants[i].best, buf); - writeCString("}", buf); - if (i != variant_names.size() -1) - writeCString(",", buf); - } - writeCString("]}", buf); - - return buf.str(); -} - -class FunctionBayesAB : public IFunction -{ -public: - static constexpr auto name = "bayesAB"; - - static FunctionPtr create(ContextPtr) - { - return std::make_shared(); - } - - String getName() const override - { - return name; - } - - bool isDeterministic() const override { return false; } - bool isDeterministicInScopeOfQuery() const override { return false; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - size_t getNumberOfArguments() const override { return 5; } - - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } - - static bool toFloat64(const ColumnConst * col_const_arr, PODArray & output) - { - Array src_arr = col_const_arr->getValue(); - - for (size_t i = 0, size = src_arr.size(); i < size; ++i) - { - switch (src_arr[i].getType()) - { - case Field::Types::Int64: - output.push_back(static_cast(src_arr[i].get())); - break; - case Field::Types::UInt64: - output.push_back(static_cast(src_arr[i].get())); - break; - case Field::Types::Float64: - output.push_back(src_arr[i].get()); - break; - default: - return false; - } - } - - return true; - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - if (input_rows_count == 0) - return ColumnString::create(); - - PODArray xs, ys; - PODArray variant_names; - String dist; - bool higher_is_better; - - if (const ColumnConst * col_dist = checkAndGetColumnConst(arguments[0].column.get())) - { - dist = col_dist->getDataAt(0).data; - dist = Poco::toLower(dist); - if (dist != BETA && dist != GAMMA) - throw Exception("First argument for function " + getName() + " cannot be " + dist, ErrorCodes::BAD_ARGUMENTS); - } - else - throw Exception("First argument for function " + getName() + " must be Constant string", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (const ColumnConst * col_higher_is_better = checkAndGetColumnConst(arguments[1].column.get())) - higher_is_better = col_higher_is_better->getBool(0); - else - throw Exception("Second argument for function " + getName() + " must be Constant boolean", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (const ColumnConst * col_const_arr = checkAndGetColumnConst(arguments[2].column.get())) - { - Array src_arr = col_const_arr->getValue(); - - for (size_t i = 0; i < src_arr.size(); ++i) - { - if (src_arr[i].getType() != Field::Types::String) - throw Exception("Third argument for function " + getName() + " must be Array of constant strings", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - variant_names.push_back(src_arr[i].get()); - } - } - else - throw Exception("Third argument for function " + getName() + " must be Array of constant strings", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (const ColumnConst * col_const_arr = checkAndGetColumnConst(arguments[3].column.get())) - { - if (!toFloat64(col_const_arr, xs)) - throw Exception("Forth and fifth Argument for function " + getName() + " must be Array of constant Numbers", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception("Forth argument for function " + getName() + " must be Array of constant numbers", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (const ColumnConst * col_const_arr = checkAndGetColumnConst(arguments[4].column.get())) - { - if (!toFloat64(col_const_arr, ys)) - throw Exception("Fifth Argument for function " + getName() + " must be Array of constant Numbers", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - else - throw Exception("Fifth argument for function " + getName() + " must be Array of constant numbers", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (variant_names.size() != xs.size() || xs.size() != ys.size()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Sizes of arguments doesn't match: variant_names: {}, xs: {}, ys: {}", variant_names.size(), xs.size(), ys.size()); - - if (variant_names.size() < 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Sizes of arguments must be larger than 1. variant_names: {}, xs: {}, ys: {}", variant_names.size(), xs.size(), ys.size()); - - if (std::count_if(xs.begin(), xs.end(), [](Float64 v) { return v < 0; }) > 0 || - std::count_if(ys.begin(), ys.end(), [](Float64 v) { return v < 0; }) > 0) - throw Exception("Negative values don't allowed", ErrorCodes::BAD_ARGUMENTS); - - Variants variants; - if (higher_is_better) - variants = bayesian_ab_test(dist, xs, ys); - else - variants = bayesian_ab_test(dist, xs, ys); - - auto dst = ColumnString::create(); - std::string result_str = convertToJson(variant_names, variants); - dst->insertData(result_str.c_str(), result_str.length()); - return dst; - } -}; - -void registerFunctionBayesAB(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} - -#else - -namespace DB -{ - -class FunctionFactory; - -void registerFunctionBayesAB(FunctionFactory & /* factory */) -{ -} - -} - -#endif diff --git a/src/Functions/abtesting.h b/src/Functions/abtesting.h deleted file mode 100644 index b1f12e79437..00000000000 --- a/src/Functions/abtesting.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include - -#if !defined(ARCADIA_BUILD) && USE_STATS - -# include -# include - -# include -# include -# include - - -namespace DB -{ - -struct Variant -{ - Float64 x; - Float64 y; - Float64 beats_control; - Float64 best; -}; - -using Variants = PODArray; - -template -Variants bayesian_ab_test(String distribution, PODArray & xs, PODArray & ys); - -String convertToJson(const PODArray & variant_names, const Variants & variants); - -} - -#endif diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 35193e9be8d..b2f038240aa 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -54,9 +54,6 @@ void registerFunctionTupleHammingDistance(FunctionFactory & factory); void registerFunctionsStringHash(FunctionFactory & factory); void registerFunctionValidateNestedArraySizes(FunctionFactory & factory); void registerFunctionsSnowflake(FunctionFactory & factory); -#if !defined(ARCADIA_BUILD) -void registerFunctionBayesAB(FunctionFactory &); -#endif void registerFunctionTid(FunctionFactory & factory); void registerFunctionLogTrace(FunctionFactory & factory); @@ -122,10 +119,6 @@ void registerFunctions() registerFunctionValidateNestedArraySizes(factory); registerFunctionsSnowflake(factory); -#if !defined(ARCADIA_BUILD) - registerFunctionBayesAB(factory); -#endif - #if USE_SSL registerFunctionEncrypt(factory); registerFunctionDecrypt(factory); diff --git a/tests/queries/0_stateless/01411_bayesian_ab_testing.reference b/tests/queries/0_stateless/01411_bayesian_ab_testing.reference deleted file mode 100644 index 98fb6a68656..00000000000 --- a/tests/queries/0_stateless/01411_bayesian_ab_testing.reference +++ /dev/null @@ -1,4 +0,0 @@ -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/01411_bayesian_ab_testing.sql b/tests/queries/0_stateless/01411_bayesian_ab_testing.sql deleted file mode 100644 index a4b03d76c51..00000000000 --- a/tests/queries/0_stateless/01411_bayesian_ab_testing.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Tags: no-fasttest - -SELECT count() FROM (SELECT bayesAB('beta', 1, ['Control', 'A', 'B'], [3000.0, 3000.0, 2000.0], [1000.0, 1100.0, 800.0])); -SELECT count() FROM (SELECT bayesAB('gamma', 1, ['Control', 'A', 'B'], [3000.0, 3000.0, 2000.0], [1000.0, 1100.0, 800.0])); -SELECT count() FROM (SELECT bayesAB('beta', 0, ['Control', 'A', 'B'], [3000.0, 3000.0, 2000.0], [1000.0, 1100.0, 800.0])); -SELECT count() FROM (SELECT bayesAB('gamma', 0, ['Control', 'A', 'B'], [3000.0, 3000.0, 2000.0], [1000.0, 1100.0, 800.0])); From 1dda59668918e894b773a6595b9738ab326a1b99 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 10 Oct 2021 17:32:01 +0300 Subject: [PATCH 053/126] Fix releasing query ID and session ID at the end of query processing. --- src/Interpreters/Session.cpp | 9 +++++++++ src/Interpreters/Session.h | 3 +++ src/Server/GRPCServer.cpp | 17 ++++++++++++++++- tests/integration/test_grpc_protocol/test.py | 2 +- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 128df040c7a..5f1b43fe1e8 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -481,5 +481,14 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t return query_context; } + +void Session::releaseSessionID() +{ + if (!named_session) + return; + named_session->release(); + named_session = nullptr; +} + } diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 772ccba7766..ab269bb619c 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -68,6 +68,9 @@ public: ContextMutablePtr makeQueryContext(const ClientInfo & query_client_info) const; ContextMutablePtr makeQueryContext(ClientInfo && query_client_info) const; + /// Releases the currently used session ID so it becomes available for reuse by another session. + void releaseSessionID(); + private: std::shared_ptr getSessionLog() const; ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const; diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 0fb9d82aca6..7aa1ca06990 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -584,6 +584,7 @@ namespace void finishQuery(); void onException(const Exception & exception); void onFatalError(); + void releaseQueryIDAndSessionID(); void close(); void readQueryInfo(); @@ -1175,6 +1176,7 @@ namespace addProgressToResult(); query_scope->logPeakMemoryUsage(); addLogsToResult(); + releaseQueryIDAndSessionID(); sendResult(); close(); @@ -1205,6 +1207,8 @@ namespace LOG_WARNING(log, "Couldn't send logs to client"); } + releaseQueryIDAndSessionID(); + try { sendException(exception); @@ -1224,7 +1228,7 @@ namespace { try { - finalize = true; + result.mutable_exception()->set_name("FatalError"); addLogsToResult(); sendResult(); } @@ -1234,6 +1238,17 @@ namespace } } + void Call::releaseQueryIDAndSessionID() + { + /// releaseQueryIDAndSessionID() should be called before sending the final result to the client + /// because the client may decide to send another query with the same query ID or session ID + /// immediately after it receives our final result, and it's prohibited to have + /// two queries executed at the same time with the same query ID or session ID. + io.process_list_entry.reset(); + if (session) + session->releaseSessionID(); + } + void Call::close() { responder.reset(); diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 79879c13c9d..7b2cdee8d76 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -211,7 +211,7 @@ def test_errors_handling(): assert "Table default.t already exists" in e.display_text def test_authentication(): - query("CREATE USER john IDENTIFIED BY 'qwe123'") + query("CREATE USER OR REPLACE john IDENTIFIED BY 'qwe123'") assert query("SELECT currentUser()", user_name="john", password="qwe123") == "john\n" def test_logs(): From af4066c255149d1ab1cef9b7fba0c9c47d1bb265 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Oct 2021 21:54:15 +0300 Subject: [PATCH 054/126] Remove printf --- base/daemon/SentryWriter.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index efd915b1e5b..ac771b9bf47 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" @@ -163,34 +164,34 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta if (stack_size > 0) { ssize_t offset = stack_trace.getOffset(); - char instruction_addr[100]; + + char instruction_addr[19] + { + '0', 'x', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', + '\0' + }; + StackTrace::Frames frames; StackTrace::symbolize(stack_trace.getFramePointers(), offset, stack_size, frames); + for (ssize_t i = stack_size - 1; i >= offset; --i) { const StackTrace::Frame & current_frame = frames[i]; sentry_value_t sentry_frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); - if (std::snprintf(instruction_addr, sizeof(instruction_addr), "0x%" PRIx64, frame_ptr) >= 0) - { - sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); - } + writeHexUIntLowercase(frame_ptr, instruction_addr + 2); + sentry_value_set_by_key(sentry_frame, "instruction_addr", sentry_value_new_string(instruction_addr)); if (current_frame.symbol.has_value()) - { sentry_value_set_by_key(sentry_frame, "function", sentry_value_new_string(current_frame.symbol.value().c_str())); - } if (current_frame.file.has_value()) - { sentry_value_set_by_key(sentry_frame, "filename", sentry_value_new_string(current_frame.file.value().c_str())); - } if (current_frame.line.has_value()) - { sentry_value_set_by_key(sentry_frame, "lineno", sentry_value_new_int32(current_frame.line.value())); - } sentry_value_append(sentry_frames, sentry_frame); } From b83655b348f93cc4f720f61921d57d59f19f0dc8 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sun, 10 Oct 2021 19:38:53 +0000 Subject: [PATCH 055/126] ru translation --- .../functions/other-functions.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 31e81b04330..c91d5caef8d 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2313,3 +2313,65 @@ SELECT count(DISTINCT t) FROM (SELECT initialQueryID() AS t FROM remote('127.0.0 └─────────┘ ``` +## shardNum {#shard-num} + +Возвращает индекс шарда, который обрабатывает часть данных распределенного запроса. Индексы начинаются с `1`. +Если запрос не распределенный, то возвращается константное значение `0`. + +**Синтаксис** + +``` sql +shardNum() +``` + +**Возвращаемое значение** + +- индекс шарда или константа `0`. + +Тип: [UInt32](../../sql-reference/data-types/int-uint.md). + +**Пример** + +В примере ниже используется конфигурация с двумя шардами. На каждом шарде выполняется запрос к таблице [system.one](../../operations/system-tables/one.md). + +Запрос: + +``` sql +CREATE TABLE shard_num_example (dummy UInt8) + ENGINE=Distributed(test_cluster_two_shards_localhost, system, one, dummy); +SELECT dummy, shardNum(), shardCount() FROM shard_num_example; +``` + +Результат: + +``` text +┌─dummy─┬─shardNum()─┬─shardCount()─┐ +│ 0 │ 2 │ 2 │ +│ 0 │ 1 │ 2 │ +└───────┴────────────┴──────────────┘ +``` + +**См. также** + +- [Distributed Table Engine](../../engines/table-engines/special/distributed.md) + +## shardCount {#shard-count} + +Возвращает общее количество шардов для распределенного запроса. +Если запрос не распределенный, то возвращается константное значение `0`. + +**Синтаксис** + +``` sql +shardCount() +``` + +**Возвращаемое значение** + +- Общее количество шардов или `0`. + +Тип: [UInt32](../../sql-reference/data-types/int-uint.md). + +**См. также** + +- Пример использования функции [shardNum()](#shard-num) также содержит вызов `shardCount()`. From cb8a66e5155b0e4c1be0a037db8c54f48d47372c Mon Sep 17 00:00:00 2001 From: olgarev Date: Sun, 10 Oct 2021 23:34:26 +0000 Subject: [PATCH 056/126] Settings and links --- docs/en/operations/settings/settings.md | 38 ++++++++++++++++++- .../statements/select/prewhere.md | 10 +++-- docs/ru/operations/settings/settings.md | 38 ++++++++++++++++++- .../statements/select/prewhere.md | 12 +++--- 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f78fbc8a2bc..0491674b701 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3783,4 +3783,40 @@ Result: │ 20 │ 20 │ 10 │ │ 10 │ 20 │ 30 │ └─────┴─────┴───────┘ -``` \ No newline at end of file +``` + +## optimize_move_to_prewhere {#optimize_move_to_prewhere} + +Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries. + +Works only for [*MergeTree](../../engines/table-engines/mergetree-family/index.md) tables. + +Possible values: + +- 0 — Automatic `PREWHERE` optimization is disabled. +- 1 — Automatic `PREWHERE` optimization is enabled. + +Default value: `1`. + +**See Also** + +- [PREWHERE](../../sql-reference/statements/select/prewhere.md) clause in `SELECT` queries + +## optimize_move_to_prewhere_if_final {#optimize_move_to_prewhere_if_final} + +Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries with [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier. + +Works only for [*MergeTree](../../engines/table-engines/mergetree-family/index.md) tables. + +Possible values: + +- 0 — Automatic `PREWHERE` optimization in `SELECT` queries with `FINAL` modifier is disabled. +- 1 — Automatic `PREWHERE` optimization in `SELECT` queries with `FINAL` modifier is enabled. + +Default value: `0`. + +**See Also** + +- [PREWHERE](../../sql-reference/statements/select/prewhere.md) clause in `SELECT` queries +- [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier in `SELECT` queries +- [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting \ No newline at end of file diff --git a/docs/en/sql-reference/statements/select/prewhere.md b/docs/en/sql-reference/statements/select/prewhere.md index ada8fff7012..646bb83e692 100644 --- a/docs/en/sql-reference/statements/select/prewhere.md +++ b/docs/en/sql-reference/statements/select/prewhere.md @@ -6,7 +6,7 @@ toc_title: PREWHERE Prewhere is an optimization to apply filtering more efficiently. It is enabled by default even if `PREWHERE` clause is not specified explicitly. It works by automatically moving part of [WHERE](../../../sql-reference/statements/select/where.md) condition to prewhere stage. The role of `PREWHERE` clause is only to control this optimization if you think that you know how to do it better than it happens by default. -With prewhere optimization, at first only the columns necessary for executing prewhere expression are read. Then the other columns are read that are needed for running the rest of the query, but only those blocks where the prewhere expression is “true” at least for some rows. If there are a lot of blocks where prewhere expression is “false” for all rows and prewhere needs less columns than other parts of query, this often allows to read a lot less data from disk for query execution. +With prewhere optimization, at first only the columns necessary for executing prewhere expression are read. Then the other columns are read that are needed for running the rest of the query, but only those blocks where the prewhere expression is `true` at least for some rows. If there are a lot of blocks where prewhere expression is `false` for all rows and prewhere needs less columns than other parts of query, this often allows to read a lot less data from disk for query execution. ## Controlling Prewhere Manually {#controlling-prewhere-manually} @@ -14,11 +14,13 @@ The clause has the same meaning as the `WHERE` clause. The difference is in whic A query may simultaneously specify `PREWHERE` and `WHERE`. In this case, `PREWHERE` precedes `WHERE`. -If the `optimize_move_to_prewhere` setting is set to 0, heuristics to automatically move parts of expressions from `WHERE` to `PREWHERE` are disabled. +If the [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) setting is set to 0, heuristics to automatically move parts of expressions from `WHERE` to `PREWHERE` are disabled. + +If query has [FINAL](from.md#select-from-final) modifier, the `PREWHERE` optimization is not always correct. It is enabled only if both settings [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) and [optimize_move_to_prewhere_if_final](../../../operations/settings/settings.md#optimize_move_to_prewhere_if_final) are turned on. !!! note "Attention" - The `PREWHERE` section is executed before` FINAL`, so the results of `FROM FINAL` queries may be skewed when using` PREWHERE` with fields not in the `ORDER BY` section of a table. + The `PREWHERE` section is executed before `FINAL`, so the results of `FROM ... FINAL` queries may be skewed when using `PREWHERE` with fields not in the `ORDER BY` section of a table. ## Limitations {#limitations} -`PREWHERE` is only supported by tables from the `*MergeTree` family. +`PREWHERE` is only supported by tables from the [*MergeTree](../../../engines/table-engines/mergetree-family/index.md) family. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 500485aea2f..887c59c3b09 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3572,4 +3572,40 @@ SELECT * FROM positional_arguments ORDER BY 2,3; │ 20 │ 20 │ 10 │ │ 10 │ 20 │ 30 │ └─────┴─────┴───────┘ -``` \ No newline at end of file +``` + +## optimize_move_to_prewhere {#optimize_move_to_prewhere} + +Включает или отключает автоматическую оптимизацию [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах [SELECT](../../sql-reference/statements/select/index.md). + +Работает только с таблицами семейства [*MergeTree](../../engines/table-engines/mergetree-family/index.md). + +Возможные значения: + +- 0 — автоматическая оптимизация `PREWHERE` отключена. +- 1 — автоматическая оптимизация `PREWHERE` включена. + +Значение по умолчанию: `1`. + +**См. также** + +- секция [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах `SELECT` + +## optimize_move_to_prewhere_if_final {#optimize_move_to_prewhere_if_final} + +Включает или отключает автоматическую оптимизацию [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах [SELECT](../../sql-reference/statements/select/index.md) с модификатором [FINAL](../../sql-reference/statements/select/from.md#select-from-final). + +Работает только с таблицами семейства [*MergeTree](../../engines/table-engines/mergetree-family/index.md). + +Возможные значения: + +- 0 — автоматическая оптимизация `PREWHERE` в запросах `SELECT` с модификатором `FINAL` отключена. +- 1 — автоматическая оптимизация `PREWHERE` в запросах `SELECT` с модификатором `FINAL` включена. + +Значение по умолчанию: `0`. + +**См. также** + +- секция [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах `SELECT` +- модификатор [FINAL](../../sql-reference/statements/select/from.md#select-from-final) в запросах `SELECT` +- настройка [optimize_move_to_prewhere](#optimize_move_to_prewhere) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index 5ba25e6fa6e..84f8869b41e 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -8,17 +8,19 @@ Prewhere — это оптимизация для более эффективн При оптимизации prewhere сначала читываются только те столбцы, которые необходимы для выполнения выражения prewhere. Затем читаются другие столбцы, необходимые для выполнения остальной части запроса, но только те блоки, в которых находится выражение prewhere «верно» по крайней мере для некоторых рядов. Если есть много блоков, где выражение prewhere «ложно» для всех строк и для выражения prewhere требуется меньше столбцов, чем для других частей запроса, это часто позволяет считывать гораздо меньше данных с диска для выполнения запроса. -## Управление prewhere вручную {#controlling-prewhere-manually} +## Управление PREWHERE вручную {#controlling-prewhere-manually} `PREWHERE` имеет смысл использовать, если есть условия фильтрации, которые использует меньшинство столбцов из тех, что есть в запросе, но достаточно сильно фильтрует данные. Таким образом, сокращается количество читаемых данных. -В запрос может быть одновременно указано и `PREWHERE` и `WHERE`. В этом случае, `PREWHERE` предшествует `WHERE`. +В запросе может быть одновременно указаны и `PREWHERE`, и `WHERE`. В этом случае `PREWHERE` предшествует `WHERE`. -Если значение параметра `optimize_move_to_prewhere` равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается. +Если значение параметра [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается. + +Если в запросе есть модификатор [FINAL](from.md#select-from-final), оптимизация `PREWHERE` не всегда корректна. Она действует только если включены обе настройки [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) и [optimize_move_to_prewhere_if_final](../../../operations/settings/settings.md#optimize_move_to_prewhere_if_final). !!! note "Внимание" - Секция `PREWHERE` выполняется до `FINAL`, поэтому результаты запросов `FROM FINAL` могут исказится при использовании `PREWHERE` с полями не входящями в `ORDER BY` таблицы. + Секция `PREWHERE` выполняется до `FINAL`, поэтому результаты запросов `FROM ... FINAL` могут исказиться при использовании `PREWHERE` с полями, не входящями в `ORDER BY` таблицы. ## Ограничения {#limitations} -`PREWHERE` поддерживается только табличными движками из семейства `*MergeTree`. +`PREWHERE` поддерживается только табличными движками из семейства [*MergeTree](../../../engines/table-engines/mergetree-family/index.md). From 3386e3a34962645fd4f4626e00ddd7cadf750625 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:24:27 +0300 Subject: [PATCH 057/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index c91d5caef8d..b05c236feac 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2353,7 +2353,7 @@ SELECT dummy, shardNum(), shardCount() FROM shard_num_example; **См. также** -- [Distributed Table Engine](../../engines/table-engines/special/distributed.md) +- Табличный движок [Distributed](../../engines/table-engines/special/distributed.md) ## shardCount {#shard-count} From dc01e86fb9798b867598c51fbc287df162bee015 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:24:46 +0300 Subject: [PATCH 058/126] Update docs/ru/engines/table-engines/special/distributed.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/engines/table-engines/special/distributed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index ff1dc7c4057..3d7b8cf32d3 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -143,7 +143,7 @@ logs - имя кластера в конфигурационном файле с !!! note "Примечание" Так как табличные функции [remote](../../../sql-reference/table-functions/remote.md) и [cluster](../../../sql-reference/table-functions/cluster.md) создают временную таблицу на движке `Distributed`, то в ней также доступен столбец `_shard_num`. -**Смотрите также** +**См. также** - общее описание [виртуальных столбцов](../../../engines/table-engines/index.md#table_engines-virtual_columns) - настройка [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) From b6cb640572f870642d1386465c0ecbe6297ca2ad Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:25:03 +0300 Subject: [PATCH 059/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index b05c236feac..029e53237a8 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2358,7 +2358,7 @@ SELECT dummy, shardNum(), shardCount() FROM shard_num_example; ## shardCount {#shard-count} Возвращает общее количество шардов для распределенного запроса. -Если запрос не распределенный, то возвращается константное значение `0`. +Если запрос не распределенный, то возвращается значение `0`. **Синтаксис** From d530a2d17b8e9742596fe9cd62795fcd0dc51a9c Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:25:19 +0300 Subject: [PATCH 060/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 029e53237a8..ca73ee75f50 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -2316,7 +2316,7 @@ SELECT count(DISTINCT t) FROM (SELECT initialQueryID() AS t FROM remote('127.0.0 ## shardNum {#shard-num} Возвращает индекс шарда, который обрабатывает часть данных распределенного запроса. Индексы начинаются с `1`. -Если запрос не распределенный, то возвращается константное значение `0`. +Если запрос не распределенный, то возвращается значение `0`. **Синтаксис** From dccfd8d2623007a7a2edc9455c260376e506a7c3 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:26:24 +0300 Subject: [PATCH 061/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index ca73ee75f50..001f704c219 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -653,8 +653,8 @@ SELECT ## buildId() {#buildid} -Возвращает ID сборки, сгенерированный компилятором для запущенного сервера ClickHouse. -Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. +Возвращает ID сборки, сгенерированный компилятором для данного сервера ClickHouse. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. ## rowNumberInBlock {#function-rownumberinblock} From 32a4c9b69cd9ee516cb2e796045af94790831cf1 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:43:10 +0300 Subject: [PATCH 062/126] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index d4777faf354..924dd559cbe 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -26,7 +26,7 @@ SELECT ## timeZone {#timezone} Возвращает часовой пояс сервера. -Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. **Синтаксис** From 294b8c01f8134592d12ff33e6a63650c6c1718b8 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:43:23 +0300 Subject: [PATCH 063/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 001f704c219..3ad9192d9a1 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -8,7 +8,7 @@ toc_title: "Прочие функции" ## hostName() {#hostname} Возвращает строку - имя хоста, на котором эта функция была выполнена. При распределённой обработке запроса, это будет имя хоста удалённого сервера, если функция выполняется на удалённом сервере. -Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. ## getMacro {#getmacro} From 0d357f34842ef9cb5a2c6aadd4b83d2a022a9ee0 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:43:34 +0300 Subject: [PATCH 064/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 3ad9192d9a1..7134a19864e 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -649,7 +649,7 @@ SELECT ## version() {#version} Возвращает версию сервера в виде строки. -Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. ## buildId() {#buildid} From e9bb7dd08ce9c45d1a430f323acf8ba38dbee3be Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 11 Oct 2021 09:43:51 +0300 Subject: [PATCH 065/126] Update docs/ru/sql-reference/functions/other-functions.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/functions/other-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 7134a19864e..925aac56968 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -644,7 +644,7 @@ SELECT ## uptime() {#uptime} Возвращает аптайм сервера в секундах. -Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями актуальными для каждого шарда. Иначе возвращается константа. +Если функция вызывается в контексте распределенной таблицы, то она генерирует обычный столбец со значениями, актуальными для каждого шарда. Иначе возвращается константа. ## version() {#version} From ebfb013ea1577aa601a0caad45af60e12ba851b8 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 8 Oct 2021 11:40:29 +0800 Subject: [PATCH 066/126] Fix potential leak of query_id_holder --- src/Processors/QueryPlan/QueryIdHolder.cpp | 1 + src/Processors/QueryPlan/QueryIdHolder.h | 5 +- .../QueryPlan/ReadFromMergeTree.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 19 ++++-- src/Storages/MergeTree/MergeTreeData.h | 7 ++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 63 ++++++++++--------- .../MergeTree/MergeTreeDataSelectExecutor.h | 2 +- .../01666_merge_tree_max_query_limit.sh | 8 +++ 8 files changed, 66 insertions(+), 41 deletions(-) diff --git a/src/Processors/QueryPlan/QueryIdHolder.cpp b/src/Processors/QueryPlan/QueryIdHolder.cpp index 87f6f892cd1..6ff238e017c 100644 --- a/src/Processors/QueryPlan/QueryIdHolder.cpp +++ b/src/Processors/QueryPlan/QueryIdHolder.cpp @@ -3,6 +3,7 @@ namespace DB { + QueryIdHolder::QueryIdHolder(const String & query_id_, const MergeTreeData & data_) : query_id(query_id_), data(data_) { } diff --git a/src/Processors/QueryPlan/QueryIdHolder.h b/src/Processors/QueryPlan/QueryIdHolder.h index ed8f9ec1d6b..1e1ee1af0a1 100644 --- a/src/Processors/QueryPlan/QueryIdHolder.h +++ b/src/Processors/QueryPlan/QueryIdHolder.h @@ -2,13 +2,16 @@ #include +#include + namespace DB { + class MergeTreeData; /// Holds the current query id and do something meaningful in destructor. /// Currently it's used for cleaning query id in the MergeTreeData query set. -struct QueryIdHolder +struct QueryIdHolder : private boost::noncopyable { QueryIdHolder(const std::string & query_id_, const MergeTreeData & data_); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index a48adc2d645..8d3005e725f 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -945,7 +945,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges); ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks); - auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result.parts_with_ranges, context); + auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result, context); if (result.parts_with_ranges.empty()) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c04e0d2e38f..646737b11ae 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5321,26 +5321,33 @@ void MergeTreeData::setDataVolume(size_t bytes, size_t rows, size_t parts) total_active_size_parts.store(parts, std::memory_order_release); } -void MergeTreeData::insertQueryIdOrThrow(const String & query_id, size_t max_queries) const +bool MergeTreeData::insertQueryIdOrThrow(const String & query_id, size_t max_queries) const { std::lock_guard lock(query_id_set_mutex); + return insertQueryIdOrThrowNoLock(query_id, max_queries, lock); +} + +bool MergeTreeData::insertQueryIdOrThrowNoLock(const String & query_id, size_t max_queries, const std::lock_guard &) const +{ if (query_id_set.find(query_id) != query_id_set.end()) - return; + return false; if (query_id_set.size() >= max_queries) throw Exception( ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for table {}. Maximum is: {}", log_name, max_queries); query_id_set.insert(query_id); + return true; } void MergeTreeData::removeQueryId(const String & query_id) const { std::lock_guard lock(query_id_set_mutex); + removeQueryIdNoLock(query_id, lock); +} + +void MergeTreeData::removeQueryIdNoLock(const String & query_id, const std::lock_guard &) const +{ if (query_id_set.find(query_id) == query_id_set.end()) - { - /// Do not throw exception, because this method is used in destructor. LOG_WARNING(log, "We have query_id removed but it's not recorded. This is a bug"); - assert(false); - } else query_id_set.erase(query_id); } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index bdebd5e9187..e33e6aa0ef4 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -794,11 +794,16 @@ public: /// section from config.xml. CompressionCodecPtr getCompressionCodecForPart(size_t part_size_compressed, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t current_time) const; + std::lock_guard getQueryIdSetLock() const { return std::lock_guard(query_id_set_mutex); } + /// Record current query id where querying the table. Throw if there are already `max_queries` queries accessing the same table. - void insertQueryIdOrThrow(const String & query_id, size_t max_queries) const; + /// Returns false if the `query_id` already exists in the running set, otherwise return true. + bool insertQueryIdOrThrow(const String & query_id, size_t max_queries) const; + bool insertQueryIdOrThrowNoLock(const String & query_id, size_t max_queries, const std::lock_guard &) const; /// Remove current query id after query finished. void removeQueryId(const String & query_id) const; + void removeQueryIdNoLock(const String & query_id, const std::lock_guard &) const; /// Return the partition expression types as a Tuple type. Return DataTypeUInt8 if partition expression is empty. DataTypePtr getPartitionValueType() const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 03d76a7f79b..44b913ea81a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -993,47 +993,48 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd std::shared_ptr MergeTreeDataSelectExecutor::checkLimits( const MergeTreeData & data, - const RangesInDataParts & parts_with_ranges, + const ReadFromMergeTree::AnalysisResult & result, const ContextPtr & context) { const auto & settings = context->getSettingsRef(); - // Check limitations. query_id is used as the quota RAII's resource key. - String query_id; + const auto data_settings = data.getSettings(); + auto max_partitions_to_read + = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data_settings->max_partitions_to_read; + if (max_partitions_to_read > 0) { - const auto data_settings = data.getSettings(); - auto max_partitions_to_read - = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data_settings->max_partitions_to_read; - if (max_partitions_to_read > 0) - { - std::set partitions; - for (const auto & part_with_ranges : parts_with_ranges) - partitions.insert(part_with_ranges.data_part->info.partition_id); - if (partitions.size() > size_t(max_partitions_to_read)) - throw Exception( - ErrorCodes::TOO_MANY_PARTITIONS, - "Too many partitions to read. Current {}, max {}", - partitions.size(), - max_partitions_to_read); - } + std::set partitions; + for (const auto & part_with_ranges : result.parts_with_ranges) + partitions.insert(part_with_ranges.data_part->info.partition_id); + if (partitions.size() > size_t(max_partitions_to_read)) + throw Exception( + ErrorCodes::TOO_MANY_PARTITIONS, + "Too many partitions to read. Current {}, max {}", + partitions.size(), + max_partitions_to_read); + } - if (data_settings->max_concurrent_queries > 0 && data_settings->min_marks_to_honor_max_concurrent_queries > 0) + if (data_settings->max_concurrent_queries > 0 && data_settings->min_marks_to_honor_max_concurrent_queries > 0 + && result.selected_marks >= data_settings->min_marks_to_honor_max_concurrent_queries) + { + auto query_id = context->getCurrentQueryId(); + if (!query_id.empty()) { - size_t sum_marks = 0; - for (const auto & part : parts_with_ranges) - sum_marks += part.getMarksCount(); - - if (sum_marks >= data_settings->min_marks_to_honor_max_concurrent_queries) + auto lock = data.getQueryIdSetLock(); + if (data.insertQueryIdOrThrowNoLock(query_id, data_settings->max_concurrent_queries, lock)) { - query_id = context->getCurrentQueryId(); - if (!query_id.empty()) - data.insertQueryIdOrThrow(query_id, data_settings->max_concurrent_queries); + try + { + return std::make_shared(query_id, data); + } + catch (...) + { + /// If we fail to construct the holder, remove query_id explicitly to avoid leak. + data.removeQueryIdNoLock(query_id, lock); + throw; + } } } } - - if (!query_id.empty()) - return std::make_shared(query_id, data); - return nullptr; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 92c4382dc90..3cc5033c9f1 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -197,7 +197,7 @@ public: /// Also, return QueryIdHolder. If not null, we should keep it until query finishes. static std::shared_ptr checkLimits( const MergeTreeData & data, - const RangesInDataParts & parts_with_ranges, + const ReadFromMergeTree::AnalysisResult & result, const ContextPtr & context); }; diff --git a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh index c5fbb35a9cd..6fb337f2ca5 100755 --- a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh +++ b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh @@ -66,6 +66,14 @@ echo "yes" ${CLICKHOUSE_CLIENT} --query "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null" wait +# Check correctness of multiple subqueries +query_id=max_concurrent_queries_$RANDOM +${CLICKHOUSE_CLIENT} --query_id "$query_id" --query "select i from simple where j in (select i from simple where i < 10)" + +# We have to grep the server's error log because the following warning message +# is generated during pipeline destruction and thus is not sent to the client. +grep -E -q "{$query_id} .*We have query_id removed but it's not recorded. This is a bug" /var/log/clickhouse-server/clickhouse-server.err.log && exit 1 + ${CLICKHOUSE_CLIENT} --multiline --multiquery --query " drop table simple " From 2d069acc220347942ac3716168ded3dc7f9ded12 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 8 Oct 2021 16:13:56 +0300 Subject: [PATCH 067/126] System table data skipping indices added size --- .../system-tables/data_skipping_indices.md | 9 +++ docs/en/operations/system-tables/parts.md | 9 +++ src/Storages/IStorage.h | 7 ++ src/Storages/MergeTree/IMergeTreeDataPart.cpp | 50 +++++++++++++- src/Storages/MergeTree/IMergeTreeDataPart.h | 21 +++++- src/Storages/MergeTree/MergeTreeData.cpp | 68 ++++++++++++++----- src/Storages/MergeTree/MergeTreeData.h | 20 ++++-- .../MergeTree/MergedBlockOutputStream.cpp | 3 +- src/Storages/MergeTree/MutateTask.cpp | 2 +- .../StorageSystemDataSkippingIndices.cpp | 19 ++++++ src/Storages/System/StorageSystemParts.cpp | 10 +++ 11 files changed, 192 insertions(+), 26 deletions(-) diff --git a/docs/en/operations/system-tables/data_skipping_indices.md b/docs/en/operations/system-tables/data_skipping_indices.md index 683666e1f77..add89ae9144 100644 --- a/docs/en/operations/system-tables/data_skipping_indices.md +++ b/docs/en/operations/system-tables/data_skipping_indices.md @@ -10,6 +10,9 @@ Columns: - `type` ([String](../../sql-reference/data-types/string.md)) — Index type. - `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation. - `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block. +- `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of compressed data, in bytes. +- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of decompressed data, in bytes. +- `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of marks, in bytes. **Example** @@ -26,6 +29,9 @@ name: clicks_idx type: minmax expr: clicks granularity: 1 +data_compressed_bytes: 58 +data_uncompressed_bytes: 6 +marks: 48 Row 2: ────── @@ -35,4 +41,7 @@ name: contacts_null_idx type: minmax expr: assumeNotNull(contacts_null) granularity: 1 +data_compressed_bytes: 58 +data_uncompressed_bytes: 6 +marks: 48 ``` diff --git a/docs/en/operations/system-tables/parts.md b/docs/en/operations/system-tables/parts.md index 51a0a1180f3..45fdcc40451 100644 --- a/docs/en/operations/system-tables/parts.md +++ b/docs/en/operations/system-tables/parts.md @@ -38,6 +38,12 @@ Columns: - `marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The size of the file with marks. +- `secondary_indices_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – Total size of compressed data for secondary indices in the data part. All the auxiliary files (for example, files with marks) are not included. + +- `secondary_indices_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – Total size of uncompressed data for secondary indices in the data part. All the auxiliary files (for example, files with marks) are not included. + +- `secondary_indices_marks_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) – The size of the file with marks for secondary indices. + - `modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) – The time the directory with the data part was modified. This usually corresponds to the time of data part creation. - `remove_time` ([DateTime](../../sql-reference/data-types/datetime.md)) – The time when the data part became inactive. @@ -119,6 +125,9 @@ rows: 6 bytes_on_disk: 310 data_compressed_bytes: 157 data_uncompressed_bytes: 91 +secondary_indices_compressed_bytes: 58 +secondary_indices_uncompressed_bytes: 6 +secondary_indices_marks_bytes: 48 marks_bytes: 144 modification_time: 2020-06-18 13:01:49 remove_time: 1970-01-01 00:00:00 diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6ce17552ba1..0a9d1113601 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -87,6 +87,8 @@ struct ColumnSize } }; +using IndexSize = ColumnSize; + /** Storage. Describes the table. Responsible for * - storage of the table data; * - the definition in which files (or not in files) the data is stored; @@ -163,6 +165,11 @@ public: using ColumnSizeByName = std::unordered_map; virtual ColumnSizeByName getColumnSizes() const { return {}; } + /// Optional size information of each secondary index. + /// Valid only for MergeTree family. + using IndexSizeByName = std::unordered_map; + virtual IndexSizeByName getSecondaryIndexSizes() const { return {}; } + /// Get mutable version (snapshot) of storage metadata. Metadata object is /// multiversion, so it can be concurrently changed, but returned copy can be /// used without any locks. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index dc2c5f8185d..1a6290580a0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -584,7 +584,7 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadColumns(require_columns_checksums); loadChecksums(require_columns_checksums); loadIndexGranularity(); - calculateColumnsSizesOnDisk(); + calculateColumnsAndSecondaryIndicesSizesOnDisk(); loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` loadRowsCount(); /// Must be called after loadIndexGranularity() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); @@ -1420,6 +1420,11 @@ void IMergeTreeDataPart::checkConsistency(bool /* require_part_metadata */) cons throw Exception("Method 'checkConsistency' is not implemented for part with type " + getType().toString(), ErrorCodes::NOT_IMPLEMENTED); } +void IMergeTreeDataPart::calculateColumnsAndSecondaryIndicesSizesOnDisk() +{ + calculateColumnsSizesOnDisk(); + calculateSecondaryIndicesSizesOnDisk(); +} void IMergeTreeDataPart::calculateColumnsSizesOnDisk() { @@ -1429,6 +1434,40 @@ void IMergeTreeDataPart::calculateColumnsSizesOnDisk() calculateEachColumnSizes(columns_sizes, total_columns_size); } +void IMergeTreeDataPart::calculateSecondaryIndicesSizesOnDisk() +{ + if (checksums.empty()) + throw Exception("Cannot calculate secondary indexes sizes when columns or checksums are not initialized", ErrorCodes::LOGICAL_ERROR); + + auto secondary_indices_descriptions = storage.getInMemoryMetadataPtr()->secondary_indices; + + for (auto & index_description : secondary_indices_descriptions) + { + ColumnSize index_size; + + auto index_ptr = MergeTreeIndexFactory::instance().get(index_description); + auto index_name = index_ptr->getFileName(); + auto index_name_escaped = escapeForFileName(index_name); + + auto index_file_name = index_name_escaped + index_ptr->getSerializedFileExtension(); + auto index_marks_file_name = index_name_escaped + index_granularity_info.marks_file_extension; + + auto bin_checksum = checksums.files.find(index_file_name); + if (bin_checksum != checksums.files.end()) + { + index_size.data_compressed = bin_checksum->second.file_size; + index_size.data_uncompressed = bin_checksum->second.uncompressed_size; + } + + auto mrk_checksum = checksums.files.find(index_marks_file_name); + if (mrk_checksum != checksums.files.end()) + index_size.marks = mrk_checksum->second.file_size; + + total_secondary_indices_size.add(index_size); + secondary_index_sizes[index_description.name] = index_size; + } +} + ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & /* type */) const { /// For some types of parts columns_size maybe not calculated @@ -1439,6 +1478,15 @@ ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const I return ColumnSize{}; } +IndexSize IMergeTreeDataPart::getSecondaryIndexSize(const String & secondary_index_name) const +{ + auto it = secondary_index_sizes.find(secondary_index_name); + if (it != secondary_index_sizes.end()) + return it->second; + + return ColumnSize{}; +} + void IMergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) const { for (const auto & [column_name, size] : columns_sizes) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index be48aed5c8b..ceb3ed64170 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -55,6 +55,8 @@ public: using ColumnSizeByName = std::unordered_map; using NameToNumber = std::unordered_map; + using IndexSizeByName = std::unordered_map; + using Type = MergeTreeDataPartType; @@ -101,9 +103,16 @@ public: /// Otherwise return information about column size on disk. ColumnSize getColumnSize(const String & column_name, const IDataType & /* type */) const; + /// NOTE: Returns zeros if secondary indexes are not found in checksums. + /// Otherwise return information about secondary index size on disk. + IndexSize getSecondaryIndexSize(const String & secondary_index_name) const; + /// Return information about column size on disk for all columns in part ColumnSize getTotalColumnsSize() const { return total_columns_size; } + /// Return information about secondary indexes size on disk for all indexes in part + IndexSize getTotalSeconaryIndicesSize() const { return total_secondary_indices_size; } + virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; virtual ~IMergeTreeDataPart(); @@ -341,7 +350,9 @@ public: /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); - void calculateColumnsSizesOnDisk(); + + /// Calculate column and secondary indices sizes on disk. + void calculateColumnsAndSecondaryIndicesSizesOnDisk(); String getRelativePathForPrefix(const String & prefix) const; @@ -396,6 +407,10 @@ protected: /// Size for each column, calculated once in calcuateColumnSizesOnDisk ColumnSizeByName columns_sizes; + ColumnSize total_secondary_indices_size; + + IndexSizeByName secondary_index_sizes; + /// Total size on disk, not only columns. May not contain size of /// checksums.txt and columns.txt. 0 - if not counted; UInt64 bytes_on_disk{0}; @@ -450,6 +465,10 @@ private: void loadPartitionAndMinMaxIndex(); + void calculateColumnsSizesOnDisk(); + + void calculateSecondaryIndicesSizesOnDisk(); + /// Load default compression codec from file default_compression_codec.txt /// if it not exists tries to deduce codec from compressed column without /// any specifial compression. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c04e0d2e38f..10e5fe9e71f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1167,7 +1167,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) } } - calculateColumnSizesImpl(); + calculateColumnAndSecondaryIndexSizesImpl(); LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); @@ -2352,7 +2352,7 @@ bool MergeTreeData::renameTempPartAndReplace( { covered_part->remove_time.store(current_time, std::memory_order_relaxed); modifyPartState(covered_part, DataPartState::Outdated); - removePartContributionToColumnSizes(covered_part); + removePartContributionToColumnAndSecondaryIndexSizes(covered_part); reduce_bytes += covered_part->getBytesOnDisk(); reduce_rows += covered_part->rows_count; ++reduce_parts; @@ -2361,7 +2361,7 @@ bool MergeTreeData::renameTempPartAndReplace( decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); modifyPartState(part_it, DataPartState::Committed); - addPartContributionToColumnSizes(part); + addPartContributionToColumnAndSecondaryIndexSizes(part); addPartContributionToDataVolume(part); } @@ -2404,7 +2404,7 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect { if (part->getState() == IMergeTreeDataPart::State::Committed) { - removePartContributionToColumnSizes(part); + removePartContributionToColumnAndSecondaryIndexSizes(part); removePartContributionToDataVolume(part); } @@ -2542,7 +2542,7 @@ restore_covered) if (part->getState() == DataPartState::Committed) { removePartContributionToDataVolume(part); - removePartContributionToColumnSizes(part); + removePartContributionToColumnAndSecondaryIndexSizes(part); } modifyPartState(it_part, DataPartState::Deleting); @@ -2590,7 +2590,7 @@ restore_covered) if ((*it)->getState() != DataPartState::Committed) { - addPartContributionToColumnSizes(*it); + addPartContributionToColumnAndSecondaryIndexSizes(*it); addPartContributionToDataVolume(*it); modifyPartState(it, DataPartState::Committed); // iterator is not invalidated here } @@ -2621,7 +2621,7 @@ restore_covered) if ((*it)->getState() != DataPartState::Committed) { - addPartContributionToColumnSizes(*it); + addPartContributionToColumnAndSecondaryIndexSizes(*it); addPartContributionToDataVolume(*it); modifyPartState(it, DataPartState::Committed); } @@ -2973,32 +2973,46 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) part->modification_time = disk->getLastModified(full_part_path).epochTime(); } -void MergeTreeData::calculateColumnSizesImpl() +void MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl() { + std::cerr << "MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl" << std::endl; + column_sizes.clear(); /// Take into account only committed parts auto committed_parts_range = getDataPartsStateRange(DataPartState::Committed); for (const auto & part : committed_parts_range) - addPartContributionToColumnSizes(part); + addPartContributionToColumnAndSecondaryIndexSizes(part); } -void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) +void MergeTreeData::addPartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part) { + std::cerr << "MergeTreeData::addPartContributionToColumnAndSecondaryIndexSizes " << part->name << std::endl; + for (const auto & column : part->getColumns()) { + std::cerr << "Column name " << column.name << std::endl; ColumnSize & total_column_size = column_sizes[column.name]; + std::cerr << "Total column size compressed " << total_column_size.data_compressed << " uncompressed size " << total_column_size.data_uncompressed << std::endl; ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); total_column_size.add(part_column_size); } + + auto indexes_descriptions = getInMemoryMetadataPtr()->secondary_indices; + for (const auto & index : indexes_descriptions) + { + IndexSize & total_secondary_index_size = secondary_index_sizes[index.name]; + IndexSize part_index_size = part->getSecondaryIndexSize(index.name); + total_secondary_index_size.add(part_index_size); + } } -void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part) +void MergeTreeData::removePartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part) { for (const auto & column : part->getColumns()) { ColumnSize & total_column_size = column_sizes[column.name]; - ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); + ColumnSize part_secondary_index_size = part->getColumnSize(column.name, *column.type); auto log_subtract = [&](size_t & from, size_t value, const char * field) { @@ -3009,9 +3023,29 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part from -= value; }; - log_subtract(total_column_size.data_compressed, part_column_size.data_compressed, ".data_compressed"); - log_subtract(total_column_size.data_uncompressed, part_column_size.data_uncompressed, ".data_uncompressed"); - log_subtract(total_column_size.marks, part_column_size.marks, ".marks"); + log_subtract(total_column_size.data_compressed, part_secondary_index_size.data_compressed, ".data_compressed"); + log_subtract(total_column_size.data_uncompressed, part_secondary_index_size.data_uncompressed, ".data_uncompressed"); + log_subtract(total_column_size.marks, part_secondary_index_size.marks, ".marks"); + } + + auto indexes_descriptions = getInMemoryMetadataPtr()->secondary_indices; + for (const auto & index : indexes_descriptions) + { + IndexSize & total_secondary_index_size = secondary_index_sizes[index.name]; + IndexSize part_secondary_index_size = part->getSecondaryIndexSize(index.name); + + auto log_subtract = [&](size_t & from, size_t value, const char * field) + { + if (value > from) + LOG_ERROR(log, "Possibly incorrect index size subtraction: {} - {} = {}, index: {}, field: {}", + from, value, from - value, index.name, field); + + from -= value; + }; + + log_subtract(total_secondary_index_size.data_compressed, part_secondary_index_size.data_compressed, ".data_compressed"); + log_subtract(total_secondary_index_size.data_uncompressed, part_secondary_index_size.data_uncompressed, ".data_uncompressed"); + log_subtract(total_secondary_index_size.marks, part_secondary_index_size.marks, ".marks"); } } @@ -4043,7 +4077,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: reduce_rows += covered_part->rows_count; data.modifyPartState(covered_part, DataPartState::Outdated); - data.removePartContributionToColumnSizes(covered_part); + data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part); } reduce_parts += covered_parts.size(); @@ -4052,7 +4086,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: ++add_parts; data.modifyPartState(part, DataPartState::Committed); - data.addPartContributionToColumnSizes(part); + data.addPartContributionToColumnAndSecondaryIndexSizes(part); } } data.decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index bdebd5e9187..0e0e84d011b 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -654,6 +654,12 @@ public: return column_sizes; } + IndexSizeByName getSecondaryIndexSizes() const override + { + auto lock = lockParts(); + return secondary_index_sizes; + } + /// For ATTACH/DETACH/DROP PARTITION. String getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr context) const; std::unordered_set getPartitionIDsFromQuery(const ASTs & asts, ContextPtr context) const; @@ -873,6 +879,9 @@ protected: /// Current column sizes in compressed and uncompressed form. ColumnSizeByName column_sizes; + /// Current secondary index sizes in compressed and uncompressed form. + IndexSizeByName secondary_index_sizes; + /// Engine-specific methods BrokenPartCallback broken_part_callback; @@ -1005,11 +1014,12 @@ protected: void checkStoragePolicy(const StoragePolicyPtr & new_storage_policy) const; - /// Calculates column sizes in compressed form for the current state of data_parts. Call with data_parts mutex locked. - void calculateColumnSizesImpl(); - /// Adds or subtracts the contribution of the part to compressed column sizes. - void addPartContributionToColumnSizes(const DataPartPtr & part); - void removePartContributionToColumnSizes(const DataPartPtr & part); + /// Calculates column and secondary indexes sizes in compressed form for the current state of data_parts. Call with data_parts mutex locked. + void calculateColumnAndSecondaryIndexSizesImpl(); + + /// Adds or subtracts the contribution of the part to compressed column and secondary indexes sizes. + void addPartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part); + void removePartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part); /// If there is no part in the partition with ID `partition_id`, returns empty ptr. Should be called under the lock. DataPartPtr getAnyPartInPartition(const String & partition_id, DataPartsLock & data_parts_lock) const; diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 5206f77290b..43146709686 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -87,7 +87,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->checksums = checksums; new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk()); new_part->index_granularity = writer->getIndexGranularity(); - new_part->calculateColumnsSizesOnDisk(); + new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); + if (default_codec != nullptr) new_part->default_codec = default_codec; new_part->storage.lockSharedData(*new_part); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index b8941fc9d84..115de043cd2 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -475,7 +475,7 @@ void finalizeMutatedPart( new_data_part->setBytesOnDisk( MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath())); new_data_part->default_codec = codec; - new_data_part->calculateColumnsSizesOnDisk(); + new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); new_data_part->storage.lockSharedData(*new_data_part); } diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp index 7a6ce4ec519..d7fc06da953 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp +++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp @@ -25,6 +25,9 @@ StorageSystemDataSkippingIndices::StorageSystemDataSkippingIndices(const Storage { "type", std::make_shared() }, { "expr", std::make_shared() }, { "granularity", std::make_shared() }, + { "data_compressed_bytes", std::make_shared() }, + { "data_uncompressed_bytes", std::make_shared() }, + { "marks", std::make_shared()} })); setInMemoryMetadata(storage_metadata); } @@ -97,6 +100,7 @@ protected: continue; const auto indices = metadata_snapshot->getSecondaryIndices(); + auto secondary_index_sizes = table->getSecondaryIndexSizes(); for (const auto & index : indices) { ++rows_count; @@ -127,6 +131,21 @@ protected: // 'granularity' column if (column_mask[src_index++]) res_columns[res_index++]->insert(index.granularity); + + auto & secondary_index_size = secondary_index_sizes[index.name]; + + // 'compressed bytes' column + if (column_mask[src_index++]) + res_columns[res_index++]->insert(secondary_index_size.data_compressed); + + // 'uncompressed bytes' column + + if (column_mask[src_index++]) + res_columns[res_index++]->insert(secondary_index_size.data_uncompressed); + + /// 'marks' column + if (column_mask[src_index++]) + res_columns[res_index++]->insert(secondary_index_size.marks); } } } diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index e79978463dd..6826082ef1d 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -30,6 +30,9 @@ StorageSystemParts::StorageSystemParts(const StorageID & table_id_) {"data_compressed_bytes", std::make_shared()}, {"data_uncompressed_bytes", std::make_shared()}, {"marks_bytes", std::make_shared()}, + {"secondary_indices_compressed_bytes", std::make_shared()}, + {"secondary_indices_uncompressed_bytes", std::make_shared()}, + {"secondary_indices_marks_bytes", std::make_shared()}, {"modification_time", std::make_shared()}, {"remove_time", std::make_shared()}, {"refcount", std::make_shared()}, @@ -98,6 +101,7 @@ void StorageSystemParts::processNextStorage( auto part_state = all_parts_state[part_number]; ColumnSize columns_size = part->getTotalColumnsSize(); + ColumnSize secondary_indexes_size = part->getTotalSeconaryIndicesSize(); size_t src_index = 0, res_index = 0; if (columns_mask[src_index++]) @@ -126,6 +130,12 @@ void StorageSystemParts::processNextStorage( columns[res_index++]->insert(columns_size.data_uncompressed); if (columns_mask[src_index++]) columns[res_index++]->insert(columns_size.marks); + if (columns_mask[src_index++]) + columns[res_index++]->insert(secondary_indexes_size.data_compressed); + if (columns_mask[src_index++]) + columns[res_index++]->insert(secondary_indexes_size.data_uncompressed); + if (columns_mask[src_index++]) + columns[res_index++]->insert(secondary_indexes_size.marks); if (columns_mask[src_index++]) columns[res_index++]->insert(static_cast(part->modification_time)); From ce0c41e1ad6a025a0117e7486ff9fd9a511c5be5 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 8 Oct 2021 23:43:16 +0300 Subject: [PATCH 068/126] Fixed tests --- src/Storages/MergeTree/MergeTreeData.cpp | 6 ------ ...1917_system_data_skipping_indices.reference | 10 +++++----- .../01932_alter_index_with_order.reference | 18 +++++++++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 10e5fe9e71f..1ede7669832 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2975,8 +2975,6 @@ static void loadPartAndFixMetadataImpl(MergeTreeData::MutableDataPartPtr part) void MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl() { - std::cerr << "MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl" << std::endl; - column_sizes.clear(); /// Take into account only committed parts @@ -2987,13 +2985,9 @@ void MergeTreeData::calculateColumnAndSecondaryIndexSizesImpl() void MergeTreeData::addPartContributionToColumnAndSecondaryIndexSizes(const DataPartPtr & part) { - std::cerr << "MergeTreeData::addPartContributionToColumnAndSecondaryIndexSizes " << part->name << std::endl; - for (const auto & column : part->getColumns()) { - std::cerr << "Column name " << column.name << std::endl; ColumnSize & total_column_size = column_sizes[column.name]; - std::cerr << "Total column size compressed " << total_column_size.data_compressed << " uncompressed size " << total_column_size.data_uncompressed << std::endl; ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); total_column_size.add(part_column_size); } diff --git a/tests/queries/0_stateless/01917_system_data_skipping_indices.reference b/tests/queries/0_stateless/01917_system_data_skipping_indices.reference index b5a4b596a97..ca7e87e017b 100644 --- a/tests/queries/0_stateless/01917_system_data_skipping_indices.reference +++ b/tests/queries/0_stateless/01917_system_data_skipping_indices.reference @@ -1,8 +1,8 @@ -default data_01917 d1_idx minmax d1 1 -default data_01917 d1_null_idx minmax assumeNotNull(d1_null) 1 -default data_01917_2 memory set frequency * length(name) 5 -default data_01917_2 sample_index1 minmax length(name), name 4 -default data_01917_2 sample_index2 ngrambf_v1 lower(name), name 4 +test data_01917 d1_idx minmax d1 1 0 0 0 +test data_01917 d1_null_idx minmax assumeNotNull(d1_null) 1 0 0 0 +test data_01917_2 memory set frequency * length(name) 5 0 0 0 +test data_01917_2 sample_index1 minmax length(name), name 4 0 0 0 +test data_01917_2 sample_index2 ngrambf_v1 lower(name), name 4 0 0 0 2 3 d1_idx diff --git a/tests/queries/0_stateless/01932_alter_index_with_order.reference b/tests/queries/0_stateless/01932_alter_index_with_order.reference index 07e1aab3df9..eff9ea7da0e 100644 --- a/tests/queries/0_stateless/01932_alter_index_with_order.reference +++ b/tests/queries/0_stateless/01932_alter_index_with_order.reference @@ -1,9 +1,9 @@ -default alter_index_test index_a set a 1 -default alter_index_test index_b minmax b 1 -default alter_index_test index_c set c 2 -default alter_index_test index_a set a 1 -default alter_index_test index_d set d 1 -default alter_index_test index_b minmax b 1 -default alter_index_test index_c set c 2 -default alter_index_test index_a set a 1 -default alter_index_test index_d set d 1 +default alter_index_test index_a set a 1 0 0 0 +default alter_index_test index_b minmax b 1 0 0 0 +default alter_index_test index_c set c 2 0 0 0 +default alter_index_test index_a set a 1 0 0 0 +default alter_index_test index_d set d 1 0 0 0 +default alter_index_test index_b minmax b 1 0 0 0 +default alter_index_test index_c set c 2 0 0 0 +default alter_index_test index_a set a 1 0 0 0 +default alter_index_test index_d set d 1 0 0 0 From 61a725f53199697451200a2d24e0173347f8b9e2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 10 Oct 2021 23:53:31 +0300 Subject: [PATCH 069/126] Fixed tests --- .../01917_system_data_skipping_indices.reference | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01917_system_data_skipping_indices.reference b/tests/queries/0_stateless/01917_system_data_skipping_indices.reference index ca7e87e017b..115d60f60cc 100644 --- a/tests/queries/0_stateless/01917_system_data_skipping_indices.reference +++ b/tests/queries/0_stateless/01917_system_data_skipping_indices.reference @@ -1,8 +1,8 @@ -test data_01917 d1_idx minmax d1 1 0 0 0 -test data_01917 d1_null_idx minmax assumeNotNull(d1_null) 1 0 0 0 -test data_01917_2 memory set frequency * length(name) 5 0 0 0 -test data_01917_2 sample_index1 minmax length(name), name 4 0 0 0 -test data_01917_2 sample_index2 ngrambf_v1 lower(name), name 4 0 0 0 +default data_01917 d1_idx minmax d1 1 0 0 0 +default data_01917 d1_null_idx minmax assumeNotNull(d1_null) 1 0 0 0 +default data_01917_2 memory set frequency * length(name) 5 0 0 0 +default data_01917_2 sample_index1 minmax length(name), name 4 0 0 0 +default data_01917_2 sample_index2 ngrambf_v1 lower(name), name 4 0 0 0 2 3 d1_idx From a07ce981214ab0f046afa23fb3f5551bc24a7270 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 11 Oct 2021 18:07:00 +0800 Subject: [PATCH 070/126] Use system.text_log in test --- .../01666_merge_tree_max_query_limit.sh | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh index 6fb337f2ca5..e04c9515009 100755 --- a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh +++ b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh @@ -4,8 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -function wait_for_query_to_start() -{ +function wait_for_query_to_start() { while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT sum(read_rows) FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done } @@ -21,14 +20,14 @@ insert into simple select number, number + 100 from numbers(5000); query_id="long_running_query-$CLICKHOUSE_DATABASE" echo "Spin up a long running query" -${CLICKHOUSE_CLIENT} --query "select sleepEachRow(0.1) from simple settings max_block_size = 1 format Null" --query_id "$query_id" > /dev/null 2>&1 & +${CLICKHOUSE_CLIENT} --query "select sleepEachRow(0.1) from simple settings max_block_size = 1 format Null" --query_id "$query_id" >/dev/null 2>&1 & wait_for_query_to_start "$query_id" # query which reads marks >= min_marks_to_honor_max_concurrent_queries is throttled echo "Check if another query with some marks to read is throttled" -${CLICKHOUSE_CLIENT} --query "select * from simple" 2> /dev/null; +${CLICKHOUSE_CLIENT} --query "select * from simple" 2>/dev/null CODE=$? -[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1; +[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1 echo "yes" # query which reads marks less than min_marks_to_honor_max_concurrent_queries is allowed @@ -41,9 +40,9 @@ ${CLICKHOUSE_CLIENT} --query "alter table simple modify setting min_marks_to_hon # Now smaller queries are also throttled echo "Check if another query with less marks to read is throttled" -${CLICKHOUSE_CLIENT} --query "select * from simple where i = 0" 2> /dev/null; +${CLICKHOUSE_CLIENT} --query "select * from simple where i = 0" 2>/dev/null CODE=$? -[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1; +[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1 echo "yes" echo "Modify max_concurrent_queries to 2" @@ -58,9 +57,9 @@ ${CLICKHOUSE_CLIENT} --query "alter table simple modify setting max_concurrent_q # Now queries are throttled again echo "Check if another query with less marks to read is throttled" -${CLICKHOUSE_CLIENT} --query "select * from simple where i = 0" 2> /dev/null; +${CLICKHOUSE_CLIENT} --query "select * from simple where i = 0" 2>/dev/null CODE=$? -[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1; +[ "$CODE" -ne "202" ] && echo "Expected error code: 202 but got: $CODE" && exit 1 echo "yes" ${CLICKHOUSE_CLIENT} --query "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null" @@ -70,10 +69,9 @@ wait query_id=max_concurrent_queries_$RANDOM ${CLICKHOUSE_CLIENT} --query_id "$query_id" --query "select i from simple where j in (select i from simple where i < 10)" -# We have to grep the server's error log because the following warning message +# We have to search the server's error log because the following warning message # is generated during pipeline destruction and thus is not sent to the client. -grep -E -q "{$query_id} .*We have query_id removed but it's not recorded. This is a bug" /var/log/clickhouse-server/clickhouse-server.err.log && exit 1 +${CLICKHOUSE_CLIENT} --query "system flush logs" +if [[ $(${CLICKHOUSE_CLIENT} --query "select count() > 0 from system.text_log where query_id = '$query_id' and level = 'Warning' and message like '%We have query_id removed but it\'s not recorded. This is a bug%' format TSVRaw") == 1 ]]; then echo "We have query_id removed but it's not recorded. This is a bug." >&2; exit 1; fi -${CLICKHOUSE_CLIENT} --multiline --multiquery --query " -drop table simple -" +${CLICKHOUSE_CLIENT} --query "drop table simple" From 9ad919d91a8da739b1aa8a3d708b608f34dbf583 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 11 Oct 2021 13:25:54 +0300 Subject: [PATCH 071/126] More timeouts in stress test --- docker/test/stress/stress | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 8fc4ade2da6..1559b084565 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -71,42 +71,42 @@ def prepare_for_hung_check(drop_databases): # FIXME this function should not exist, but... # ThreadFuzzer significantly slows down server and causes false-positive hung check failures - call("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'", shell=True, stderr=STDOUT, timeout=30) # We attach gdb to clickhouse-server before running tests # to print stacktraces of all crashes even if clickhouse cannot print it for some reason. # However, it obstruct checking for hung queries. logging.info("Will terminate gdb (if any)") - call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT) + call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT, timeout=30) # Some tests set too low memory limit for default user and forget to reset in back. # It may cause SYSTEM queries to fail, let's disable memory limit. - call("clickhouse client --max_memory_usage_for_user=0 -q 'SELECT 1 FORMAT Null'", shell=True, stderr=STDOUT) + call("clickhouse client --max_memory_usage_for_user=0 -q 'SELECT 1 FORMAT Null'", shell=True, stderr=STDOUT, timeout=30) # Some tests execute SYSTEM STOP MERGES or similar queries. # It may cause some ALTERs to hang. # Possibly we should fix tests and forbid to use such queries without specifying table. - call("clickhouse client -q 'SYSTEM START MERGES'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START TTL MERGES'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START MOVES'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START FETCHES'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT) - call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START MERGES'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START TTL MERGES'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START MOVES'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START FETCHES'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT, timeout=30) + call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT, timeout=30) # Issue #21004, live views are experimental, so let's just suppress it - call("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """, shell=True, stderr=STDOUT) + call("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """, shell=True, stderr=STDOUT, timeout=30) # Kill other queries which known to be slow # It's query from 01232_preparing_sets_race_condition_long, it may take up to 1000 seconds in slow builds - call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'insert into tableB select %'" """, shell=True, stderr=STDOUT) + call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'insert into tableB select %'" """, shell=True, stderr=STDOUT, timeout=30) # Long query from 00084_external_agregation - call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'" """, shell=True, stderr=STDOUT) + call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'" """, shell=True, stderr=STDOUT, timeout=30) if drop_databases: # Here we try to drop all databases in async mode. If some queries really hung, than drop will hung too. # Otherwise we will get rid of queries which wait for background pool. It can take a long time on slow builds (more than 900 seconds). - databases = check_output('clickhouse client -q "SHOW DATABASES"', shell=True).decode('utf-8').strip().split() + databases = check_output('clickhouse client -q "SHOW DATABASES"', shell=True, timeout=30).decode('utf-8').strip().split() for db in databases: if db == "system": continue @@ -117,13 +117,13 @@ def prepare_for_hung_check(drop_databases): # Wait for last queries to finish if any, not longer than 300 seconds call("""clickhouse client -q "select sleepEachRow(( select maxOrDefault(300 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 300 - ) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT) + ) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT, timeout=30) # Even if all clickhouse-test processes are finished, there are probably some sh scripts, # which still run some new queries. Let's ignore them. try: query = """clickhouse client -q "SELECT count() FROM system.processes where where elapsed > 300" """ - output = check_output(query, shell=True, stderr=STDOUT).decode('utf-8').strip() + output = check_output(query, shell=True, stderr=STDOUT, timeout=30).decode('utf-8').strip() if int(output) == 0: return False except: @@ -176,6 +176,7 @@ if __name__ == "__main__": if res != 0 and have_long_running_queries: logging.info("Hung check failed with exit code {}".format(res)) hung_check_status = "Hung check failed\tFAIL\n" - open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status) + with open(os.path.join(args.output_folder, "test_results.tsv"), 'w+') as results: + results.write(hung_check_status) logging.info("Stress test finished") From 83717b7c3b7e64ad752f53d2041bc8360e609715 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 29 Sep 2021 18:05:57 +0800 Subject: [PATCH 072/126] Get rid of naming limitation of projections. --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 8 ++++++-- src/Storages/MergeTree/MergeTask.cpp | 16 +++++++++++----- src/Storages/MergeTree/MergeTask.h | 6 +++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- .../MergeTree/MergeTreeDataMergerMutator.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +--- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 8 ++++---- src/Storages/MergeTree/checkDataPart.cpp | 8 +++++--- src/Storages/ProjectionsDescription.cpp | 17 +++++++---------- 10 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index dc2c5f8185d..6ac53c68e84 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -439,9 +439,13 @@ void IMergeTreeDataPart::removeIfNeeded() if (file_name.empty()) throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); - if (!startsWith(file_name, "tmp")) + if (!startsWith(file_name, "tmp") && !endsWith(file_name, ".tmp_proj")) { - LOG_ERROR(storage.log, "~DataPart() should remove part {} but its name doesn't start with tmp. Too suspicious, keeping the part.", path); + LOG_ERROR( + storage.log, + "~DataPart() should remove part {} but its name doesn't start with \"tmp\" or end with \".tmp_proj\". Too " + "suspicious, keeping the part.", + path); return; } } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 357659b3bbb..c6e8dafd8b0 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -89,7 +89,10 @@ static void extractMergingAndGatheringColumns( bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() { - const String local_tmp_prefix = global_ctx->parent_part ? ctx->prefix : "tmp_merge_"; + // projection parts have different prefix and suffix compared to normal parts. + // E.g. `proj_a.proj` for a normal projection merge and `proj_a.tmp_proj` for a projection materialization merge. + const String local_tmp_prefix = global_ctx->parent_part ? "" : "tmp_merge_"; + const String local_tmp_suffix = global_ctx->parent_part ? ctx->suffix : ""; if (global_ctx->merges_blocker->isCancelled()) throw Exception("Cancelled merging parts", ErrorCodes::ABORTED); @@ -114,7 +117,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() } ctx->disk = global_ctx->space_reservation->getDisk(); - auto local_new_part_tmp_path = global_ctx->data->relative_data_path + local_tmp_prefix + global_ctx->future_part->name + (global_ctx->parent_part ? ".proj" : "") + "/"; + auto local_new_part_relative_tmp_path = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix + "/"; + auto local_new_part_tmp_path = global_ctx->data->relative_data_path + local_new_part_relative_tmp_path; if (ctx->disk->exists(local_new_part_tmp_path)) throw Exception("Directory " + fullPath(ctx->disk, local_new_part_tmp_path) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); @@ -138,7 +142,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->future_part->type, global_ctx->future_part->part_info, local_single_disk_volume, - local_tmp_prefix + global_ctx->future_part->name + (global_ctx->parent_part ? ".proj" : ""), + local_new_part_relative_tmp_path, global_ctx->parent_part); global_ctx->new_data_part->uuid = global_ctx->future_part->uuid; @@ -526,7 +530,9 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c auto projection_future_part = std::make_shared(); projection_future_part->assign(std::move(projection_parts)); projection_future_part->name = projection.name; - projection_future_part->path = global_ctx->future_part->path + "/" + projection.name + ".proj/"; + // TODO (ab): path in future_part is only for merge process introspection, which is not available for merges of projection parts. + // Let's comment this out to avoid code inconsistency and add it back after we implement projection merge introspection. + // projection_future_part->path = global_ctx->future_part->path + "/" + projection.name + ".proj/"; projection_future_part->part_info = {"all", 0, 0, 0}; MergeTreeData::MergingParams projection_merging_params; @@ -553,7 +559,7 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c global_ctx->deduplicate_by_columns, projection_merging_params, global_ctx->new_data_part.get(), - "", // empty string for projection + ".proj", global_ctx->data, global_ctx->merges_blocker, global_ctx->ttl_merges_blocker)); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 05903f94c91..22dc70bd78c 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -58,7 +58,7 @@ public: Names deduplicate_by_columns_, MergeTreeData::MergingParams merging_params_, const IMergeTreeDataPart * parent_part_, - String prefix_, + String suffix_, MergeTreeData * data_, ActionBlocker * merges_blocker_, ActionBlocker * ttl_merges_blocker_) @@ -83,7 +83,7 @@ public: auto prepare_stage_ctx = std::make_shared(); - prepare_stage_ctx->prefix = std::move(prefix_); + prepare_stage_ctx->suffix = std::move(suffix_); prepare_stage_ctx->merging_params = std::move(merging_params_); (*stages.begin())->setRuntimeContext(std::move(prepare_stage_ctx), global_ctx); @@ -170,7 +170,7 @@ private: struct ExecuteAndFinalizeHorizontalPartRuntimeContext : public IStageRuntimeContext //-V730 { /// Dependencies - String prefix; + String suffix; MergeTreeData::MergingParams merging_params{}; DiskPtr tmp_disk{nullptr}; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 903f4cd27fc..5d97c64b49b 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -428,7 +428,7 @@ MergeTaskPtr MergeTreeDataMergerMutator::mergePartsToTemporaryPart( const Names & deduplicate_by_columns, const MergeTreeData::MergingParams & merging_params, const IMergeTreeDataPart * parent_part, - const String & prefix) + const String & suffix) { return std::make_shared( future_part, @@ -442,7 +442,7 @@ MergeTaskPtr MergeTreeDataMergerMutator::mergePartsToTemporaryPart( deduplicate_by_columns, merging_params, parent_part, - prefix, + suffix, &data, &merges_blocker, &ttl_merges_blocker); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 9eb91d7fbf8..22650ac4eca 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -108,7 +108,7 @@ public: const Names & deduplicate_by_columns, const MergeTreeData::MergingParams & merging_params, const IMergeTreeDataPart * parent_part = nullptr, - const String & prefix = ""); + const String & suffix = ""); /// Mutate a single data part with the specified commands. Will create and return a temporary part. MutateTaskPtr mutatePartToTemporaryPart( diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 03d76a7f79b..77a91af037e 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -173,9 +173,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( auto projection_plan = std::make_unique(); if (query_info.projection->desc->is_minmax_count_projection) { - Pipe pipe(std::make_shared( - query_info.minmax_count_projection_block.cloneEmpty(), - Chunk(query_info.minmax_count_projection_block.getColumns(), query_info.minmax_count_projection_block.rows()))); + Pipe pipe(std::make_shared(query_info.minmax_count_projection_block)); auto read_from_pipe = std::make_unique(std::move(pipe)); projection_plan->addStep(std::move(read_from_pipe)); } diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index d939312c0bb..752f85a1290 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -575,7 +575,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempProjectionPart( return writeProjectionPartImpl( part_name, part_type, - "tmp_insert_" + part_name + ".proj" /* relative_path */, + part_name + ".tmp_proj" /* relative_path */, true /* is_temp */, parent_part, data, diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index b8941fc9d84..fbc2f58d424 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -654,7 +654,7 @@ public: {}, projection_merging_params, ctx->new_data_part.get(), - "tmp_merge_"); + ".tmp_proj"); next_level_parts.push_back(executeHere(tmp_part_merge_task)); @@ -832,8 +832,8 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() auto projection_block = projection_squash.add({}); if (projection_block) { - projection_parts[projection.name].emplace_back( - MergeTreeDataWriter::writeTempProjectionPart(*ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); + projection_parts[projection.name].emplace_back(MergeTreeDataWriter::writeTempProjectionPart( + *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); } } @@ -1082,7 +1082,7 @@ private: if (!ctx->disk->isDirectory(it->path())) ctx->disk->createHardLink(it->path(), destination); - else if (!startsWith("tmp_", it->name())) // ignore projection tmp merge dir + else if (!endsWith(".tmp_proj", it->name())) // ignore projection tmp merge dir { // it's a projection part directory ctx->disk->createDirectories(destination); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 8a234833da7..0af395fd1bd 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -102,7 +102,7 @@ IMergeTreeDataPart::Checksums checkDataPart( /// It also calculates checksum of projections. auto checksum_file = [&](const String & file_path, const String & file_name) { - if (disk->isDirectory(file_path) && endsWith(file_name, ".proj") && !startsWith(file_name, "tmp_")) // ignore projection tmp merge dir + if (disk->isDirectory(file_path) && endsWith(file_name, ".proj")) { auto projection_name = file_name.substr(0, file_name.size() - sizeof(".proj") + 1); auto pit = data_part->getProjectionParts().find(projection_name); @@ -124,7 +124,8 @@ IMergeTreeDataPart::Checksums checkDataPart( auto file_buf = disk->readFile(proj_path); HashingReadBuffer hashing_buf(*file_buf); hashing_buf.ignoreAll(); - projection_checksums_data.files[MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION] = IMergeTreeDataPart::Checksums::Checksum(hashing_buf.count(), hashing_buf.getHash()); + projection_checksums_data.files[MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION] + = IMergeTreeDataPart::Checksums::Checksum(hashing_buf.count(), hashing_buf.getHash()); } else { @@ -140,7 +141,8 @@ IMergeTreeDataPart::Checksums checkDataPart( [&](const ISerialization::SubstreamPath & substream_path) { String projection_file_name = ISerialization::getFileNameForStream(projection_column, substream_path) + ".bin"; - checksums_data.files[projection_file_name] = checksum_compressed_file(disk, projection_path + projection_file_name); + checksums_data.files[projection_file_name] + = checksum_compressed_file(disk, projection_path + projection_file_name); }, {}); } diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index c0b96bd9f54..42294b8152c 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -89,9 +89,6 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const if (projection_definition->name.empty()) throw Exception("Projection must have name in definition.", ErrorCodes::INCORRECT_QUERY); - if (startsWith(projection_definition->name, "tmp_")) - throw Exception("Projection's name cannot start with 'tmp_'", ErrorCodes::INCORRECT_QUERY); - if (!projection_definition->query) throw Exception("QUERY is required for projection", ErrorCodes::INCORRECT_QUERY); @@ -220,13 +217,13 @@ void ProjectionDescription::recalculateWithNewColumns(const ColumnsDescription & Block ProjectionDescription::calculate(const Block & block, ContextPtr context) const { auto builder = InterpreterSelectQuery( - query_ast, - context, - Pipe(std::make_shared(block, Chunk(block.getColumns(), block.rows()))), - SelectQueryOptions{ - type == ProjectionDescription::Type::Normal ? QueryProcessingStage::FetchColumns - : QueryProcessingStage::WithMergeableState}) - .buildQueryPipeline(); + query_ast, + context, + Pipe(std::make_shared(block, Chunk(block.getColumns(), block.rows()))), + SelectQueryOptions{ + type == ProjectionDescription::Type::Normal ? QueryProcessingStage::FetchColumns + : QueryProcessingStage::WithMergeableState}) + .buildQueryPipeline(); builder.resize(1); builder.addTransform(std::make_shared(builder.getHeader(), block.rows(), 0)); From b0d887a0fef89fb529cff4f7c02cfab8cf75c280 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 11 Oct 2021 14:00:10 +0300 Subject: [PATCH 073/126] Added tests --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 + ...28_system_data_skipping_indices_size.reference | 1 + .../2028_system_data_skipping_indices_size.sql | 15 +++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/2028_system_data_skipping_indices_size.reference create mode 100644 tests/queries/0_stateless/2028_system_data_skipping_indices_size.sql diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 1a6290580a0..0f701cc4adf 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1452,6 +1452,7 @@ void IMergeTreeDataPart::calculateSecondaryIndicesSizesOnDisk() auto index_file_name = index_name_escaped + index_ptr->getSerializedFileExtension(); auto index_marks_file_name = index_name_escaped + index_granularity_info.marks_file_extension; + /// If part does not contain index auto bin_checksum = checksums.files.find(index_file_name); if (bin_checksum != checksums.files.end()) { diff --git a/tests/queries/0_stateless/2028_system_data_skipping_indices_size.reference b/tests/queries/0_stateless/2028_system_data_skipping_indices_size.reference new file mode 100644 index 00000000000..d0378511850 --- /dev/null +++ b/tests/queries/0_stateless/2028_system_data_skipping_indices_size.reference @@ -0,0 +1 @@ +default test_table value_index minmax value 1 38 12 24 diff --git a/tests/queries/0_stateless/2028_system_data_skipping_indices_size.sql b/tests/queries/0_stateless/2028_system_data_skipping_indices_size.sql new file mode 100644 index 00000000000..e77f88aa36f --- /dev/null +++ b/tests/queries/0_stateless/2028_system_data_skipping_indices_size.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test_table; + +CREATE TABLE test_table +( + key UInt64, + value String, + INDEX value_index value TYPE minmax GRANULARITY 1 +) +Engine=MergeTree() +ORDER BY key; + +INSERT INTO test_table VALUES (0, 'Value'); +SELECT * FROM system.data_skipping_indices WHERE database = currentDatabase(); + +DROP TABLE test_table; From 95a69b9f4b6478f902c7a970296a8c5b2c193a9c Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 11 Oct 2021 14:05:52 +0300 Subject: [PATCH 074/126] Make test non endless --- .../01509_check_many_parallel_quorum_inserts_long.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh index c2682cd8cfa..6533eeb12f5 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh @@ -18,9 +18,10 @@ for i in $(seq 1 $NUM_REPLICAS); do done function thread { - while true - do + i=0 retries=300 + while [[ $i -lt $retries ]]; do # server can be dead $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 1 --query "INSERT INTO r$1 SELECT $2" && break + ((++i)) sleep 0.1 done } From 72bccaa50141cd1206d1d064bc2d767a68cb9f99 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 11 Oct 2021 19:12:08 +0800 Subject: [PATCH 075/126] Fix path name --- src/Storages/MergeTree/IMergeTreeDataPart.h | 1 + src/Storages/MergeTree/MergeTask.cpp | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index be48aed5c8b..b74b2ca3321 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -175,6 +175,7 @@ public: /// A directory path (relative to storage's path) where part data is actually stored /// Examples: 'detached/tmp_fetch_', 'tmp_', '' + /// NOTE: Cannot have trailing slash. mutable String relative_path; MergeTreeIndexGranularityInfo index_granularity_info; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index c6e8dafd8b0..aa3f91a4f00 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -117,8 +117,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() } ctx->disk = global_ctx->space_reservation->getDisk(); - auto local_new_part_relative_tmp_path = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix + "/"; - auto local_new_part_tmp_path = global_ctx->data->relative_data_path + local_new_part_relative_tmp_path; + auto local_new_part_relative_tmp_path_name = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; + auto local_new_part_tmp_path = global_ctx->data->relative_data_path + local_new_part_relative_tmp_path_name + "/"; if (ctx->disk->exists(local_new_part_tmp_path)) throw Exception("Directory " + fullPath(ctx->disk, local_new_part_tmp_path) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); @@ -142,7 +142,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->future_part->type, global_ctx->future_part->part_info, local_single_disk_volume, - local_new_part_relative_tmp_path, + local_new_part_relative_tmp_path_name, global_ctx->parent_part); global_ctx->new_data_part->uuid = global_ctx->future_part->uuid; From 59a78830f9b73f9ff366449874047828019ad58e Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 11 Oct 2021 14:46:01 +0300 Subject: [PATCH 076/126] Better timeouts in clickhouse-test --- tests/clickhouse-test | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f10e38b87e5..061333297e2 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -89,10 +89,13 @@ def make_clickhouse_client(base_args): # hence we should use 'system'. database='system', settings=get_additional_client_options_dict(base_args)) + def clickhouse_execute_one(base_args, *args, **kwargs): return make_clickhouse_client(base_args).execute_one(*args, **kwargs) + def clickhouse_execute(base_args, *args, **kwargs): return make_clickhouse_client(base_args).execute(*args, **kwargs) + def clickhouse_execute_pandas(base_args, *args, **kwargs): return make_clickhouse_client(base_args).execute_pandas(*args, **kwargs) @@ -109,6 +112,7 @@ def stop_tests(): global restarted_tests with stop_tests_triggered_lock: + print("Stopping tests") if not stop_tests_triggered.is_set(): stop_tests_triggered.set() @@ -875,7 +879,7 @@ def run_tests_array(all_tests_with_params): while True: if is_concurrent: - case = queue.get() + case = queue.get(timeout=args.timeout) if not case: break else: @@ -1076,10 +1080,10 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel): pool.map_async(run_tests_array, parallel_tests_array) for suit in test_suite.parallel_tests: - queue.put(suit) + queue.put(suit, timeout=args.timeout) for _ in range(jobs): - queue.put(None) + queue.put(None, timeout=args.timeout) queue.close() From f4269ce41750648e3be629b08ce73e38afa273ae Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 8 Sep 2021 20:10:49 +0800 Subject: [PATCH 077/126] Allow optimize_arithmetic_operations_in_aggregate_functions when alias is used. --- .../ArithmeticOperationsInAgrFuncOptimize.cpp | 12 +++++++----- ...etic_operations_in_aggr_func_with_alias.reference | 10 ++++++++++ ...arithmetic_operations_in_aggr_func_with_alias.sql | 4 ++++ .../0_stateless/01470_columns_transformers.reference | 4 ++-- 4 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.reference create mode 100644 tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql diff --git a/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp b/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp index bdd9ce32707..70a58971d3f 100644 --- a/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp +++ b/src/Interpreters/ArithmeticOperationsInAgrFuncOptimize.cpp @@ -107,10 +107,7 @@ ASTPtr tryExchangeFunctions(const ASTFunction & func) || !supported.find(lower_name)->second.count(child_func->name)) return {}; - /// Cannot rewrite function with alias cause alias could become undefined - if (!func.tryGetAlias().empty() || !child_func->tryGetAlias().empty()) - return {}; - + auto original_alias = func.tryGetAlias(); const auto & child_func_args = child_func->arguments->children; const auto * first_literal = child_func_args[0]->as(); const auto * second_literal = child_func_args[1]->as(); @@ -132,7 +129,12 @@ ASTPtr tryExchangeFunctions(const ASTFunction & func) optimized_ast = exchangeExtractSecondArgument(new_name, *child_func); } - return optimized_ast; + if (optimized_ast) + { + optimized_ast->setAlias(original_alias); + return optimized_ast; + } + return {}; } } diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.reference b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.reference new file mode 100644 index 00000000000..9e0d871041b --- /dev/null +++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.reference @@ -0,0 +1,10 @@ +SELECT min(n AS a) + (1 AS b) AS c +FROM +( + SELECT number AS n + FROM numbers(10) + WHERE (1 > 0) AND (n > 0) +) +WHERE (a > 0) AND (b > 0) +HAVING c > 0 +2 diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql new file mode 100644 index 00000000000..73b87817bb3 --- /dev/null +++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql @@ -0,0 +1,4 @@ +set optimize_arithmetic_operations_in_aggregate_functions = 1; + +explain syntax select min((n as a) + (1 as b)) c from (select number n from numbers(10)) where a > 0 and b > 0 having c > 0; +select min((n as a) + (1 as b)) c from (select number n from numbers(10)) where a > 0 and b > 0 having c > 0; diff --git a/tests/queries/0_stateless/01470_columns_transformers.reference b/tests/queries/0_stateless/01470_columns_transformers.reference index ae0adb3ba60..8fa86582018 100644 --- a/tests/queries/0_stateless/01470_columns_transformers.reference +++ b/tests/queries/0_stateless/01470_columns_transformers.reference @@ -54,8 +54,8 @@ SELECT sum(k) FROM columns_transformers SELECT - avg(i + 1 AS i), - avg(j + 2 AS j), + avg(i) + 1, + avg(j) + 2, avg(k) FROM columns_transformers SELECT From 635783fb663267180f25e92637b7915399b884b6 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 24 Sep 2021 00:23:17 +0800 Subject: [PATCH 078/126] Only do TreeOptimizer for initial queries --- src/Interpreters/TreeOptimizer.cpp | 4 ---- src/Interpreters/TreeRewriter.cpp | 8 +++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 3236418fe6f..8fb72f74c65 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -710,9 +709,6 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, if (settings.optimize_arithmetic_operations_in_aggregate_functions) optimizeAggregationFunctions(query); - /// Push the predicate expression down to the subqueries. - result.rewrite_subqueries = PredicateExpressionsOptimizer(context, tables_with_columns, settings).optimize(*select_query); - /// GROUP BY injective function elimination. optimizeGroupBy(select_query, result.source_columns_set, context); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 8f923d82b27..9bcddb6b982 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -1036,7 +1037,12 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (settings.legacy_column_name_of_tuple_literal) markTupleLiteralsAsLegacy(query); - TreeOptimizer::apply(query, result, tables_with_columns, getContext()); + /// Push the predicate expression down to subqueries. The optimization should be applied to both initial and secondary queries. + result.rewrite_subqueries = PredicateExpressionsOptimizer(getContext(), tables_with_columns, settings).optimize(*select_query); + + /// Only apply AST optimization for initial queries. + if (getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + TreeOptimizer::apply(query, result, tables_with_columns, getContext()); /// array_join_alias_to_name, array_join_result_to_source. getArrayJoinedColumns(query, result, select_query, result.source_columns, source_columns_set); From 3ae960e04b7675b54d5e386573c387c72ad1e5cd Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 11 Oct 2021 16:40:12 +0300 Subject: [PATCH 079/126] Review fixes --- docker/test/stress/stress | 2 +- tests/clickhouse-test | 6 +++--- ...4_cancel_http_readonly_queries_on_client_close.sh | 5 +++-- .../0_stateless/01085_max_distributed_connections.sh | 5 +++-- .../01085_max_distributed_connections_http.sh | 5 +++-- .../0_stateless/01600_quota_by_forwarded_ip.sh | 12 +++++++++--- .../0_stateless/01602_max_distributed_connections.sh | 8 ++++++-- .../queries/0_stateless/01675_data_type_coroutine.sh | 6 ++++-- .../0_stateless/01681_hyperscan_debug_assertion.sh | 5 +++-- .../01834_alias_columns_laziness_filimonov.sh | 5 +++-- tests/queries/0_stateless/02044_url_glob_parallel.sh | 6 ++++-- 11 files changed, 42 insertions(+), 23 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 1559b084565..5e98c67d8e1 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -117,7 +117,7 @@ def prepare_for_hung_check(drop_databases): # Wait for last queries to finish if any, not longer than 300 seconds call("""clickhouse client -q "select sleepEachRow(( select maxOrDefault(300 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 300 - ) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT, timeout=30) + ) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT, timeout=330) # Even if all clickhouse-test processes are finished, there are probably some sh scripts, # which still run some new queries. Let's ignore them. diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 061333297e2..62860a36fc7 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -879,7 +879,7 @@ def run_tests_array(all_tests_with_params): while True: if is_concurrent: - case = queue.get(timeout=args.timeout) + case = queue.get(timeout=args.timeout * 1.1) if not case: break else: @@ -1080,10 +1080,10 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel): pool.map_async(run_tests_array, parallel_tests_array) for suit in test_suite.parallel_tests: - queue.put(suit, timeout=args.timeout) + queue.put(suit, timeout=args.timeout * 1.1) for _ in range(jobs): - queue.put(None, timeout=args.timeout) + queue.put(None, timeout=args.timeout * 1.1) queue.close() diff --git a/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh b/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh index 340df58e473..74b4c4052f8 100755 --- a/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh +++ b/tests/queries/0_stateless/00834_cancel_http_readonly_queries_on_client_close.sh @@ -7,9 +7,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} --max-time 1 -sS "${CLICKHOUSE_URL}&query_id=cancel_http_readonly_queries_on_client_close&cancel_http_readonly_queries_on_client_close=1&query=SELECT+count()+FROM+system.numbers" 2>&1 | grep -cF 'curl: (28)' -while true -do +i=0 retries=300 +while [[ $i -lt $retries ]]; do ${CLICKHOUSE_CURL} -sS --data "SELECT count() FROM system.processes WHERE query_id = 'cancel_http_readonly_queries_on_client_close'" "${CLICKHOUSE_URL}" | grep '0' && break + ((++i)) sleep 0.2 done diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.sh b/tests/queries/0_stateless/01085_max_distributed_connections.sh index 4ffcd980956..34862289d1e 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections.sh @@ -5,10 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +i=0 retries=300 # Sometimes five seconds are not enough due to system overload. # But if it can run in less than five seconds at least sometimes - it is enough for the test. -while true -do +while [[ $i -lt $retries ]]; do opts=( --max_distributed_connections 20 --max_threads 1 @@ -19,4 +19,5 @@ do # "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break + ((++i)) done diff --git a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh index 3edf70f31b8..0e40918257d 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh @@ -8,9 +8,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Sometimes 1.8 seconds are not enough due to system overload. # But if it can run in less than five seconds at least sometimes - it is enough for the test. -while true -do +i=0 retries=100 +while [[ $i -lt $retries ]]; do query="SELECT sleepEachRow(1) FROM remote('127.{2,3}', system.one) FORMAT Null" # 1.8 less then 2 seconds, but long enough to cover possible load peaks timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" && break + ((++i)) done diff --git a/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh b/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh index 97e4da5f9e3..1d768c8b027 100755 --- a/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh +++ b/tests/queries/0_stateless/01600_quota_by_forwarded_ip.sh @@ -21,7 +21,9 @@ CREATE QUOTA quota_by_forwarded_ip_${CLICKHOUSE_DATABASE} KEYED BY forwarded_ip_ echo '--- Test with quota by immediate IP ---' -while true; do +i=0 retries=300 +while [[ $i -lt $retries ]]; do + ((++i)) ${CLICKHOUSE_CURL} --fail -sS "${CLICKHOUSE_URL}&user=quoted_by_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" 2>/dev/null || break done | uniq @@ -33,14 +35,18 @@ ${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 1.2.3.4' -sS "${CLICKHOUSE_URL}&user=quo echo '--- Test with quota by forwarded IP ---' -while true; do +i=0 retries=300 +while [[ $i -lt $retries ]]; do + ((++i)) ${CLICKHOUSE_CURL} --fail -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" 2>/dev/null || break done | uniq ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" | grep -oF 'exceeded' +i=0 retries=300 # X-Forwarded-For is respected for quota by forwarded IP address -while true; do +while [[ $i -lt $retries ]]; do + ((++i)) ${CLICKHOUSE_CURL} -H 'X-Forwarded-For: 1.2.3.4' -sS "${CLICKHOUSE_URL}&user=quoted_by_forwarded_ip_${CLICKHOUSE_DATABASE}" -d "SELECT count() FROM numbers(10)" | grep -oP '^10$' || break done | uniq diff --git a/tests/queries/0_stateless/01602_max_distributed_connections.sh b/tests/queries/0_stateless/01602_max_distributed_connections.sh index 51ff803ad5e..ed835a8768f 100755 --- a/tests/queries/0_stateless/01602_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01602_max_distributed_connections.sh @@ -13,14 +13,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # If concurrency is 10 (good), the query may take less than 10 second with non-zero probability # and the following loops will finish with probability 1 assuming independent random variables. -while true; do +i=0 retries=30 +while [[ $i -lt $retries ]]; do timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && break + ((++i)) done -while true; do +i=0 retries=30 +while [[ $i -lt $retries ]]; do timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && break + ((++i)) done # If max_distributed_connections is low and async_socket_for_remote is disabled, diff --git a/tests/queries/0_stateless/01675_data_type_coroutine.sh b/tests/queries/0_stateless/01675_data_type_coroutine.sh index 781e43e4134..8e80d722a4c 100755 --- a/tests/queries/0_stateless/01675_data_type_coroutine.sh +++ b/tests/queries/0_stateless/01675_data_type_coroutine.sh @@ -4,12 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh + +counter=0 retries=60 I=0 -while true -do +while [[ $counter -lt $retries ]]; do I=$((I + 1)) TYPE=$(perl -e "print 'Array(' x $I; print 'UInt8'; print ')' x $I") ${CLICKHOUSE_CLIENT} --max_parser_depth 1000000 --query "SELECT * FROM remote('127.0.0.{1,2}', generateRandom('x $TYPE', 1, 1, 1)) LIMIT 1 FORMAT Null" 2>&1 | grep -q -F 'Maximum parse depth' && break; + ((++counter)) done #echo "I = ${I}" diff --git a/tests/queries/0_stateless/01681_hyperscan_debug_assertion.sh b/tests/queries/0_stateless/01681_hyperscan_debug_assertion.sh index 2b4cd1a5f01..62469da0b3e 100755 --- a/tests/queries/0_stateless/01681_hyperscan_debug_assertion.sh +++ b/tests/queries/0_stateless/01681_hyperscan_debug_assertion.sh @@ -13,13 +13,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) M=1000000 -while true -do +i=0 retries=300 +while [[ $i -lt $retries ]]; do $CLICKHOUSE_CLIENT --allow_hyperscan 1 --max_memory_usage $M --format Null --query " SELECT [1, 2, 3, 11] = arraySort(multiMatchAllIndices('фабрикант', ['', 'рикан', 'а', 'f[a${RANDOM}e]b[ei]rl', 'ф[иа${RANDOM}эе]б[еэи][рпл]', 'афиукд', 'a[f${RANDOM}t],th', '^ф[аие${RANDOM}э]?б?[еэи]?$', 'бе${RANDOM}рлик', 'fa${RANDOM}b', 'фа[беьв]+е?[рл${RANDOM}ко]'])) " 2>&1 | grep -q 'Memory limit' || break; M=$((M + 100000)) + ((++i)) done echo 'Ok' diff --git a/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh b/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh index 793f477b3cb..1d70ba1df7c 100755 --- a/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh +++ b/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh @@ -16,9 +16,10 @@ insert into aliases_lazyness(x) select * from numbers(40); # The exact time is not guaranteed, so we check in a loop that at least once # the query will process in less than one second, that proves that the behaviour is not like it was long time ago. -while true -do +i=0 retries=300 +while [[ $i -lt $retries ]]; do timeout 1 ${CLICKHOUSE_CLIENT} --query "SELECT x, y FROM aliases_lazyness WHERE x = 1 FORMAT Null" && break + ((++i)) done ${CLICKHOUSE_CLIENT} --multiquery --query " diff --git a/tests/queries/0_stateless/02044_url_glob_parallel.sh b/tests/queries/0_stateless/02044_url_glob_parallel.sh index 6491a661201..c9c779a9ddb 100755 --- a/tests/queries/0_stateless/02044_url_glob_parallel.sh +++ b/tests/queries/0_stateless/02044_url_glob_parallel.sh @@ -5,9 +5,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh + +i=0 retries=60 # Sometimes five seconds are not enough due to system overload. # But if it can run in less than five seconds at least sometimes - it is enough for the test. -while true -do +while [[ $i -lt $retries ]]; do timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://127.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null && break + ((++i)) done From ab4b2295749a900c803dd8620d19f8d46a7d023d Mon Sep 17 00:00:00 2001 From: Haavard Kvaalen Date: Mon, 11 Oct 2021 16:20:51 +0200 Subject: [PATCH 080/126] Make sure we update position on commit Make sure we update GTID set on QueryEvents with "COMMIT" or "XA COMMIT". Without this we could have to redo the last transaction if e.g. ClickHouse was restarted. Note that this did not affect normal transactions on InnoDB, since they are terminated with a XID_EVENT. --- src/Core/MySQL/MySQLReplication.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index b5468d15edc..b5adab67e3a 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -815,6 +815,7 @@ namespace MySQLReplication { event = std::make_shared(std::move(event_header)); event->parseEvent(event_payload); + position.update(event); auto query = std::static_pointer_cast(event); switch (query->typ) @@ -826,7 +827,7 @@ namespace MySQLReplication break; } default: - position.update(event); + break; } break; } From 362bcb2f6662c7c05731efed229d0a9273bc307f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 30 Aug 2021 14:04:59 +0300 Subject: [PATCH 081/126] Introduce ProfileEvents packet --- programs/client/Client.cpp | 4 + src/Client/ClientBase.cpp | 9 ++ src/Client/ClientBase.h | 1 + src/Client/Connection.cpp | 21 ++++ src/Client/Connection.h | 3 + src/Client/HedgedConnections.cpp | 2 + src/Client/LocalConnection.cpp | 2 + src/Client/MultiplexedConnections.cpp | 2 + src/Client/Suggest.cpp | 3 + src/Common/CurrentMetrics.h | 6 ++ src/Common/MemoryTracker.h | 5 + src/Core/Protocol.h | 3 +- src/DataStreams/ConnectionCollector.cpp | 2 + src/DataStreams/RemoteQueryExecutor.cpp | 4 + src/Server/TCPHandler.cpp | 137 ++++++++++++++++++++++++ src/Server/TCPHandler.h | 4 + 16 files changed, 207 insertions(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 04193036872..da910430985 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -15,6 +15,7 @@ #include #include #include "Client.h" +#include "Core/Protocol.h" #include #include @@ -377,6 +378,9 @@ std::vector Client::loadWarningMessages() case Protocol::Server::EndOfStream: return messages; + case Protocol::Server::ProfileEvents: + continue; + default: throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from server {}", packet.type, connection->getDescription()); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 56d9993d14b..ee5f3580050 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "Core/Protocol.h" #if !defined(ARCADIA_BUILD) # include @@ -611,6 +612,10 @@ bool ClientBase::receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled) onEndOfStream(); return false; + case Protocol::Server::ProfileEvents: + onProfileEvents(); + return true; + default: throw Exception( ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from server {}", packet.type, connection->getDescription()); @@ -651,6 +656,10 @@ void ClientBase::onEndOfStream() } +void ClientBase::onProfileEvents() +{} + + /// Flush all buffers. void ClientBase::resetOutput() { diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index b122803e1db..0fa205a4d6e 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -114,6 +114,7 @@ private: void onReceiveExceptionFromServer(std::unique_ptr && e); void onProfileInfo(const BlockStreamProfileInfo & profile_info); void onEndOfStream(); + void onProfileEvents(); void sendData(Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); void sendDataFrom(ReadBuffer & buf, Block & sample, diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 2aa157bb318..1aabe449ed5 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -870,6 +871,10 @@ Packet Connection::receivePacket() case Protocol::Server::ReadTaskRequest: return res; + case Protocol::Server::ProfileEvents: + res.block = receiveProfileEvents(); + return res; + default: /// In unknown state, disconnect - to not leave unsynchronised connection. disconnect(); @@ -923,6 +928,13 @@ Block Connection::receiveDataImpl(NativeReader & reader) } +Block Connection::receiveProfileEvents() +{ + initBlockProfileEventsInput(); + return receiveDataImpl(*block_profile_events_in); +} + + void Connection::initInputBuffers() { @@ -956,6 +968,15 @@ void Connection::initBlockLogsInput() } +void Connection::initBlockProfileEventsInput() +{ + if (!block_profile_events_in) + { + block_profile_events_in = std::make_unique(*in, server_revision); + } +} + + void Connection::setDescription() { auto resolved_address = getResolvedAddress(); diff --git a/src/Client/Connection.h b/src/Client/Connection.h index a5130d876ea..b6054941aeb 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -206,6 +206,7 @@ private: std::shared_ptr maybe_compressed_in; std::unique_ptr block_in; std::unique_ptr block_logs_in; + std::unique_ptr block_profile_events_in; /// Where to write data for INSERT. std::shared_ptr maybe_compressed_out; @@ -249,6 +250,7 @@ private: Block receiveData(); Block receiveLogData(); Block receiveDataImpl(NativeReader & reader); + Block receiveProfileEvents(); std::vector receiveMultistringMessage(UInt64 msg_type) const; std::unique_ptr receiveException() const; @@ -258,6 +260,7 @@ private: void initInputBuffers(); void initBlockInput(); void initBlockLogsInput(); + void initBlockProfileEventsInput(); [[noreturn]] void throwUnexpectedPacket(UInt64 packet_type, const char * expected) const; }; diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index b833241b2bc..1ca890f40f9 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -1,3 +1,4 @@ +#include "Core/Protocol.h" #if defined(OS_LINUX) #include @@ -412,6 +413,7 @@ Packet HedgedConnections::receivePacketFromReplica(const ReplicaLocation & repli case Protocol::Server::Totals: case Protocol::Server::Extremes: case Protocol::Server::Log: + case Protocol::Server::ProfileEvents: replica_with_last_received_packet = replica_location; break; diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 29bc0c84437..efd302622dd 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -5,6 +5,7 @@ #include #include #include +#include "Core/Protocol.h" namespace DB @@ -328,6 +329,7 @@ Packet LocalConnection::receivePacket() case Protocol::Server::Extremes: [[fallthrough]]; case Protocol::Server::Log: [[fallthrough]]; case Protocol::Server::Data: + case Protocol::Server::ProfileEvents: { if (state->block && state->block.value()) { diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index a4e1eb09253..a27f7709555 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -2,6 +2,7 @@ #include #include #include +#include "Core/Protocol.h" namespace DB @@ -320,6 +321,7 @@ Packet MultiplexedConnections::receivePacketUnlocked(AsyncCallback async_callbac case Protocol::Server::Totals: case Protocol::Server::Extremes: case Protocol::Server::Log: + case Protocol::Server::ProfileEvents: break; case Protocol::Server::EndOfStream: diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index f500332b616..38aeae76a38 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "Core/Protocol.h" #include #include #include @@ -162,6 +163,8 @@ void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & t continue; case Protocol::Server::Log: continue; + case Protocol::Server::ProfileEvents: + continue; case Protocol::Server::Exception: packet.exception->rethrow(); diff --git a/src/Common/CurrentMetrics.h b/src/Common/CurrentMetrics.h index f6f4785a95a..21c3f704872 100644 --- a/src/Common/CurrentMetrics.h +++ b/src/Common/CurrentMetrics.h @@ -41,6 +41,12 @@ namespace CurrentMetrics values[metric].store(value, std::memory_order_relaxed); } + /// Get value of specified metric. + inline Value get(Metric metric) + { + return values[metric].load(std::memory_order_relaxed); + } + /// Add value for specified metric. You must subtract value later; or see class Increment below. inline void add(Metric metric, Value value = 1) { diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index b860c611be2..7da70db0876 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -143,6 +143,11 @@ public: metric.store(metric_, std::memory_order_relaxed); } + CurrentMetrics::Metric getMetric() + { + return metric.load(std::memory_order_relaxed); + } + void setDescription(const char * description) { description_ptr.store(description, std::memory_order_relaxed); diff --git a/src/Core/Protocol.h b/src/Core/Protocol.h index 9ec792a6230..4958f343bbc 100644 --- a/src/Core/Protocol.h +++ b/src/Core/Protocol.h @@ -80,7 +80,8 @@ namespace Protocol ReadTaskRequest = 13, /// String (UUID) describes a request for which next task is needed /// This is such an inverted logic, where server sends requests /// And client returns back response - MAX = ReadTaskRequest, + ProfileEvents = 14, + MAX = ProfileEvents, }; /// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10 diff --git a/src/DataStreams/ConnectionCollector.cpp b/src/DataStreams/ConnectionCollector.cpp index 8e700c0ab7f..df206478e91 100644 --- a/src/DataStreams/ConnectionCollector.cpp +++ b/src/DataStreams/ConnectionCollector.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "Core/Protocol.h" #include namespace CurrentMetrics @@ -81,6 +82,7 @@ void ConnectionCollector::drainConnections(IConnections & connections) noexcept { case Protocol::Server::EndOfStream: case Protocol::Server::Log: + case Protocol::Server::ProfileEvents: break; case Protocol::Server::Exception: diff --git a/src/DataStreams/RemoteQueryExecutor.cpp b/src/DataStreams/RemoteQueryExecutor.cpp index 3c78fddfd39..fc2db2f3f6f 100644 --- a/src/DataStreams/RemoteQueryExecutor.cpp +++ b/src/DataStreams/RemoteQueryExecutor.cpp @@ -4,6 +4,7 @@ #include #include +#include "Core/Protocol.h" #include #include #include @@ -390,6 +391,9 @@ std::optional RemoteQueryExecutor::processPacket(Packet packet) log_queue->pushBlock(std::move(packet.block)); break; + case Protocol::Server::ProfileEvents: + break; + default: got_unknown_packet_from_replica = true; throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}", diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index f3247e7bc2b..fbb5c755142 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -812,6 +813,128 @@ void TCPHandler::sendExtremes(const Block & extremes) } +namespace +{ + using namespace ProfileEvents; + + enum ProfileEventTypes : int8_t + { + INCREMENT = 1, + GAUGE = 2, + }; + + constexpr size_t NAME_COLUMN_INDEX = 4; + constexpr size_t VALUE_COLUMN_INDEX = 5; + + /* + * Add records about provided non-zero ProfileEvents::Counters. + */ + void dumpProfileEvents( + ProfileEvents::Counters const & snapshot, + MutableColumns & columns, + String const & host_name, + time_t current_time, + UInt64 thread_id) + { + size_t rows = 0; + auto & name_column = columns[NAME_COLUMN_INDEX]; + auto & value_column = columns[VALUE_COLUMN_INDEX]; + for (ProfileEvents::Event event = 0; event < ProfileEvents::Counters::num_counters; ++event) + { + UInt64 value = snapshot[event].load(std::memory_order_relaxed); + + if (value == 0) + continue; + + const char * desc = ProfileEvents::getName(event); + name_column->insertData(desc, strlen(desc)); + value_column->insert(value); + rows++; + } + + // Fill the rest of the columns with data + for (size_t row = 0; row < rows; ++row) + { + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(current_time)); + columns[i++]->insert(UInt64{thread_id}); + columns[i++]->insert(ProfileEventTypes::INCREMENT); + } + } + + void dumpMemoryTracker( + MemoryTracker * memoryTracker, + MutableColumns & columns, + String const & host_name, + time_t current_time, + UInt64 thread_id) + { + auto metric = memoryTracker->getMetric(); + if (metric == CurrentMetrics::end()) + return; + + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(current_time)); + columns[i++]->insert(UInt64{thread_id}); + columns[i++]->insert(ProfileEventTypes::GAUGE); + + auto const * metric_name = CurrentMetrics::getName(metric); + columns[i++]->insertData(metric_name, strlen(metric_name)); + auto metric_value = CurrentMetrics::get(metric); + columns[i++]->insert(metric_value); + } +} + + +void TCPHandler::sendProfileEvents() +{ + auto thread_group = CurrentThread::getGroup(); + auto const counters_snapshot = CurrentThread::getProfileEvents().getPartiallyAtomicSnapshot(); + auto current_time = time(nullptr); + auto * memory_tracker = CurrentThread::getMemoryTracker(); + + auto const thread_id = CurrentThread::get().thread_id; + + auto profile_event_type = std::make_shared( + DataTypeEnum8::Values + { + { "increment", static_cast(INCREMENT)}, + { "gauge", static_cast(GAUGE)}, + }); + + NamesAndTypesList column_names_and_types = { + { "host_name", std::make_shared() }, + { "current_time", std::make_shared() }, + { "thread_id", std::make_shared() }, + { "type", profile_event_type }, + { "name", std::make_shared() }, + { "value", std::make_shared() }, + }; + + ColumnsWithTypeAndName temp_columns; + for (auto const & name_and_type : column_names_and_types) + temp_columns.emplace_back(name_and_type.type, name_and_type.name); + + Block block(std::move(temp_columns)); + + MutableColumns columns = block.mutateColumns(); + dumpProfileEvents(counters_snapshot, columns, server_display_name, current_time, thread_id); + dumpMemoryTracker(memory_tracker, columns, server_display_name, current_time, thread_id); + + block.setColumns(std::move(columns)); + + initProfileEventsBlockOutput(block); + + writeVarUInt(Protocol::Server::ProfileEvents, *out); + writeStringBinary("", *out); + + state.logs_block_out->write(block); + out->next(); +} + + bool TCPHandler::receiveProxyHeader() { if (in->eof()) @@ -1453,6 +1576,20 @@ void TCPHandler::initLogsBlockOutput(const Block & block) } +void TCPHandler::initProfileEventsBlockOutput(const Block & block) +{ + if (!state.profile_events_block_out) + { + const Settings & query_settings = query_context->getSettingsRef(); + state.profile_events_block_out = std::make_unique( + *out, + client_tcp_protocol_version, + block.cloneEmpty(), + !query_settings.low_cardinality_allow_in_native_format); + } +} + + bool TCPHandler::isQueryCancelled() { if (state.is_cancelled || state.sent_all_data) diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index d001b12ee66..9ff061e096b 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -48,6 +48,8 @@ struct QueryState InternalTextLogsQueuePtr logs_queue; std::unique_ptr logs_block_out; + std::unique_ptr profile_events_block_out; + /// From where to read data for INSERT. std::shared_ptr maybe_compressed_in; std::unique_ptr block_in; @@ -228,11 +230,13 @@ private: void sendProfileInfo(const BlockStreamProfileInfo & info); void sendTotals(const Block & totals); void sendExtremes(const Block & extremes); + void sendProfileEvents(); /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled. void initBlockInput(); void initBlockOutput(const Block & block); void initLogsBlockOutput(const Block & block); + void initProfileEventsBlockOutput(const Block & block); bool isQueryCancelled(); From e9b1e0546179c38fe748b5df947e96bcd95771b4 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 30 Aug 2021 18:35:25 +0300 Subject: [PATCH 082/126] Send profile events from all threads of current group --- src/Common/ThreadStatus.h | 4 +++- src/Interpreters/ThreadStatusExt.cpp | 1 + src/Server/TCPHandler.cpp | 23 +++++++++++++---------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 9e8d8f637b8..dbd0b4e5664 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace Poco @@ -41,7 +42,7 @@ struct ViewRuntimeData; class QueryViewsLog; using InternalTextLogsQueuePtr = std::shared_ptr; using InternalTextLogsQueueWeakPtr = std::weak_ptr; - +using ThreadStatusPtr = ThreadStatus *; /** Thread group is a collection of threads dedicated to single task * (query or other process like background merge). @@ -66,6 +67,7 @@ public: std::function fatal_error_callback; std::vector thread_ids; + std::unordered_set threads; /// The first thread created this thread group UInt64 master_thread_id = 0; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 465b8e31b08..81a745ef430 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -123,6 +123,7 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) /// NOTE: thread may be attached multiple times if it is reused from a thread pool. thread_group->thread_ids.emplace_back(thread_id); + thread_group->threads.insert(this); logs_queue_ptr = thread_group->logs_queue_ptr; fatal_error_callback = thread_group->fatal_error_callback; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index fbb5c755142..4c6d01c564c 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -666,6 +666,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() /// Some time passed and there is a progress. after_send_progress.restart(); sendProgress(); + sendProfileEvents(); } sendLogs(); @@ -691,6 +692,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendProfileInfo(executor.getProfileInfo()); sendProgress(); sendLogs(); + sendProfileEvents(); } if (state.is_connection_closed) @@ -867,12 +869,12 @@ namespace MemoryTracker * memoryTracker, MutableColumns & columns, String const & host_name, - time_t current_time, UInt64 thread_id) { auto metric = memoryTracker->getMetric(); if (metric == CurrentMetrics::end()) return; + time_t current_time = time(nullptr); size_t i = 0; columns[i++]->insertData(host_name.data(), host_name.size()); @@ -890,13 +892,6 @@ namespace void TCPHandler::sendProfileEvents() { - auto thread_group = CurrentThread::getGroup(); - auto const counters_snapshot = CurrentThread::getProfileEvents().getPartiallyAtomicSnapshot(); - auto current_time = time(nullptr); - auto * memory_tracker = CurrentThread::getMemoryTracker(); - - auto const thread_id = CurrentThread::get().thread_id; - auto profile_event_type = std::make_shared( DataTypeEnum8::Values { @@ -920,9 +915,17 @@ void TCPHandler::sendProfileEvents() Block block(std::move(temp_columns)); MutableColumns columns = block.mutateColumns(); - dumpProfileEvents(counters_snapshot, columns, server_display_name, current_time, thread_id); - dumpMemoryTracker(memory_tracker, columns, server_display_name, current_time, thread_id); + auto thread_group = CurrentThread::getGroup(); + for (auto * thread : thread_group->threads) + { + auto const counters_snapshot = thread->performance_counters.getPartiallyAtomicSnapshot(); + auto current_time = time(nullptr); + auto * memory_tracker = &thread->memory_tracker; + auto const thread_id = CurrentThread::get().thread_id; + dumpProfileEvents(counters_snapshot, columns, server_display_name, current_time, thread_id); + dumpMemoryTracker(memory_tracker, columns, server_display_name, thread_id); + } block.setColumns(std::move(columns)); initProfileEventsBlockOutput(block); From 4c5a77457076dff8041c3baaeb89feef6c853d6f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 30 Aug 2021 18:48:22 +0300 Subject: [PATCH 083/126] Add comment --- src/Core/Protocol.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Protocol.h b/src/Core/Protocol.h index 4958f343bbc..b2957e4ae30 100644 --- a/src/Core/Protocol.h +++ b/src/Core/Protocol.h @@ -80,7 +80,7 @@ namespace Protocol ReadTaskRequest = 13, /// String (UUID) describes a request for which next task is needed /// This is such an inverted logic, where server sends requests /// And client returns back response - ProfileEvents = 14, + ProfileEvents = 14, /// Packet with profile events from server. MAX = ProfileEvents, }; From 803b8623c1b03dcb5bd4591d67dea421cd121b2e Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 31 Aug 2021 16:50:56 +0300 Subject: [PATCH 084/126] Fix TCPHandler::sendProfileEvents --- src/Interpreters/ThreadStatusExt.cpp | 5 +++++ src/Server/TCPHandler.cpp | 13 +++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 81a745ef430..7b7bfec006c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -398,6 +399,10 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) finalizePerformanceCounters(); /// Detach from thread group + { + std::lock_guard guard(thread_group->mutex); + thread_group->threads.erase(this); + } performance_counters.setParent(&ProfileEvents::global_counters); memory_tracker.reset(); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4c6d01c564c..9c1b107c513 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1,4 +1,8 @@ +#include #include +#include +#include +#include #include #include #include @@ -916,7 +920,12 @@ void TCPHandler::sendProfileEvents() MutableColumns columns = block.mutateColumns(); auto thread_group = CurrentThread::getGroup(); - for (auto * thread : thread_group->threads) + std::vector threads; + { + std::lock_guard guard(thread_group->mutex); + std::copy(thread_group->threads.begin(), thread_group->threads.end(), std::back_inserter(threads)); + } + for (auto * thread : threads) { auto const counters_snapshot = thread->performance_counters.getPartiallyAtomicSnapshot(); auto current_time = time(nullptr); @@ -933,7 +942,7 @@ void TCPHandler::sendProfileEvents() writeVarUInt(Protocol::Server::ProfileEvents, *out); writeStringBinary("", *out); - state.logs_block_out->write(block); + state.profile_events_block_out->write(block); out->next(); } From 74cdaba7fa90bf0e87c24c7c6a6c384249f33d5c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 1 Sep 2021 17:47:12 +0300 Subject: [PATCH 085/126] WIP on profile events forwarding --- src/DataStreams/RemoteQueryExecutor.cpp | 1 + src/Server/GRPCServer.cpp | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/DataStreams/RemoteQueryExecutor.cpp b/src/DataStreams/RemoteQueryExecutor.cpp index fc2db2f3f6f..dc97e577513 100644 --- a/src/DataStreams/RemoteQueryExecutor.cpp +++ b/src/DataStreams/RemoteQueryExecutor.cpp @@ -392,6 +392,7 @@ std::optional RemoteQueryExecutor::processPacket(Packet packet) break; case Protocol::Server::ProfileEvents: + /// Pass profile events from remote server to client break; default: diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index cc3c7085dfd..fc712916372 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -596,6 +596,7 @@ namespace void addExtremesToResult(const Block & extremes); void addProfileInfoToResult(const BlockStreamProfileInfo & info); void addLogsToResult(); + void addProfileEventsToResult(); void sendResult(); void throwIfFailedToSendResult(); void sendException(const Exception & exception); @@ -1123,6 +1124,7 @@ namespace if (after_send_progress.elapsedMicroseconds() >= interactive_delay) { addProgressToResult(); + addProfileEventsToResult(); after_send_progress.restart(); } @@ -1174,6 +1176,7 @@ namespace finalize = true; io.onFinish(); addProgressToResult(); + addProfileEventsToResult(); query_scope->logPeakMemoryUsage(); addLogsToResult(); sendResult(); @@ -1437,6 +1440,11 @@ namespace } } + void Call::addProfileEventsToResult() + { + + } + void Call::sendResult() { /// gRPC doesn't allow to write anything to a finished responder. From 356723427df530d834dd9f0bae1977b95f1ccc84 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 2 Sep 2021 17:27:19 +0300 Subject: [PATCH 086/126] WIP on ProfileEvents forwarding --- src/Client/Connection.cpp | 2 ++ src/Common/CurrentThread.cpp | 18 +++++++++++ src/Common/CurrentThread.h | 3 ++ src/Common/ThreadStatus.cpp | 12 ++++++++ src/Common/ThreadStatus.h | 15 +++++++++ src/Core/Protocol.h | 3 +- src/DataStreams/RemoteQueryExecutor.cpp | 9 +++++- src/Interpreters/ThreadStatusExt.cpp | 1 + src/Server/TCPHandler.cpp | 41 ++++++++++++++++++++----- src/Server/TCPHandler.h | 1 + 10 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 1aabe449ed5..c6badf96bf9 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -22,6 +22,7 @@ #include #include #include +#include "Core/Block.h" #include #include #include @@ -872,6 +873,7 @@ Packet Connection::receivePacket() return res; case Protocol::Server::ProfileEvents: + LOG_DEBUG(log_wrapper.get(), "Connection received ProfileEvents"); res.block = receiveProfileEvents(); return res; diff --git a/src/Common/CurrentThread.cpp b/src/Common/CurrentThread.cpp index c6b9e027c48..10d9f4d07df 100644 --- a/src/Common/CurrentThread.cpp +++ b/src/Common/CurrentThread.cpp @@ -91,6 +91,24 @@ std::shared_ptr CurrentThread::getInternalTextLogsQueue() return current_thread->getInternalTextLogsQueue(); } +void CurrentThread::attachInternalProfileEventsQueue(const InternalProfileEventsQueuePtr & queue) +{ + if (unlikely(!current_thread)) + return; + current_thread->attachInternalProfileEventsQueue(queue); +} + +InternalProfileEventsQueuePtr CurrentThread::getInternalProfileEventsQueue() +{ + if (unlikely(!current_thread)) + return nullptr; + + if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died) + return nullptr; + + return current_thread->getInternalProfileEventsQueue(); +} + ThreadGroupStatusPtr CurrentThread::getGroup() { if (unlikely(!current_thread)) diff --git a/src/Common/CurrentThread.h b/src/Common/CurrentThread.h index 96ea7f7e795..9dbe8d355d6 100644 --- a/src/Common/CurrentThread.h +++ b/src/Common/CurrentThread.h @@ -46,6 +46,9 @@ public: LogsLevel client_logs_level); static std::shared_ptr getInternalTextLogsQueue(); + static void attachInternalProfileEventsQueue(const InternalProfileEventsQueuePtr & queue); + static InternalProfileEventsQueuePtr getInternalProfileEventsQueue(); + static void setFatalErrorCallback(std::function callback); /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index b1d76c4660e..4c49e9b1d0d 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace DB @@ -197,6 +198,17 @@ void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & thread_group->client_logs_level = client_logs_level; } +void ThreadStatus::attachInternalProfileEventsQueue(const InternalProfileEventsQueuePtr & profile_queue) +{ + profile_queue_ptr = profile_queue; + + if (!thread_group) + return; + + std::lock_guard lock(thread_group->mutex); + thread_group->profile_queue_ptr = profile_queue; +} + void ThreadStatus::setFatalErrorCallback(std::function callback) { fatal_error_callback = std::move(callback); diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index dbd0b4e5664..16a47a21184 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -42,6 +43,10 @@ struct ViewRuntimeData; class QueryViewsLog; using InternalTextLogsQueuePtr = std::shared_ptr; using InternalTextLogsQueueWeakPtr = std::weak_ptr; + +using InternalProfileEventsQueue = ConcurrentBoundedQueue; +using InternalProfileEventsQueuePtr = std::shared_ptr; +using InternalProfileEventsQueueWeakPtr = std::weak_ptr; using ThreadStatusPtr = ThreadStatus *; /** Thread group is a collection of threads dedicated to single task @@ -64,6 +69,7 @@ public: ContextWeakPtr global_context; InternalTextLogsQueueWeakPtr logs_queue_ptr; + InternalProfileEventsQueueWeakPtr profile_queue_ptr; std::function fatal_error_callback; std::vector thread_ids; @@ -134,6 +140,8 @@ protected: /// A logs queue used by TCPHandler to pass logs to a client InternalTextLogsQueueWeakPtr logs_queue_ptr; + InternalProfileEventsQueueWeakPtr profile_queue_ptr; + bool performance_counters_finalized = false; UInt64 query_start_time_nanoseconds = 0; UInt64 query_start_time_microseconds = 0; @@ -208,6 +216,13 @@ public: void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, LogsLevel client_logs_level); + InternalProfileEventsQueuePtr getInternalProfileEventsQueue() const + { + return thread_state == Died ? nullptr : profile_queue_ptr.lock(); + } + + void attachInternalProfileEventsQueue(const InternalProfileEventsQueuePtr & profile_queue); + /// Callback that is used to trigger sending fatal error messages to client. void setFatalErrorCallback(std::function callback); void onFatalError(); diff --git a/src/Core/Protocol.h b/src/Core/Protocol.h index b2957e4ae30..fb18e1135a5 100644 --- a/src/Core/Protocol.h +++ b/src/Core/Protocol.h @@ -104,7 +104,8 @@ namespace Protocol "Log", "TableColumns", "PartUUIDs", - "ReadTaskRequest" + "ReadTaskRequest", + "ProfileEvents", }; return packet <= MAX ? data[packet] diff --git a/src/DataStreams/RemoteQueryExecutor.cpp b/src/DataStreams/RemoteQueryExecutor.cpp index dc97e577513..51c5c2edc57 100644 --- a/src/DataStreams/RemoteQueryExecutor.cpp +++ b/src/DataStreams/RemoteQueryExecutor.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -393,7 +395,12 @@ std::optional RemoteQueryExecutor::processPacket(Packet packet) case Protocol::Server::ProfileEvents: /// Pass profile events from remote server to client - break; + { + LOG_DEBUG(log, "RemoteQueryExecutor received ProfileEvents"); + auto profile_queue = CurrentThread::getInternalProfileEventsQueue(); + profile_queue->emplace(std::move(packet.block)); + break; + } default: got_unknown_packet_from_replica = true; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 7b7bfec006c..7ff74a0618c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -129,6 +129,7 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) logs_queue_ptr = thread_group->logs_queue_ptr; fatal_error_callback = thread_group->fatal_error_callback; query_context = thread_group->query_context; + profile_queue_ptr = thread_group->profile_queue_ptr; if (global_context.expired()) global_context = thread_group->global_context; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 9c1b107c513..f1014d611fd 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -243,6 +244,8 @@ void TCPHandler::runImpl() sendLogs(); }); } + state.profile_queue = std::make_shared(std::numeric_limits::max()); + CurrentThread::attachInternalProfileEventsQueue(state.profile_queue); query_context->setExternalTablesInitializer([this] (ContextPtr context) { @@ -670,10 +673,10 @@ void TCPHandler::processOrdinaryQueryWithProcessors() /// Some time passed and there is a progress. after_send_progress.restart(); sendProgress(); - sendProfileEvents(); } sendLogs(); + sendProfileEvents(); if (block) { @@ -696,7 +699,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendProfileInfo(executor.getProfileInfo()); sendProgress(); sendLogs(); - sendProfileEvents(); + // sendProfileEvents(); } if (state.is_connection_closed) @@ -935,15 +938,37 @@ void TCPHandler::sendProfileEvents() dumpProfileEvents(counters_snapshot, columns, server_display_name, current_time, thread_id); dumpMemoryTracker(memory_tracker, columns, server_display_name, thread_id); } - block.setColumns(std::move(columns)); - initProfileEventsBlockOutput(block); + MutableColumns logs_columns; + Block curr_block; + size_t rows = 0; - writeVarUInt(Protocol::Server::ProfileEvents, *out); - writeStringBinary("", *out); + bool from_queue = false; + for (; state.profile_queue->tryPop(curr_block); ++rows) + { + from_queue = true; + auto curr_columns = curr_block.getColumns(); + for (size_t j = 0; j < curr_columns.size(); ++j) + columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size()); + } - state.profile_events_block_out->write(block); - out->next(); + bool empty = true; + for (auto & column : columns) + empty = empty && column->empty(); + + if (!empty) + { + block.setColumns(std::move(columns)); + + initProfileEventsBlockOutput(block); + + writeVarUInt(Protocol::Server::ProfileEvents, *out); + writeStringBinary("", *out); + + state.profile_events_block_out->write(block); + out->next(); + LOG_DEBUG(log, "Sent ProfileEvents packet {} data from queue", (from_queue ? "with" : "without")); + } } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 9ff061e096b..b5d7d1f0776 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -48,6 +48,7 @@ struct QueryState InternalTextLogsQueuePtr logs_queue; std::unique_ptr logs_block_out; + InternalProfileEventsQueuePtr profile_queue; std::unique_ptr profile_events_block_out; /// From where to read data for INSERT. From 9071a7151ef7171c22c27570d95ddb2a018f12f6 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 7 Sep 2021 15:07:24 +0300 Subject: [PATCH 087/126] Fix communication & race conditions --- src/Common/ThreadStatus.cpp | 6 +++++ src/Server/TCPHandler.cpp | 46 ++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 4c49e9b1d0d..d521106a29b 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -142,6 +142,12 @@ ThreadStatus::~ThreadStatus() /// We've already allocated a little bit more than the limit and cannot track it in the thread memory tracker or its parent. } + if (thread_group) + { + std::lock_guard guard(thread_group->mutex); + thread_group->threads.erase(this); + } + #if !defined(ARCADIA_BUILD) /// It may cause segfault if query_context was destroyed, but was not detached auto query_context_ptr = query_context.lock(); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index f1014d611fd..581cba91356 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -835,6 +837,14 @@ namespace constexpr size_t NAME_COLUMN_INDEX = 4; constexpr size_t VALUE_COLUMN_INDEX = 5; + struct ProfileEventsSnapshot + { + UInt64 thread_id; + ProfileEvents::Counters counters; + CurrentMetrics::Metric metric; + time_t current_time; + }; + /* * Add records about provided non-zero ProfileEvents::Counters. */ @@ -873,12 +883,11 @@ namespace } void dumpMemoryTracker( - MemoryTracker * memoryTracker, + CurrentMetrics::Metric metric, MutableColumns & columns, String const & host_name, UInt64 thread_id) { - auto metric = memoryTracker->getMetric(); if (metric == CurrentMetrics::end()) return; time_t current_time = time(nullptr); @@ -923,20 +932,28 @@ void TCPHandler::sendProfileEvents() MutableColumns columns = block.mutateColumns(); auto thread_group = CurrentThread::getGroup(); - std::vector threads; + std::vector snapshots; { std::lock_guard guard(thread_group->mutex); - std::copy(thread_group->threads.begin(), thread_group->threads.end(), std::back_inserter(threads)); + for (auto * thread : thread_group->threads) + { + auto current_time = time(nullptr); + auto counters = thread->performance_counters.getPartiallyAtomicSnapshot(); + auto metric = thread->memory_tracker.getMetric(); + auto const thread_id = CurrentThread::get().thread_id; + snapshots.push_back(ProfileEventsSnapshot{thread_id, std::move(counters), metric, current_time}); + } } - for (auto * thread : threads) - { - auto const counters_snapshot = thread->performance_counters.getPartiallyAtomicSnapshot(); - auto current_time = time(nullptr); - auto * memory_tracker = &thread->memory_tracker; - auto const thread_id = CurrentThread::get().thread_id; - dumpProfileEvents(counters_snapshot, columns, server_display_name, current_time, thread_id); - dumpMemoryTracker(memory_tracker, columns, server_display_name, thread_id); + for (auto & snapshot : snapshots) + { + dumpProfileEvents( + snapshot.counters, + columns, + server_display_name, + snapshot.current_time, + snapshot.thread_id); + dumpMemoryTracker(snapshot.metric, columns, server_display_name, snapshot.thread_id); } MutableColumns logs_columns; @@ -952,10 +969,7 @@ void TCPHandler::sendProfileEvents() columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size()); } - bool empty = true; - for (auto & column : columns) - empty = empty && column->empty(); - + bool empty = columns[0]->empty(); if (!empty) { block.setColumns(std::move(columns)); From 15ac65aa33fa77f6fa7ff0d67d40db3043f4d634 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 14 Sep 2021 14:06:00 +0300 Subject: [PATCH 088/126] Add thread usage info on client side --- src/Client/ClientBase.cpp | 16 +++++++++++++--- src/Client/ClientBase.h | 2 +- src/Common/ProgressIndication.cpp | 9 +++++++++ src/Common/ProgressIndication.h | 8 ++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index ee5f3580050..988f008fef7 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,8 @@ #include #include #include +#include "Columns/ColumnsNumber.h" +#include "Core/Block.h" #include "Core/Protocol.h" #if !defined(ARCADIA_BUILD) @@ -613,7 +615,7 @@ bool ClientBase::receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled) return false; case Protocol::Server::ProfileEvents: - onProfileEvents(); + onProfileEvents(packet.block); return true; default: @@ -656,8 +658,16 @@ void ClientBase::onEndOfStream() } -void ClientBase::onProfileEvents() -{} +void ClientBase::onProfileEvents(Block & block) +{ + if (block.rows() == 0) + return; + const auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); + for (size_t i = 0; i < block.rows(); ++i) + { + progress_indication.addThreadIdToList(array_thread_id[i]); + } +} /// Flush all buffers. diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 0fa205a4d6e..070b676366c 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -114,7 +114,7 @@ private: void onReceiveExceptionFromServer(std::unique_ptr && e); void onProfileInfo(const BlockStreamProfileInfo & profile_info); void onEndOfStream(); - void onProfileEvents(); + void onProfileEvents(Block & block); void sendData(Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query); void sendDataFrom(ReadBuffer & buf, Block & sample, diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 0d65eaece86..02bb7d202d7 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -29,6 +29,7 @@ void ProgressIndication::resetProgress() show_progress_bar = false; written_progress_chars = 0; write_progress_on_update = false; + thread_ids.clear(); } void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update_) @@ -43,6 +44,11 @@ void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool }); } +void ProgressIndication::addThreadIdToList(UInt64 thread_id) +{ + thread_ids.insert(thread_id); +} + void ProgressIndication::writeFinalProgress() { if (progress.read_rows < 1000) @@ -57,6 +63,9 @@ void ProgressIndication::writeFinalProgress() << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; else std::cout << ". "; + size_t used_threads = getUsedThreadsCount(); + if (used_threads != 0) + std::cout << "\nUsed threads to process: " << used_threads << "."; } void ProgressIndication::writeProgress() diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 044d8cb1a89..ba7889c7326 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include #include @@ -41,6 +43,10 @@ public: /// How much seconds passed since query execution start. double elapsedSeconds() const { return watch.elapsedSeconds(); } + void addThreadIdToList(UInt64 thread_id); + + size_t getUsedThreadsCount() const { return thread_ids.size(); } + private: /// This flag controls whether to show the progress bar. We start showing it after /// the query has been executing for 0.5 seconds, and is still less than half complete. @@ -58,6 +64,8 @@ private: Stopwatch watch; bool write_progress_on_update = false; + + std::unordered_set thread_ids; }; } From 4c6b3c40f2f3b854b85c5ff640ad9ee0f3bbe704 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 14 Sep 2021 16:24:57 +0300 Subject: [PATCH 089/126] Calculate approximate cores number used --- src/Client/ClientBase.cpp | 26 +++++++++++++++++++- src/Common/ProgressIndication.cpp | 41 ++++++++++++++++++++++++++++--- src/Common/ProgressIndication.h | 17 +++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 988f008fef7..5e0e11d103e 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "Columns/ColumnString.h" #include "Columns/ColumnsNumber.h" #include "Core/Block.h" #include "Core/Protocol.h" @@ -75,6 +76,12 @@ namespace ErrorCodes } +namespace ProfileEvents +{ + extern const Event UserTimeMicroseconds; + extern const Event SystemTimeMicroseconds; +} + namespace DB { @@ -663,9 +670,26 @@ void ClientBase::onProfileEvents(Block & block) if (block.rows() == 0) return; const auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); + const auto & names = typeid_cast(*block.getByName("name").column); + const auto & array_values = typeid_cast(*block.getByName("value").column).getData(); + + auto const * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds); + auto const * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds); + for (size_t i = 0; i < block.rows(); ++i) { - progress_indication.addThreadIdToList(array_thread_id[i]); + auto thread_id = array_thread_id[i]; + progress_indication.addThreadIdToList(thread_id); + auto event_name = names.getDataAt(i); + auto value = array_values[i]; + if (event_name == user_time_name) + { + progress_indication.updateThreadUserTime(thread_id, value); + } + else if (event_name == system_time_name) + { + progress_indication.updateThreadSystemTime(thread_id, value); + } } } diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 02bb7d202d7..ceb039b15f5 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -1,5 +1,8 @@ #include "ProgressIndication.h" +#include +#include #include +#include #include #include #include @@ -29,7 +32,7 @@ void ProgressIndication::resetProgress() show_progress_bar = false; written_progress_chars = 0; write_progress_on_update = false; - thread_ids.clear(); + thread_times.clear(); } void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update_) @@ -46,7 +49,28 @@ void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool void ProgressIndication::addThreadIdToList(UInt64 thread_id) { - thread_ids.insert(thread_id); + if (thread_times.contains(thread_id)) + return; + thread_times[thread_id] = {}; +} + +void ProgressIndication::updateThreadUserTime(UInt64 thread_id, UInt64 value) +{ + thread_times[thread_id].user_ms = value; +} + +void ProgressIndication::updateThreadSystemTime(UInt64 thread_id, UInt64 value) +{ + thread_times[thread_id].system_ms = value; +} + +UInt64 ProgressIndication::getAccumulatedThreadTime() const +{ + return std::accumulate(thread_times.cbegin(), thread_times.cend(), static_cast(0), + [](UInt64 acc, auto const & elem) + { + return acc + elem.second.user_ms + elem.second.system_ms; + }); } void ProgressIndication::writeFinalProgress() @@ -63,9 +87,20 @@ void ProgressIndication::writeFinalProgress() << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; else std::cout << ". "; + size_t used_threads = getUsedThreadsCount(); if (used_threads != 0) - std::cout << "\nUsed threads to process: " << used_threads << "."; + { + std::cout << "\nUsed threads to process: " << used_threads; + + auto elapsed_ms = watch.elapsedMicroseconds(); + auto accumulated_thread_times = getAccumulatedThreadTime(); + auto approximate_core_number = (accumulated_thread_times + elapsed_ms - 1) / elapsed_ms; + if (approximate_core_number != 0) + std::cout << " and cores: " << approximate_core_number << "."; + else + std::cout << "."; + } } void ProgressIndication::writeProgress() diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index ba7889c7326..7517853f74d 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -45,9 +45,16 @@ public: void addThreadIdToList(UInt64 thread_id); - size_t getUsedThreadsCount() const { return thread_ids.size(); } + void updateThreadUserTime(UInt64 thread_id, UInt64 value); + + void updateThreadSystemTime(UInt64 thread_id, UInt64 value); private: + + size_t getUsedThreadsCount() const { return thread_times.size(); } + + UInt64 getAccumulatedThreadTime() const; + /// This flag controls whether to show the progress bar. We start showing it after /// the query has been executing for 0.5 seconds, and is still less than half complete. bool show_progress_bar = false; @@ -65,7 +72,13 @@ private: bool write_progress_on_update = false; - std::unordered_set thread_ids; + struct ThreadTime + { + UInt64 user_ms = 0; + UInt64 system_ms = 0; + }; + + std::unordered_map thread_times; }; } From 7e3caf96bec1afefe26eff8639515ad869e635de Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 15 Sep 2021 18:45:43 +0300 Subject: [PATCH 090/126] Fix cores approximation --- src/Client/ClientBase.cpp | 9 +++-- src/Common/ProgressIndication.cpp | 56 +++++++++++++++++++++++-------- src/Common/ProgressIndication.h | 16 +++++---- src/Server/TCPHandler.cpp | 11 +++++- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5e0e11d103e..6a05ebd7c1b 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -671,6 +671,7 @@ void ClientBase::onProfileEvents(Block & block) return; const auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); const auto & names = typeid_cast(*block.getByName("name").column); + const auto & host_names = typeid_cast(*block.getByName("host_name").column); const auto & array_values = typeid_cast(*block.getByName("value").column).getData(); auto const * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds); @@ -679,16 +680,18 @@ void ClientBase::onProfileEvents(Block & block) for (size_t i = 0; i < block.rows(); ++i) { auto thread_id = array_thread_id[i]; - progress_indication.addThreadIdToList(thread_id); + auto host_name = host_names.getDataAt(i).toString(); + if (thread_id != 0) + progress_indication.addThreadIdToList(host_name, thread_id); auto event_name = names.getDataAt(i); auto value = array_values[i]; if (event_name == user_time_name) { - progress_indication.updateThreadUserTime(thread_id, value); + progress_indication.updateThreadUserTime(host_name, thread_id, value); } else if (event_name == system_time_name) { - progress_indication.updateThreadSystemTime(thread_id, value); + progress_indication.updateThreadSystemTime(host_name, thread_id, value); } } } diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index ceb039b15f5..b06df1bba15 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -1,4 +1,5 @@ #include "ProgressIndication.h" +#include #include #include #include @@ -8,6 +9,11 @@ #include +namespace +{ + constexpr UInt64 ZERO = 0; +} + namespace DB { @@ -47,29 +53,53 @@ void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool }); } -void ProgressIndication::addThreadIdToList(UInt64 thread_id) +void ProgressIndication::addThreadIdToList(String const & host, UInt64 thread_id) { - if (thread_times.contains(thread_id)) + auto & thread_to_times = thread_times[host]; + if (thread_to_times.contains(thread_id)) return; - thread_times[thread_id] = {}; + thread_to_times[thread_id] = {}; } -void ProgressIndication::updateThreadUserTime(UInt64 thread_id, UInt64 value) +void ProgressIndication::updateThreadUserTime(String const & host, UInt64 thread_id, UInt64 value) { - thread_times[thread_id].user_ms = value; + thread_times[host][thread_id].user_ms = value; } -void ProgressIndication::updateThreadSystemTime(UInt64 thread_id, UInt64 value) +void ProgressIndication::updateThreadSystemTime(String const & host, UInt64 thread_id, UInt64 value) { - thread_times[thread_id].system_ms = value; + thread_times[host][thread_id].system_ms = value; } -UInt64 ProgressIndication::getAccumulatedThreadTime() const +size_t ProgressIndication::getUsedThreadsCount() const { - return std::accumulate(thread_times.cbegin(), thread_times.cend(), static_cast(0), - [](UInt64 acc, auto const & elem) + return std::accumulate(thread_times.cbegin(), thread_times.cend(), 0, + [] (size_t acc, auto const & threads) { - return acc + elem.second.user_ms + elem.second.system_ms; + return acc + threads.second.size(); + }); +} + +UInt64 ProgressIndication::getApproximateCoresNumber() const +{ + return std::accumulate(thread_times.cbegin(), thread_times.cend(), ZERO, + [](UInt64 acc, auto const & threads) + { + auto total_time = std::accumulate(threads.second.cbegin(), threads.second.cend(), ZERO, + [] (UInt64 temp, auto const & elem) + { + if (elem.first == 0) + return temp; + return temp + elem.second.user_ms + elem.second.system_ms; + }); + // Zero thread_id represents thread group which execute query + // (including thread of TCPHandler). + auto const & accumulated_time = threads.second.find(ZERO)->second; + // Performance events of TCPHandler thread are not transmitted, but + // we can calculate it's working time which shows how long the query + // is being processed. + auto io_time = accumulated_time.user_ms + accumulated_time.system_ms - total_time; + return acc + (total_time + io_time - 1) / io_time; }); } @@ -93,9 +123,7 @@ void ProgressIndication::writeFinalProgress() { std::cout << "\nUsed threads to process: " << used_threads; - auto elapsed_ms = watch.elapsedMicroseconds(); - auto accumulated_thread_times = getAccumulatedThreadTime(); - auto approximate_core_number = (accumulated_thread_times + elapsed_ms - 1) / elapsed_ms; + auto approximate_core_number = getApproximateCoresNumber(); if (approximate_core_number != 0) std::cout << " and cores: " << approximate_core_number << "."; else diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 7517853f74d..f1d7d214f4f 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -43,17 +44,17 @@ public: /// How much seconds passed since query execution start. double elapsedSeconds() const { return watch.elapsedSeconds(); } - void addThreadIdToList(UInt64 thread_id); + void addThreadIdToList(String const & host, UInt64 thread_id); - void updateThreadUserTime(UInt64 thread_id, UInt64 value); + void updateThreadUserTime(String const & host, UInt64 thread_id, UInt64 value); - void updateThreadSystemTime(UInt64 thread_id, UInt64 value); + void updateThreadSystemTime(String const & host, UInt64 thread_id, UInt64 value); private: - size_t getUsedThreadsCount() const { return thread_times.size(); } + size_t getUsedThreadsCount() const; - UInt64 getAccumulatedThreadTime() const; + UInt64 getApproximateCoresNumber() const; /// This flag controls whether to show the progress bar. We start showing it after /// the query has been executing for 0.5 seconds, and is still less than half complete. @@ -78,7 +79,10 @@ private: UInt64 system_ms = 0; }; - std::unordered_map thread_times; + using ThreadIdToTimeMap = std::unordered_map; + using HostToThreadTimesMap = std::unordered_map; + + HostToThreadTimesMap thread_times; }; } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 581cba91356..c24bf599527 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -932,19 +932,28 @@ void TCPHandler::sendProfileEvents() MutableColumns columns = block.mutateColumns(); auto thread_group = CurrentThread::getGroup(); + auto const current_thread_id = CurrentThread::get().thread_id; std::vector snapshots; + ProfileEventsSnapshot group_snapshot; { std::lock_guard guard(thread_group->mutex); for (auto * thread : thread_group->threads) { + auto const thread_id = thread->thread_id; + if (thread_id == current_thread_id) + continue; auto current_time = time(nullptr); auto counters = thread->performance_counters.getPartiallyAtomicSnapshot(); auto metric = thread->memory_tracker.getMetric(); - auto const thread_id = CurrentThread::get().thread_id; snapshots.push_back(ProfileEventsSnapshot{thread_id, std::move(counters), metric, current_time}); } + group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); + group_snapshot.metric = thread_group->memory_tracker.getMetric(); + group_snapshot.current_time = time(nullptr); } + dumpProfileEvents(group_snapshot.counters, columns, server_display_name, group_snapshot.current_time, 0); + dumpMemoryTracker(group_snapshot.metric, columns, server_display_name, 0); for (auto & snapshot : snapshots) { dumpProfileEvents( From 1d2e2d73057ebd9dd791cf9e4f98107da2fd5e88 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 15 Sep 2021 23:35:04 +0300 Subject: [PATCH 091/126] cleanup --- src/DataStreams/RemoteQueryExecutor.cpp | 1 - src/Server/TCPHandler.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/DataStreams/RemoteQueryExecutor.cpp b/src/DataStreams/RemoteQueryExecutor.cpp index 51c5c2edc57..aa316e54e6f 100644 --- a/src/DataStreams/RemoteQueryExecutor.cpp +++ b/src/DataStreams/RemoteQueryExecutor.cpp @@ -396,7 +396,6 @@ std::optional RemoteQueryExecutor::processPacket(Packet packet) case Protocol::Server::ProfileEvents: /// Pass profile events from remote server to client { - LOG_DEBUG(log, "RemoteQueryExecutor received ProfileEvents"); auto profile_queue = CurrentThread::getInternalProfileEventsQueue(); profile_queue->emplace(std::move(packet.block)); break; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index c24bf599527..04b85125d66 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -675,10 +675,10 @@ void TCPHandler::processOrdinaryQueryWithProcessors() /// Some time passed and there is a progress. after_send_progress.restart(); sendProgress(); + sendProfileEvents(); } sendLogs(); - sendProfileEvents(); if (block) { @@ -701,7 +701,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendProfileInfo(executor.getProfileInfo()); sendProgress(); sendLogs(); - // sendProfileEvents(); + sendProfileEvents(); } if (state.is_connection_closed) From 73df6190df357ecf219be454d116321a89c3319e Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 17 Sep 2021 18:00:13 +0300 Subject: [PATCH 092/126] Cleanup code --- src/Client/ClientBase.cpp | 6 ++- src/Client/Connection.cpp | 1 - src/Common/ProgressIndication.cpp | 69 ++++++++++++++++++++----------- src/Common/ProgressIndication.h | 25 +++++------ src/Server/TCPHandler.cpp | 12 +++--- 5 files changed, 67 insertions(+), 46 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 6a05ebd7c1b..b3148525f8c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -677,6 +677,7 @@ void ClientBase::onProfileEvents(Block & block) auto const * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds); auto const * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds); + HostToThreadTimesMap thread_times; for (size_t i = 0; i < block.rows(); ++i) { auto thread_id = array_thread_id[i]; @@ -687,13 +688,14 @@ void ClientBase::onProfileEvents(Block & block) auto value = array_values[i]; if (event_name == user_time_name) { - progress_indication.updateThreadUserTime(host_name, thread_id, value); + thread_times[host_name][thread_id].user_ms = value; } else if (event_name == system_time_name) { - progress_indication.updateThreadSystemTime(host_name, thread_id, value); + thread_times[host_name][thread_id].system_ms = value; } } + progress_indication.updateThreadTimes(thread_times); } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index c6badf96bf9..40f74bcf9a7 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -873,7 +873,6 @@ Packet Connection::receivePacket() return res; case Protocol::Server::ProfileEvents: - LOG_DEBUG(log_wrapper.get(), "Connection received ProfileEvents"); res.block = receiveProfileEvents(); return res; diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index b06df1bba15..9a87a86c76b 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -12,6 +12,28 @@ namespace { constexpr UInt64 ZERO = 0; + + UInt64 calculateNewCoresNumber(DB::ThreadIdToTimeMap const & prev, DB::ThreadIdToTimeMap const& next) + { + if (next.find(ZERO) == next.end()) + return ZERO; + auto accumulated = std::accumulate(next.cbegin(), next.cend(), ZERO, + [&prev](UInt64 acc, auto const & elem) + { + if (elem.first == ZERO) + return acc; + auto thread_time = elem.second.time(); + auto it = prev.find(elem.first); + if (it != prev.end()) + thread_time -= it->second.time(); + return acc + thread_time; + }); + + auto elapsed = next.at(ZERO).time() - (prev.contains(ZERO) ? prev.at(ZERO).time() : ZERO); + if (elapsed == ZERO) + return ZERO; + return (accumulated + elapsed - 1) / elapsed; + } } namespace DB @@ -38,6 +60,7 @@ void ProgressIndication::resetProgress() show_progress_bar = false; written_progress_chars = 0; write_progress_on_update = false; + host_active_cores.clear(); thread_times.clear(); } @@ -61,14 +84,15 @@ void ProgressIndication::addThreadIdToList(String const & host, UInt64 thread_id thread_to_times[thread_id] = {}; } -void ProgressIndication::updateThreadUserTime(String const & host, UInt64 thread_id, UInt64 value) +void ProgressIndication::updateThreadTimes(HostToThreadTimesMap & new_thread_times) { - thread_times[host][thread_id].user_ms = value; -} - -void ProgressIndication::updateThreadSystemTime(String const & host, UInt64 thread_id, UInt64 value) -{ - thread_times[host][thread_id].system_ms = value; + for (auto & new_host_map : new_thread_times) + { + auto & host_map = thread_times[new_host_map.first]; + auto new_cores = calculateNewCoresNumber(host_map, new_host_map.second); + host_active_cores[new_host_map.first] = new_cores; + host_map = std::move(new_host_map.second); + } } size_t ProgressIndication::getUsedThreadsCount() const @@ -82,24 +106,10 @@ size_t ProgressIndication::getUsedThreadsCount() const UInt64 ProgressIndication::getApproximateCoresNumber() const { - return std::accumulate(thread_times.cbegin(), thread_times.cend(), ZERO, - [](UInt64 acc, auto const & threads) + return std::accumulate(host_active_cores.cbegin(), host_active_cores.cend(), ZERO, + [](UInt64 acc, auto const & elem) { - auto total_time = std::accumulate(threads.second.cbegin(), threads.second.cend(), ZERO, - [] (UInt64 temp, auto const & elem) - { - if (elem.first == 0) - return temp; - return temp + elem.second.user_ms + elem.second.system_ms; - }); - // Zero thread_id represents thread group which execute query - // (including thread of TCPHandler). - auto const & accumulated_time = threads.second.find(ZERO)->second; - // Performance events of TCPHandler thread are not transmitted, but - // we can calculate it's working time which shows how long the query - // is being processed. - auto io_time = accumulated_time.user_ms + accumulated_time.system_ms - total_time; - return acc + (total_time + io_time - 1) / io_time; + return acc + elem.second; }); } @@ -220,6 +230,17 @@ void ProgressIndication::writeProgress() message << ' ' << (99 * current_count / max_count) << '%'; } + // If approximate cores number is known, display it. + auto cores_number = getApproximateCoresNumber(); + if (cores_number != 0) + { + // Calculated cores number may be not accurate + // so it's better to print min(threads, cores). + auto threads_number = getUsedThreadsCount(); + message << " Running " << threads_number << " threads on " + << std::min(cores_number, threads_number) << " cores."; + } + message << CLEAR_TO_END_OF_LINE; ++increment; diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index f1d7d214f4f..4a98b5e849b 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -14,6 +14,17 @@ namespace DB { +struct ThreadTime +{ + UInt64 time() const noexcept { return user_ms + system_ms; } + + UInt64 user_ms = 0; + UInt64 system_ms = 0; +}; + +using ThreadIdToTimeMap = std::unordered_map; +using HostToThreadTimesMap = std::unordered_map; + class ProgressIndication { public: @@ -46,9 +57,7 @@ public: void addThreadIdToList(String const & host, UInt64 thread_id); - void updateThreadUserTime(String const & host, UInt64 thread_id, UInt64 value); - - void updateThreadSystemTime(String const & host, UInt64 thread_id, UInt64 value); + void updateThreadTimes(HostToThreadTimesMap & new_thread_times); private: @@ -73,15 +82,7 @@ private: bool write_progress_on_update = false; - struct ThreadTime - { - UInt64 user_ms = 0; - UInt64 system_ms = 0; - }; - - using ThreadIdToTimeMap = std::unordered_map; - using HostToThreadTimesMap = std::unordered_map; - + std::unordered_map host_active_cores; HostToThreadTimesMap thread_times; }; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 04b85125d66..9bca044617a 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -947,13 +947,12 @@ void TCPHandler::sendProfileEvents() auto metric = thread->memory_tracker.getMetric(); snapshots.push_back(ProfileEventsSnapshot{thread_id, std::move(counters), metric, current_time}); } - group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); - group_snapshot.metric = thread_group->memory_tracker.getMetric(); + group_snapshot.current_time = time(nullptr); + group_snapshot.metric = thread_group->memory_tracker.getMetric(); + group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); } - dumpProfileEvents(group_snapshot.counters, columns, server_display_name, group_snapshot.current_time, 0); - dumpMemoryTracker(group_snapshot.metric, columns, server_display_name, 0); for (auto & snapshot : snapshots) { dumpProfileEvents( @@ -964,15 +963,15 @@ void TCPHandler::sendProfileEvents() snapshot.thread_id); dumpMemoryTracker(snapshot.metric, columns, server_display_name, snapshot.thread_id); } + dumpProfileEvents(group_snapshot.counters, columns, server_display_name, group_snapshot.current_time, 0); + dumpMemoryTracker(group_snapshot.metric, columns, server_display_name, 0); MutableColumns logs_columns; Block curr_block; size_t rows = 0; - bool from_queue = false; for (; state.profile_queue->tryPop(curr_block); ++rows) { - from_queue = true; auto curr_columns = curr_block.getColumns(); for (size_t j = 0; j < curr_columns.size(); ++j) columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size()); @@ -990,7 +989,6 @@ void TCPHandler::sendProfileEvents() state.profile_events_block_out->write(block); out->next(); - LOG_DEBUG(log, "Sent ProfileEvents packet {} data from queue", (from_queue ? "with" : "without")); } } From 9f9af28b5ec5eb58f73c16f4d2737c3f284d163f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 17 Sep 2021 19:47:54 +0300 Subject: [PATCH 093/126] Output memory usage with progress --- src/Client/ClientBase.cpp | 6 ++- src/Common/MemoryTracker.h | 3 ++ src/Common/ProgressIndication.cpp | 45 ++++++++++------- src/Common/ProgressIndication.h | 15 +++--- src/Server/TCPHandler.cpp | 82 ++++++++++++++++++------------- 5 files changed, 91 insertions(+), 60 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index b3148525f8c..1e104292e06 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -694,8 +694,12 @@ void ClientBase::onProfileEvents(Block & block) { thread_times[host_name][thread_id].system_ms = value; } + else if (event_name == MemoryTracker::USAGE_EVENT_NAME) + { + thread_times[host_name][thread_id].memory_usage = value; + } } - progress_indication.updateThreadTimes(thread_times); + progress_indication.updateThreadEventData(thread_times); } diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 7da70db0876..ce0eef52e17 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -64,6 +64,9 @@ private: void setOrRaiseProfilerLimit(Int64 value); public: + + static constexpr auto USAGE_EVENT_NAME = "MemoryTrackerUsage"; + explicit MemoryTracker(VariableContext level_ = VariableContext::Thread); explicit MemoryTracker(MemoryTracker * parent_, VariableContext level_ = VariableContext::Thread); diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 9a87a86c76b..189af2e9972 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -61,7 +61,7 @@ void ProgressIndication::resetProgress() written_progress_chars = 0; write_progress_on_update = false; host_active_cores.clear(); - thread_times.clear(); + thread_data.clear(); } void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update_) @@ -78,17 +78,17 @@ void ProgressIndication::setFileProgressCallback(ContextMutablePtr context, bool void ProgressIndication::addThreadIdToList(String const & host, UInt64 thread_id) { - auto & thread_to_times = thread_times[host]; + auto & thread_to_times = thread_data[host]; if (thread_to_times.contains(thread_id)) return; thread_to_times[thread_id] = {}; } -void ProgressIndication::updateThreadTimes(HostToThreadTimesMap & new_thread_times) +void ProgressIndication::updateThreadEventData(HostToThreadTimesMap & new_thread_data) { - for (auto & new_host_map : new_thread_times) + for (auto & new_host_map : new_thread_data) { - auto & host_map = thread_times[new_host_map.first]; + auto & host_map = thread_data[new_host_map.first]; auto new_cores = calculateNewCoresNumber(host_map, new_host_map.second); host_active_cores[new_host_map.first] = new_cores; host_map = std::move(new_host_map.second); @@ -97,7 +97,7 @@ void ProgressIndication::updateThreadTimes(HostToThreadTimesMap & new_thread_tim size_t ProgressIndication::getUsedThreadsCount() const { - return std::accumulate(thread_times.cbegin(), thread_times.cend(), 0, + return std::accumulate(thread_data.cbegin(), thread_data.cend(), 0, [] (size_t acc, auto const & threads) { return acc + threads.second.size(); @@ -113,6 +113,19 @@ UInt64 ProgressIndication::getApproximateCoresNumber() const }); } +UInt64 ProgressIndication::getMemoryUsage() const +{ + return std::accumulate(thread_data.cbegin(), thread_data.cend(), ZERO, + [](UInt64 acc, auto const & host_data) + { + return acc + std::accumulate(host_data.second.cbegin(), host_data.second.cend(), ZERO, + [](UInt64 memory, auto const & data) + { + return memory + data.second.memory_usage; + }); + }); +} + void ProgressIndication::writeFinalProgress() { if (progress.read_rows < 1000) @@ -127,18 +140,6 @@ void ProgressIndication::writeFinalProgress() << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; else std::cout << ". "; - - size_t used_threads = getUsedThreadsCount(); - if (used_threads != 0) - { - std::cout << "\nUsed threads to process: " << used_threads; - - auto approximate_core_number = getApproximateCoresNumber(); - if (approximate_core_number != 0) - std::cout << " and cores: " << approximate_core_number << "."; - else - std::cout << "."; - } } void ProgressIndication::writeProgress() @@ -238,7 +239,13 @@ void ProgressIndication::writeProgress() // so it's better to print min(threads, cores). auto threads_number = getUsedThreadsCount(); message << " Running " << threads_number << " threads on " - << std::min(cores_number, threads_number) << " cores."; + << std::min(cores_number, threads_number) << " cores"; + + auto memory_usage = getMemoryUsage(); + if (memory_usage != 0) + message << " with " << formatReadableSizeWithDecimalSuffix(memory_usage) << " RAM used."; + else + message << "."; } message << CLEAR_TO_END_OF_LINE; diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 4a98b5e849b..3d9bbc7f3ff 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -14,15 +14,16 @@ namespace DB { -struct ThreadTime +struct ThreadEventData { UInt64 time() const noexcept { return user_ms + system_ms; } - UInt64 user_ms = 0; - UInt64 system_ms = 0; + UInt64 user_ms = 0; + UInt64 system_ms = 0; + UInt64 memory_usage = 0; }; -using ThreadIdToTimeMap = std::unordered_map; +using ThreadIdToTimeMap = std::unordered_map; using HostToThreadTimesMap = std::unordered_map; class ProgressIndication @@ -57,7 +58,7 @@ public: void addThreadIdToList(String const & host, UInt64 thread_id); - void updateThreadTimes(HostToThreadTimesMap & new_thread_times); + void updateThreadEventData(HostToThreadTimesMap & new_thread_data); private: @@ -65,6 +66,8 @@ private: UInt64 getApproximateCoresNumber() const; + UInt64 getMemoryUsage() const; + /// This flag controls whether to show the progress bar. We start showing it after /// the query has been executing for 0.5 seconds, and is still less than half complete. bool show_progress_bar = false; @@ -83,7 +86,7 @@ private: bool write_progress_on_update = false; std::unordered_map host_active_cores; - HostToThreadTimesMap thread_times; + HostToThreadTimesMap thread_data; }; } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 9bca044617a..45a65f990df 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -842,6 +842,7 @@ namespace UInt64 thread_id; ProfileEvents::Counters counters; CurrentMetrics::Metric metric; + Int64 memory_usage; time_t current_time; }; @@ -849,18 +850,16 @@ namespace * Add records about provided non-zero ProfileEvents::Counters. */ void dumpProfileEvents( - ProfileEvents::Counters const & snapshot, + ProfileEventsSnapshot const & snapshot, MutableColumns & columns, - String const & host_name, - time_t current_time, - UInt64 thread_id) + String const & host_name) { size_t rows = 0; auto & name_column = columns[NAME_COLUMN_INDEX]; auto & value_column = columns[VALUE_COLUMN_INDEX]; for (ProfileEvents::Event event = 0; event < ProfileEvents::Counters::num_counters; ++event) { - UInt64 value = snapshot[event].load(std::memory_order_relaxed); + UInt64 value = snapshot.counters[event].load(std::memory_order_relaxed); if (value == 0) continue; @@ -876,32 +875,43 @@ namespace { size_t i = 0; columns[i++]->insertData(host_name.data(), host_name.size()); - columns[i++]->insert(UInt64(current_time)); - columns[i++]->insert(UInt64{thread_id}); + columns[i++]->insert(UInt64(snapshot.current_time)); + columns[i++]->insert(UInt64{snapshot.thread_id}); columns[i++]->insert(ProfileEventTypes::INCREMENT); } } void dumpMemoryTracker( - CurrentMetrics::Metric metric, + ProfileEventsSnapshot const & snapshot, MutableColumns & columns, - String const & host_name, - UInt64 thread_id) + String const & host_name) { - if (metric == CurrentMetrics::end()) - return; - time_t current_time = time(nullptr); + { + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(snapshot.current_time)); + columns[i++]->insert(UInt64{snapshot.thread_id}); + columns[i++]->insert(ProfileEventTypes::GAUGE); - size_t i = 0; - columns[i++]->insertData(host_name.data(), host_name.size()); - columns[i++]->insert(UInt64(current_time)); - columns[i++]->insert(UInt64{thread_id}); - columns[i++]->insert(ProfileEventTypes::GAUGE); + columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME)); + columns[i++]->insert(snapshot.memory_usage); + } - auto const * metric_name = CurrentMetrics::getName(metric); - columns[i++]->insertData(metric_name, strlen(metric_name)); - auto metric_value = CurrentMetrics::get(metric); - columns[i++]->insert(metric_value); + if (snapshot.metric != CurrentMetrics::end()) + { + time_t current_time = time(nullptr); + + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(current_time)); + columns[i++]->insert(UInt64{snapshot.thread_id}); + columns[i++]->insert(ProfileEventTypes::GAUGE); + + auto const * metric_name = CurrentMetrics::getName(snapshot.metric); + columns[i++]->insertData(metric_name, strlen(metric_name)); + auto metric_value = CurrentMetrics::get(snapshot.metric); + columns[i++]->insert(metric_value); + } } } @@ -945,26 +955,30 @@ void TCPHandler::sendProfileEvents() auto current_time = time(nullptr); auto counters = thread->performance_counters.getPartiallyAtomicSnapshot(); auto metric = thread->memory_tracker.getMetric(); - snapshots.push_back(ProfileEventsSnapshot{thread_id, std::move(counters), metric, current_time}); + auto memory_usage = thread->memory_tracker.get(); + snapshots.push_back(ProfileEventsSnapshot{ + thread_id, + std::move(counters), + metric, + memory_usage, + current_time + }); } + group_snapshot.thread_id = 0; group_snapshot.current_time = time(nullptr); - group_snapshot.metric = thread_group->memory_tracker.getMetric(); - group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); + group_snapshot.metric = thread_group->memory_tracker.getMetric(); + group_snapshot.memory_usage = thread_group->memory_tracker.get(); + group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); } for (auto & snapshot : snapshots) { - dumpProfileEvents( - snapshot.counters, - columns, - server_display_name, - snapshot.current_time, - snapshot.thread_id); - dumpMemoryTracker(snapshot.metric, columns, server_display_name, snapshot.thread_id); + dumpProfileEvents(snapshot, columns, server_display_name); + dumpMemoryTracker(snapshot, columns, server_display_name); } - dumpProfileEvents(group_snapshot.counters, columns, server_display_name, group_snapshot.current_time, 0); - dumpMemoryTracker(group_snapshot.metric, columns, server_display_name, 0); + dumpProfileEvents(group_snapshot, columns, server_display_name); + dumpMemoryTracker(group_snapshot, columns, server_display_name); MutableColumns logs_columns; Block curr_block; From 0bdabf46f281a14d1138e2bd0956f10e746bc436 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Sep 2021 18:59:27 +0300 Subject: [PATCH 094/126] Send ProfileEvents only to supported clients --- src/Core/ProtocolDefines.h | 4 +++- src/Server/TCPHandler.cpp | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 37875305e75..6c62b969ff9 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -36,6 +36,8 @@ #define DBMS_MIN_PROTOCOL_VERSION_WITH_DISTRIBUTED_DEPTH 54448 +#define DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS 54450 + /// Version of ClickHouse TCP protocol. /// /// Should be incremented manually on protocol changes. @@ -43,6 +45,6 @@ /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -#define DBMS_TCP_PROTOCOL_VERSION 54449 +#define DBMS_TCP_PROTOCOL_VERSION 54450 #define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449 diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 45a65f990df..c036715aea7 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -918,6 +918,9 @@ namespace void TCPHandler::sendProfileEvents() { + if (client_tcp_protocol_version < DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS) + return; + auto profile_event_type = std::make_shared( DataTypeEnum8::Values { From c1f3e7e0bb8f5073c056c635cf101bdae4c7a941 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Sep 2021 00:47:48 +0300 Subject: [PATCH 095/126] Fix build --- src/Common/ProgressIndication.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 189af2e9972..0fe40b306cb 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -237,7 +237,7 @@ void ProgressIndication::writeProgress() { // Calculated cores number may be not accurate // so it's better to print min(threads, cores). - auto threads_number = getUsedThreadsCount(); + UInt64 threads_number = getUsedThreadsCount(); message << " Running " << threads_number << " threads on " << std::min(cores_number, threads_number) << " cores"; From 3a0764634e0be034c7e7ae93a4ec44a76d0e2b64 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Sep 2021 00:52:01 +0300 Subject: [PATCH 096/126] Update InternalProfileEventsQueue usage --- src/DataStreams/RemoteQueryExecutor.cpp | 6 ++---- src/Server/TCPHandler.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/DataStreams/RemoteQueryExecutor.cpp b/src/DataStreams/RemoteQueryExecutor.cpp index aa316e54e6f..08d3db748b7 100644 --- a/src/DataStreams/RemoteQueryExecutor.cpp +++ b/src/DataStreams/RemoteQueryExecutor.cpp @@ -395,11 +395,9 @@ std::optional RemoteQueryExecutor::processPacket(Packet packet) case Protocol::Server::ProfileEvents: /// Pass profile events from remote server to client - { - auto profile_queue = CurrentThread::getInternalProfileEventsQueue(); + if (auto profile_queue = CurrentThread::getInternalProfileEventsQueue()) profile_queue->emplace(std::move(packet.block)); - break; - } + break; default: got_unknown_packet_from_replica = true; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index c036715aea7..c25cb1dddfc 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -246,8 +246,11 @@ void TCPHandler::runImpl() sendLogs(); }); } - state.profile_queue = std::make_shared(std::numeric_limits::max()); - CurrentThread::attachInternalProfileEventsQueue(state.profile_queue); + if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS) + { + state.profile_queue = std::make_shared(std::numeric_limits::max()); + CurrentThread::attachInternalProfileEventsQueue(state.profile_queue); + } query_context->setExternalTablesInitializer([this] (ContextPtr context) { From 9590c97a19f2330983e9ccbba5ba5fb7e5f16076 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Sep 2021 15:29:04 +0300 Subject: [PATCH 097/126] Create ProfileEventsQueue in GRPCServer --- src/Server/GRPCServer.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index fc712916372..d0e054677c0 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -1,4 +1,6 @@ #include "GRPCServer.h" +#include +#include #if USE_GRPC #include @@ -622,6 +624,7 @@ namespace BlockIO io; Progress progress; InternalTextLogsQueuePtr logs_queue; + InternalProfileEventsQueuePtr profile_queue; GRPCQueryInfo query_info; /// We reuse the same messages multiple times. GRPCResult result; @@ -773,6 +776,8 @@ namespace CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this]{ onFatalError(); }); } + profile_queue = std::make_shared(std::numeric_limits::max()); + CurrentThread::attachInternalProfileEventsQueue(profile_queue); /// Set the current database if specified. if (!query_info.database().empty()) From 7c3192735adda0ec7067dc4cfef92de47b9af938 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 6 Oct 2021 16:52:59 +0300 Subject: [PATCH 098/126] Reset profile events stream in Connection::sendQuery --- src/Client/Connection.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 40f74bcf9a7..1531e6c1e91 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -537,6 +537,7 @@ void Connection::sendQuery( maybe_compressed_out.reset(); block_in.reset(); block_logs_in.reset(); + block_profile_events_in.reset(); block_out.reset(); /// Send empty block which means end of data. From bfdd34c13d24466e96e06b185690960d07d33bb4 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 11 Oct 2021 16:55:08 +0300 Subject: [PATCH 099/126] code cleanup --- src/Client/ClientBase.cpp | 9 +++++---- src/Server/GRPCServer.cpp | 11 ----------- src/Server/TCPHandler.cpp | 21 +-------------------- 3 files changed, 6 insertions(+), 35 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 1e104292e06..baf082a3541 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -667,18 +667,19 @@ void ClientBase::onEndOfStream() void ClientBase::onProfileEvents(Block & block) { - if (block.rows() == 0) + const auto rows = block.rows(); + if (rows == 0) return; const auto & array_thread_id = typeid_cast(*block.getByName("thread_id").column).getData(); const auto & names = typeid_cast(*block.getByName("name").column); const auto & host_names = typeid_cast(*block.getByName("host_name").column); const auto & array_values = typeid_cast(*block.getByName("value").column).getData(); - auto const * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds); - auto const * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds); + const auto * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds); + const auto * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds); HostToThreadTimesMap thread_times; - for (size_t i = 0; i < block.rows(); ++i) + for (size_t i = 0; i < rows; ++i) { auto thread_id = array_thread_id[i]; auto host_name = host_names.getDataAt(i).toString(); diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index d0e054677c0..3b01f3aedde 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -598,7 +598,6 @@ namespace void addExtremesToResult(const Block & extremes); void addProfileInfoToResult(const BlockStreamProfileInfo & info); void addLogsToResult(); - void addProfileEventsToResult(); void sendResult(); void throwIfFailedToSendResult(); void sendException(const Exception & exception); @@ -624,7 +623,6 @@ namespace BlockIO io; Progress progress; InternalTextLogsQueuePtr logs_queue; - InternalProfileEventsQueuePtr profile_queue; GRPCQueryInfo query_info; /// We reuse the same messages multiple times. GRPCResult result; @@ -776,8 +774,6 @@ namespace CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); CurrentThread::setFatalErrorCallback([this]{ onFatalError(); }); } - profile_queue = std::make_shared(std::numeric_limits::max()); - CurrentThread::attachInternalProfileEventsQueue(profile_queue); /// Set the current database if specified. if (!query_info.database().empty()) @@ -1129,7 +1125,6 @@ namespace if (after_send_progress.elapsedMicroseconds() >= interactive_delay) { addProgressToResult(); - addProfileEventsToResult(); after_send_progress.restart(); } @@ -1181,7 +1176,6 @@ namespace finalize = true; io.onFinish(); addProgressToResult(); - addProfileEventsToResult(); query_scope->logPeakMemoryUsage(); addLogsToResult(); sendResult(); @@ -1445,11 +1439,6 @@ namespace } } - void Call::addProfileEventsToResult() - { - - } - void Call::sendResult() { /// gRPC doesn't allow to write anything to a finished responder. diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index c25cb1dddfc..2401b8614fa 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -844,7 +844,6 @@ namespace { UInt64 thread_id; ProfileEvents::Counters counters; - CurrentMetrics::Metric metric; Int64 memory_usage; time_t current_time; }; @@ -899,22 +898,6 @@ namespace columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME)); columns[i++]->insert(snapshot.memory_usage); } - - if (snapshot.metric != CurrentMetrics::end()) - { - time_t current_time = time(nullptr); - - size_t i = 0; - columns[i++]->insertData(host_name.data(), host_name.size()); - columns[i++]->insert(UInt64(current_time)); - columns[i++]->insert(UInt64{snapshot.thread_id}); - columns[i++]->insert(ProfileEventTypes::GAUGE); - - auto const * metric_name = CurrentMetrics::getName(snapshot.metric); - columns[i++]->insertData(metric_name, strlen(metric_name)); - auto metric_value = CurrentMetrics::get(snapshot.metric); - columns[i++]->insert(metric_value); - } } } @@ -953,6 +936,7 @@ void TCPHandler::sendProfileEvents() ProfileEventsSnapshot group_snapshot; { std::lock_guard guard(thread_group->mutex); + snapshots.reserve(thread_group->threads.size()); for (auto * thread : thread_group->threads) { auto const thread_id = thread->thread_id; @@ -960,12 +944,10 @@ void TCPHandler::sendProfileEvents() continue; auto current_time = time(nullptr); auto counters = thread->performance_counters.getPartiallyAtomicSnapshot(); - auto metric = thread->memory_tracker.getMetric(); auto memory_usage = thread->memory_tracker.get(); snapshots.push_back(ProfileEventsSnapshot{ thread_id, std::move(counters), - metric, memory_usage, current_time }); @@ -973,7 +955,6 @@ void TCPHandler::sendProfileEvents() group_snapshot.thread_id = 0; group_snapshot.current_time = time(nullptr); - group_snapshot.metric = thread_group->memory_tracker.getMetric(); group_snapshot.memory_usage = thread_group->memory_tracker.get(); group_snapshot.counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); } From e6c088fe056b919374ec4bd53c10b2c3ef9794b0 Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Mon, 11 Oct 2021 13:03:00 +0300 Subject: [PATCH 100/126] Mark join_engine_deadlock as long test --- ...adlock.reference => 02033_join_engine_deadlock_long.reference} | 0 ...join_engine_deadlock.sh => 02033_join_engine_deadlock_long.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02033_join_engine_deadlock.reference => 02033_join_engine_deadlock_long.reference} (100%) rename tests/queries/0_stateless/{02033_join_engine_deadlock.sh => 02033_join_engine_deadlock_long.sh} (100%) diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock.reference b/tests/queries/0_stateless/02033_join_engine_deadlock_long.reference similarity index 100% rename from tests/queries/0_stateless/02033_join_engine_deadlock.reference rename to tests/queries/0_stateless/02033_join_engine_deadlock_long.reference diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock.sh b/tests/queries/0_stateless/02033_join_engine_deadlock_long.sh similarity index 100% rename from tests/queries/0_stateless/02033_join_engine_deadlock.sh rename to tests/queries/0_stateless/02033_join_engine_deadlock_long.sh From 706e2b6b8816a3e9a5022341331227a1986987da Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 11 Oct 2021 21:42:46 +0300 Subject: [PATCH 101/126] more strict check for intersecting parts --- src/Storages/MergeTree/MergeTreePartInfo.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index df3d9cb9237..82fe0b860c8 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -60,11 +60,16 @@ struct MergeTreePartInfo /// True if contains rhs (this part is obtained by merging rhs with some other parts or mutating rhs) bool contains(const MergeTreePartInfo & rhs) const { + /// Containing part may have equal level iff block numbers are equal (unless level is MAX_LEVEL) + /// (e.g. all_0_5_2 does not contain all_0_4_2, but all_0_5_3 or all_0_4_2_9 do) + bool strictly_contains_block_range = (min_block == rhs.min_block && max_block == rhs.max_block) || level > rhs.level + || level == MAX_LEVEL || level == LEGACY_MAX_LEVEL; return partition_id == rhs.partition_id /// Parts for different partitions are not merged && min_block <= rhs.min_block && max_block >= rhs.max_block && level >= rhs.level - && mutation >= rhs.mutation; + && mutation >= rhs.mutation + && strictly_contains_block_range; } /// Return part mutation version, if part wasn't mutated return zero From f1791ddc44e9dfe6644aa1f406e6b3924374e725 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 11 Oct 2021 15:49:30 -0300 Subject: [PATCH 102/126] explanation for volume/max_data_part_size_bytes --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index f7118f7557e..4f473279067 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -688,7 +688,7 @@ Tags: - `policy_name_N` — Policy name. Policy names must be unique. - `volume_name_N` — Volume name. Volume names must be unique. - `disk` — a disk within a volume. -- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. +- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks. If the a size of a merged part estimated to be bigger than `max_data_part_size_bytes` then this part will be written to a next volume. Basically this feature allows to keep new/small parts on a hot (SSD) volume and move them to a cold (HDD) volume when they reach large size. Do not use this setting if your policy has only one volume. - `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1). - `prefer_not_to_merge` — Disables merging of data parts on this volume. When this setting is enabled, merging data on this volume is not allowed. This allows controlling how ClickHouse works with slow disks. From fb83d2ddd593f914a465274566eb9bdca429ed55 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 11 Oct 2021 15:56:02 -0300 Subject: [PATCH 103/126] Update mergetree.md --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index e8152441101..bef14924d36 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -668,7 +668,7 @@ TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); - `policy_name_N` — название политики. Названия политик должны быть уникальны. - `volume_name_N` — название тома. Названия томов должны быть уникальны. - `disk` — диск, находящийся внутри тома. -- `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома. +- `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома. Если в результате слияния размер куска ожидается больше, чем max_data_part_size_bytes, то этот кусок будет записан в следующий том. В основном эта функция позволяет хранить новые / мелкие куски на горячем (SSD) томе и перемещать их на холодный (HDD) том, когда они достигают большого размера. Не используйте этот параметр, если политика имеет только один том. - `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1). - `prefer_not_to_merge` — Отключает слияние кусков данных, хранящихся на данном томе. Если данная настройка включена, то слияние данных, хранящихся на данном томе, не допускается. Это позволяет контролировать работу ClickHouse с медленными дисками. From 381d666e4b368ba3d750bafea5399a076a27a5f5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 11 Oct 2021 23:09:31 +0300 Subject: [PATCH 104/126] Ignore parallel removing warning in 00992_system_parts_race_condition_zookeeper_long CI: https://clickhouse-test-reports.s3.yandex.net/0/6c7fbf0b888db9c1272478189f0ff40212a3e7c9/functional_stateless_tests_(release,_databaseordinary)/test_run.txt.out.log --- .../00992_system_parts_race_condition_zookeeper_long.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index 273b39961af..aee8a7727e5 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -79,7 +79,7 @@ timeout $TIMEOUT bash -c thread5 2> /dev/null & wait check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum(c))" -$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" & -$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" & +$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & +$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & wait From 83a9a8d4dc8239035c4a0ff9c85658d17c78131b Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 11 Oct 2021 23:23:56 +0300 Subject: [PATCH 105/126] Update KeeperStateMachine.cpp --- src/Coordination/KeeperStateMachine.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index be4f73cf5ae..682a523fcaf 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -125,10 +125,6 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n } else { - LOG_TEST(log, "Commit request for session {} with type {}, log id {}{}", - request_for_session.session_id, toString(request_for_session.request->getOpNum()), log_idx, - request_for_session.request->getPath().empty() ? "" : ", path " + request_for_session.request->getPath()); - std::lock_guard lock(storage_and_responses_lock); KeeperStorage::ResponsesForSessions responses_for_sessions = storage->processRequest(request_for_session.request, request_for_session.session_id, log_idx); for (auto & response_for_session : responses_for_sessions) From d4e496c31424bfe428626ac3a18e5de1b13e13dc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 01:47:34 +0300 Subject: [PATCH 106/126] Add support for PowerPC build --- cmake/linux/default_libs.cmake | 2 +- cmake/target.cmake | 13 +- contrib/CMakeLists.txt | 2 +- .../internal/jemalloc_internal_defs.h.in | 4 +- contrib/libuv-cmake/CMakeLists.txt | 160 ++++++++++++++++++ src/Interpreters/ITokenExtractor.cpp | 2 +- utils/CMakeLists.txt | 2 +- utils/memcpy-bench/CMakeLists.txt | 2 +- 8 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 contrib/libuv-cmake/CMakeLists.txt diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index c1e4d450389..a2da7ba1915 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs") # We need builtins from Clang's RT even without libcxx - for ubsan+int128. # See https://bugs.llvm.org/show_bug.cgi?id=16404 -if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64)) +if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING) execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) else () set (BUILTINS_LIBRARY "-lgcc") diff --git a/cmake/target.cmake b/cmake/target.cmake index d1a0b8f9cbf..ca6009e68d3 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -34,11 +34,14 @@ if (CMAKE_CROSSCOMPILING) # FIXME: broken dependencies set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") - - set (ENABLE_PARQUET OFF CACHE INTERNAL "") - set (ENABLE_ORC OFF CACHE INTERNAL "") - - set (ENABLE_MYSQL OFF CACHE INTERNAL "") + set (USE_SENTRY OFF CACHE INTERNAL "") +# set (ENABLE_ROCKSDB OFF CACHE INTERNAL "") + endif () + elseif (ARCH_PPC64LE) + set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_GRPC OFF CACHE INTERNAL "") + set (USE_SENTRY OFF CACHE INTERNAL "") +# set (ENABLE_ROCKSDB OFF CACHE INTERNAL "") endif () elseif (OS_FREEBSD) # FIXME: broken dependencies diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4d817c4c6e2..5ff85fa85c2 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -278,7 +278,7 @@ if (USE_FASTOPS) endif() if (USE_AMQPCPP OR USE_CASSANDRA) - add_subdirectory (libuv) + add_subdirectory (libuv-cmake) endif() if (USE_AMQPCPP) add_subdirectory (amqpcpp-cmake) diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 8068861041f..97d0d4d8471 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -81,7 +81,7 @@ /* #undef JEMALLOC_HAVE_ISSETUGID */ /* Defined if pthread_atfork(3) is available. */ -#define JEMALLOC_HAVE_PTHREAD_ATFORK +/* #undef JEMALLOC_HAVE_PTHREAD_ATFORK */ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP @@ -284,7 +284,7 @@ #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS /* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ -/* #undef JEMALLOC_DEFINE_MADVISE_FREE */ +#define JEMALLOC_DEFINE_MADVISE_FREE /* * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt new file mode 100644 index 00000000000..4fbd0575b55 --- /dev/null +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -0,0 +1,160 @@ +# This file is a modified version of contrib/libuv/CMakeLists.txt + +include(CMakeDependentOption) + +set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv") +set (BINARY_DIR "${CMAKE_BINARY_DIR}/contrib/libuv") + + +if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") + list(APPEND uv_cflags -fvisibility=hidden --std=gnu89) + list(APPEND uv_cflags -Wall -Wextra -Wstrict-prototypes) + list(APPEND uv_cflags -Wno-unused-parameter) +endif() + +set(uv_sources + src/fs-poll.c + src/idna.c + src/inet.c + src/random.c + src/strscpy.c + src/threadpool.c + src/timer.c + src/uv-common.c + src/uv-data-getter-setters.c + src/version.c + src/unix/async.c + src/unix/core.c + src/unix/dl.c + src/unix/fs.c + src/unix/getaddrinfo.c + src/unix/getnameinfo.c + src/unix/loop-watcher.c + src/unix/loop.c + src/unix/pipe.c + src/unix/poll.c + src/unix/process.c + src/unix/random-devurandom.c + src/unix/signal.c + src/unix/stream.c + src/unix/tcp.c + src/unix/thread.c + src/unix/tty.c + src/unix/udp.c) + +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "Android|Linux|OS/390") + list(APPEND uv_sources src/unix/proctitle.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD") + list(APPEND uv_sources src/unix/freebsd.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") + list(APPEND uv_sources src/unix/posix-hrtime.c src/unix/bsd-proctitle.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "DragonFly|FreeBSD|NetBSD|OpenBSD") + list(APPEND uv_sources src/unix/bsd-ifaddrs.c src/unix/kqueue.c) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + list(APPEND uv_sources src/unix/random-getrandom.c) +endif() + +if(APPLE OR CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") + list(APPEND uv_sources src/unix/random-getentropy.c) +endif() + +if(APPLE) + list(APPEND uv_defines _DARWIN_UNLIMITED_SELECT=1 _DARWIN_USE_64_BIT_INODE=1) + list(APPEND uv_sources + src/unix/darwin-proctitle.c + src/unix/darwin.c + src/unix/fsevents.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112) + list(APPEND uv_libraries dl rt) + list(APPEND uv_sources + src/unix/linux-core.c + src/unix/linux-inotify.c + src/unix/linux-syscalls.c + src/unix/procfs-exepath.c + src/unix/random-getrandom.c + src/unix/random-sysctl-linux.c + src/unix/sysinfo-loadavg.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") + list(APPEND uv_sources src/unix/netbsd.c) + list(APPEND uv_libraries kvm) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") + list(APPEND uv_sources src/unix/openbsd.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "OS/390") + list(APPEND uv_defines PATH_MAX=255) + list(APPEND uv_defines _AE_BIMODAL) + list(APPEND uv_defines _ALL_SOURCE) + list(APPEND uv_defines _LARGE_TIME_API) + list(APPEND uv_defines _OPEN_MSGQ_EXT) + list(APPEND uv_defines _OPEN_SYS_FILE_EXT) + list(APPEND uv_defines _OPEN_SYS_IF_EXT) + list(APPEND uv_defines _OPEN_SYS_SOCK_EXT3) + list(APPEND uv_defines _OPEN_SYS_SOCK_IPV6) + list(APPEND uv_defines _UNIX03_SOURCE) + list(APPEND uv_defines _UNIX03_THREADS) + list(APPEND uv_defines _UNIX03_WITHDRAWN) + list(APPEND uv_defines _XOPEN_SOURCE_EXTENDED) + list(APPEND uv_sources + src/unix/pthread-fixes.c + src/unix/pthread-barrier.c + src/unix/os390.c + src/unix/os390-syscalls.c) +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "SunOS") + list(APPEND uv_defines __EXTENSIONS__ _XOPEN_SOURCE=500) + list(APPEND uv_libraries kstat nsl sendfile socket) + list(APPEND uv_sources src/unix/no-proctitle.c src/unix/sunos.c) +endif() + +set(uv_sources_tmp "") +foreach(file ${uv_sources}) + list(APPEND uv_sources_tmp "${SOURCE_DIR}/${file}") +endforeach(file) +set(uv_sources "${uv_sources_tmp}") + +list(APPEND uv_defines CLICKHOUSE_GLIBC_COMPATIBILITY) + +add_library(uv ${uv_sources}) +target_compile_definitions(uv + INTERFACE USING_UV_SHARED=1 + PRIVATE ${uv_defines} BUILDING_UV_SHARED=1) +target_compile_options(uv PRIVATE ${uv_cflags}) +target_include_directories(uv PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src) +target_link_libraries(uv ${uv_libraries}) + +add_library(uv_a STATIC ${uv_sources}) +target_compile_definitions(uv_a PRIVATE ${uv_defines}) +target_compile_options(uv_a PRIVATE ${uv_cflags}) +target_include_directories(uv_a PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src) +target_link_libraries(uv_a ${uv_libraries}) + +if(UNIX) + # Now for some gibbering horrors from beyond the stars... + foreach(x ${uv_libraries}) + set(LIBS "${LIBS} -l${x}") + endforeach(x) + file(STRINGS ${SOURCE_DIR}/configure.ac configure_ac REGEX ^AC_INIT) + string(REGEX MATCH [0-9]+[.][0-9]+[.][0-9]+ PACKAGE_VERSION "${configure_ac}") + string(REGEX MATCH ^[0-9]+ UV_VERSION_MAJOR "${PACKAGE_VERSION}") + # The version in the filename is mirroring the behaviour of autotools. + set_target_properties(uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0 + SOVERSION ${UV_VERSION_MAJOR}) +endif() + diff --git a/src/Interpreters/ITokenExtractor.cpp b/src/Interpreters/ITokenExtractor.cpp index 83166079e89..8c1af130f71 100644 --- a/src/Interpreters/ITokenExtractor.cpp +++ b/src/Interpreters/ITokenExtractor.cpp @@ -6,7 +6,7 @@ #include #if defined(__SSE2__) -#include +#include #if defined(__SSE4_2__) #include diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index a6bf2843e9a..8309b6bcb53 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -38,7 +38,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) endif () # memcpy_jart.S contains position dependent code - if (NOT CMAKE_POSITION_INDEPENDENT_CODE AND NOT OS_DARWIN AND NOT OS_SUNOS AND NOT ARCH_AARCH64) + if (NOT CMAKE_POSITION_INDEPENDENT_CODE AND OS_LINUX AND ARCH_AMD64) add_subdirectory (memcpy-bench) endif () endif () diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt index 5353b6fb68e..593a359a876 100644 --- a/utils/memcpy-bench/CMakeLists.txt +++ b/utils/memcpy-bench/CMakeLists.txt @@ -16,7 +16,7 @@ add_executable (memcpy-bench add_compile_options(memcpy-bench PRIVATE -fno-tree-loop-distribute-patterns) if (OS_SUNOS) - target_compile_options(memcpy-bench PRIVATE "-Wa,--divide") + target_compile_options(memcpy-bench PRIVATE "-Wa,--divide") endif() set_source_files_properties(FastMemcpy.cpp PROPERTIES COMPILE_FLAGS "-Wno-old-style-cast") From 0c6c716bdd7c18bb2fe7cabcadaa0189eb8e7457 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 01:49:29 +0300 Subject: [PATCH 107/126] Update submodule --- contrib/sysroot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/sysroot b/contrib/sysroot index 611d3315e9e..002415524b5 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit 611d3315e9e369a338de4ffa128eb87b4fb87dec +Subproject commit 002415524b5d14124bb8a61a3ce7ac65774f5479 From 54f3d0d2d977d62bee1a75510974326b9dc91047 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 02:02:44 +0300 Subject: [PATCH 108/126] Fix error --- cmake/target.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/target.cmake b/cmake/target.cmake index ca6009e68d3..872202f2f29 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -36,7 +36,6 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_GRPC OFF CACHE INTERNAL "") set (USE_SENTRY OFF CACHE INTERNAL "") # set (ENABLE_ROCKSDB OFF CACHE INTERNAL "") - endif () elseif (ARCH_PPC64LE) set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") From fa14dbdf429c06c3acc64311f0b49948028b6f51 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 02:03:00 +0300 Subject: [PATCH 109/126] Update submodules --- contrib/boost | 2 +- contrib/libuv | 2 +- contrib/s2geometry | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/boost b/contrib/boost index 66d17f060c4..311cfd49896 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 66d17f060c4867aeea99fa2a20cfdae89ae2a2ec +Subproject commit 311cfd498966d4f77742703d605d9c2e7b4cc6a8 diff --git a/contrib/libuv b/contrib/libuv index e2e9b7e9f97..95081e7c16c 160000 --- a/contrib/libuv +++ b/contrib/libuv @@ -1 +1 @@ -Subproject commit e2e9b7e9f978ce8a1367b5fe781d97d1ce9f94ab +Subproject commit 95081e7c16c9857babe6d4e2bc1c779198ea89ae diff --git a/contrib/s2geometry b/contrib/s2geometry index 20ea540d81f..38b7a290f92 160000 --- a/contrib/s2geometry +++ b/contrib/s2geometry @@ -1 +1 @@ -Subproject commit 20ea540d81f4575a3fc0aea585aac611bcd03ede +Subproject commit 38b7a290f927cc372218c2094602b83e35b18c05 From e1409c143b7d05a31bd26b8ebf063a160ce8b87d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 02:06:22 +0300 Subject: [PATCH 110/126] Add toolchain file --- cmake/linux/toolchain-ppc64le.cmake | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 cmake/linux/toolchain-ppc64le.cmake diff --git a/cmake/linux/toolchain-ppc64le.cmake b/cmake/linux/toolchain-ppc64le.cmake new file mode 100644 index 00000000000..cf85fc20fc4 --- /dev/null +++ b/cmake/linux/toolchain-ppc64le.cmake @@ -0,0 +1,32 @@ +set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set (CMAKE_SYSTEM_NAME "Linux") +set (CMAKE_SYSTEM_PROCESSOR "ppc64le") +set (CMAKE_C_COMPILER_TARGET "ppc64le-linux-gnu") +set (CMAKE_CXX_COMPILER_TARGET "ppc64le-linux-gnu") +set (CMAKE_ASM_COMPILER_TARGET "ppc64le-linux-gnu") + +set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le") + +set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc") + +find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") +find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9") + +set (CMAKE_AR "${LLVM_AR_PATH}" CACHE FILEPATH "" FORCE) +set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}" CACHE FILEPATH "" FORCE) + +set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") +set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") +set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") + +set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE) + +set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") +set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") + +set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) + +set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) From cb9bdf9666b8e1efac8b1d96e62359622c5bfd65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 02:07:03 +0300 Subject: [PATCH 111/126] Minor change --- cmake/target.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/target.cmake b/cmake/target.cmake index 872202f2f29..e8932a893c0 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -55,7 +55,7 @@ if (CMAKE_CROSSCOMPILING) endif () # Don't know why but CXX_STANDARD doesn't work for cross-compilation - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}") endif () From 75547e64a5ce564bcac8fd1446af5155f6a25822 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 02:10:58 +0300 Subject: [PATCH 112/126] Add to packager --- docker/packager/packager | 9 +++++++-- tests/ci/ci_config.json | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index f8e5fd717cf..ae7b99200ee 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -61,6 +61,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ DARWIN_ARM_SUFFIX = "-darwin-aarch64" ARM_SUFFIX = "-aarch64" FREEBSD_SUFFIX = "-freebsd" + PPC_SUFFIX = '-ppc64le' result = [] cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1'] @@ -69,8 +70,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX) is_cross_arm = compiler.endswith(ARM_SUFFIX) + is_cross_ppc = compiler.endswith(PPC_SUFFIX) is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX) - is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd + is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd or is_cross_ppc # Explicitly use LLD with Clang by default. # Don't force linker for cross-compilation. @@ -97,6 +99,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ elif is_cross_freebsd: cc = compiler[:-len(FREEBSD_SUFFIX)] cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake") + elif is_cross_ppc: + cc = compiler[:-len(PPC_SUFFIX)] + cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake") else: cc = compiler @@ -205,7 +210,7 @@ if __name__ == "__main__": parser.add_argument("--build-type", choices=("debug", ""), default="") parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64", "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64", - "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", + "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le", "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 6222e4f61bc..4feae56b93c 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -162,6 +162,16 @@ "splitted": "unsplitted", "tidy": "disable", "with_coverage": false + }, + { + "compiler": "clang-13-ppc64le", + "build-type": "", + "sanitizer": "", + "package-type": "binary", + "bundled": "bundled", + "splitted": "unsplitted", + "tidy": "disable", + "with_coverage": false } ], "tests_config": { From 0d076468666df08c950c7630d8b126d693bfc32e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 03:17:35 +0300 Subject: [PATCH 113/126] Fix strange code --- cmake/find/ssl.cmake | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmake/find/ssl.cmake b/cmake/find/ssl.cmake index fd6ed56dcdb..fdc0bfb27d3 100644 --- a/cmake/find/ssl.cmake +++ b/cmake/find/ssl.cmake @@ -53,12 +53,7 @@ endif () if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY) set (USE_INTERNAL_SSL_LIBRARY 1) set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/boringssl") - - if (ARCH_AMD64) - set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") - elseif (ARCH_AARCH64) - set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") - endif () + set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") set (OPENSSL_CRYPTO_LIBRARY crypto) set (OPENSSL_SSL_LIBRARY ssl) set (OPENSSL_FOUND 1) From b5d69d599e3b9f7fd8654a8e5cbd2d1dba374273 Mon Sep 17 00:00:00 2001 From: feng lv Date: Tue, 12 Oct 2021 04:13:02 +0000 Subject: [PATCH 114/126] fix sample by tuple() add tests fix fix fix --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++++ tests/queries/0_stateless/02096_sample_by_tuple.reference | 0 tests/queries/0_stateless/02096_sample_by_tuple.sql | 7 +++++++ 3 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/02096_sample_by_tuple.reference create mode 100644 tests/queries/0_stateless/02096_sample_by_tuple.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c04e0d2e38f..51b68eed951 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -132,10 +132,14 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int SUPPORT_IS_DISABLED; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; + extern const int INCORRECT_QUERY; } static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key, bool check_sample_column_is_correct) { + if (metadata.sampling_key.column_names.empty()) + throw Exception("There are no columns in sampling expression", ErrorCodes::INCORRECT_QUERY); + const auto & pk_sample_block = metadata.getPrimaryKey().sample_block; if (!pk_sample_block.has(metadata.sampling_key.column_names[0]) && !allow_sampling_expression_not_in_primary_key) throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); diff --git a/tests/queries/0_stateless/02096_sample_by_tuple.reference b/tests/queries/0_stateless/02096_sample_by_tuple.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02096_sample_by_tuple.sql b/tests/queries/0_stateless/02096_sample_by_tuple.sql new file mode 100644 index 00000000000..4996c9b8384 --- /dev/null +++ b/tests/queries/0_stateless/02096_sample_by_tuple.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS t; + +CREATE TABLE t (n UInt8) ENGINE=MergeTree ORDER BY n SAMPLE BY tuple(); -- { serverError 80 } + +CREATE TABLE t (n UInt8) ENGINE=MergeTree ORDER BY tuple(); + +ALTER TABLE t MODIFY SAMPLE BY tuple(); -- { serverError 80 } From ee1fd495ea77015db254a42108b630175c5930af Mon Sep 17 00:00:00 2001 From: feng lv Date: Tue, 12 Oct 2021 05:30:35 +0000 Subject: [PATCH 115/126] remove redundant dot in exception message --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 3965945b3ca..c098c6e0506 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -999,7 +999,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, } else throw Exception(storage_already_exists_error_code, - "{} {}.{} already exists.", storage_name, backQuoteIfNeed(create.database), backQuoteIfNeed(create.table)); + "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.database), backQuoteIfNeed(create.table)); } data_path = database->getTableDataPath(create); From c15b67c18264919df7c8048ea36ea5058a185d36 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 12 Oct 2021 11:42:24 +0300 Subject: [PATCH 116/126] Fix naming --- src/Storages/MergeTree/MergeTreeData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 1ede7669832..f9c26225440 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3006,7 +3006,7 @@ void MergeTreeData::removePartContributionToColumnAndSecondaryIndexSizes(const D for (const auto & column : part->getColumns()) { ColumnSize & total_column_size = column_sizes[column.name]; - ColumnSize part_secondary_index_size = part->getColumnSize(column.name, *column.type); + ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); auto log_subtract = [&](size_t & from, size_t value, const char * field) { @@ -3017,9 +3017,9 @@ void MergeTreeData::removePartContributionToColumnAndSecondaryIndexSizes(const D from -= value; }; - log_subtract(total_column_size.data_compressed, part_secondary_index_size.data_compressed, ".data_compressed"); - log_subtract(total_column_size.data_uncompressed, part_secondary_index_size.data_uncompressed, ".data_uncompressed"); - log_subtract(total_column_size.marks, part_secondary_index_size.marks, ".marks"); + log_subtract(total_column_size.data_compressed, part_column_size.data_compressed, ".data_compressed"); + log_subtract(total_column_size.data_uncompressed, part_column_size.data_uncompressed, ".data_uncompressed"); + log_subtract(total_column_size.marks, part_column_size.marks, ".marks"); } auto indexes_descriptions = getInMemoryMetadataPtr()->secondary_indices; From cb176cf9addf71b94b58cbe23b2c542d7417896d Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Tue, 12 Oct 2021 12:33:54 +0300 Subject: [PATCH 117/126] Add `long` tag to 02033_join_engine_deadlock_long --- tests/queries/0_stateless/02033_join_engine_deadlock_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02033_join_engine_deadlock_long.sh b/tests/queries/0_stateless/02033_join_engine_deadlock_long.sh index f4ae564e2a7..2a887cbbcae 100755 --- a/tests/queries/0_stateless/02033_join_engine_deadlock_long.sh +++ b/tests/queries/0_stateless/02033_join_engine_deadlock_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: deadlock +# Tags: long, deadlock CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From b2f8cd07e68107db07ffe79abdf7948d48f2f90d Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 12 Oct 2021 12:54:02 +0300 Subject: [PATCH 118/126] Update normalizeString.cpp --- src/Functions/normalizeString.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 5beca566cd1..2b3a869e452 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -10,9 +10,9 @@ #include #include #include -#include "common/logger_useful.h" -#include "Columns/ColumnString.h" -#include "Parsers/IAST_fwd.h" +#include +#include +#include namespace DB { From 5bf64c62c271cef95f5cfda1796eef82311c6bdd Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 12 Oct 2021 10:19:21 +0000 Subject: [PATCH 119/126] Delete test --- src/Functions/tests/gtest_abtesting.cpp | 105 ------------------------ 1 file changed, 105 deletions(-) delete mode 100644 src/Functions/tests/gtest_abtesting.cpp diff --git a/src/Functions/tests/gtest_abtesting.cpp b/src/Functions/tests/gtest_abtesting.cpp deleted file mode 100644 index e7ef5b5c3cf..00000000000 --- a/src/Functions/tests/gtest_abtesting.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include - -#if !defined(ARCADIA_BUILD) && USE_STATS - -# include - -using namespace DB; - -Variants test_bayesab(std::string dist, PODArray xs, PODArray ys, size_t & max, size_t & min) -{ - Variants variants; - - //std::cout << std::fixed; - if (dist == "beta") - { -/* std::cout << dist << "\nclicks: "; - for (auto x : xs) - std::cout << x << " "; - - std::cout <<"\tconversions: "; - for (auto y : ys) - std::cout << y << " "; - - std::cout << "\n";*/ - - variants = bayesian_ab_test(dist, xs, ys); - } - else if (dist == "gamma") - { -/* std::cout << dist << "\nclicks: "; - for (auto x : xs) - std::cout << x << " "; - - std::cout <<"\tcost: "; - for (auto y : ys) - std::cout << y << " "; - - std::cout << "\n";*/ - - variants = bayesian_ab_test(dist, xs, ys); - } - -/* for (size_t i = 0; i < variants.size(); ++i) - std::cout << i << " beats 0: " << variants[i].beats_control << std::endl; - - for (size_t i = 0; i < variants.size(); ++i) - std::cout << i << " to be best: " << variants[i].best << std::endl; - - std::cout << convertToJson({"0", "1", "2"}, variants) << std::endl; -*/ - Float64 max_val = 0.0, min_val = 2.0; - for (size_t i = 0; i < variants.size(); ++i) - { - if (variants[i].best > max_val) - { - max_val = variants[i].best; - max = i; - } - - if (variants[i].best < min_val) - { - min_val = variants[i].best; - min = i; - } - } - - return variants; -} - - -TEST(BayesAB, beta) -{ - size_t max = 0, min = 0; - - auto variants = test_bayesab("beta", {10000, 1000, 900}, {600, 110, 90}, max, min); - ASSERT_EQ(1, max); - - variants = test_bayesab("beta", {3000, 3000, 3000}, {600, 100, 90}, max, min); - ASSERT_EQ(0, max); - - variants = test_bayesab("beta", {3000, 3000, 3000}, {100, 90, 110}, max, min); - ASSERT_EQ(2, max); - - variants = test_bayesab("beta", {3000, 3000, 3000}, {110, 90, 100}, max, min); - ASSERT_EQ(0, max); -} - - -TEST(BayesAB, gamma) -{ - size_t max = 0, min = 0; - auto variants = test_bayesab("gamma", {10000, 1000, 900}, {600, 110, 90}, max, min); - ASSERT_EQ(1, max); - - variants = test_bayesab("gamma", {3000, 3000, 3000}, {600, 100, 90}, max, min); - ASSERT_EQ(0, max); - - variants = test_bayesab("gamma", {3000, 3000, 3000}, {100, 90, 110}, max, min); - ASSERT_EQ(2, max); - - variants = test_bayesab("gamma", {3000, 3000, 3000}, {110, 90, 100}, max, min); - ASSERT_EQ(0, max); -} - -#endif From dca1f8e7f5c017ebaf263af542f2075e437412c4 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 Oct 2021 15:38:40 +0300 Subject: [PATCH 120/126] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d024100a27e..e12238577a7 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ [![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.com) -ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real time. +ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time. ## Useful Links -* [Official website](https://clickhouse.com/) has quick high-level overview of ClickHouse on main page. -* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster. +* [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page. +* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster. * [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. +* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. * [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser](https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. From 89ecbfdfe78dd5d02de2fd006b0d882db4259567 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 Oct 2021 15:51:19 +0300 Subject: [PATCH 121/126] Update distinctive-features.md --- docs/en/introduction/distinctive-features.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/introduction/distinctive-features.md b/docs/en/introduction/distinctive-features.md index 34ba4b89415..daf43dc0da8 100644 --- a/docs/en/introduction/distinctive-features.md +++ b/docs/en/introduction/distinctive-features.md @@ -39,9 +39,9 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of ClickHouse supports a [declarative query language based on SQL](../sql-reference/index.md) that is identical to the ANSI SQL standard in [many cases](../sql-reference/ansi.md). -Supported queries include [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), subqueries in [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md) clause, [IN](../sql-reference/operators/in.md) operator, and scalar subqueries. +Supported queries include [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), subqueries in [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md) clause, [IN](../sql-reference/operators/in.md) operator, [window functions](../sql-reference/window-functions.md) and scalar subqueries. -Correlated (dependent) subqueries and window functions are not supported at the time of writing but might become available in the future. +Correlated (dependent) subqueries are not supported at the time of writing but might become available in the future. ## Vector Computation Engine {#vector-engine} From b3610134fdfa41590fbd5b4b1850bab8c47db8b8 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Tue, 12 Oct 2021 16:31:51 +0300 Subject: [PATCH 122/126] Update docs/ru/sql-reference/statements/select/prewhere.md Co-authored-by: gyuton <40863448+gyuton@users.noreply.github.com> --- docs/ru/sql-reference/statements/select/prewhere.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index 84f8869b41e..4376cbeb295 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -14,7 +14,7 @@ Prewhere — это оптимизация для более эффективн В запросе может быть одновременно указаны и `PREWHERE`, и `WHERE`. В этом случае `PREWHERE` предшествует `WHERE`. -Если значение параметра [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) равно 0, эвристика по автоматическому перемещнию части выражений из `WHERE` к `PREWHERE` отключается. +Если значение параметра [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) равно 0, эвристика по автоматическому перемещению части выражений из `WHERE` к `PREWHERE` отключается. Если в запросе есть модификатор [FINAL](from.md#select-from-final), оптимизация `PREWHERE` не всегда корректна. Она действует только если включены обе настройки [optimize_move_to_prewhere](../../../operations/settings/settings.md#optimize_move_to_prewhere) и [optimize_move_to_prewhere_if_final](../../../operations/settings/settings.md#optimize_move_to_prewhere_if_final). From 0cfaf9c50861aa931b1b01e90d7e0fcc8a700472 Mon Sep 17 00:00:00 2001 From: olgarev Date: Tue, 12 Oct 2021 13:44:00 +0000 Subject: [PATCH 123/126] Unnecessary links removed --- docs/en/operations/settings/settings.md | 6 ------ docs/ru/operations/settings/settings.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 0491674b701..aa70eb4f721 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3798,10 +3798,6 @@ Possible values: Default value: `1`. -**See Also** - -- [PREWHERE](../../sql-reference/statements/select/prewhere.md) clause in `SELECT` queries - ## optimize_move_to_prewhere_if_final {#optimize_move_to_prewhere_if_final} Enables or disables automatic [PREWHERE](../../sql-reference/statements/select/prewhere.md) optimization in [SELECT](../../sql-reference/statements/select/index.md) queries with [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier. @@ -3817,6 +3813,4 @@ Default value: `0`. **See Also** -- [PREWHERE](../../sql-reference/statements/select/prewhere.md) clause in `SELECT` queries -- [FINAL](../../sql-reference/statements/select/from.md#select-from-final) modifier in `SELECT` queries - [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting \ No newline at end of file diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 887c59c3b09..bccbbf69e39 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3587,10 +3587,6 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Значение по умолчанию: `1`. -**См. также** - -- секция [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах `SELECT` - ## optimize_move_to_prewhere_if_final {#optimize_move_to_prewhere_if_final} Включает или отключает автоматическую оптимизацию [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах [SELECT](../../sql-reference/statements/select/index.md) с модификатором [FINAL](../../sql-reference/statements/select/from.md#select-from-final). @@ -3606,6 +3602,4 @@ SELECT * FROM positional_arguments ORDER BY 2,3; **См. также** -- секция [PREWHERE](../../sql-reference/statements/select/prewhere.md) в запросах `SELECT` -- модификатор [FINAL](../../sql-reference/statements/select/from.md#select-from-final) в запросах `SELECT` - настройка [optimize_move_to_prewhere](#optimize_move_to_prewhere) \ No newline at end of file From 63cfc2311bb564aebfd3bb0804aca9fd20b7bfff Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 12 Oct 2021 19:38:17 +0300 Subject: [PATCH 124/126] Update distinctive-features.md --- docs/en/introduction/distinctive-features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/distinctive-features.md b/docs/en/introduction/distinctive-features.md index daf43dc0da8..951a8a9d3e5 100644 --- a/docs/en/introduction/distinctive-features.md +++ b/docs/en/introduction/distinctive-features.md @@ -39,7 +39,7 @@ In ClickHouse, data can reside on different shards. Each shard can be a group of ClickHouse supports a [declarative query language based on SQL](../sql-reference/index.md) that is identical to the ANSI SQL standard in [many cases](../sql-reference/ansi.md). -Supported queries include [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), subqueries in [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md) clause, [IN](../sql-reference/operators/in.md) operator, [window functions](../sql-reference/window-functions.md) and scalar subqueries. +Supported queries include [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), subqueries in [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md) clause, [IN](../sql-reference/operators/in.md) operator, [window functions](../sql-reference/window-functions/index.md) and scalar subqueries. Correlated (dependent) subqueries are not supported at the time of writing but might become available in the future. From 65b63a67da556ca671ad7c8c1a3c28a305097b1d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 23:03:20 +0300 Subject: [PATCH 125/126] Add a script for convenient install on multiple OS --- docs/_includes/install/universal.sh | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 docs/_includes/install/universal.sh diff --git a/docs/_includes/install/universal.sh b/docs/_includes/install/universal.sh new file mode 100755 index 00000000000..b5f833b19a6 --- /dev/null +++ b/docs/_includes/install/universal.sh @@ -0,0 +1,59 @@ +#!/bin/sh -e + +OS=$(uname -s) +ARCH=$(uname -m) + +DIR= + +if [ "${OS}" = "Linux" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="amd64" + elif [ "${ARCH}" = "aarch64" ] + then + DIR="aarch64" + elif [ "${ARCH}" = "powerpc64le" ] + then + DIR="powerpc64le" + fi +elif [ "${OS}" = "FreeBSD" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="freebsd" + elif [ "${ARCH}" = "aarch64" ] + then + #DIR="freebsd-aarch64" + elif [ "${ARCH}" = "powerpc64le" ] + then + #DIR="freebsd-powerpc64le" + fi +elif [ "${OS}" = "Darwin" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="macos" + elif [ "${ARCH}" = "aarch64" ] + then + DIR="macos-aarch64" + fi +fi + +if [ -z "${DIR}" ] +then + echo "The '${OS}' operating system with the '${ARCH}' architecture is not supported." + exit 1 +fi + +URL="https://builds.clickhouse.com/master/${DIR}/clickhouse" +echo "Will download ${URL}" +curl -O "${URL}" && chmod a+x clickhouse && +echo "Successfully downloaded the ClickHouse binary, you can run it as: + ./clickhouse" + +if [ "${OS}" = "Linux" ] +then + echo "You can also install it: + ./clickhouse install" +fi From ac403b1df259df7084c89d57ea77bfad4c914783 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 12 Oct 2021 23:09:26 +0300 Subject: [PATCH 126/126] Publish the install script --- docs/_includes/install/universal.sh | 2 +- docs/tools/website.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/_includes/install/universal.sh b/docs/_includes/install/universal.sh index b5f833b19a6..db1072f149f 100755 --- a/docs/_includes/install/universal.sh +++ b/docs/_includes/install/universal.sh @@ -55,5 +55,5 @@ echo "Successfully downloaded the ClickHouse binary, you can run it as: if [ "${OS}" = "Linux" ] then echo "You can also install it: - ./clickhouse install" + sudo ./clickhouse install" fi diff --git a/docs/tools/website.py b/docs/tools/website.py index 5e4f48e3441..54804ae3f36 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -156,6 +156,11 @@ def build_website(args): os.path.join(args.src_dir, 'utils', 'list-versions', 'version_date.tsv'), os.path.join(args.output_dir, 'data', 'version_date.tsv')) + # This file can be requested to install ClickHouse. + shutil.copy2( + os.path.join(args.src_dir, 'docs', '_includes', 'install', 'universal.sh'), + os.path.join(args.output_dir, 'data', 'install.sh')) + for root, _, filenames in os.walk(args.output_dir): for filename in filenames: if filename == 'main.html': @@ -218,7 +223,7 @@ def minify_file(path, css_digest, js_digest): # TODO: restore cssmin # elif path.endswith('.css'): # content = cssmin.cssmin(content) -# TODO: restore jsmin +# TODO: restore jsmin # elif path.endswith('.js'): # content = jsmin.jsmin(content) with open(path, 'wb') as f: