From c1e80683c444512a446e4579b000f512996237d5 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Fri, 3 Mar 2023 17:57:10 +0100 Subject: [PATCH 01/49] Refine OptimizeRegularExpression Function --- src/Common/OptimizedRegularExpression.cpp | 107 +++++++++++++++------- src/Common/OptimizedRegularExpression.h | 3 +- src/Common/tests/gtest_optimize_re.cpp | 31 +++++++ 3 files changed, 107 insertions(+), 34 deletions(-) create mode 100644 src/Common/tests/gtest_optimize_re.cpp diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index 7d96feba1f3..ff87cd7ef86 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -47,6 +47,25 @@ void OptimizedRegularExpressionImpl::analyze( Substrings trivial_substrings(1); Substring * last_substring = &trivial_substrings.back(); + std::string bracket_string; + bool appending_bracket_string = false; + + auto finish_last_substring = [&]() + { + if (depth != 0) + return; + /// combine last substr and bracket string + last_substring->first += bracket_string; + bracket_string = ""; + /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg + if (!last_substring->first.empty() && !appending_bracket_string) + { + trivial_substrings.resize(trivial_substrings.size() + 1); + last_substring = &trivial_substrings.back(); + } + appending_bracket_string = false; + }; + bool in_curly_braces = false; bool in_square_braces = false; @@ -83,15 +102,21 @@ void OptimizedRegularExpressionImpl::analyze( last_substring->second = pos - begin; last_substring->first.push_back(*pos); } + else if (depth == 1 && appending_bracket_string) + { + bracket_string += *pos; + } break; default: /// all other escape sequences are not supported is_trivial = false; - if (!last_substring->first.empty()) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } + appending_bracket_string = false; + //if (!last_substring->first.empty()) + //{ + // trivial_substrings.resize(trivial_substrings.size() + 1); + // last_substring = &trivial_substrings.back(); + //} + finish_last_substring(); break; } @@ -102,8 +127,13 @@ void OptimizedRegularExpressionImpl::analyze( case '|': if (depth == 0) has_alternative_on_depth_0 = true; + if (depth == 1) + { + appending_bracket_string = false; + bracket_string = ""; + } is_trivial = false; - if (!in_square_braces && !last_substring->first.empty()) + if (!in_square_braces && !last_substring->first.empty() && depth == 0) { trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); @@ -116,11 +146,10 @@ void OptimizedRegularExpressionImpl::analyze( { ++depth; is_trivial = false; - if (!last_substring->first.empty()) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } + /// we dont change the value of appending_bracket_string when depth > 1 + /// e.g. (de(fg)) should capture defg + if (depth == 1) + appending_bracket_string = true; /// Check for case-insensitive flag. if (pos + 1 < end && pos[1] == '?') @@ -143,6 +172,10 @@ void OptimizedRegularExpressionImpl::analyze( break; } } + if (pos + 2 < end && pos[1] == '?' && pos[2] == ':') + { + pos += 2; + } } ++pos; break; @@ -151,11 +184,8 @@ void OptimizedRegularExpressionImpl::analyze( in_square_braces = true; ++depth; is_trivial = false; - if (!last_substring->first.empty()) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } + appending_bracket_string = false; + finish_last_substring(); ++pos; break; @@ -166,11 +196,12 @@ void OptimizedRegularExpressionImpl::analyze( in_square_braces = false; --depth; is_trivial = false; - if (!last_substring->first.empty()) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } + finish_last_substring(); + //if (!last_substring->first.empty()) + //{ + // trivial_substrings.resize(trivial_substrings.size() + 1); + // last_substring = &trivial_substrings.back(); + //} ++pos; break; @@ -179,22 +210,21 @@ void OptimizedRegularExpressionImpl::analyze( { --depth; is_trivial = false; - if (!last_substring->first.empty()) + if (pos + 1 < end && (pos[1] == '?' || pos[1] == '*')) { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); + /// TODO: (abc(def)?) should remain the abc part. + bracket_string = ""; + appending_bracket_string = false; } + finish_last_substring(); } ++pos; break; case '^': case '$': case '.': case '+': is_trivial = false; - if (!last_substring->first.empty() && !in_square_braces) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } + appending_bracket_string = false; + finish_last_substring(); ++pos; break; @@ -206,12 +236,20 @@ void OptimizedRegularExpressionImpl::analyze( [[fallthrough]]; case '*': is_trivial = false; - if (!last_substring->first.empty() && !in_square_braces) + if (depth == 0 && !last_substring->first.empty() && !in_square_braces) { last_substring->first.resize(last_substring->first.size() - 1); - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); } + if (depth >= 1 && appending_bracket_string) + { + /// ab(*cd) should be ab + appending_bracket_string = false; + if (!bracket_string.empty()) + { + bracket_string.resize(bracket_string.size() - 1); + } + } + finish_last_substring(); ++pos; break; @@ -232,11 +270,16 @@ void OptimizedRegularExpressionImpl::analyze( last_substring->second = pos - begin; last_substring->first.push_back(*pos); } + else if (depth >= 1 && appending_bracket_string) + bracket_string += *pos; ++pos; break; } } + appending_bracket_string = false; + finish_last_substring(); + if (last_substring && last_substring->first.empty()) trivial_substrings.pop_back(); diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index d8ed1e205c8..d8b54520bf3 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -95,6 +95,7 @@ public: out_required_substring_is_prefix = required_substring_is_prefix; } + static void analyze(std::string_view regexp_, std::string & required_substring, bool & is_trivial, bool & required_substring_is_prefix); private: bool is_trivial; bool required_substring_is_prefix; @@ -104,8 +105,6 @@ private: std::optional case_insensitive_substring_searcher; std::unique_ptr re2; unsigned number_of_subpatterns; - - static void analyze(std::string_view regexp_, std::string & required_substring, bool & is_trivial, bool & required_substring_is_prefix); }; using OptimizedRegularExpression = OptimizedRegularExpressionImpl; diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp new file mode 100644 index 00000000000..e68f699ee80 --- /dev/null +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -0,0 +1,31 @@ +#include + +#include + +TEST(OptimizeRE, analyze) +{ + auto test_f = [](const std::string & regexp, const std::string & answer) + { + std::string required; + bool is_trivial; + bool is_prefix; + OptimizedRegularExpression::analyze(regexp, required, is_trivial, is_prefix); + EXPECT_EQ(required, answer); + }; + test_f("abc", "abc"); + test_f("abc(de)fg", "abcdefg"); + test_f("abc(de|xyz)fg", "abc"); + test_f("abc(de?f|xyz)fg", "abc"); + test_f("abc|fg", ""); + test_f("(abc)", "abc"); + test_f("(abc|fg)", ""); + test_f("abc(abc|fg)xyzz", "xyzz"); + test_f("abc[k]xyzz", "xyzz"); + /// actually the best answer should be xyzz + test_f("(abc[k]xyzz)", "abc"); + test_f("abc((de)fg(hi))jk", "abcdefghijk"); + test_f("abc((de)fghi+zzz)jk", "abcdefghi"); + test_f("abc((de)fg(hi))?jk", "abc"); + test_f("abc((de)fghi?zzz)jk", "abcdefgh"); + test_f("abc(*cd)jk", "abc"); +} From 420108a7a05f3cbc5d4230b2fcf1dad0168a9070 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 6 Mar 2023 19:10:36 +0100 Subject: [PATCH 02/49] support alternatives --- src/Common/OptimizedRegularExpression.cpp | 197 ++++++++++++---------- src/Common/OptimizedRegularExpression.h | 8 +- src/Common/tests/gtest_optimize_re.cpp | 26 ++- 3 files changed, 131 insertions(+), 100 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index ff87cd7ef86..b1fc9a2174c 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -16,11 +18,13 @@ namespace DB template -void OptimizedRegularExpressionImpl::analyze( +const char * OptimizedRegularExpressionImpl::analyze( std::string_view regexp, + const char * pos, std::string & required_substring, bool & is_trivial, - bool & required_substring_is_prefix) + bool & required_substring_is_prefix, + std::vector & alternatives) { /** The expression is trivial if all the metacharacters in it are escaped. * The non-alternative string is @@ -30,9 +34,9 @@ void OptimizedRegularExpressionImpl::analyze( * and also avoid substrings of the form `http://` or `www` and some other * (this is the hack for typical use case in web analytics applications). */ - const char * begin = regexp.data(); - const char * pos = begin; + const char * begin = pos; const char * end = regexp.data() + regexp.size(); + bool first_call = begin == regexp.data(); int depth = 0; is_trivial = true; required_substring_is_prefix = false; @@ -47,23 +51,46 @@ void OptimizedRegularExpressionImpl::analyze( Substrings trivial_substrings(1); Substring * last_substring = &trivial_substrings.back(); - std::string bracket_string; - bool appending_bracket_string = false; - - auto finish_last_substring = [&]() + auto finish_non_trivial_char = [&]() { if (depth != 0) return; - /// combine last substr and bracket string - last_substring->first += bracket_string; - bracket_string = ""; - /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg - if (!last_substring->first.empty() && !appending_bracket_string) + + if (!last_substring->first.empty()) { trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); } - appending_bracket_string = false; + }; + + + auto finish_group = [&](std::string group_required_string, bool group_is_trivial, bool group_is_prefix, std::vector & group_alternatives) + { + if (alternatives.empty() && !group_alternatives.empty()) + { + /// Check if group alternatives has empty strings + bool has_empty_str = false; + for (const std::string & alter : group_alternatives) + has_empty_str |= alter.empty(); + if (!has_empty_str) + alternatives = std::move(group_alternatives); + } + + if (group_is_prefix) + last_substring->first += group_required_string; + else + { + finish_non_trivial_char(); + last_substring->first = group_required_string; + } + /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg + if (!last_substring->first.empty() && !group_is_trivial) + { + trivial_substrings.resize(trivial_substrings.size() + 1); + last_substring = &trivial_substrings.back(); + } + if (!group_is_trivial) + is_trivial = false; }; bool in_curly_braces = false; @@ -92,31 +119,19 @@ void OptimizedRegularExpressionImpl::analyze( case '$': case '.': case '[': + case ']': case '?': case '*': case '+': + case '-': case '{': - if (depth == 0 && !in_curly_braces && !in_square_braces) - { - if (last_substring->first.empty()) - last_substring->second = pos - begin; - last_substring->first.push_back(*pos); - } - else if (depth == 1 && appending_bracket_string) - { - bracket_string += *pos; - } - break; + case '}': + case '/': + goto ordinary; default: /// all other escape sequences are not supported is_trivial = false; - appending_bracket_string = false; - //if (!last_substring->first.empty()) - //{ - // trivial_substrings.resize(trivial_substrings.size() + 1); - // last_substring = &trivial_substrings.back(); - //} - finish_last_substring(); + finish_non_trivial_char(); break; } @@ -125,32 +140,18 @@ void OptimizedRegularExpressionImpl::analyze( } case '|': - if (depth == 0) - has_alternative_on_depth_0 = true; - if (depth == 1) - { - appending_bracket_string = false; - bracket_string = ""; - } is_trivial = false; - if (!in_square_braces && !last_substring->first.empty() && depth == 0) - { - trivial_substrings.resize(trivial_substrings.size() + 1); - last_substring = &trivial_substrings.back(); - } ++pos; + if (depth == 0) + { + has_alternative_on_depth_0 = true; + goto finish; + } break; case '(': if (!in_square_braces) { - ++depth; - is_trivial = false; - /// we dont change the value of appending_bracket_string when depth > 1 - /// e.g. (de(fg)) should capture defg - if (depth == 1) - appending_bracket_string = true; - /// Check for case-insensitive flag. if (pos + 1 < end && pos[1] == '?') { @@ -176,6 +177,23 @@ void OptimizedRegularExpressionImpl::analyze( { pos += 2; } + std::string group_required_substr; + bool group_is_trival; + bool group_is_prefix; + std::vector group_alters; + pos = analyze(regexp, pos + 1, group_required_substr, group_is_trival, group_is_prefix, group_alters); + /// pos should be ')', if not, then it is not a valid regular expression + if (pos == end) + return pos; + + /// For ()? ()* (){0,1}, we can just ignore the whole group. + if ((pos + 1 < end && (pos[1] == '?' || pos[1] == '*')) || + (pos + 2 < end && pos[1] == '{' && pos[2] == '0')) + { + finish_non_trivial_char(); + } + else + finish_group(group_required_substr, group_is_trival, group_is_prefix, group_alters); } ++pos; break; @@ -184,8 +202,7 @@ void OptimizedRegularExpressionImpl::analyze( in_square_braces = true; ++depth; is_trivial = false; - appending_bracket_string = false; - finish_last_substring(); + finish_non_trivial_char(); ++pos; break; @@ -193,38 +210,25 @@ void OptimizedRegularExpressionImpl::analyze( if (!in_square_braces) goto ordinary; - in_square_braces = false; --depth; + if (depth == 0) + in_square_braces = false; is_trivial = false; - finish_last_substring(); - //if (!last_substring->first.empty()) - //{ - // trivial_substrings.resize(trivial_substrings.size() + 1); - // last_substring = &trivial_substrings.back(); - //} + finish_non_trivial_char(); ++pos; break; case ')': if (!in_square_braces) { - --depth; - is_trivial = false; - if (pos + 1 < end && (pos[1] == '?' || pos[1] == '*')) - { - /// TODO: (abc(def)?) should remain the abc part. - bracket_string = ""; - appending_bracket_string = false; - } - finish_last_substring(); + goto finish; } ++pos; break; case '^': case '$': case '.': case '+': is_trivial = false; - appending_bracket_string = false; - finish_last_substring(); + finish_non_trivial_char(); ++pos; break; @@ -240,16 +244,7 @@ void OptimizedRegularExpressionImpl::analyze( { last_substring->first.resize(last_substring->first.size() - 1); } - if (depth >= 1 && appending_bracket_string) - { - /// ab(*cd) should be ab - appending_bracket_string = false; - if (!bracket_string.empty()) - { - bracket_string.resize(bracket_string.size() - 1); - } - } - finish_last_substring(); + finish_non_trivial_char(); ++pos; break; @@ -270,22 +265,19 @@ void OptimizedRegularExpressionImpl::analyze( last_substring->second = pos - begin; last_substring->first.push_back(*pos); } - else if (depth >= 1 && appending_bracket_string) - bracket_string += *pos; ++pos; break; } } - - appending_bracket_string = false; - finish_last_substring(); - +finish: if (last_substring && last_substring->first.empty()) trivial_substrings.pop_back(); if (!is_trivial) { - if (!has_alternative_on_depth_0 && !has_case_insensitive_flag) + /// we calculate required substring even though has_alternative_on_depth_0. + /// we will clear the required substring after putting it to alternatives. + if (!has_case_insensitive_flag) { /// We choose the non-alternative substring of the maximum length for first search. @@ -305,7 +297,7 @@ void OptimizedRegularExpressionImpl::analyze( } } - if (max_length >= MIN_LENGTH_FOR_STRSTR) + if (max_length >= MIN_LENGTH_FOR_STRSTR || !first_call) { required_substring = candidate_it->first; required_substring_is_prefix = candidate_it->second == 0; @@ -318,6 +310,30 @@ void OptimizedRegularExpressionImpl::analyze( required_substring_is_prefix = trivial_substrings.front().second == 0; } + /// if it is xxx|xxx|xxx, we should call the next xxx|xxx recursively and collect the result. + if (has_alternative_on_depth_0) + { + if (alternatives.empty()) + alternatives.push_back(required_substring); + std::vector next_alternatives; + /// this two vals are useless, xxx|xxx cannot be trivial nor prefix. + bool next_is_trivial; + bool next_is_prefix; + pos = analyze(regexp, pos, required_substring, next_is_trivial, next_is_prefix, next_alternatives); + /// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring. + if (next_alternatives.empty()) + { + alternatives.push_back(required_substring); + } + else + { + alternatives.insert(alternatives.end(), next_alternatives.begin(), next_alternatives.end()); + } + required_substring.clear(); + } + + return pos; + /* std::cerr << "regexp: " << regexp << ", is_trivial: " << is_trivial @@ -330,7 +346,8 @@ void OptimizedRegularExpressionImpl::analyze( template OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(const std::string & regexp_, int options) { - analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix); + std::vector alternatives; /// this vector collects patterns in (xx|xx|xx). for now it's not used. + analyze(regexp_, regexp_.data(), required_substring, is_trivial, required_substring_is_prefix, alternatives); /// Just three following options are supported if (options & (~(RE_CASELESS | RE_NO_CAPTURE | RE_DOT_NL))) diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index d8b54520bf3..a19ce22deab 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -95,7 +95,13 @@ public: out_required_substring_is_prefix = required_substring_is_prefix; } - static void analyze(std::string_view regexp_, std::string & required_substring, bool & is_trivial, bool & required_substring_is_prefix); + static const char * analyze( + std::string_view regexp_, + const char * pos, + std::string & required_substring, + bool & is_trivial, + bool & required_substring_is_prefix, + std::vector & alternatives); private: bool is_trivial; bool required_substring_is_prefix; diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index e68f699ee80..ebd6a28900b 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -4,28 +4,36 @@ TEST(OptimizeRE, analyze) { - auto test_f = [](const std::string & regexp, const std::string & answer) + auto test_f = [](const std::string & regexp, const std::string & answer, std::vector expect_alternatives = {}) { std::string required; bool is_trivial; bool is_prefix; - OptimizedRegularExpression::analyze(regexp, required, is_trivial, is_prefix); + std::vector alternatives; + OptimizedRegularExpression::analyze(regexp, regexp.data(), required, is_trivial, is_prefix, alternatives); + std::cerr << regexp << std::endl; EXPECT_EQ(required, answer); + EXPECT_EQ(alternatives, expect_alternatives); }; test_f("abc", "abc"); test_f("abc(de)fg", "abcdefg"); - test_f("abc(de|xyz)fg", "abc"); - test_f("abc(de?f|xyz)fg", "abc"); - test_f("abc|fg", ""); + test_f("abc(de|xyz)fg", "abc", {"de", "xyz"}); + test_f("abc(de?f|xyz)fg", "abc", {"d", "xyz"}); + test_f("abc|fgk|xyz", "", {"abc","fgk", "xyz"}); test_f("(abc)", "abc"); - test_f("(abc|fg)", ""); - test_f("abc(abc|fg)xyzz", "xyzz"); + test_f("(abc|fgk)", "", {"abc","fgk"}); + test_f("abc(abc|fg)xyzz", "xyzz", {"abc","fg"}); test_f("abc[k]xyzz", "xyzz"); - /// actually the best answer should be xyzz - test_f("(abc[k]xyzz)", "abc"); + test_f("(abc[k]xyzz)", "xyzz"); test_f("abc((de)fg(hi))jk", "abcdefghijk"); test_f("abc((de)fghi+zzz)jk", "abcdefghi"); test_f("abc((de)fg(hi))?jk", "abc"); test_f("abc((de)fghi?zzz)jk", "abcdefgh"); test_f("abc(*cd)jk", "abc"); + test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"de", "xyz", "{xx}"}); + test_f("abc(abc|fg)?xyzz", "xyzz"); + test_f("abc(abc|fg){0,1}xyzz", "xyzz"); + test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abc", "fg", "bcd", "bc"}); + test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abc", "fg", "d", "k", "f", "bc"}); + test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc", "efg", "xyz"}); } From 39a11854862d914033ba3f9271cd0c57da75ae11 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Fri, 10 Mar 2023 15:30:29 +0100 Subject: [PATCH 03/49] fix test --- src/Common/OptimizedRegularExpression.cpp | 1 + src/Common/tests/gtest_optimize_re.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index b1fc9a2174c..efa33f27ed4 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -150,6 +150,7 @@ const char * OptimizedRegularExpressionImpl::analyze( break; case '(': + is_trivial = false; if (!in_square_braces) { /// Check for case-insensitive flag. diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index ebd6a28900b..c90b5086b1c 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -25,7 +25,7 @@ TEST(OptimizeRE, analyze) test_f("abc(abc|fg)xyzz", "xyzz", {"abc","fg"}); test_f("abc[k]xyzz", "xyzz"); test_f("(abc[k]xyzz)", "xyzz"); - test_f("abc((de)fg(hi))jk", "abcdefghijk"); + test_f("abc((de)fg(hi))jk", "abcdefghi"); test_f("abc((de)fghi+zzz)jk", "abcdefghi"); test_f("abc((de)fg(hi))?jk", "abc"); test_f("abc((de)fghi?zzz)jk", "abcdefgh"); From de8d0040a4452e72c7d8d00444ce787000e0ce21 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 13 Mar 2023 18:34:47 +0100 Subject: [PATCH 04/49] refine code --- src/Common/OptimizedRegularExpression.cpp | 170 ++++++++++++++++------ src/Common/OptimizedRegularExpression.h | 3 +- src/Common/tests/gtest_optimize_re.cpp | 23 +-- 3 files changed, 139 insertions(+), 57 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index efa33f27ed4..4d0cdae1b99 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -16,15 +16,42 @@ namespace DB } } +namespace +{ -template -const char * OptimizedRegularExpressionImpl::analyze( +struct Literal +{ + std::string literal; + bool prefix; /// this literal string is the prefix of the whole string. + bool suffix; /// this literal string is the suffic of the whole string. + void clear() + { + literal.clear(); + prefix = false; + suffix = false; + } +}; + +using Literals = std::vector; + +size_t shortest_alter_length(const Literals & literals) +{ + if (literals.empty()) return 0; + size_t shortest = ~(0); + for (const auto & lit : literals) + { + if (shortest > lit.literal.size()) + shortest = lit.literal.size(); + } + return shortest; +} + +const char * analyzeImpl( std::string_view regexp, const char * pos, - std::string & required_substring, + Literal & required_substring, bool & is_trivial, - bool & required_substring_is_prefix, - std::vector & alternatives) + Literals & global_alters) { /** The expression is trivial if all the metacharacters in it are escaped. * The non-alternative string is @@ -39,7 +66,7 @@ const char * OptimizedRegularExpressionImpl::analyze( bool first_call = begin == regexp.data(); int depth = 0; is_trivial = true; - required_substring_is_prefix = false; + ///required_substring_is_prefix = false; required_substring.clear(); bool has_alternative_on_depth_0 = false; bool has_case_insensitive_flag = false; @@ -51,46 +78,79 @@ const char * OptimizedRegularExpressionImpl::analyze( Substrings trivial_substrings(1); Substring * last_substring = &trivial_substrings.back(); - auto finish_non_trivial_char = [&]() + Literals cur_alters; + + auto finish_cur_alters = [&]() + { + if (cur_alters.empty()) + return; + + if (global_alters.empty()) + { + global_alters = std::move(cur_alters); + return; + } + if (shortest_alter_length(global_alters) > shortest_alter_length(cur_alters)) + { + cur_alters.clear(); + } + else + { + global_alters.clear(); + global_alters = std::move(cur_alters); + } + }; + + auto finish_non_trivial_char = [&](bool create_new_substr = true) { if (depth != 0) return; - if (!last_substring->first.empty()) + for (auto & alter : cur_alters) + { + if (alter.suffix) + { + alter.literal += last_substring->first; + } + } + + finish_cur_alters(); + + if (!last_substring->first.empty() && create_new_substr) { trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); } }; - - auto finish_group = [&](std::string group_required_string, bool group_is_trivial, bool group_is_prefix, std::vector & group_alternatives) + /// Resolve the string or alters in a group (xxxxx) + auto finish_group = [&](Literal & group_required_string, Literals & group_alters) { - if (alternatives.empty() && !group_alternatives.empty()) + for (auto & alter : group_alters) { - /// Check if group alternatives has empty strings - bool has_empty_str = false; - for (const std::string & alter : group_alternatives) - has_empty_str |= alter.empty(); - if (!has_empty_str) - alternatives = std::move(group_alternatives); + if (alter.prefix) + { + alter.literal = last_substring->first + alter.literal; + } } - if (group_is_prefix) - last_substring->first += group_required_string; + if (group_required_string.prefix) + last_substring->first += group_required_string.literal; else { finish_non_trivial_char(); - last_substring->first = group_required_string; + last_substring->first = std::move(group_required_string).literal; } /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg - if (!last_substring->first.empty() && !group_is_trivial) + if (!last_substring->first.empty() && !group_required_string.suffix) { trivial_substrings.resize(trivial_substrings.size() + 1); last_substring = &trivial_substrings.back(); } - if (!group_is_trivial) - is_trivial = false; + + /// assign group alters to current alters. + finish_cur_alters(); + cur_alters = std::move(group_alters); }; bool in_curly_braces = false; @@ -178,23 +238,24 @@ const char * OptimizedRegularExpressionImpl::analyze( { pos += 2; } - std::string group_required_substr; - bool group_is_trival; - bool group_is_prefix; - std::vector group_alters; - pos = analyze(regexp, pos + 1, group_required_substr, group_is_trival, group_is_prefix, group_alters); + Literal group_required_substr; + bool group_is_trival = true; + Literals group_alters; + pos = analyzeImpl(regexp, pos + 1, group_required_substr, group_is_trival, group_alters); /// pos should be ')', if not, then it is not a valid regular expression if (pos == end) return pos; - /// For ()? ()* (){0,1}, we can just ignore the whole group. + /// For ()? or ()* or (){0,1}, we can just ignore the whole group. if ((pos + 1 < end && (pos[1] == '?' || pos[1] == '*')) || (pos + 2 < end && pos[1] == '{' && pos[2] == '0')) { finish_non_trivial_char(); } else - finish_group(group_required_substr, group_is_trival, group_is_prefix, group_alters); + { + finish_group(group_required_substr, group_alters); + } } ++pos; break; @@ -271,8 +332,8 @@ const char * OptimizedRegularExpressionImpl::analyze( } } finish: - if (last_substring && last_substring->first.empty()) - trivial_substrings.pop_back(); + + finish_non_trivial_char(false); if (!is_trivial) { @@ -300,35 +361,37 @@ finish: if (max_length >= MIN_LENGTH_FOR_STRSTR || !first_call) { - required_substring = candidate_it->first; - required_substring_is_prefix = candidate_it->second == 0; + required_substring.literal = candidate_it->first; + required_substring.prefix = candidate_it->second == 0; + required_substring.suffix = candidate_it + 1 == trivial_substrings.end(); } } } else if (!trivial_substrings.empty()) { - required_substring = trivial_substrings.front().first; - required_substring_is_prefix = trivial_substrings.front().second == 0; + required_substring.literal = trivial_substrings.front().first; + required_substring.prefix = trivial_substrings.front().second == 0; + required_substring.suffix = true; } /// if it is xxx|xxx|xxx, we should call the next xxx|xxx recursively and collect the result. if (has_alternative_on_depth_0) { - if (alternatives.empty()) - alternatives.push_back(required_substring); - std::vector next_alternatives; + /// compare the quality of required substring and alternatives and choose the better one. + if (shortest_alter_length(global_alters) < required_substring.literal.size()) + global_alters = {required_substring}; + Literals next_alternatives; /// this two vals are useless, xxx|xxx cannot be trivial nor prefix. - bool next_is_trivial; - bool next_is_prefix; - pos = analyze(regexp, pos, required_substring, next_is_trivial, next_is_prefix, next_alternatives); + bool next_is_trivial = true; + pos = analyzeImpl(regexp, pos, required_substring, next_is_trivial, next_alternatives); /// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring. if (next_alternatives.empty()) { - alternatives.push_back(required_substring); + global_alters.push_back(required_substring); } else { - alternatives.insert(alternatives.end(), next_alternatives.begin(), next_alternatives.end()); + global_alters.insert(global_alters.end(), next_alternatives.begin(), next_alternatives.end()); } required_substring.clear(); } @@ -342,13 +405,30 @@ finish: << ", required_substring_is_prefix: " << required_substring_is_prefix << std::endl;*/ } +} +template +void OptimizedRegularExpressionImpl::analyze( + std::string_view regexp_, + std::string & required_substring, + bool & is_trivial, + bool & required_substring_is_prefix, + std::vector & alternatives) +{ + Literals alter_literals; + Literal required_lit; + analyzeImpl(regexp_, regexp_.data(), required_lit, is_trivial, alter_literals); + required_substring = std::move(required_lit.literal); + required_substring_is_prefix = required_lit.prefix; + for (auto & lit : alter_literals) + alternatives.push_back(std::move(lit.literal)); +} template OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(const std::string & regexp_, int options) { std::vector alternatives; /// this vector collects patterns in (xx|xx|xx). for now it's not used. - analyze(regexp_, regexp_.data(), required_substring, is_trivial, required_substring_is_prefix, alternatives); + analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternatives); /// Just three following options are supported if (options & (~(RE_CASELESS | RE_NO_CAPTURE | RE_DOT_NL))) diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index a19ce22deab..566bedc5549 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -95,9 +95,8 @@ public: out_required_substring_is_prefix = required_substring_is_prefix; } - static const char * analyze( + static void analyze( std::string_view regexp_, - const char * pos, std::string & required_substring, bool & is_trivial, bool & required_substring_is_prefix, diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index c90b5086b1c..9de652da365 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -10,30 +10,33 @@ TEST(OptimizeRE, analyze) bool is_trivial; bool is_prefix; std::vector alternatives; - OptimizedRegularExpression::analyze(regexp, regexp.data(), required, is_trivial, is_prefix, alternatives); + OptimizedRegularExpression::analyze(regexp, required, is_trivial, is_prefix, alternatives); std::cerr << regexp << std::endl; EXPECT_EQ(required, answer); EXPECT_EQ(alternatives, expect_alternatives); }; test_f("abc", "abc"); test_f("abc(de)fg", "abcdefg"); - test_f("abc(de|xyz)fg", "abc", {"de", "xyz"}); - test_f("abc(de?f|xyz)fg", "abc", {"d", "xyz"}); + test_f("abc(de|xyz)fg", "abc", {"abcdefg", "abcxyzfg"}); + test_f("abc(de?f|xyz)fg", "abc", {"abcd", "abcxyzfg"}); test_f("abc|fgk|xyz", "", {"abc","fgk", "xyz"}); test_f("(abc)", "abc"); test_f("(abc|fgk)", "", {"abc","fgk"}); - test_f("abc(abc|fg)xyzz", "xyzz", {"abc","fg"}); + test_f("(abc|fgk)(e|f|zkh|)", "", {"abc","fgk"}); + test_f("abc(abc|fg)xyzz", "xyzz", {"abcabcxyzz","abcfgxyzz"}); test_f("abc[k]xyzz", "xyzz"); test_f("(abc[k]xyzz)", "xyzz"); - test_f("abc((de)fg(hi))jk", "abcdefghi"); + test_f("abc((de)fg(hi))jk", "abcdefghijk"); + test_f("abc((?:de)fg(?:hi))jk", "abcdefghijk"); test_f("abc((de)fghi+zzz)jk", "abcdefghi"); test_f("abc((de)fg(hi))?jk", "abc"); test_f("abc((de)fghi?zzz)jk", "abcdefgh"); - test_f("abc(*cd)jk", "abc"); - test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"de", "xyz", "{xx}"}); + test_f("abc(*cd)jk", "cdjk"); + test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"abcdefg", "abcxyzfg", "abc{xx}fg"}); test_f("abc(abc|fg)?xyzz", "xyzz"); test_f("abc(abc|fg){0,1}xyzz", "xyzz"); - test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abc", "fg", "bcd", "bc"}); - test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abc", "fg", "d", "k", "f", "bc"}); - test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc", "efg", "xyz"}); + test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bcfz", "bcgz", "bcz"}); + test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bcfz", "bcgz", "bcz"}); + test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc/", "efg/", "xyz/"}); + test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "Googlebot-", "Googlebot"}); } From 01be209e431a4921de7629c54ab069e3a92fc779 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 14 Mar 2023 17:44:02 +0100 Subject: [PATCH 05/49] fix test --- src/Common/OptimizedRegularExpression.cpp | 4 +++- src/Common/tests/gtest_optimize_re.cpp | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index 4d0cdae1b99..22cd0958c42 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -359,7 +360,7 @@ finish: } } - if (max_length >= MIN_LENGTH_FOR_STRSTR || !first_call) + if (max_length >= MIN_LENGTH_FOR_STRSTR || (!first_call && max_length > 0)) { required_substring.literal = candidate_it->first; required_substring.prefix = candidate_it->second == 0; @@ -430,6 +431,7 @@ OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(cons std::vector alternatives; /// this vector collects patterns in (xx|xx|xx). for now it's not used. analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternatives); + /// Just three following options are supported if (options & (~(RE_CASELESS | RE_NO_CAPTURE | RE_DOT_NL))) throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "OptimizedRegularExpression: Unsupported option."); diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index 9de652da365..eb837a2f344 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -4,7 +4,7 @@ TEST(OptimizeRE, analyze) { - auto test_f = [](const std::string & regexp, const std::string & answer, std::vector expect_alternatives = {}) + auto test_f = [](const std::string & regexp, const std::string & answer, std::vector expect_alternatives = {}, bool trival_expected = false) { std::string required; bool is_trivial; @@ -14,8 +14,10 @@ TEST(OptimizeRE, analyze) std::cerr << regexp << std::endl; EXPECT_EQ(required, answer); EXPECT_EQ(alternatives, expect_alternatives); + EXPECT_EQ(is_trivial, trival_expected); }; - test_f("abc", "abc"); + test_f("abc", "abc", {}, true); + test_f("c([^k]*)de", ""); test_f("abc(de)fg", "abcdefg"); test_f("abc(de|xyz)fg", "abc", {"abcdefg", "abcxyzfg"}); test_f("abc(de?f|xyz)fg", "abc", {"abcd", "abcxyzfg"}); @@ -35,8 +37,8 @@ TEST(OptimizeRE, analyze) test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"abcdefg", "abcxyzfg", "abc{xx}fg"}); test_f("abc(abc|fg)?xyzz", "xyzz"); test_f("abc(abc|fg){0,1}xyzz", "xyzz"); - test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bcfz", "bcgz", "bcz"}); - test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bcfz", "bcgz", "bcz"}); + test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bcfz", "bcgz", ""}); + test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bcfz", "bcgz", ""}); test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc/", "efg/", "xyz/"}); test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "Googlebot-", "Googlebot"}); } From 076d33bb03e13e6dfae1c111ba591196b651ab77 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Tue, 14 Mar 2023 18:15:42 +0100 Subject: [PATCH 06/49] refine a little bit --- src/Common/OptimizedRegularExpression.cpp | 2 +- src/Common/tests/gtest_optimize_re.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index 22cd0958c42..c2354f14bcb 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -386,7 +386,7 @@ finish: bool next_is_trivial = true; pos = analyzeImpl(regexp, pos, required_substring, next_is_trivial, next_alternatives); /// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring. - if (next_alternatives.empty()) + if (next_alternatives.empty() || shortest_alter_length(next_alternatives) < required_substring.literal.size()) { global_alters.push_back(required_substring); } diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index eb837a2f344..088993cfa84 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -37,8 +37,10 @@ TEST(OptimizeRE, analyze) test_f(R"(abc(de|xyz|(\{xx\}))fg)", "abc", {"abcdefg", "abcxyzfg", "abc{xx}fg"}); test_f("abc(abc|fg)?xyzz", "xyzz"); test_f("abc(abc|fg){0,1}xyzz", "xyzz"); - test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bcfz", "bcgz", ""}); - test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bcfz", "bcgz", ""}); + test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bc"}); + test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bc"}); test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc/", "efg/", "xyz/"}); test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "Googlebot-", "Googlebot"}); + test_f("abc|(:?xx|yy|zz|x?)def", "", {"abc", "def"}); + test_f("abc|(:?xx|yy|zz|x?){1,2}def", "", {"abc", "def"}); } From d78a9e03ad0381049b8d0322528b6d2261bce048 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 15 Mar 2023 15:38:11 +0100 Subject: [PATCH 07/49] refine --- src/Common/OptimizedRegularExpression.cpp | 18 +- src/Core/Settings.h | 2 +- src/Dictionaries/RegExpTreeDictionary.cpp | 192 +++++++++++++++------- src/Dictionaries/RegExpTreeDictionary.h | 4 + 4 files changed, 149 insertions(+), 67 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index c2354f14bcb..a8aadd2fecd 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -1,6 +1,3 @@ -#include -#include - #include #include #include @@ -88,18 +85,17 @@ const char * analyzeImpl( if (global_alters.empty()) { - global_alters = std::move(cur_alters); + global_alters = cur_alters; + cur_alters.clear(); return; } - if (shortest_alter_length(global_alters) > shortest_alter_length(cur_alters)) - { - cur_alters.clear(); - } - else + /// that means current alternatives have better quality. + if (shortest_alter_length(global_alters) < shortest_alter_length(cur_alters)) { global_alters.clear(); - global_alters = std::move(cur_alters); + global_alters = cur_alters; } + cur_alters.clear(); }; auto finish_non_trivial_char = [&](bool create_new_substr = true) @@ -140,7 +136,7 @@ const char * analyzeImpl( else { finish_non_trivial_char(); - last_substring->first = std::move(group_required_string).literal; + last_substring->first = group_required_string.literal; } /// if we can still append, no need to finish it. e.g. abc(de)fg should capture abcdefg if (!last_substring->first.empty() && !group_required_string.suffix) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3908254b6f1..0f5e83389c2 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -917,7 +917,7 @@ class IColumn; M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \ \ M(Bool, regexp_dict_allow_other_sources, false, "Allow regexp_tree dictionary to use sources other than yaml source.", 0) \ - M(Bool, regexp_dict_allow_hyperscan, false, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ + M(Bool, regexp_dict_allow_hyperscan, true, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index caba2a52a51..4e8ae4260d0 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -4,14 +4,16 @@ #include #include +#include #include #include #include -#include "Common/Exception.h" #include +#include #include +#include #include #include #include @@ -172,10 +174,6 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) auto keys_column = block.getByName(kKeys).column; auto values_column = block.getByName(kValues).column; -#ifdef USE_VECTORSCAN - SlowWithHyperscanChecker checker; -#endif - size_t size = block.rows(); for (size_t i = 0; i < size; i++) { @@ -219,12 +217,36 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } } regex_nodes.emplace(id, node); -#if USE_VECTORSCAN - if (use_vectorscan && !checker.isSlow(regex)) + +#ifdef USE_VECTORSCAN + String required_substring; + bool is_trivial, required_substring_is_prefix; + std::vector alternatives; + + if (use_vectorscan) + OptimizedRegularExpression::analyze(regex, required_substring, is_trivial, required_substring_is_prefix, alternatives); + + for (auto & alter : alternatives) { - simple_regexps.push_back(regex); + if (alter.size() < 3) + { + alternatives.clear(); + break; + } + } + if (!required_substring.empty()) + { + simple_regexps.push_back(required_substring); regexp_ids.push_back(id); } + else if (!alternatives.empty()) + { + for (auto & alter : alternatives) + { + simple_regexps.push_back(alter); + regexp_ids.push_back(id); + } + } else #endif complex_regexp_nodes.push_back(node); @@ -284,20 +306,52 @@ void RegExpTreeDictionary::loadData() use_vectorscan = false; if (!use_vectorscan) return; - #if USE_VECTORSCAN - try + +#ifdef USE_VECTORSCAN + std::vector patterns; + std::vector flags; + std::vector lens; + + for (const std::string & ref : simple_regexps) { - std::vector regexps_views(simple_regexps.begin(), simple_regexps.end()); - hyperscan_regex = MultiRegexps::getOrSet(regexps_views, std::nullopt); - hyperscan_regex->get(); + patterns.push_back(ref.data()); + lens.push_back(ref.size()); + flags.push_back(HS_FLAG_SINGLEMATCH); } - catch (Exception & e) + + hs_database_t * db = nullptr; + hs_compile_error_t * compile_error; + + std::unique_ptr ids; + ids.reset(new unsigned int[patterns.size()]); + for (size_t i = 0; i < patterns.size(); i++) + ids[i] = static_cast(i+1); + + hs_error_t err = hs_compile_lit_multi(patterns.data(), flags.data(), ids.get(), lens.data(), static_cast(patterns.size()), HS_MODE_BLOCK, nullptr, &db, &compile_error); + origin_db = (db); + if (err != HS_SUCCESS) { - /// Some compile errors will be thrown as LOGICAL ERROR and cause crash, e.g. empty expression or expressions are too large. - /// We catch the error here and rethrow again. - throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Error occurs when compiling regular expressions, reason: {}", e.message()); + /// CompilerError is a unique_ptr, so correct memory free after the exception is thrown. + MultiRegexps::CompilerErrorPtr error(compile_error); + + if (error->expression < 0) + throw Exception::createRuntime(ErrorCodes::LOGICAL_ERROR, String(error->message)); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Pattern '{}' failed with error '{}'", patterns[error->expression], String(error->message)); } - #endif + + ProfileEvents::increment(ProfileEvents::RegexpCreated); + + /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch + /// function which is faster than allocating scratch space each time in each thread. + hs_scratch_t * scratch = nullptr; + err = hs_alloc_scratch(db, &scratch); + origin_scratch.reset(scratch); + /// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch. + if (err != HS_SUCCESS) + throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not allocate scratch space for vectorscan"); +#endif + } else { @@ -396,47 +450,70 @@ bool RegExpTreeDictionary::setAttributes( return attributes_to_set.size() == attributes.size(); } -namespace +/// a temp struct to store all the matched result. +struct MatchContext { - struct MatchContext + std::set matched_idx_set; + std::vector> matched_idx_sorted_list; + + const std::vector & regexp_ids ; + const std::unordered_map & topology_order; + const char * data; + size_t length; + const std::map & regex_nodes; + + size_t pre_match_counter = 0; + size_t match_counter = 0; + + MatchContext( + const std::vector & regexp_ids_, + const std::unordered_map & topology_order_, + const char * data_, size_t length_, + const std::map & regex_nodes_ + ) + : regexp_ids(regexp_ids_), + topology_order(topology_order_), + data(data_), + length(length_), + regex_nodes(regex_nodes_) + {} + + [[maybe_unused]] + void insertIdx(unsigned int idx) { - std::set matched_idx_set; - std::vector> matched_idx_sorted_list; - - const std::vector & regexp_ids ; - const std::unordered_map & topology_order; - - MatchContext(const std::vector & regexp_ids_, const std::unordered_map & topology_order_) - : regexp_ids(regexp_ids_), topology_order(topology_order_) {} - - [[maybe_unused]] - void insertIdx(unsigned int idx) + UInt64 node_id = regexp_ids[idx-1]; + pre_match_counter++; + if (!regex_nodes.at(node_id)->match(data, length)) { - UInt64 node_id = regexp_ids[idx-1]; - UInt64 topological_order = topology_order.at(node_id); - matched_idx_set.emplace(node_id); - matched_idx_sorted_list.push_back(std::make_pair(topological_order, node_id)); + return; } + match_counter++; + matched_idx_set.emplace(node_id); - void insertNodeID(UInt64 id) - { - UInt64 topological_order = topology_order.at(id); - matched_idx_set.emplace(id); - matched_idx_sorted_list.push_back(std::make_pair(topological_order, id)); - } + UInt64 topological_order = topology_order.at(node_id); + matched_idx_sorted_list.push_back(std::make_pair(topological_order, node_id)); + } - /// Sort by topological order, which indicates the matching priorities. - void sort() - { - std::sort(matched_idx_sorted_list.begin(), matched_idx_sorted_list.end()); - } + [[maybe_unused]] + void insertNodeID(UInt64 id) + { + matched_idx_set.emplace(id); - bool contains(UInt64 idx) const - { - return matched_idx_set.contains(idx); - } - }; -} + UInt64 topological_order = topology_order.at(id); + matched_idx_sorted_list.push_back(std::make_pair(topological_order, id)); + } + + /// Sort by topological order, which indicates the matching priorities. + void sort() + { + std::sort(matched_idx_sorted_list.begin(), matched_idx_sorted_list.end()); + } + + bool contains(UInt64 idx) const + { + return matched_idx_set.contains(idx); + } +}; std::unordered_map RegExpTreeDictionary::match( const ColumnString::Chars & keys_data, @@ -449,7 +526,7 @@ std::unordered_map RegExpTreeDictionary::match( hs_scratch_t * scratch = nullptr; if (use_vectorscan) { - hs_error_t err = hs_clone_scratch(hyperscan_regex->get()->getScratch(), &scratch); + hs_error_t err = hs_clone_scratch(origin_scratch.get(), &scratch); if (err != HS_SUCCESS) { @@ -476,11 +553,14 @@ std::unordered_map RegExpTreeDictionary::match( auto key_offset = keys_offsets[key_idx]; UInt64 length = key_offset - offset - 1; - MatchContext match_result(regexp_ids, topology_order); + const char * begin = reinterpret_cast(keys_data.data()) + offset; + + MatchContext match_result(regexp_ids, topology_order, begin, length, regex_nodes); #if USE_VECTORSCAN if (use_vectorscan) { + /// pre-select all the possible matches auto on_match = [](unsigned int id, unsigned long long /* from */, // NOLINT unsigned long long /* to */, // NOLINT @@ -490,8 +570,9 @@ std::unordered_map RegExpTreeDictionary::match( static_cast(context)->insertIdx(id); return 0; }; + hs_error_t err = hs_scan( - hyperscan_regex->get()->getDB(), + origin_db, reinterpret_cast(keys_data.data()) + offset, static_cast(length), 0, @@ -501,6 +582,7 @@ std::unordered_map RegExpTreeDictionary::match( if (err != HS_SUCCESS) throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); + } #endif diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 32206f25429..87cdfb808d1 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -33,6 +34,7 @@ namespace ErrorCodes class RegExpTreeDictionary : public IDictionary { + friend struct MatchContext; public: struct Configuration { @@ -162,6 +164,8 @@ private: std::unordered_map topology_order; #if USE_VECTORSCAN MultiRegexps::DeferredConstructedRegexpsPtr hyperscan_regex; + MultiRegexps::ScratchPtr origin_scratch; + hs_database_t* origin_db; #endif Poco::Logger * logger; From 424e8df9ad8493f904f54ca2d9d63f0a32b6b8d7 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 15 Mar 2023 16:01:12 +0100 Subject: [PATCH 08/49] fix style --- src/Dictionaries/RegExpTreeDictionary.cpp | 5 ++--- src/Dictionaries/RegExpTreeDictionary.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 4e8ae4260d0..b0f3dcb0249 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -36,6 +35,7 @@ #if USE_VECTORSCAN # include +# include #endif namespace DB @@ -48,6 +48,7 @@ namespace ErrorCodes extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int UNSUPPORTED_METHOD; extern const int INCORRECT_DICTIONARY_DEFINITION; + extern const int LOGICAL_ERROR; } const std::string kRegExp = "regexp"; @@ -340,8 +341,6 @@ void RegExpTreeDictionary::loadData() throw Exception(ErrorCodes::BAD_ARGUMENTS, "Pattern '{}' failed with error '{}'", patterns[error->expression], String(error->message)); } - ProfileEvents::increment(ProfileEvents::RegexpCreated); - /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch /// function which is faster than allocating scratch space each time in each thread. hs_scratch_t * scratch = nullptr; diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 87cdfb808d1..17a0c6bbef3 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include From e0954ce7beb9b38c16ebe222fc6abc354d1cd919 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Thu, 16 Mar 2023 00:22:05 +0100 Subject: [PATCH 09/49] fix compile --- src/Dictionaries/RegExpTreeDictionary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index b0f3dcb0249..5618da3505b 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -219,7 +219,7 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } regex_nodes.emplace(id, node); -#ifdef USE_VECTORSCAN +#if USE_VECTORSCAN String required_substring; bool is_trivial, required_substring_is_prefix; std::vector alternatives; @@ -308,7 +308,7 @@ void RegExpTreeDictionary::loadData() if (!use_vectorscan) return; -#ifdef USE_VECTORSCAN +#if USE_VECTORSCAN std::vector patterns; std::vector flags; std::vector lens; From 10980554091440304b8a235079947e685f2c3ad5 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:21:10 +0000 Subject: [PATCH 10/49] Stop `wait for quorum` retries on shutdown --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 035cbdac55e..43e7e1e3979 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -1073,13 +1073,26 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( } }, [&zookeeper]() { zookeeper->cleanupEphemeralNodes(); }); + if (!conflict_block_ids.empty()) return conflict_block_ids; + if (isQuorumEnabled()) { ZooKeeperRetriesControl quorum_retries_ctl("waitForQuorum", zookeeper_retries_info); quorum_retries_ctl.retryLoop([&]() { + if (storage.is_readonly) + { + /// stop retries if in shutdown + if (storage.shutdown_called) + throw Exception( + ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + + quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); + return; + } + zookeeper->setKeeper(storage.getZooKeeper()); if (is_already_existing_part) From 795a1c84e1d77390dede017756af701250058039 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:48:04 +0000 Subject: [PATCH 11/49] Actually stop retries in case of storage shutdown --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 43e7e1e3979..600e2f408ef 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -649,8 +649,11 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) + { + retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + } /// When we attach existing parts it's okay to be in read-only mode /// For example during RESTORE REPLICA. @@ -1086,8 +1089,11 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) + { + quorum_retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + } quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); return; From 1073c7f3c942949d1f5de6956fab9b3bcde8b34b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 20 Mar 2023 13:54:25 +0000 Subject: [PATCH 12/49] Revert "Actually stop retries in case of storage shutdown" This was unnecessary since, as for exceptions, retries done only on zkutil::KeeperException This reverts commit 795a1c84e1d77390dede017756af701250058039. --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 600e2f408ef..43e7e1e3979 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -649,11 +649,8 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) - { - retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); - } /// When we attach existing parts it's okay to be in read-only mode /// For example during RESTORE REPLICA. @@ -1089,11 +1086,8 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( { /// stop retries if in shutdown if (storage.shutdown_called) - { - quorum_retries_ctl.stopRetries(); throw Exception( ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); - } quorum_retries_ctl.setUserError(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode: replica_path={}", storage.replica_path); return; From 038bfb40ab7fd66c693317f957dd4be1a43eb280 Mon Sep 17 00:00:00 2001 From: AVMusorin Date: Mon, 27 Feb 2023 15:42:38 +0100 Subject: [PATCH 13/49] dynamic update system.backups --- src/Backups/BackupUtils.cpp | 137 --------------------------- src/Backups/BackupUtils.h | 9 -- src/Backups/BackupsWorker.cpp | 171 +++++++++++++++++++++++++++++++--- src/Backups/BackupsWorker.h | 12 +++ 4 files changed, 172 insertions(+), 157 deletions(-) diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index c6a0840964b..89b75a103c2 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -1,10 +1,7 @@ #include -#include -#include #include #include #include -#include #include @@ -60,140 +57,6 @@ DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & e } -void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool) -{ - size_t num_active_jobs = 0; - std::mutex mutex; - std::condition_variable event; - std::exception_ptr exception; - - bool always_single_threaded = !backup->supportsWritingInMultipleThreads(); - auto thread_group = CurrentThread::getGroup(); - - for (auto & name_and_entry : backup_entries) - { - auto & name = name_and_entry.first; - auto & entry = name_and_entry.second; - - { - std::unique_lock lock{mutex}; - if (exception) - break; - ++num_active_jobs; - } - - auto job = [&](bool async) - { - SCOPE_EXIT_SAFE( - std::lock_guard lock{mutex}; - if (!--num_active_jobs) - event.notify_all(); - if (async) - CurrentThread::detachFromGroupIfNotDetached(); - ); - - try - { - if (async && thread_group) - CurrentThread::attachToGroup(thread_group); - - if (async) - setThreadName("BackupWorker"); - - { - std::lock_guard lock{mutex}; - if (exception) - return; - } - - backup->writeFile(name, std::move(entry)); - } - catch (...) - { - std::lock_guard lock{mutex}; - if (!exception) - exception = std::current_exception(); - } - }; - - if (always_single_threaded || !thread_pool.trySchedule([job] { job(true); })) - job(false); - } - - { - std::unique_lock lock{mutex}; - event.wait(lock, [&] { return !num_active_jobs; }); - if (exception) - std::rethrow_exception(exception); - } -} - - -void restoreTablesData(DataRestoreTasks && tasks, ThreadPool & thread_pool) -{ - size_t num_active_jobs = 0; - std::mutex mutex; - std::condition_variable event; - std::exception_ptr exception; - - auto thread_group = CurrentThread::getGroup(); - - for (auto & task : tasks) - { - { - std::unique_lock lock{mutex}; - if (exception) - break; - ++num_active_jobs; - } - - auto job = [&](bool async) - { - SCOPE_EXIT_SAFE( - std::lock_guard lock{mutex}; - if (!--num_active_jobs) - event.notify_all(); - if (async) - CurrentThread::detachFromGroupIfNotDetached(); - ); - - try - { - if (async && thread_group) - CurrentThread::attachToGroup(thread_group); - - if (async) - setThreadName("RestoreWorker"); - - { - std::lock_guard lock{mutex}; - if (exception) - return; - } - - std::move(task)(); - } - catch (...) - { - std::lock_guard lock{mutex}; - if (!exception) - exception = std::current_exception(); - } - }; - - if (!thread_pool.trySchedule([job] { job(true); })) - job(false); - } - - { - std::unique_lock lock{mutex}; - event.wait(lock, [&] { return !num_active_jobs; }); - if (exception) - std::rethrow_exception(exception); - } -} - - /// Returns access required to execute BACKUP query. AccessRightsElements getRequiredAccessToBackup(const ASTBackupQuery::Elements & elements) { diff --git a/src/Backups/BackupUtils.h b/src/Backups/BackupUtils.h index cda9121b1fa..f451b003652 100644 --- a/src/Backups/BackupUtils.h +++ b/src/Backups/BackupUtils.h @@ -7,21 +7,12 @@ namespace DB { class IBackup; -using BackupMutablePtr = std::shared_ptr; -class IBackupEntry; -using BackupEntries = std::vector>>; -using DataRestoreTasks = std::vector>; class AccessRightsElements; class DDLRenamingMap; /// Initializes a DDLRenamingMap from a BACKUP or RESTORE query. DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & elements); -/// Write backup entries to an opened backup. -void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool); - -/// Run data restoring tasks which insert data to tables. -void restoreTablesData(DataRestoreTasks && tasks, ThreadPool & thread_pool); /// Returns access required to execute BACKUP query. AccessRightsElements getRequiredAccessToBackup(const ASTBackupQuery::Elements & elements); diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index bdcff249e7d..58185053124 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace DB @@ -346,7 +347,7 @@ void BackupsWorker::doBackup( } /// Write the backup entries to the backup. - writeBackupEntries(backup, std::move(backup_entries), backups_thread_pool); + writeBackupEntries(backup_id, backup, std::move(backup_entries), backups_thread_pool, backup_settings.internal); /// We have written our backup entries, we need to tell other hosts (they could be waiting for it). backup_coordination->setStage(backup_settings.host_id, Stage::COMPLETED, ""); @@ -374,6 +375,7 @@ void BackupsWorker::doBackup( LOG_INFO(log, "{} {} was created successfully", (backup_settings.internal ? "Internal backup" : "Backup"), backup_name_for_logging); setStatus(backup_id, BackupStatus::BACKUP_CREATED); + /// NOTE: we need to update metadata again after backup->finalizeWriting(), because backup metadata is written there. setNumFilesAndSize(backup_id, num_files, total_size, num_entries, uncompressed_size, compressed_size, 0, 0); } catch (...) @@ -394,6 +396,88 @@ void BackupsWorker::doBackup( } +void BackupsWorker::writeBackupEntries(const OperationID & backup_id, BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool, bool internal) +{ + size_t num_active_jobs = 0; + std::mutex mutex; + std::condition_variable event; + std::exception_ptr exception; + + bool always_single_threaded = !backup->supportsWritingInMultipleThreads(); + auto thread_group = CurrentThread::getGroup(); + + for (auto & name_and_entry : backup_entries) + { + auto & name = name_and_entry.first; + auto & entry = name_and_entry.second; + + { + std::unique_lock lock{mutex}; + if (exception) + break; + ++num_active_jobs; + } + + auto job = [&](bool async) + { + SCOPE_EXIT_SAFE( + std::lock_guard lock{mutex}; + if (!--num_active_jobs) + event.notify_all(); + if (async) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + try + { + if (async && thread_group) + CurrentThread::attachToGroup(thread_group); + + if (async) + setThreadName("BackupWorker"); + + { + std::lock_guard lock{mutex}; + if (exception) + return; + } + + backup->writeFile(name, std::move(entry)); + // Update metadata + if (!internal) + { + setNumFilesAndSize( + backup_id, + backup->getNumFiles(), + backup->getTotalSize(), + backup->getNumEntries(), + backup->getUncompressedSize(), + backup->getCompressedSize(), + 0, 0); + } + + } + catch (...) + { + std::lock_guard lock{mutex}; + if (!exception) + exception = std::current_exception(); + } + }; + + if (always_single_threaded || !thread_pool.trySchedule([job] { job(true); })) + job(false); + } + + { + std::unique_lock lock{mutex}; + event.wait(lock, [&] { return !num_active_jobs; }); + if (exception) + std::rethrow_exception(exception); + } +} + + OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr context) { auto restore_query = std::static_pointer_cast(query->clone()); @@ -578,7 +662,7 @@ void BackupsWorker::doRestore( } /// Execute the data restoring tasks. - restoreTablesData(std::move(data_restore_tasks), restores_thread_pool); + restoreTablesData(restore_id, backup, std::move(data_restore_tasks), restores_thread_pool); /// We have restored everything, we need to tell other hosts (they could be waiting for it). restore_coordination->setStage(restore_settings.host_id, Stage::COMPLETED, ""); @@ -586,15 +670,6 @@ void BackupsWorker::doRestore( LOG_INFO(log, "Restored from {} {} successfully", (restore_settings.internal ? "internal backup" : "backup"), backup_name_for_logging); setStatus(restore_id, BackupStatus::RESTORED); - setNumFilesAndSize( - restore_id, - backup->getNumFiles(), - backup->getTotalSize(), - backup->getNumEntries(), - backup->getUncompressedSize(), - backup->getCompressedSize(), - backup->getNumReadFiles(), - backup->getNumReadBytes()); } catch (...) { @@ -614,6 +689,80 @@ void BackupsWorker::doRestore( } +void BackupsWorker::restoreTablesData(const OperationID & restore_id, BackupPtr backup, DataRestoreTasks && tasks, ThreadPool & thread_pool) +{ + size_t num_active_jobs = 0; + std::mutex mutex; + std::condition_variable event; + std::exception_ptr exception; + + auto thread_group = CurrentThread::getGroup(); + + for (auto & task : tasks) + { + { + std::unique_lock lock{mutex}; + if (exception) + break; + ++num_active_jobs; + } + + auto job = [&](bool async) + { + SCOPE_EXIT_SAFE( + std::lock_guard lock{mutex}; + if (!--num_active_jobs) + event.notify_all(); + if (async) + CurrentThread::detachFromGroupIfNotDetached(); + ); + + try + { + if (async && thread_group) + CurrentThread::attachToGroup(thread_group); + + if (async) + setThreadName("RestoreWorker"); + + { + std::lock_guard lock{mutex}; + if (exception) + return; + } + + std::move(task)(); + setNumFilesAndSize( + restore_id, + backup->getNumFiles(), + backup->getTotalSize(), + backup->getNumEntries(), + backup->getUncompressedSize(), + backup->getCompressedSize(), + backup->getNumReadFiles(), + backup->getNumReadBytes()); + } + catch (...) + { + std::lock_guard lock{mutex}; + if (!exception) + exception = std::current_exception(); + } + }; + + if (!thread_pool.trySchedule([job] { job(true); })) + job(false); + } + + { + std::unique_lock lock{mutex}; + event.wait(lock, [&] { return !num_active_jobs; }); + if (exception) + std::rethrow_exception(exception); + } +} + + void BackupsWorker::addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status) { Info info; diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 0f5c16cd71f..c36b58da14f 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -17,6 +17,12 @@ struct RestoreSettings; struct BackupInfo; class IBackupCoordination; class IRestoreCoordination; +class IBackup; +using BackupMutablePtr = std::shared_ptr; +using BackupPtr = std::shared_ptr; +class IBackupEntry; +using BackupEntries = std::vector>>; +using DataRestoreTasks = std::vector>; /// Manager of backups and restores: executes backups and restores' threads in the background. /// Keeps information about backups and restores started in this session. @@ -99,6 +105,9 @@ private: ContextMutablePtr mutable_context, bool called_async); + /// Write backup entries to an opened backup. + void writeBackupEntries(const OperationID & backup_id, BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool, bool internal); + OperationID startRestoring(const ASTPtr & query, ContextMutablePtr context); void doRestore( @@ -111,6 +120,9 @@ private: ContextMutablePtr context, bool called_async); + /// Run data restoring tasks which insert data to tables. + void restoreTablesData(const OperationID & restore_id, BackupPtr backup, DataRestoreTasks && tasks, ThreadPool & thread_pool); + void addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status); void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true); void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } From 575c4263a3dea3865b6e87bbe54d3f64ca6faa73 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 22 Mar 2023 17:47:25 +0100 Subject: [PATCH 14/49] address comments --- src/Common/OptimizedRegularExpression.cpp | 70 +++++++++++------------ src/Common/OptimizedRegularExpression.h | 3 + src/Common/tests/gtest_optimize_re.cpp | 2 +- src/Dictionaries/RegExpTreeDictionary.cpp | 14 ++--- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index a8aadd2fecd..77acfdda935 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -21,7 +22,7 @@ struct Literal { std::string literal; bool prefix; /// this literal string is the prefix of the whole string. - bool suffix; /// this literal string is the suffic of the whole string. + bool suffix; /// this literal string is the suffix of the whole string. void clear() { literal.clear(); @@ -35,12 +36,10 @@ using Literals = std::vector; size_t shortest_alter_length(const Literals & literals) { if (literals.empty()) return 0; - size_t shortest = ~(0); + size_t shortest = std::numeric_limits::max(); for (const auto & lit : literals) - { if (shortest > lit.literal.size()) shortest = lit.literal.size(); - } return shortest; } @@ -49,7 +48,7 @@ const char * analyzeImpl( const char * pos, Literal & required_substring, bool & is_trivial, - Literals & global_alters) + Literals & global_alternatives) { /** The expression is trivial if all the metacharacters in it are escaped. * The non-alternative string is @@ -61,10 +60,9 @@ const char * analyzeImpl( */ const char * begin = pos; const char * end = regexp.data() + regexp.size(); - bool first_call = begin == regexp.data(); + bool is_first_call = begin == regexp.data(); int depth = 0; is_trivial = true; - ///required_substring_is_prefix = false; required_substring.clear(); bool has_alternative_on_depth_0 = false; bool has_case_insensitive_flag = false; @@ -76,26 +74,26 @@ const char * analyzeImpl( Substrings trivial_substrings(1); Substring * last_substring = &trivial_substrings.back(); - Literals cur_alters; + Literals cur_alternatives; - auto finish_cur_alters = [&]() + auto finish_cur_alternatives = [&]() { - if (cur_alters.empty()) + if (cur_alternatives.empty()) return; - if (global_alters.empty()) + if (global_alternatives.empty()) { - global_alters = cur_alters; - cur_alters.clear(); + global_alternatives = cur_alternatives; + cur_alternatives.clear(); return; } /// that means current alternatives have better quality. - if (shortest_alter_length(global_alters) < shortest_alter_length(cur_alters)) + if (shortest_alter_length(global_alternatives) < shortest_alter_length(cur_alternatives)) { - global_alters.clear(); - global_alters = cur_alters; + global_alternatives.clear(); + global_alternatives = cur_alternatives; } - cur_alters.clear(); + cur_alternatives.clear(); }; auto finish_non_trivial_char = [&](bool create_new_substr = true) @@ -103,7 +101,7 @@ const char * analyzeImpl( if (depth != 0) return; - for (auto & alter : cur_alters) + for (auto & alter : cur_alternatives) { if (alter.suffix) { @@ -111,7 +109,7 @@ const char * analyzeImpl( } } - finish_cur_alters(); + finish_cur_alternatives(); if (!last_substring->first.empty() && create_new_substr) { @@ -121,9 +119,9 @@ const char * analyzeImpl( }; /// Resolve the string or alters in a group (xxxxx) - auto finish_group = [&](Literal & group_required_string, Literals & group_alters) + auto finish_group = [&](Literal & group_required_string, Literals & group_alternatives) { - for (auto & alter : group_alters) + for (auto & alter : group_alternatives) { if (alter.prefix) { @@ -146,8 +144,8 @@ const char * analyzeImpl( } /// assign group alters to current alters. - finish_cur_alters(); - cur_alters = std::move(group_alters); + finish_cur_alternatives(); + cur_alternatives = std::move(group_alternatives); }; bool in_curly_braces = false; @@ -356,7 +354,7 @@ finish: } } - if (max_length >= MIN_LENGTH_FOR_STRSTR || (!first_call && max_length > 0)) + if (max_length >= MIN_LENGTH_FOR_STRSTR || (!is_first_call && max_length > 0)) { required_substring.literal = candidate_it->first; required_substring.prefix = candidate_it->second == 0; @@ -375,8 +373,8 @@ finish: if (has_alternative_on_depth_0) { /// compare the quality of required substring and alternatives and choose the better one. - if (shortest_alter_length(global_alters) < required_substring.literal.size()) - global_alters = {required_substring}; + if (shortest_alter_length(global_alternatives) < required_substring.literal.size()) + global_alternatives = {required_substring}; Literals next_alternatives; /// this two vals are useless, xxx|xxx cannot be trivial nor prefix. bool next_is_trivial = true; @@ -384,11 +382,11 @@ finish: /// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring. if (next_alternatives.empty() || shortest_alter_length(next_alternatives) < required_substring.literal.size()) { - global_alters.push_back(required_substring); + global_alternatives.push_back(required_substring); } else { - global_alters.insert(global_alters.end(), next_alternatives.begin(), next_alternatives.end()); + global_alternatives.insert(global_alternatives.end(), next_alternatives.begin(), next_alternatives.end()); } required_substring.clear(); } @@ -412,20 +410,20 @@ void OptimizedRegularExpressionImpl::analyze( bool & required_substring_is_prefix, std::vector & alternatives) { - Literals alter_literals; - Literal required_lit; - analyzeImpl(regexp_, regexp_.data(), required_lit, is_trivial, alter_literals); - required_substring = std::move(required_lit.literal); - required_substring_is_prefix = required_lit.prefix; - for (auto & lit : alter_literals) + Literals alternative_literals; + Literal required_literal; + analyzeImpl(regexp_, regexp_.data(), required_literal, is_trivial, alternative_literals); + required_substring = std::move(required_literal.literal); + required_substring_is_prefix = required_literal.prefix; + for (auto & lit : alternative_literals) alternatives.push_back(std::move(lit.literal)); } template OptimizedRegularExpressionImpl::OptimizedRegularExpressionImpl(const std::string & regexp_, int options) { - std::vector alternatives; /// this vector collects patterns in (xx|xx|xx). for now it's not used. - analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternatives); + std::vector alternativesDummy; /// this vector extracts patterns a,b,c from pattern (a|b|c). for now it's not used. + analyze(regexp_, required_substring, is_trivial, required_substring_is_prefix, alternativesDummy); /// Just three following options are supported diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index 566bedc5549..f6b59f0a465 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -95,12 +95,15 @@ public: out_required_substring_is_prefix = required_substring_is_prefix; } + /// analyze function will extract the longest string literal or multiple alternative string literals from regexp for pre-checking if + /// a string contains the string literal(s). If not, we can tell this string can never match the regexp. static void analyze( std::string_view regexp_, std::string & required_substring, bool & is_trivial, bool & required_substring_is_prefix, std::vector & alternatives); + private: bool is_trivial; bool required_substring_is_prefix; diff --git a/src/Common/tests/gtest_optimize_re.cpp b/src/Common/tests/gtest_optimize_re.cpp index 088993cfa84..556700f1fcc 100644 --- a/src/Common/tests/gtest_optimize_re.cpp +++ b/src/Common/tests/gtest_optimize_re.cpp @@ -40,7 +40,7 @@ TEST(OptimizeRE, analyze) test_f("abc(abc|fg)xyzz|bcdd?k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bc"}); test_f("abc(abc|fg)xyzz|bc(dd?x|kk?y|(f))k|bc(f|g|h?)z", "", {"abcabcxyzz", "abcfgxyzz", "bcd", "bck", "bcfk", "bc"}); test_f("((?:abc|efg|xyz)/[a-zA-Z0-9]{1-50})(/?[^ ]*|)", "", {"abc/", "efg/", "xyz/"}); - test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "Googlebot-", "Googlebot"}); + test_f(R"([Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Daumoa(?:-feedfetcher|)|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|))", "", {"pider-", "bingbot", "Yeti-", "Yeti", "Catchpoint bot", "Catchpoint", "harlotte", "Daumoa-feedfetcher", "Daumoa", "-Googlebot", "Googlebot"}); test_f("abc|(:?xx|yy|zz|x?)def", "", {"abc", "def"}); test_f("abc|(:?xx|yy|zz|x?){1,2}def", "", {"abc", "def"}); } diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 5618da3505b..c072ba78d46 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -242,9 +242,9 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } else if (!alternatives.empty()) { - for (auto & alter : alternatives) + for (auto & alternative : alternatives) { - simple_regexps.push_back(alter); + simple_regexps.push_back(alternative); regexp_ids.push_back(id); } } @@ -311,12 +311,12 @@ void RegExpTreeDictionary::loadData() #if USE_VECTORSCAN std::vector patterns; std::vector flags; - std::vector lens; + std::vector lengths; - for (const std::string & ref : simple_regexps) + for (const std::string & simple_regexp : simple_regexps) { - patterns.push_back(ref.data()); - lens.push_back(ref.size()); + patterns.push_back(simple_regexp.data()); + lengths.push_back(simple_regexp.size()); flags.push_back(HS_FLAG_SINGLEMATCH); } @@ -328,7 +328,7 @@ void RegExpTreeDictionary::loadData() for (size_t i = 0; i < patterns.size(); i++) ids[i] = static_cast(i+1); - hs_error_t err = hs_compile_lit_multi(patterns.data(), flags.data(), ids.get(), lens.data(), static_cast(patterns.size()), HS_MODE_BLOCK, nullptr, &db, &compile_error); + hs_error_t err = hs_compile_lit_multi(patterns.data(), flags.data(), ids.get(), lengths.data(), static_cast(patterns.size()), HS_MODE_BLOCK, nullptr, &db, &compile_error); origin_db = (db); if (err != HS_SUCCESS) { From 02de4ad6dfaecff1ba722a449dd3ca4db058bfe1 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 22 Mar 2023 17:50:19 +0100 Subject: [PATCH 15/49] address comments --- src/Common/OptimizedRegularExpression.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common/OptimizedRegularExpression.cpp b/src/Common/OptimizedRegularExpression.cpp index 77acfdda935..68f5b86877e 100644 --- a/src/Common/OptimizedRegularExpression.cpp +++ b/src/Common/OptimizedRegularExpression.cpp @@ -33,7 +33,7 @@ struct Literal using Literals = std::vector; -size_t shortest_alter_length(const Literals & literals) +size_t shortest_literal_length(const Literals & literals) { if (literals.empty()) return 0; size_t shortest = std::numeric_limits::max(); @@ -88,7 +88,7 @@ const char * analyzeImpl( return; } /// that means current alternatives have better quality. - if (shortest_alter_length(global_alternatives) < shortest_alter_length(cur_alternatives)) + if (shortest_literal_length(global_alternatives) < shortest_literal_length(cur_alternatives)) { global_alternatives.clear(); global_alternatives = cur_alternatives; @@ -373,14 +373,14 @@ finish: if (has_alternative_on_depth_0) { /// compare the quality of required substring and alternatives and choose the better one. - if (shortest_alter_length(global_alternatives) < required_substring.literal.size()) + if (shortest_literal_length(global_alternatives) < required_substring.literal.size()) global_alternatives = {required_substring}; Literals next_alternatives; /// this two vals are useless, xxx|xxx cannot be trivial nor prefix. bool next_is_trivial = true; pos = analyzeImpl(regexp, pos, required_substring, next_is_trivial, next_alternatives); /// For xxx|xxx|xxx, we only conbine the alternatives and return a empty required_substring. - if (next_alternatives.empty() || shortest_alter_length(next_alternatives) < required_substring.literal.size()) + if (next_alternatives.empty() || shortest_literal_length(next_alternatives) < required_substring.literal.size()) { global_alternatives.push_back(required_substring); } From 5e95a37c526875ad1791240895f0f5cb72632e41 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 23 Mar 2023 07:05:51 -0700 Subject: [PATCH 16/49] Fix decimal-256 text output issue on s390x --- base/base/wide_integer_impl.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 4a54c0fb2a4..874a66440a7 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -732,9 +732,10 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(rhs); + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, big(i)); + base_type rhs_item = get_item(t, big(i)); if (lhs.items[big(i)] != rhs_item) return lhs.items[big(i)] > rhs_item; @@ -757,9 +758,10 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(lhs); + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, big(i)); + base_type rhs_item = get_item(t, big(i)); if (lhs.items[big(i)] != rhs_item) return lhs.items[big(i)] < rhs_item; @@ -779,9 +781,10 @@ public: { if constexpr (should_keep_size()) { + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, any(i)); + base_type rhs_item = get_item(t, any(i)); if (lhs.items[any(i)] != rhs_item) return false; From 08583c840535529d297f7eab6d72271c4b7752b6 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 23 Mar 2023 07:17:37 -0700 Subject: [PATCH 17/49] Fixed the style issue --- base/base/wide_integer_impl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 874a66440a7..30d08be2e4f 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -732,7 +732,7 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(rhs); - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, big(i)); @@ -758,7 +758,7 @@ public: if (std::numeric_limits::is_signed && (is_negative(lhs) != is_negative(rhs))) return is_negative(lhs); - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, big(i)); @@ -781,7 +781,7 @@ public: { if constexpr (should_keep_size()) { - integer t = rhs; + integer t = rhs; for (unsigned i = 0; i < item_count; ++i) { base_type rhs_item = get_item(t, any(i)); From 9003b7d7893d9d8596dd6215d7cb78a9c2d92599 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:14:32 +0100 Subject: [PATCH 18/49] Fix tsan error lock-order-inversion --- src/Interpreters/ProcessList.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 5db39ece2e5..c7916f3eed2 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -434,11 +434,16 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { - std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); - assert(it != executors.end()); + ExecutorHolderPtr executor_holder; + { + std::lock_guard lock(executors_mutex); + auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + assert(it != executors.end()); + executor_holder = *it; + } /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) - (*it)->remove(); + /// We should do it with released executors_mutex to avoid possible lock order inversion. + executor_holder->remove(); } bool QueryStatus::checkTimeLimit() From f353561204c5707fd229fcaeefc880096006cff2 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 14:37:40 +0000 Subject: [PATCH 19/49] Avoid breaking batches with read requests --- src/Coordination/KeeperDispatcher.cpp | 32 +++++++++++++++++++++---- src/Coordination/KeeperDispatcher.h | 16 +++++++++++++ src/Coordination/KeeperServer.cpp | 4 +++- src/Coordination/KeeperServer.h | 3 ++- src/Coordination/KeeperStateMachine.cpp | 5 +++- src/Coordination/KeeperStateMachine.h | 4 ++++ 6 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 2aa11dd9eed..fc97bc5e0e1 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -90,7 +90,7 @@ void KeeperDispatcher::requestThread() KeeperStorage::RequestsForSessions current_batch; - bool has_read_request = false; + bool has_read_request{false}; /// If new request is not read request or we must to process it through quorum. /// Otherwise we will process it locally. @@ -98,6 +98,8 @@ void KeeperDispatcher::requestThread() { current_batch.emplace_back(request); + size_t read_requests = 0; + const auto try_get_request = [&] { /// Trying to get batch requests as fast as possible @@ -106,7 +108,12 @@ void KeeperDispatcher::requestThread() CurrentMetrics::sub(CurrentMetrics::KeeperOutstandingRequets); /// Don't append read request into batch, we have to process them separately if (!coordination_settings->quorum_reads && request.request->isReadRequest()) - has_read_request = true; + { + ++read_requests; + std::pair key{current_batch.back().session_id, current_batch.back().request->xid}; + std::lock_guard lock(read_mutex); + related_read_requests[key].push_back(request); + } else current_batch.emplace_back(request); @@ -118,7 +125,7 @@ void KeeperDispatcher::requestThread() /// If we have enough requests in queue, we will try to batch at least max_quick_batch_size of them. size_t max_quick_batch_size = coordination_settings->max_requests_quick_batch_size; - while (!shutdown_called && !has_read_request && current_batch.size() < max_quick_batch_size && try_get_request()) + while (!shutdown_called && current_batch.size() + read_requests < max_quick_batch_size && try_get_request()) ; const auto prev_result_done = [&] @@ -129,7 +136,7 @@ void KeeperDispatcher::requestThread() }; /// Waiting until previous append will be successful, or batch is big enough - while (!shutdown_called && !has_read_request && !prev_result_done() && current_batch.size() <= max_batch_size) + while (!shutdown_called && !prev_result_done() && current_batch.size() <= max_batch_size) { try_get_request(); } @@ -319,7 +326,22 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf snapshot_s3.startup(config, macros); - server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3); + server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3, [this](const KeeperStorage::RequestForSession & request_for_session) + { + std::lock_guard lock(read_mutex); + if (auto it = related_read_requests.find(std::pair{request_for_session.session_id, request_for_session.request->xid}); it != related_read_requests.end()) + { + for (const auto & read_request : it->second) + { + if (server->isLeaderAlive()) + server->putLocalReadRequest(read_request); + else + addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); + } + + related_read_requests.erase(it); + } + }); try { diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index 9371d2fbbac..e7570727b9a 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -1,5 +1,6 @@ #pragma once +#include "Common/ZooKeeper/ZooKeeperCommon.h" #include "config.h" #if USE_NURAFT @@ -103,6 +104,21 @@ private: void forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions); public: + std::mutex read_mutex; + + struct PairHash + { + auto operator()(std::pair pair) const + { + SipHash hash; + hash.update(pair.first); + hash.update(pair.second); + return hash.get64(); + } + }; + + std::unordered_map, KeeperStorage::RequestsForSessions, PairHash> related_read_requests; + /// Just allocate some objects, real initialization is done by `intialize method` KeeperDispatcher(); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 78a095f8c8d..56ed8f4eafe 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -107,7 +107,8 @@ KeeperServer::KeeperServer( const Poco::Util::AbstractConfiguration & config, ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, - KeeperSnapshotManagerS3 & snapshot_manager_s3) + KeeperSnapshotManagerS3 & snapshot_manager_s3, + KeeperStateMachine::CommitCallback commit_callback) : server_id(configuration_and_settings_->server_id) , coordination_settings(configuration_and_settings_->coordination_settings) , log(&Poco::Logger::get("KeeperServer")) @@ -128,6 +129,7 @@ KeeperServer::KeeperServer( coordination_settings, keeper_context, config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr, + commit_callback, checkAndGetSuperdigest(configuration_and_settings_->super_digest)); state_manager = nuraft::cs_new( diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index bcff81f66f2..db4e9c1962e 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -72,7 +72,8 @@ public: const Poco::Util::AbstractConfiguration & config_, ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, - KeeperSnapshotManagerS3 & snapshot_manager_s3); + KeeperSnapshotManagerS3 & snapshot_manager_s3, + KeeperStateMachine::CommitCallback commit_callback); /// Load state machine from the latest snapshot and load log storage. Start NuRaft with required settings. void startup(const Poco::Util::AbstractConfiguration & config, bool enable_ipv6 = true); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index d0bd18b63e2..0b69b00bf0e 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -46,8 +46,10 @@ KeeperStateMachine::KeeperStateMachine( const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, + CommitCallback commit_callback_, const std::string & superdigest_) - : coordination_settings(coordination_settings_) + : commit_callback(commit_callback_) + , coordination_settings(coordination_settings_) , snapshot_manager( snapshots_path_, coordination_settings->snapshots_to_keep, @@ -274,6 +276,7 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n ProfileEvents::increment(ProfileEvents::KeeperCommits); last_committed_idx = log_idx; + commit_callback(request_for_session); return nullptr; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index d8181532f09..6babf741dbd 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -22,6 +22,8 @@ using SnapshotsQueue = ConcurrentBoundedQueue; class KeeperStateMachine : public nuraft::state_machine { public: + using CommitCallback = std::function; + KeeperStateMachine( ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, @@ -29,6 +31,7 @@ public: const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, + CommitCallback commit_callback_, const std::string & superdigest_ = ""); /// Read state from the latest snapshot @@ -105,6 +108,7 @@ public: void recalculateStorageStats(); private: + CommitCallback commit_callback; /// In our state machine we always have a single snapshot which is stored /// in memory in compressed (serialized) format. SnapshotMetadataPtr latest_snapshot_meta = nullptr; From f7c0cca297c0e093fe24eaa225f125c2f0cfd31e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 14:37:51 +0000 Subject: [PATCH 20/49] Maybe better keeper-bench --- utils/keeper-bench/Generator.cpp | 21 ++++++++++++++++++++- utils/keeper-bench/Generator.h | 14 ++++++++++++++ utils/keeper-bench/Runner.cpp | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 5d1d0f8a491..b6d8223862c 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -162,6 +162,19 @@ ZooKeeperRequestPtr SetRequestGenerator::generate() return request; } +void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +{ + for (auto & generator : generators) + generator->startup(zookeeper); +} + +ZooKeeperRequestPtr MixedRequestGenerator::generate() +{ + pcg64 rng(randomSeed()); + std::uniform_int_distribution distribution(0, generators.size() - 1); + + return generators[distribution(rng)]->generate(); +} void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) { @@ -315,7 +328,13 @@ std::unique_ptr getGenerator(const std::string & name) { return std::make_unique("/set_generator", 5); } - + else if (name == "mixed_small_data") + { + std::vector> generators; + generators.push_back(std::make_unique("/set_generator", 5)); + generators.push_back(std::make_unique("/get_generator", 10, 32)); + return std::make_unique(std::move(generators)); + } throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index 1ff01b25ed4..e2c546e4bce 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -122,5 +122,19 @@ private: uint64_t data_size; }; +class MixedRequestGenerator final : public IGenerator +{ +public: + explicit MixedRequestGenerator(std::vector> generators_) + : generators(std::move(generators_)) + {} + + void startup(Coordination::ZooKeeper & zookeeper) override; + Coordination::ZooKeeperRequestPtr generate() override; + +private: + std::vector> generators; +}; + std::unique_ptr getGenerator(const std::string & name); diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 2f3cf4b0620..c858b476483 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -159,9 +159,9 @@ void Runner::runBenchmark() std::cerr << "Prepared\n"; try { + auto connections = getConnections(); for (size_t i = 0; i < concurrency; ++i) { - auto connections = getConnections(); pool.scheduleOrThrowOnError([this, connections]() mutable { thread(connections); }); } } From 7a82830f1086548f348b398b6e2c33adbb12ba9c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 24 Mar 2023 15:04:05 +0000 Subject: [PATCH 21/49] Automatic style fix --- docker/test/performance-comparison/perf.py | 2 ++ docker/test/performance-comparison/report.py | 1 - tests/ci/clickhouse_helper.py | 1 - tests/ci/docker_images_check.py | 1 - tests/ci/get_previous_release_tag.py | 1 - tests/ci/report.py | 2 +- tests/integration/helpers/cluster.py | 3 +-- tests/integration/helpers/network.py | 2 -- .../pytest_xdist_logging_to_separate_files.py | 1 + .../test_detach_part_wrong_partition_id.py | 1 - .../test_cluster_copier/test_three_nodes.py | 1 - .../test_cluster_copier/test_two_nodes.py | 1 - tests/integration/test_composable_protocols/test.py | 1 - .../test_create_query_constraints/test.py | 2 -- .../common.py | 1 - tests/integration/test_disks_app_func/test.py | 1 - .../test_distributed_ddl_parallel/test.py | 1 + tests/integration/test_fetch_memory_usage/test.py | 1 - .../scripts/stress_test.py | 1 - tests/integration/test_jbod_balancer/test.py | 1 - .../test_keeper_and_access_storage/test.py | 1 + tests/integration/test_keeper_back_to_back/test.py | 2 +- tests/integration/test_keeper_persistent_log/test.py | 1 - .../test_keeper_zookeeper_converter/test.py | 1 - tests/integration/test_merge_tree_load_parts/test.py | 6 +++--- .../s3_endpoint/endpoint.py | 1 - .../test_merge_tree_settings_constraints/test.py | 1 - .../test_old_parts_finally_removed/test.py | 1 - tests/integration/test_partition/test.py | 4 +++- tests/integration/test_password_constraints/test.py | 1 - tests/integration/test_read_only_table/test.py | 1 - .../test_reload_auxiliary_zookeepers/test.py | 1 - .../s3_endpoint/endpoint.py | 1 + tests/integration/test_s3_with_proxy/test.py | 1 + .../integration/test_ssl_cert_authentication/test.py | 1 - tests/integration/test_storage_kafka/kafka_pb2.py | 1 - .../test_storage_kafka/message_with_repeated_pb2.py | 1 - tests/integration/test_storage_kafka/social_pb2.py | 1 - tests/integration/test_storage_kafka/test.py | 12 ++---------- tests/integration/test_storage_nats/nats_pb2.py | 1 - .../test_storage_postgresql_replica/test.py | 1 - .../test_storage_rabbitmq/rabbitmq_pb2.py | 1 - tests/integration/test_storage_rabbitmq/test.py | 3 --- tests/integration/test_storage_s3/test.py | 1 + .../test_storage_s3/test_invalid_env_credentials.py | 1 + tests/integration/test_system_merges/test.py | 1 - tests/integration/test_ttl_move/test.py | 2 +- tests/integration/test_zero_copy_fetch/test.py | 1 - utils/changelog-simple/format-changelog.py | 1 + utils/keeper-overload/keeper-overload.py | 2 +- 50 files changed, 23 insertions(+), 57 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 65bf49c2914..7a4e6386d0d 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -26,6 +26,7 @@ logging.basicConfig( total_start_seconds = time.perf_counter() stage_start_seconds = total_start_seconds + # Thread executor that does not hides exception that happens during function # execution, and rethrows it after join() class SafeThread(Thread): @@ -158,6 +159,7 @@ for e in subst_elems: available_parameters[name] = values + # Takes parallel lists of templates, substitutes them with all combos of # parameters. The set of parameters is determined based on the first list. # Note: keep the order of queries -- sometimes we have DROP IF EXISTS diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 782cf29863c..214f2d550b4 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -670,7 +670,6 @@ if args.report == "main": ) elif args.report == "all-queries": - print((header_template.format())) add_tested_commits() diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index d60a9e6afd1..64b64896f66 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -141,7 +141,6 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" base_ref = "master" head_ref = "master" diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 192d216614e..f2b1105b3b0 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -96,7 +96,6 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( pr_info: PRInfo, images_dict: ImagesDict ) -> Set[DockerImage]: - if not images_dict: return set() diff --git a/tests/ci/get_previous_release_tag.py b/tests/ci/get_previous_release_tag.py index c6fe6cd5fb5..c2d279f7fec 100755 --- a/tests/ci/get_previous_release_tag.py +++ b/tests/ci/get_previous_release_tag.py @@ -51,7 +51,6 @@ def find_previous_release( for release in releases: if release.version < server_version: - # Check if the artifact exists on GitHub. # It can be not true for a short period of time # after creating a tag for a new release before uploading the packages. diff --git a/tests/ci/report.py b/tests/ci/report.py index 947fb33d905..ddee035d26f 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -473,7 +473,7 @@ def create_build_html_report( commit_url: str, ) -> str: rows = "" - for (build_result, build_log_url, artifact_urls) in zip( + for build_result, build_log_url, artifact_urls in zip( build_results, build_logs_urls, artifact_urls_list ): row = "" diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index dc5ada81995..a9a996e0a5f 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -63,6 +63,7 @@ DEFAULT_ENV_NAME = ".env" SANITIZER_SIGN = "==================" + # to create docker-compose env file def _create_env_file(path, variables): logging.debug(f"Env {variables} stored in {path}") @@ -1454,7 +1455,6 @@ class ClickHouseCluster: config_root_name="clickhouse", extra_configs=[], ) -> "ClickHouseInstance": - """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -3089,7 +3089,6 @@ class ClickHouseInstance: config_root_name="clickhouse", extra_configs=[], ): - self.name = name self.base_cmd = cluster.base_cmd self.docker_id = cluster.get_instance_docker_id(self.name) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e408c9beec1..471aa2bdc2e 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -216,7 +216,6 @@ class _NetworkManager: container_exit_timeout=60, docker_api_version=os.environ.get("DOCKER_API_VERSION"), ): - self.container_expire_timeout = container_expire_timeout self.container_exit_timeout = container_exit_timeout @@ -232,7 +231,6 @@ class _NetworkManager: def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): - for i in range(5): if self._container is not None: try: diff --git a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py index d424ad58fa4..370aa23a014 100644 --- a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py +++ b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py @@ -1,6 +1,7 @@ import logging import os.path + # Makes the parallel workers of pytest-xdist to log to separate files. # Without this function all workers will log to the same log file # and mix everything together making it much more difficult for troubleshooting. diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py index 02fccfae4e5..a6f7a8653da 100644 --- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -24,7 +24,6 @@ def start_cluster(): def test_detach_part_wrong_partition_id(start_cluster): - # Here we create table with partition by UUID. node_21_6.query( "create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()" diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index 31d6c0448f4..e7d07757adb 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -19,7 +19,6 @@ cluster = ClickHouseCluster(__file__) def started_cluster(): global cluster try: - for name in ["first", "second", "third"]: cluster.add_instance( name, diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py index 10ab7d03b00..2b6fcf6cac2 100644 --- a/tests/integration/test_cluster_copier/test_two_nodes.py +++ b/tests/integration/test_cluster_copier/test_two_nodes.py @@ -19,7 +19,6 @@ cluster = ClickHouseCluster(__file__) def started_cluster(): global cluster try: - for name in ["first_of_two", "second_of_two"]: instance = cluster.add_instance( name, diff --git a/tests/integration/test_composable_protocols/test.py b/tests/integration/test_composable_protocols/test.py index bc87fea5296..df74cfffa54 100644 --- a/tests/integration/test_composable_protocols/test.py +++ b/tests/integration/test_composable_protocols/test.py @@ -63,7 +63,6 @@ def netcat(hostname, port, content): def test_connections(): - client = Client(server.ip_address, 9000, command=cluster.client_bin_path) assert client.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_create_query_constraints/test.py b/tests/integration/test_create_query_constraints/test.py index 8df043fd24b..33c41b4f161 100644 --- a/tests/integration/test_create_query_constraints/test.py +++ b/tests/integration/test_create_query_constraints/test.py @@ -25,7 +25,6 @@ def start_cluster(): def test_create_query_const_constraints(): - instance.query("CREATE USER u_const SETTINGS max_threads = 1 CONST") instance.query("GRANT ALL ON *.* TO u_const") @@ -57,7 +56,6 @@ def test_create_query_const_constraints(): def test_create_query_minmax_constraints(): - instance.query("CREATE USER u_minmax SETTINGS max_threads = 4 MIN 2 MAX 6") instance.query("GRANT ALL ON *.* TO u_minmax") diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py index b38e81b0227..01addae2542 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py @@ -348,7 +348,6 @@ class RangedLayoutTester(BaseLayoutTester): self.layouts = LAYOUTS_RANGED def execute(self, layout_name, node): - if layout_name not in self.layout_to_dictionary: raise RuntimeError("Source doesn't support layout: {}".format(layout_name)) diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 027ef8feed0..2428c53854e 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -7,7 +7,6 @@ import pytest def started_cluster(): global cluster try: - cluster = ClickHouseCluster(__file__) cluster.add_instance( "disks_app_test", main_configs=["config.xml"], with_minio=True diff --git a/tests/integration/test_distributed_ddl_parallel/test.py b/tests/integration/test_distributed_ddl_parallel/test.py index 6ebfe472e09..eb98dd3e230 100644 --- a/tests/integration/test_distributed_ddl_parallel/test.py +++ b/tests/integration/test_distributed_ddl_parallel/test.py @@ -10,6 +10,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) + # By default the exceptions that was throwed in threads will be ignored # (they will not mark the test as failed, only printed to stderr). # diff --git a/tests/integration/test_fetch_memory_usage/test.py b/tests/integration/test_fetch_memory_usage/test.py index a4371140150..7591cc0e8a9 100644 --- a/tests/integration/test_fetch_memory_usage/test.py +++ b/tests/integration/test_fetch_memory_usage/test.py @@ -18,7 +18,6 @@ def started_cluster(): def test_huge_column(started_cluster): - if ( node.is_built_with_thread_sanitizer() or node.is_built_with_memory_sanitizer() diff --git a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py index b8bafb3d0c1..fe69d72c1c7 100644 --- a/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py +++ b/tests/integration/test_host_regexp_multiple_ptr_records_concurrent/scripts/stress_test.py @@ -13,7 +13,6 @@ number_of_iterations = 100 def perform_request(): - buffer = BytesIO() crl = pycurl.Curl() crl.setopt(pycurl.INTERFACE, client_ip) diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index e746698611a..df34a075d5a 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -45,7 +45,6 @@ def start_cluster(): def check_balance(node, table): - partitions = node.query( """ WITH diff --git a/tests/integration/test_keeper_and_access_storage/test.py b/tests/integration/test_keeper_and_access_storage/test.py index 6ec307f7082..0314825b6b7 100644 --- a/tests/integration/test_keeper_and_access_storage/test.py +++ b/tests/integration/test_keeper_and_access_storage/test.py @@ -10,6 +10,7 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/keeper.xml"], stay_alive=True ) + # test that server is able to start @pytest.fixture(scope="module") def started_cluster(): diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 73fface02b4..b737ac284d2 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -546,7 +546,6 @@ def test_random_requests(started_cluster): def test_end_of_session(started_cluster): - fake_zk1 = None fake_zk2 = None genuine_zk1 = None @@ -685,6 +684,7 @@ def test_concurrent_watches(started_cluster): nonlocal watches_created nonlocal all_paths_created fake_zk.ensure_path(global_path + "/" + str(i)) + # new function each time def dumb_watch(event): nonlocal dumb_watch_triggered_counter diff --git a/tests/integration/test_keeper_persistent_log/test.py b/tests/integration/test_keeper_persistent_log/test.py index 70cc14fe26d..4164ffb33d3 100644 --- a/tests/integration/test_keeper_persistent_log/test.py +++ b/tests/integration/test_keeper_persistent_log/test.py @@ -163,7 +163,6 @@ def test_state_duplicate_restart(started_cluster): # http://zookeeper-user.578899.n2.nabble.com/Why-are-ephemeral-nodes-written-to-disk-tp7583403p7583418.html def test_ephemeral_after_restart(started_cluster): - try: node_zk = None node_zk2 = None diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index 063421bf922..de5a9416119 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -114,7 +114,6 @@ def start_clickhouse(): def copy_zookeeper_data(make_zk_snapshots): - if make_zk_snapshots: # force zookeeper to create snapshot generate_zk_snapshot() else: diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py index 777b6f14fc6..dfbe00c8e28 100644 --- a/tests/integration/test_merge_tree_load_parts/test.py +++ b/tests/integration/test_merge_tree_load_parts/test.py @@ -148,17 +148,17 @@ def test_merge_tree_load_parts_corrupted(started_cluster): node1.query("SYSTEM WAIT LOADING PARTS mt_load_parts_2") def check_parts_loading(node, partition, loaded, failed, skipped): - for (min_block, max_block) in loaded: + for min_block, max_block in loaded: part_name = f"{partition}_{min_block}_{max_block}" assert node.contains_in_log(f"Loading Active part {part_name}") assert node.contains_in_log(f"Finished loading Active part {part_name}") - for (min_block, max_block) in failed: + for min_block, max_block in failed: part_name = f"{partition}_{min_block}_{max_block}" assert node.contains_in_log(f"Loading Active part {part_name}") assert not node.contains_in_log(f"Finished loading Active part {part_name}") - for (min_block, max_block) in skipped: + for min_block, max_block in skipped: part_name = f"{partition}_{min_block}_{max_block}" assert not node.contains_in_log(f"Loading Active part {part_name}") assert not node.contains_in_log(f"Finished loading Active part {part_name}") diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index b6567dfebc5..4613fdb850b 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -42,7 +42,6 @@ def delete(_bucket): @route("/<_bucket>/<_path:path>", ["GET", "POST", "PUT", "DELETE"]) def server(_bucket, _path): - # It's delete query for failed part if _path.endswith("delete"): response.set_header("Location", "http://minio1:9001/" + _bucket + "/" + _path) diff --git a/tests/integration/test_merge_tree_settings_constraints/test.py b/tests/integration/test_merge_tree_settings_constraints/test.py index 0bb0179108d..be6e2a31873 100644 --- a/tests/integration/test_merge_tree_settings_constraints/test.py +++ b/tests/integration/test_merge_tree_settings_constraints/test.py @@ -20,7 +20,6 @@ def start_cluster(): def test_merge_tree_settings_constraints(): - assert "Setting storage_policy should not be changed" in instance.query_and_get_error( f"CREATE TABLE wrong_table (number Int64) engine = MergeTree() ORDER BY number SETTINGS storage_policy = 'secret_policy'" ) diff --git a/tests/integration/test_old_parts_finally_removed/test.py b/tests/integration/test_old_parts_finally_removed/test.py index 108b72c5ccd..5347d433419 100644 --- a/tests/integration/test_old_parts_finally_removed/test.py +++ b/tests/integration/test_old_parts_finally_removed/test.py @@ -63,7 +63,6 @@ def test_part_finally_removed(started_cluster): ) for i in range(60): - if ( node1.query( "SELECT count() from system.parts WHERE table = 'drop_outdated_part'" diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index ae4393fc6f6..a34141c6189 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -528,7 +528,9 @@ def test_make_clone_in_detached(started_cluster): ["cp", "-r", path + "all_0_0_0", path + "detached/broken_all_0_0_0"] ) assert_eq_with_retry(instance, "select * from clone_in_detached", "\n") - assert ["broken_all_0_0_0",] == sorted( + assert [ + "broken_all_0_0_0", + ] == sorted( instance.exec_in_container(["ls", path + "detached/"]).strip().split("\n") ) diff --git a/tests/integration/test_password_constraints/test.py b/tests/integration/test_password_constraints/test.py index e3628861b28..9cdff51caa1 100644 --- a/tests/integration/test_password_constraints/test.py +++ b/tests/integration/test_password_constraints/test.py @@ -17,7 +17,6 @@ def start_cluster(): def test_complexity_rules(start_cluster): - error_message = "DB::Exception: Invalid password. The password should: be at least 12 characters long, contain at least 1 numeric character, contain at least 1 lowercase character, contain at least 1 uppercase character, contain at least 1 special character" assert error_message in node.query_and_get_error( "CREATE USER u_1 IDENTIFIED WITH plaintext_password BY ''" diff --git a/tests/integration/test_read_only_table/test.py b/tests/integration/test_read_only_table/test.py index 914c6a99508..df084f9dbbd 100644 --- a/tests/integration/test_read_only_table/test.py +++ b/tests/integration/test_read_only_table/test.py @@ -49,7 +49,6 @@ def start_cluster(): def test_restart_zookeeper(start_cluster): - for table_id in range(NUM_TABLES): node1.query( f"INSERT INTO test_table_{table_id} VALUES (1), (2), (3), (4), (5);" diff --git a/tests/integration/test_reload_auxiliary_zookeepers/test.py b/tests/integration/test_reload_auxiliary_zookeepers/test.py index bb1455333fc..476c5dee99e 100644 --- a/tests/integration/test_reload_auxiliary_zookeepers/test.py +++ b/tests/integration/test_reload_auxiliary_zookeepers/test.py @@ -20,7 +20,6 @@ def start_cluster(): def test_reload_auxiliary_zookeepers(start_cluster): - node.query( "CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', 'node') ORDER BY tuple() PARTITION BY date;" ) diff --git a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py index d6a732cc681..1d33ca02f86 100644 --- a/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py +++ b/tests/integration/test_s3_aws_sdk_has_slightly_unreliable_behaviour/s3_endpoint/endpoint.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from bottle import request, route, run, response + # Handle for MultipleObjectsDelete. @route("/<_bucket>", ["POST"]) def delete(_bucket): diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 1102d190a87..1af040c3c30 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -5,6 +5,7 @@ import time import pytest from helpers.cluster import ClickHouseCluster + # Runs simple proxy resolver in python env container. def run_resolver(cluster): container_id = cluster.get_container_id("resolver") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 7c62ca0d8b6..b3570b6e281 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -87,7 +87,6 @@ config = """ def execute_query_native(node, query, user, cert_name): - config_path = f"{SCRIPT_DIR}/configs/client.xml" formatted = config.format( diff --git a/tests/integration/test_storage_kafka/kafka_pb2.py b/tests/integration/test_storage_kafka/kafka_pb2.py index 7de1363bbf1..3e47af6c1e0 100644 --- a/tests/integration/test_storage_kafka/kafka_pb2.py +++ b/tests/integration/test_storage_kafka/kafka_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.kafka_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _KEYVALUEPAIR._serialized_start = 46 _KEYVALUEPAIR._serialized_end = 88 diff --git a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py index 4d1a23c0b43..3715a9bea04 100644 --- a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py +++ b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.message_with_repeated_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"H\001" _MESSAGE._serialized_start = 62 diff --git a/tests/integration/test_storage_kafka/social_pb2.py b/tests/integration/test_storage_kafka/social_pb2.py index 830ade81d33..f91a7bd0539 100644 --- a/tests/integration/test_storage_kafka/social_pb2.py +++ b/tests/integration/test_storage_kafka/social_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.social_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _USER._serialized_start = 47 _USER._serialized_end = 90 diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 51952ac1eb7..3a4fa6c6bfe 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -121,7 +121,7 @@ def kafka_create_topic( def kafka_delete_topic(admin_client, topic, max_retries=50): result = admin_client.delete_topics([topic]) - for (topic, e) in result.topic_error_codes: + for topic, e in result.topic_error_codes: if e == 0: logging.debug(f"Topic {topic} deleted") else: @@ -917,9 +917,7 @@ def describe_consumer_group(kafka_cluster, name): member_info["client_id"] = client_id member_info["client_host"] = client_host member_topics_assignment = [] - for (topic, partitions) in MemberAssignment.decode( - member_assignment - ).assignment: + for topic, partitions in MemberAssignment.decode(member_assignment).assignment: member_topics_assignment.append({"topic": topic, "partitions": partitions}) member_info["assignment"] = member_topics_assignment res.append(member_info) @@ -1537,7 +1535,6 @@ def test_kafka_protobuf_no_delimiter(kafka_cluster): def test_kafka_materialized_view(kafka_cluster): - instance.query( """ DROP TABLE IF EXISTS test.view; @@ -2315,7 +2312,6 @@ def test_kafka_virtual_columns2(kafka_cluster): def test_kafka_produce_key_timestamp(kafka_cluster): - admin_client = KafkaAdminClient( bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) @@ -2444,7 +2440,6 @@ def test_kafka_insert_avro(kafka_cluster): def test_kafka_produce_consume_avro(kafka_cluster): - admin_client = KafkaAdminClient( bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) @@ -4031,7 +4026,6 @@ def test_kafka_predefined_configuration(kafka_cluster): # https://github.com/ClickHouse/ClickHouse/issues/26643 def test_issue26643(kafka_cluster): - # for backporting: # admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") admin_client = KafkaAdminClient( @@ -4313,7 +4307,6 @@ def test_row_based_formats(kafka_cluster): "RowBinaryWithNamesAndTypes", "MsgPack", ]: - print(format_name) kafka_create_topic(admin_client, format_name) @@ -4438,7 +4431,6 @@ def test_block_based_formats_2(kafka_cluster): "ORC", "JSONCompactColumns", ]: - kafka_create_topic(admin_client, format_name) instance.query( diff --git a/tests/integration/test_storage_nats/nats_pb2.py b/tests/integration/test_storage_nats/nats_pb2.py index 4330ff57950..e9e5cb72363 100644 --- a/tests/integration/test_storage_nats/nats_pb2.py +++ b/tests/integration/test_storage_nats/nats_pb2.py @@ -31,7 +31,6 @@ ProtoKeyValue = _reflection.GeneratedProtocolMessageType( _sym_db.RegisterMessage(ProtoKeyValue) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _PROTOKEYVALUE._serialized_start = 45 _PROTOKEYVALUE._serialized_end = 88 diff --git a/tests/integration/test_storage_postgresql_replica/test.py b/tests/integration/test_storage_postgresql_replica/test.py index 5df8b9029e6..8666d7ae58c 100644 --- a/tests/integration/test_storage_postgresql_replica/test.py +++ b/tests/integration/test_storage_postgresql_replica/test.py @@ -706,7 +706,6 @@ def test_abrupt_connection_loss_while_heavy_replication(started_cluster): def test_abrupt_server_restart_while_heavy_replication(started_cluster): - # FIXME (kssenii) temporary disabled if instance.is_built_with_sanitizer(): pytest.skip("Temporary disabled (FIXME)") diff --git a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py index e017b4e66c2..a5845652eef 100644 --- a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py +++ b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py @@ -21,7 +21,6 @@ _builder.BuildTopDescriptorsAndMessages( DESCRIPTOR, "clickhouse_path.format_schemas.rabbitmq_pb2", globals() ) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None _KEYVALUEPROTO._serialized_start = 49 _KEYVALUEPROTO._serialized_end = 92 diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 2e54f21787a..53b6c4109ef 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2864,7 +2864,6 @@ def test_rabbitmq_predefined_configuration(rabbitmq_cluster): def test_rabbitmq_msgpack(rabbitmq_cluster): - instance.query( """ drop table if exists rabbit_in; @@ -2908,7 +2907,6 @@ def test_rabbitmq_msgpack(rabbitmq_cluster): def test_rabbitmq_address(rabbitmq_cluster): - instance2.query( """ drop table if exists rabbit_in; @@ -3243,7 +3241,6 @@ def test_block_based_formats_2(rabbitmq_cluster): "ORC", "JSONCompactColumns", ]: - print(format_name) instance.query( diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 8b20727a7b5..4d493d9526b 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -18,6 +18,7 @@ MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(started_cluster): # Allows read-write access for bucket without authorization. diff --git a/tests/integration/test_storage_s3/test_invalid_env_credentials.py b/tests/integration/test_storage_s3/test_invalid_env_credentials.py index 2f5d9349904..aa6479a2ed3 100644 --- a/tests/integration/test_storage_s3/test_invalid_env_credentials.py +++ b/tests/integration/test_storage_s3/test_invalid_env_credentials.py @@ -11,6 +11,7 @@ MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(started_cluster): # Allows read-write access for bucket without authorization. diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 0a469bd7bbd..ff303afe19e 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -171,7 +171,6 @@ def test_mutation_simple(started_cluster, replicated): starting_block = 0 if replicated else 1 try: - for node in nodes: node.query( f"create table {name} (a Int64) engine={engine} order by tuple()" diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 99978cbf6dc..89824293320 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1863,7 +1863,7 @@ def test_ttl_move_if_exists(started_cluster, name, dest_type): ) ) - for (node, policy) in zip( + for node, policy in zip( [node1, node2], ["only_jbod_1", "small_jbod_with_external"] ): node.query( diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py index b71752528d3..9b9aa5e0da7 100644 --- a/tests/integration/test_zero_copy_fetch/test.py +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -16,7 +16,6 @@ cluster = ClickHouseCluster(__file__) @pytest.fixture(scope="module") def started_cluster(): try: - cluster.add_instance( "node1", main_configs=["configs/storage_conf.xml"], diff --git a/utils/changelog-simple/format-changelog.py b/utils/changelog-simple/format-changelog.py index d5e1518270e..01f2694dd0f 100755 --- a/utils/changelog-simple/format-changelog.py +++ b/utils/changelog-simple/format-changelog.py @@ -20,6 +20,7 @@ parser.add_argument( ) args = parser.parse_args() + # This function mirrors the PR description checks in ClickhousePullRequestTrigger. # Returns False if the PR should not be mentioned changelog. def parse_one_pull_request(item): diff --git a/utils/keeper-overload/keeper-overload.py b/utils/keeper-overload/keeper-overload.py index bdb4563c713..0a059b10588 100755 --- a/utils/keeper-overload/keeper-overload.py +++ b/utils/keeper-overload/keeper-overload.py @@ -166,7 +166,7 @@ def main(args): keeper_bench_path = args.keeper_bench_path keepers = [] - for (port, server_id) in zip(PORTS, SERVER_IDS): + for port, server_id in zip(PORTS, SERVER_IDS): keepers.append( Keeper( keeper_binary_path, server_id, port, workdir, args.with_thread_fuzzer From ee0fd39f2a1959a792977a114523bdd477c7ed16 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 24 Mar 2023 15:20:19 +0000 Subject: [PATCH 22/49] Fix data-race --- src/Interpreters/ProcessList.cpp | 10 ++++++++-- src/Interpreters/ProcessList.h | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index c7916f3eed2..d7ba0b8330f 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -388,6 +388,12 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } +bool QueryStatus::ExecutorHolder::equals(const PipelineExecutor * e) +{ + std::lock_guard lock(mutex); + return executor == e; +} + CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -428,7 +434,7 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }) == executors.end()); + assert(std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }) == executors.end()); executors.push_back(std::make_shared(e)); } @@ -437,7 +443,7 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) ExecutorHolderPtr executor_holder; { std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + auto it = std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }); assert(it != executors.end()); executor_holder = *it; } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 30bfde4e218..98d7daa0949 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -127,6 +127,8 @@ protected: void remove(); + bool equals(const PipelineExecutor * e); + PipelineExecutor * executor; std::mutex mutex; }; From d00587202dd2b1aad5188b09acdb4fe40c950cde Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 24 Mar 2023 19:25:07 +0000 Subject: [PATCH 23/49] Make better --- src/Interpreters/ProcessList.cpp | 17 +++++++---------- src/Interpreters/ProcessList.h | 3 +-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index d7ba0b8330f..a86377a37d5 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -388,12 +388,6 @@ void QueryStatus::ExecutorHolder::remove() executor = nullptr; } -bool QueryStatus::ExecutorHolder::equals(const PipelineExecutor * e) -{ - std::lock_guard lock(mutex); - return executor == e; -} - CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -434,19 +428,22 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }) == executors.end()); + assert(!executor_indexes.contains(e)); executors.push_back(std::make_shared(e)); + executor_indexes[e] = executors.size() - 1; } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { ExecutorHolderPtr executor_holder; + { std::lock_guard lock(executors_mutex); - auto it = std::find_if(executors.begin(), executors.end(), [e](ExecutorHolderPtr & x){ return x->equals(e); }); - assert(it != executors.end()); - executor_holder = *it; + assert(executor_indexes.contains(e)); + executor_holder = executors[executor_indexes[e]]; + executor_indexes.erase(e); } + /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) /// We should do it with released executors_mutex to avoid possible lock order inversion. executor_holder->remove(); diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 98d7daa0949..9e5c91bf2ed 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -127,8 +127,6 @@ protected: void remove(); - bool equals(const PipelineExecutor * e); - PipelineExecutor * executor; std::mutex mutex; }; @@ -137,6 +135,7 @@ protected: /// Array of PipelineExecutors to be cancelled when a cancelQuery is received std::vector executors; + std::unordered_map executor_indexes; enum QueryStreamsStatus { From 6c3765c8b663554efde86816395385b97dcabb9c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 24 Mar 2023 20:40:45 +0000 Subject: [PATCH 24/49] Small fix --- src/Coordination/KeeperDispatcher.cpp | 48 +++++++++++++++---------- src/Coordination/KeeperDispatcher.h | 16 ++------- src/Coordination/KeeperStateMachine.cpp | 4 ++- src/Coordination/KeeperStateMachine.h | 2 +- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index fc97bc5e0e1..69abf85ae4e 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -110,9 +110,9 @@ void KeeperDispatcher::requestThread() if (!coordination_settings->quorum_reads && request.request->isReadRequest()) { ++read_requests; - std::pair key{current_batch.back().session_id, current_batch.back().request->xid}; - std::lock_guard lock(read_mutex); - related_read_requests[key].push_back(request); + const auto & last_request = current_batch.back(); + std::lock_guard lock(read_request_queue_mutex); + read_request_queue[last_request.session_id][last_request.request->xid].push_back(request); } else current_batch.emplace_back(request); @@ -328,18 +328,24 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf server = std::make_unique(configuration_and_settings, config, responses_queue, snapshots_queue, snapshot_s3, [this](const KeeperStorage::RequestForSession & request_for_session) { - std::lock_guard lock(read_mutex); - if (auto it = related_read_requests.find(std::pair{request_for_session.session_id, request_for_session.request->xid}); it != related_read_requests.end()) + /// check if we have queue of read requests depending on this request to be committed + std::lock_guard lock(read_request_queue_mutex); + if (auto it = read_request_queue.find(request_for_session.session_id); it != read_request_queue.end()) { - for (const auto & read_request : it->second) - { - if (server->isLeaderAlive()) - server->putLocalReadRequest(read_request); - else - addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); - } + auto & xid_to_request_queue = it->second; - related_read_requests.erase(it); + if (auto request_queue_it = xid_to_request_queue.find(request_for_session.request->xid); request_queue_it != xid_to_request_queue.end()) + { + for (const auto & read_request : request_queue_it->second) + { + if (server->isLeaderAlive()) + server->putLocalReadRequest(read_request); + else + addErrorResponses({read_request}, Coordination::Error::ZCONNECTIONLOSS); + } + + xid_to_request_queue.erase(request_queue_it); + } } }); @@ -554,12 +560,18 @@ void KeeperDispatcher::sessionCleanerTask() void KeeperDispatcher::finishSession(int64_t session_id) { - std::lock_guard lock(session_to_response_callback_mutex); - auto session_it = session_to_response_callback.find(session_id); - if (session_it != session_to_response_callback.end()) { - session_to_response_callback.erase(session_it); - CurrentMetrics::sub(CurrentMetrics::KeeperAliveConnections); + std::lock_guard lock(session_to_response_callback_mutex); + auto session_it = session_to_response_callback.find(session_id); + if (session_it != session_to_response_callback.end()) + { + session_to_response_callback.erase(session_it); + CurrentMetrics::sub(CurrentMetrics::KeeperAliveConnections); + } + } + { + std::lock_guard lock(read_request_queue_mutex); + read_request_queue.erase(session_id); } } diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index e7570727b9a..77b5510cbb3 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -104,20 +104,10 @@ private: void forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions); public: - std::mutex read_mutex; + std::mutex read_request_queue_mutex; - struct PairHash - { - auto operator()(std::pair pair) const - { - SipHash hash; - hash.update(pair.first); - hash.update(pair.second); - return hash.get64(); - } - }; - - std::unordered_map, KeeperStorage::RequestsForSessions, PairHash> related_read_requests; + /// queue of read requests that can be processed after a request with specific session ID and XID is committed + std::unordered_map> read_request_queue; /// Just allocate some objects, real initialization is done by `intialize method` KeeperDispatcher(); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 0b69b00bf0e..632aaec6b54 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -276,7 +276,9 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n ProfileEvents::increment(ProfileEvents::KeeperCommits); last_committed_idx = log_idx; - commit_callback(request_for_session); + + if (commit_callback) + commit_callback(request_for_session); return nullptr; } diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 6babf741dbd..e4f0295db99 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -31,7 +31,7 @@ public: const CoordinationSettingsPtr & coordination_settings_, const KeeperContextPtr & keeper_context_, KeeperSnapshotManagerS3 * snapshot_manager_s3_, - CommitCallback commit_callback_, + CommitCallback commit_callback_ = {}, const std::string & superdigest_ = ""); /// Read state from the latest snapshot From b143a1a3ebabb9333192fcbbf98bbcff791d1819 Mon Sep 17 00:00:00 2001 From: Sergei Solomatov Date: Sun, 26 Mar 2023 11:20:53 +0000 Subject: [PATCH 25/49] fix query --- src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 4d8de325902..2c97c92ba99 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -302,11 +302,12 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "and a.attnum = ANY(ix.indkey) " "and t.relkind in ('r', 'p') " /// simple tables "and t.relname = {} " /// Connection is already done to a needed database, only table name is needed. - "{}" + "and t.relnamespace = (select oid from pg_namespace where nspname = {}) " "and ix.indisreplident = 't' " /// index is is replica identity index "ORDER BY a.attname", /// column name - (postgres_schema.empty() ? "" : "and t.relnamespace = " + quoteString(postgres_schema)) + " ", - quoteString(postgres_table)); + quoteString(postgres_table), + (postgres_schema.empty() ? quoteString("public") : quoteString(postgres_schema)) + ); table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } From 5d5308e060bcfe1d7a07cbebd2cc3eae833afe2d Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Mar 2023 17:59:39 +0000 Subject: [PATCH 26/49] CMake cleanup: Remove configuration of CMAKE_SHARED_LINKER_FLAGS Follow-up to #44828 --- CMakeLists.txt | 1 - PreLoad.cmake | 5 ++--- cmake/linux/toolchain-riscv64.cmake | 1 - cmake/linux/toolchain-x86_64.cmake | 1 - cmake/tools.cmake | 2 -- 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49367705a08..5550a19b699 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,7 +180,6 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. if (LINKER_NAME MATCHES "lld" AND OS_LINUX) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") message (STATUS "Adding .gdb-index via --gdb-index linker option.") endif () endif() diff --git a/PreLoad.cmake b/PreLoad.cmake index 0e1ee70fc8f..b456c724cc6 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -19,8 +19,8 @@ endif() if (NOT "$ENV{CFLAGS}" STREQUAL "" OR NOT "$ENV{CXXFLAGS}" STREQUAL "" OR NOT "$ENV{LDFLAGS}" STREQUAL "" - OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_SHARED_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS - OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_SHARED_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT) + OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS + OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT) # if $ENV message("CFLAGS: $ENV{CFLAGS}") @@ -36,7 +36,6 @@ if (NOT "$ENV{CFLAGS}" STREQUAL "" message("CMAKE_C_FLAGS_INIT: ${CMAKE_C_FLAGS_INIT}") message("CMAKE_CXX_FLAGS_INIT: ${CMAKE_CXX_FLAGS_INIT}") message("CMAKE_EXE_LINKER_FLAGS_INIT: ${CMAKE_EXE_LINKER_FLAGS_INIT}") - message("CMAKE_SHARED_LINKER_FLAGS_INIT: ${CMAKE_SHARED_LINKER_FLAGS_INIT}") message("CMAKE_MODULE_LINKER_FLAGS_INIT: ${CMAKE_MODULE_LINKER_FLAGS_INIT}") message(FATAL_ERROR " diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake index 49a036c2972..ea57c3b2c42 100644 --- a/cmake/linux/toolchain-riscv64.cmake +++ b/cmake/linux/toolchain-riscv64.cmake @@ -22,7 +22,6 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=bfd") -set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=bfd") # Currently, lld does not work with the error: # ld.lld: error: section size decrease is too large diff --git a/cmake/linux/toolchain-x86_64.cmake b/cmake/linux/toolchain-x86_64.cmake index e73d779284a..55b9df79f70 100644 --- a/cmake/linux/toolchain-x86_64.cmake +++ b/cmake/linux/toolchain-x86_64.cmake @@ -30,7 +30,6 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index f17ab4da5cd..974b0bd1d3d 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -95,10 +95,8 @@ if (LINKER_NAME) configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ld.lld.in" "${LLD_WRAPPER}" @ONLY) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") else () set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") endif () endif () From f9d473a94d025a831d51ae579bbd31d0a4678187 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Mar 2023 20:00:04 +0000 Subject: [PATCH 27/49] parseDateTime(): Fix UB (signed integer overflow) --- src/Functions/parseDateTime.cpp | 81 +++++++++++-------- .../02668_parse_datetime_in_joda_syntax.sql | 3 + 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 7799520b8e5..abee7e0d8f8 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -1032,11 +1032,12 @@ namespace bool allow_negative, bool allow_plus_sign, bool is_year, - int repetitions, - int max_digits_to_read, + size_t repetitions, + size_t max_digits_to_read, const String & fragment, Int32 & result) { + bool negative = false; if (allow_negative && cur < end && *cur == '-') { @@ -1051,6 +1052,15 @@ namespace Int64 number = 0; const Pos start = cur; + + /// Avoid integer overflow in (*) + if (max_digits_to_read >= std::numeric_limits::digits10) [[unlikely]] + throw Exception( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Unable to parse fragment {} from {} because max_digits_to_read is too big", + fragment, + std::string_view(start, cur - start)); + if (is_year && repetitions == 2) { // If abbreviated two year digit is provided in format string, try to read @@ -1059,10 +1069,10 @@ namespace // [70, 99] -> [1970, 1999] // If more than two digits are provided, then simply read in full year // normally without conversion - int count = 0; + size_t count = 0; while (cur < end && cur < start + max_digits_to_read && *cur >= '0' && *cur <= '9') { - number = number * 10 + (*cur - '0'); + number = number * 10 + (*cur - '0'); /// (*) ++cur; ++count; } @@ -1077,7 +1087,7 @@ namespace { while (cur < end && cur < start + max_digits_to_read && *cur >= '0' && *cur <= '9') { - number = number * 10 + (*cur - '0'); + number = number * 10 + (*cur - '0'); /// (*) ++cur; } } @@ -1091,24 +1101,25 @@ namespace } } + if (negative) + number *= -1; + /// Need to have read at least one digit. - if (cur == start) + if (cur == start) [[unlikely]] throw Exception( ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to parse fragment {} from {} because read number failed", fragment, std::string_view(cur, end - cur)); - if (negative) - number *= -1; - /// Check if number exceeds the range of Int32 - if (number < std::numeric_limits::lowest() || number > std::numeric_limits::max()) + if (number < std::numeric_limits::min() || number > std::numeric_limits::max()) [[unlikely]] throw Exception( ErrorCodes::CANNOT_PARSE_DATETIME, "Unable to parse fragment {} from {} because number is out of range of Int32", fragment, std::string_view(start, cur - start)); + result = static_cast(number); return cur; @@ -1125,7 +1136,7 @@ namespace return cur; } - static Pos jodaCenturyOfEra(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaCenturyOfEra(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 century; cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, repetitions, fragment, century); @@ -1133,7 +1144,7 @@ namespace return cur; } - static Pos jodaYearOfEra(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaYearOfEra(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 year_of_era; cur = readNumberWithVariableLength(cur, end, false, false, true, repetitions, repetitions, fragment, year_of_era); @@ -1141,7 +1152,7 @@ namespace return cur; } - static Pos jodaWeekYear(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaWeekYear(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 week_year; cur = readNumberWithVariableLength(cur, end, true, true, true, repetitions, repetitions, fragment, week_year); @@ -1149,15 +1160,15 @@ namespace return cur; } - static Pos jodaWeekOfWeekYear(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaWeekOfWeekYear(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 week; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, week); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, week); date.setWeek(week); return cur; } - static Pos jodaDayOfWeek1Based(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaDayOfWeek1Based(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 day_of_week; cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, repetitions, fragment, day_of_week); @@ -1197,7 +1208,7 @@ namespace return cur; } - static Pos jodaYear(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaYear(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 year; cur = readNumberWithVariableLength(cur, end, true, true, true, repetitions, repetitions, fragment, year); @@ -1205,15 +1216,15 @@ namespace return cur; } - static Pos jodaDayOfYear(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaDayOfYear(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 day_of_year; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 3), fragment, day_of_year); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 3uz), fragment, day_of_year); date.setDayOfYear(day_of_year); return cur; } - static Pos jodaMonthOfYear(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaMonthOfYear(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 month; cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, 2, fragment, month); @@ -1251,11 +1262,11 @@ namespace return cur; } - static Pos jodaDayOfMonth(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaDayOfMonth(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 day_of_month; cur = readNumberWithVariableLength( - cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, day_of_month); + cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, day_of_month); date.setDayOfMonth(day_of_month); return cur; } @@ -1271,50 +1282,50 @@ namespace return cur; } - static Pos jodaHourOfHalfDay(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaHourOfHalfDay(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 hour; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, hour); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, hour); date.setHour(hour, true, false); return cur; } - static Pos jodaClockHourOfHalfDay(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaClockHourOfHalfDay(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 hour; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, hour); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, hour); date.setHour(hour, true, true); return cur; } - static Pos jodaHourOfDay(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaHourOfDay(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 hour; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, hour); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, hour); date.setHour(hour, false, false); return cur; } - static Pos jodaClockHourOfDay(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaClockHourOfDay(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 hour; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, hour); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, hour); date.setHour(hour, false, true); return cur; } - static Pos jodaMinuteOfHour(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaMinuteOfHour(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 minute; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, minute); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, minute); date.setMinute(minute); return cur; } - static Pos jodaSecondOfMinute(int repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) + static Pos jodaSecondOfMinute(size_t repetitions, Pos cur, Pos end, const String & fragment, DateTime & date) { Int32 second; - cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2), fragment, second); + cur = readNumberWithVariableLength(cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, second); date.setSecond(second); return cur; } @@ -1612,7 +1623,7 @@ namespace } else { - int repetitions = 1; + size_t repetitions = 1; ++pos; while (pos < end && *cur_token == *pos) { diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql index 27b4a8bf83d..7ce5c1a4fdd 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql @@ -229,4 +229,7 @@ select parseDateTimeInJodaSyntax('60', 's', 'UTC'); -- { serverError CANNOT_PARS select parseDateTimeInJodaSyntax('-1', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTimeInJodaSyntax('123456789', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } +-- integer overflow in AST Fuzzer +select parseDateTimeInJodaSyntax('19191919191919191919191919191919', 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } + -- { echoOff } From 79e22d13fbaf495b83e10b4ca87363eb81f2635c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 27 Mar 2023 07:36:39 +0000 Subject: [PATCH 28/49] small fix --- src/Coordination/KeeperDispatcher.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 12248950c32..1828182751d 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -92,7 +92,7 @@ void KeeperDispatcher::requestThread() KeeperStorage::RequestsForSessions current_batch; size_t current_batch_bytes_size = 0; - bool has_read_request{false}; + bool has_read_request = false; /// If new request is not read request or we must to process it through quorum. /// Otherwise we will process it locally. @@ -101,8 +101,6 @@ void KeeperDispatcher::requestThread() current_batch_bytes_size += request.request->bytesSize(); current_batch.emplace_back(request); - size_t read_requests = 0; - const auto try_get_request = [&] { /// Trying to get batch requests as fast as possible @@ -112,7 +110,6 @@ void KeeperDispatcher::requestThread() /// Don't append read request into batch, we have to process them separately if (!coordination_settings->quorum_reads && request.request->isReadRequest()) { - ++read_requests; const auto & last_request = current_batch.back(); std::lock_guard lock(read_request_queue_mutex); read_request_queue[last_request.session_id][last_request.request->xid].push_back(request); From 4fdce459ebdda82c65f9eefecbad76cc25de4004 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 08:34:18 +0000 Subject: [PATCH 29/49] Fix expected result --- .../0_stateless/02668_parse_datetime_in_joda_syntax.reference | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference index 9391af957a1..7402f104ae4 100644 --- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference +++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference @@ -342,3 +342,5 @@ select parseDateTimeInJodaSyntax('0/', 's/', 'UTC') = toDateTime('1970-01-01 00: select parseDateTimeInJodaSyntax('60', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTimeInJodaSyntax('-1', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTimeInJodaSyntax('123456789', 's', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } +-- integer overflow in AST Fuzzer +select parseDateTimeInJodaSyntax('19191919191919191919191919191919', 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC', 'UTC'); -- { serverError CANNOT_PARSE_DATETIME } From 2981890ab4d583216f2a28722a0faddaef9a0329 Mon Sep 17 00:00:00 2001 From: Tyler Hannan Date: Mon, 27 Mar 2023 12:03:47 +0200 Subject: [PATCH 30/49] Fix Sample... --- docs/en/sql-reference/statements/alter/projection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/projection.md b/docs/en/sql-reference/statements/alter/projection.md index 626d71709ac..030e9352a00 100644 --- a/docs/en/sql-reference/statements/alter/projection.md +++ b/docs/en/sql-reference/statements/alter/projection.md @@ -128,7 +128,7 @@ SELECT user_agent, sum(pages_visited) FROM visits -GROUP BY user_id +GROUP BY user_agent ``` As mentioned before, we could review the `system.query_log` table. On the `projections` field we have the name of the projection used or empty if none has been used: From 78eb71a49b58a89e5fa3268c216bf0217fdd0884 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 12:11:32 +0200 Subject: [PATCH 31/49] Remove the old changelog script --- utils/changelog-simple/.gitignore | 2 - utils/changelog-simple/README.md | 21 --- utils/changelog-simple/changelog.sh | 96 ------------ utils/changelog-simple/format-changelog.py | 165 --------------------- 4 files changed, 284 deletions(-) delete mode 100644 utils/changelog-simple/.gitignore delete mode 100644 utils/changelog-simple/README.md delete mode 100755 utils/changelog-simple/changelog.sh delete mode 100755 utils/changelog-simple/format-changelog.py diff --git a/utils/changelog-simple/.gitignore b/utils/changelog-simple/.gitignore deleted file mode 100644 index 78caa68e38e..00000000000 --- a/utils/changelog-simple/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.txt -*.json diff --git a/utils/changelog-simple/README.md b/utils/changelog-simple/README.md deleted file mode 100644 index cd8f8da9b61..00000000000 --- a/utils/changelog-simple/README.md +++ /dev/null @@ -1,21 +0,0 @@ -## How To Generate Changelog - -Generate github token: -* https://github.com/settings/tokens - keep all checkboxes unchecked, no scopes need to be enabled. - -Dependencies: -``` -sudo apt-get install git curl jq python3 python3-fuzzywuzzy -``` - -Update information about tags: -``` -git fetch --tags -``` - -Usage example: - -``` -export GITHUB_USER=... GITHUB_TOKEN=ghp_... -./changelog.sh v21.5.6.6-stable v21.6.2.7-prestable -``` diff --git a/utils/changelog-simple/changelog.sh b/utils/changelog-simple/changelog.sh deleted file mode 100755 index 52817acfae4..00000000000 --- a/utils/changelog-simple/changelog.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -set -e - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -from="$1" -to="$2" -log_command=(git log "$from..$to" --first-parent) - -"${log_command[@]}" > "changelog-log.txt" - -# Check for diamond merges. -if "${log_command[@]}" --oneline --grep "Merge branch '" | grep '' -then - # DO NOT ADD automated handling of diamond merges to this script. - # It is an unsustainable way to work with git, and it MUST be visible. - echo Warning: suspected diamond merges above. - echo Some commits will be missed, review these manually. -fi - -# Search for PR numbers in commit messages. First variant is normal merge, and second -# variant is squashed. Next are some backport message variants. -find_prs=(sed -n "s/^.*merg[eding]*.*#\([[:digit:]]\+\).*$/\1/Ip; - s/^.*(#\([[:digit:]]\+\))$/\1/p; - s/^.*back[- ]*port[ed of]*.*#\([[:digit:]]\+\).*$/\1/Ip; - s/^.*cherry[- ]*pick[ed of]*.*#\([[:digit:]]\+\).*$/\1/Ip") - -# awk is to filter out small task numbers from different task tracker, which are -# referenced by documentation commits like '* DOCSUP-824: query log (#115)'. -"${find_prs[@]}" "changelog-log.txt" | sort -rn | uniq | awk '$0 > 1000 { print $0 }' > "changelog-prs.txt" - -echo "$(wc -l < "changelog-prs.txt") PRs added between $from and $to." -if [ $(wc -l < "changelog-prs.txt") -eq 0 ] ; then exit 0 ; fi - -function github_download() -{ - local url=${1} - local file=${2} - if ! [ -f "$file" ] - then - echo "curl -u \"$GITHUB_USER:***\" -sSf \"$url\" > \"$file\"" - - if ! curl -u "$GITHUB_USER:$GITHUB_TOKEN" \ - -sSf "$url" \ - > "$file" - then - >&2 echo "Failed to download '$url' to '$file'. Contents: '$(cat "$file")'." - rm "$file" - return 1 - fi - sleep 0.1 - fi -} - -rm changelog-prs-filtered.txt &> /dev/null ||: -for pr in $(cat "changelog-prs.txt") -do - # Download PR info from github. - file="pr$pr.json" - github_download "https://api.github.com/repos/ClickHouse/ClickHouse/pulls/$pr" "$file" || continue - - if ! [ "$pr" == "$(jq -r .number "$file")" ] - then - >&2 echo "Got wrong data for PR #$pr (please check and remove '$file')." - continue - fi - - # Filter out PRs by bots. - user_login=$(jq -r .user.login "$file") - - filter_bot=$(echo "$user_login" | grep -q "\[bot\]$" && echo "Skip." || echo "Ok." ||:) - filter_robot=$(echo "$user_login" | grep -q "robot-clickhouse" && echo "Skip." || echo "Ok." ||:) - - if [ "Skip." == "$filter_robot" ] || [ "Skip." == "$filter_bot" ] - then - continue - fi - - # Download author info from github. - user_id=$(jq -r .user.id "$file") - user_file="user$user_id.json" - github_download "$(jq -r .user.url "$file")" "$user_file" || continue - - if ! [ "$user_id" == "$(jq -r .id "$user_file")" ] - then - >&2 echo "Got wrong data for user #$user_id (please check and remove '$user_file')." - continue - fi - - echo "$pr" >> changelog-prs-filtered.txt -done - -echo "### ClickHouse release $to FIXME as compared to $from -" > changelog.md -"$script_dir/format-changelog.py" changelog-prs-filtered.txt >> changelog.md -cat changelog.md diff --git a/utils/changelog-simple/format-changelog.py b/utils/changelog-simple/format-changelog.py deleted file mode 100755 index 01f2694dd0f..00000000000 --- a/utils/changelog-simple/format-changelog.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import collections -import fuzzywuzzy.fuzz -import itertools -import json -import os -import re -import sys - -parser = argparse.ArgumentParser(description="Format changelog for given PRs.") -parser.add_argument( - "file", - metavar="FILE", - type=argparse.FileType("r", encoding="utf-8"), - nargs="?", - default=sys.stdin, - help="File with PR numbers, one per line.", -) -args = parser.parse_args() - - -# This function mirrors the PR description checks in ClickhousePullRequestTrigger. -# Returns False if the PR should not be mentioned changelog. -def parse_one_pull_request(item): - description = item["body"] - # Don't skip empty lines because they delimit parts of description - lines = [ - line - for line in [ - x.strip() for x in (description.split("\n") if description else []) - ] - ] - lines = [re.sub(r"\s+", " ", l) for l in lines] - - category = "" - entry = "" - - if lines: - i = 0 - while i < len(lines): - if re.match(r"(?i).*change\s*log\s*category", lines[i]): - i += 1 - if i >= len(lines): - break - # Can have one empty line between header and the category itself. Filter it out. - if not lines[i]: - i += 1 - if i >= len(lines): - break - category = re.sub(r"^[-*\s]*", "", lines[i]) - i += 1 - - elif re.match(r"(?i).*change\s*log\s*entry", lines[i]): - i += 1 - # Can have one empty line between header and the entry itself. Filter it out. - if i < len(lines) and not lines[i]: - i += 1 - # All following lines until empty one are the changelog entry. - entry_lines = [] - while i < len(lines) and lines[i]: - entry_lines.append(lines[i]) - i += 1 - entry = " ".join(entry_lines) - else: - i += 1 - - if not category: - # Shouldn't happen, because description check in CI should catch such PRs. - # Fall through, so that it shows up in output and the user can fix it. - category = "NO CL CATEGORY" - - # Filter out the PR categories that are not for changelog. - if re.match( - r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", - category, - ): - return False - - if not entry: - # Shouldn't happen, because description check in CI should catch such PRs. - category = "NO CL ENTRY" - entry = "NO CL ENTRY: '" + item["title"] + "'" - - entry = entry.strip() - if entry[-1] != ".": - entry += "." - - item["entry"] = entry - item["category"] = category - - return True - - -# This array gives the preferred category order, and is also used to -# normalize category names. -categories_preferred_order = [ - "Backward Incompatible Change", - "New Feature", - "Performance Improvement", - "Improvement", - "Bug Fix", - "Build/Testing/Packaging Improvement", - "Other", -] - -category_to_pr = collections.defaultdict(lambda: []) -users = {} -for line in args.file: - pr = json.loads(open(f"pr{line.strip()}.json").read()) - assert pr["number"] - if not parse_one_pull_request(pr): - continue - - assert pr["category"] - - # Normalize category name - for c in categories_preferred_order: - if fuzzywuzzy.fuzz.ratio(pr["category"].lower(), c.lower()) >= 90: - pr["category"] = c - break - - category_to_pr[pr["category"]].append(pr) - user_id = pr["user"]["id"] - users[user_id] = json.loads(open(f"user{user_id}.json").read()) - - -def print_category(category): - print(("#### " + category)) - print() - for pr in category_to_pr[category]: - user = users[pr["user"]["id"]] - user_name = user["name"] if user["name"] else user["login"] - - # Substitute issue links. - # 1) issue number w/o markdown link - pr["entry"] = re.sub( - r"([^[])#([0-9]{4,})", - r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", - pr["entry"], - ) - # 2) issue URL w/o markdown link - pr["entry"] = re.sub( - r"([^(])https://github.com/ClickHouse/ClickHouse/issues/([0-9]{4,})", - r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", - pr["entry"], - ) - - print( - f'* {pr["entry"]} [#{pr["number"]}]({pr["html_url"]}) ([{user_name}]({user["html_url"]})).' - ) - - print() - - -# Print categories in preferred order -for category in categories_preferred_order: - if category in category_to_pr: - print_category(category) - category_to_pr.pop(category) - -# Print the rest of the categories -for category in category_to_pr: - print_category(category) From 1963268fecad6fa100b4945172dff1604645d22e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 13:17:00 +0300 Subject: [PATCH 32/49] Update PULL_REQUEST_TEMPLATE.md There is no "prestable" version. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5d09d3a9ef3..449abc9484d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ tests/ci/run_check.py ### Changelog category (leave one): - New Feature - Improvement -- Bug Fix (user-visible misbehavior in official stable or prestable release) +- Bug Fix (user-visible misbehavior in an official stable release) - Performance Improvement - Backward Incompatible Change - Build/Testing/Packaging Improvement From e0252db8d457297ef64d849abd62be7b7efc8307 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 12:19:32 +0200 Subject: [PATCH 33/49] No prestable --- CHANGELOG.md | 2 +- docs/changelogs/v21.10.1.8013-prestable.md | 2 +- docs/changelogs/v21.10.2.15-stable.md | 2 +- docs/changelogs/v21.10.3.9-stable.md | 4 ++-- docs/changelogs/v21.10.4.26-stable.md | 2 +- docs/changelogs/v21.10.5.3-stable.md | 2 +- docs/changelogs/v21.10.6.2-stable.md | 4 ++-- docs/changelogs/v21.11.1.8636-prestable.md | 4 ++-- docs/changelogs/v21.11.11.1-stable.md | 2 +- docs/changelogs/v21.11.2.2-stable.md | 2 +- docs/changelogs/v21.11.3.6-stable.md | 2 +- docs/changelogs/v21.11.4.14-stable.md | 2 +- docs/changelogs/v21.11.5.33-stable.md | 2 +- docs/changelogs/v21.11.6.7-stable.md | 2 +- docs/changelogs/v21.11.7.9-stable.md | 4 ++-- docs/changelogs/v21.11.8.4-stable.md | 2 +- docs/changelogs/v21.11.9.1-stable.md | 2 +- docs/changelogs/v21.12.1.9017-prestable.md | 2 +- docs/changelogs/v21.12.2.17-stable.md | 4 ++-- docs/changelogs/v21.12.3.32-stable.md | 2 +- docs/changelogs/v21.12.4.1-stable.md | 2 +- docs/changelogs/v21.3.16.5-lts.md | 2 +- docs/changelogs/v21.3.17.2-lts.md | 2 +- docs/changelogs/v21.3.18.4-lts.md | 2 +- docs/changelogs/v21.3.19.1-lts.md | 4 ++-- docs/changelogs/v21.3.20.1-lts.md | 2 +- docs/changelogs/v21.6.9.7-stable.md | 2 +- docs/changelogs/v21.7.10.4-stable.md | 2 +- docs/changelogs/v21.7.11.3-stable.md | 2 +- docs/changelogs/v21.7.9.7-stable.md | 2 +- docs/changelogs/v21.8.10.19-lts.md | 2 +- docs/changelogs/v21.8.11.4-lts.md | 4 ++-- docs/changelogs/v21.8.12.29-lts.md | 2 +- docs/changelogs/v21.8.13.6-lts.md | 4 ++-- docs/changelogs/v21.8.14.5-lts.md | 2 +- docs/changelogs/v21.8.15.7-lts.md | 2 +- docs/changelogs/v21.8.5.7-lts.md | 2 +- docs/changelogs/v21.8.6.15-lts.md | 2 +- docs/changelogs/v21.8.7.22-lts.md | 2 +- docs/changelogs/v21.8.8.29-lts.md | 2 +- docs/changelogs/v21.8.9.13-lts.md | 2 +- docs/changelogs/v21.9.2.17-stable.md | 2 +- docs/changelogs/v21.9.3.30-stable.md | 2 +- docs/changelogs/v21.9.4.35-stable.md | 2 +- docs/changelogs/v21.9.5.16-stable.md | 2 +- docs/changelogs/v21.9.6.24-stable.md | 4 ++-- docs/changelogs/v22.1.1.2542-prestable.md | 4 ++-- docs/changelogs/v22.1.3.7-stable.md | 2 +- docs/changelogs/v22.1.4.30-stable.md | 2 +- docs/changelogs/v22.10.1.1877-stable.md | 4 ++-- docs/changelogs/v22.10.2.11-stable.md | 2 +- docs/changelogs/v22.10.3.27-stable.md | 2 +- docs/changelogs/v22.10.4.23-stable.md | 2 +- docs/changelogs/v22.10.5.54-stable.md | 3 +-- docs/changelogs/v22.10.6.3-stable.md | 3 +-- docs/changelogs/v22.10.7.13-stable.md | 3 +-- docs/changelogs/v22.11.1.1360-stable.md | 2 +- docs/changelogs/v22.11.2.30-stable.md | 2 +- docs/changelogs/v22.11.3.47-stable.md | 3 +-- docs/changelogs/v22.11.4.3-stable.md | 3 +-- docs/changelogs/v22.11.5.15-stable.md | 3 +-- docs/changelogs/v22.11.6.44-stable.md | 3 +-- docs/changelogs/v22.12.1.1752-stable.md | 2 +- docs/changelogs/v22.12.2.25-stable.md | 2 +- docs/changelogs/v22.12.3.5-stable.md | 3 +-- docs/changelogs/v22.12.4.76-stable.md | 3 +-- docs/changelogs/v22.12.5.34-stable.md | 3 +-- docs/changelogs/v22.2.1.2139-prestable.md | 2 +- docs/changelogs/v22.2.3.5-stable.md | 2 +- docs/changelogs/v22.3.1.1262-prestable.md | 2 +- docs/changelogs/v22.3.10.22-lts.md | 4 ++-- docs/changelogs/v22.3.11.12-lts.md | 2 +- docs/changelogs/v22.3.12.19-lts.md | 2 +- docs/changelogs/v22.3.13.80-lts.md | 4 ++-- docs/changelogs/v22.3.14.18-lts.md | 2 +- docs/changelogs/v22.3.14.23-lts.md | 2 +- docs/changelogs/v22.3.15.33-lts.md | 2 +- docs/changelogs/v22.3.16.1190-lts.md | 3 +-- docs/changelogs/v22.3.18.37-lts.md | 3 +-- docs/changelogs/v22.3.19.6-lts.md | 3 +-- docs/changelogs/v22.3.3.44-lts.md | 2 +- docs/changelogs/v22.3.4.20-lts.md | 2 +- docs/changelogs/v22.3.5.5-lts.md | 2 +- docs/changelogs/v22.3.6.5-lts.md | 2 +- docs/changelogs/v22.3.7.28-lts.md | 4 ++-- docs/changelogs/v22.3.8.39-lts.md | 4 ++-- docs/changelogs/v22.3.9.19-lts.md | 2 +- docs/changelogs/v22.4.1.2305-prestable.md | 2 +- docs/changelogs/v22.4.3.3-stable.md | 2 +- docs/changelogs/v22.4.4.7-stable.md | 2 +- docs/changelogs/v22.4.5.9-stable.md | 2 +- docs/changelogs/v22.4.6.53-stable.md | 4 ++-- docs/changelogs/v22.5.1.2079-stable.md | 4 ++-- docs/changelogs/v22.5.2.53-stable.md | 2 +- docs/changelogs/v22.5.3.21-stable.md | 2 +- docs/changelogs/v22.5.4.19-stable.md | 2 +- docs/changelogs/v22.6.1.1985-stable.md | 4 ++-- docs/changelogs/v22.6.2.12-stable.md | 2 +- docs/changelogs/v22.6.3.35-stable.md | 2 +- docs/changelogs/v22.6.4.35-stable.md | 2 +- docs/changelogs/v22.6.5.22-stable.md | 2 +- docs/changelogs/v22.6.6.16-stable.md | 2 +- docs/changelogs/v22.6.7.7-stable.md | 2 +- docs/changelogs/v22.6.8.35-stable.md | 2 +- docs/changelogs/v22.6.9.11-stable.md | 2 +- docs/changelogs/v22.7.1.2484-stable.md | 4 ++-- docs/changelogs/v22.7.2.15-stable.md | 2 +- docs/changelogs/v22.7.3.5-stable.md | 2 +- docs/changelogs/v22.7.4.16-stable.md | 2 +- docs/changelogs/v22.7.5.13-stable.md | 2 +- docs/changelogs/v22.7.6.74-stable.md | 2 +- docs/changelogs/v22.7.7.24-stable.md | 2 +- docs/changelogs/v22.8.1.2097-lts.md | 2 +- docs/changelogs/v22.8.10.29-lts.md | 2 +- docs/changelogs/v22.8.11.15-lts.md | 2 +- docs/changelogs/v22.8.12.45-lts.md | 3 +-- docs/changelogs/v22.8.13.20-lts.md | 3 +-- docs/changelogs/v22.8.14.53-lts.md | 3 +-- docs/changelogs/v22.8.15.23-lts.md | 3 +-- docs/changelogs/v22.8.3.13-lts.md | 2 +- docs/changelogs/v22.8.4.7-lts.md | 2 +- docs/changelogs/v22.8.5.29-lts.md | 2 +- docs/changelogs/v22.8.6.71-lts.md | 2 +- docs/changelogs/v22.8.7.34-lts.md | 2 +- docs/changelogs/v22.8.8.3-lts.md | 2 +- docs/changelogs/v22.8.9.24-lts.md | 2 +- docs/changelogs/v22.9.2.7-stable.md | 2 +- docs/changelogs/v22.9.3.18-stable.md | 2 +- docs/changelogs/v22.9.4.32-stable.md | 2 +- docs/changelogs/v22.9.5.25-stable.md | 2 +- docs/changelogs/v22.9.6.20-stable.md | 2 +- docs/changelogs/v22.9.7.34-stable.md | 2 +- docs/changelogs/v23.1.1.3077-stable.md | 3 +-- docs/changelogs/v23.1.2.9-stable.md | 3 +-- docs/changelogs/v23.1.3.5-stable.md | 3 +-- docs/changelogs/v23.1.4.58-stable.md | 3 +-- docs/changelogs/v23.1.5.24-stable.md | 3 +-- docs/changelogs/v23.2.1.2537-stable.md | 3 +-- docs/changelogs/v23.2.2.20-stable.md | 3 +-- docs/changelogs/v23.2.3.17-stable.md | 3 +-- docs/changelogs/v23.2.4.12-stable.md | 3 +-- tests/ci/run_check.py | 4 ++-- utils/changelog/changelog.py | 8 ++++++++ 143 files changed, 170 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e22377e2332..01bbafe2f16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,7 +140,7 @@ * Upgrade Intel QPL from v0.3.0 to v1.0.0 2. Build libaccel-config and link it statically to QPL library instead of dynamically. [#45809](https://github.com/ClickHouse/ClickHouse/pull/45809) ([jasperzhu](https://github.com/jinjunzh)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Flush data exactly by `rabbitmq_flush_interval_ms` or by `rabbitmq_max_block_size` in `StorageRabbitMQ`. Closes [#42389](https://github.com/ClickHouse/ClickHouse/issues/42389). Closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#44404](https://github.com/ClickHouse/ClickHouse/pull/44404) ([Kseniia Sumarokova](https://github.com/kssenii)). * Use PODArray to render in sparkBar function, so we can control the memory usage. Close [#44467](https://github.com/ClickHouse/ClickHouse/issues/44467). [#44489](https://github.com/ClickHouse/ClickHouse/pull/44489) ([Duc Canh Le](https://github.com/canhld94)). diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md index 02ea593e02a..1c40aa67711 100644 --- a/docs/changelogs/v21.10.1.8013-prestable.md +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -108,7 +108,7 @@ sidebar_label: 2022 * Print out git status information at CMake configure stage. [#28047](https://github.com/ClickHouse/ClickHouse/pull/28047) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). * Add new log level `` for testing environments. [#28559](https://github.com/ClickHouse/ClickHouse/pull/28559) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md index 4ae5c8f5072..42402808260 100644 --- a/docs/changelogs/v21.10.2.15-stable.md +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#29970](https://github.com/ClickHouse/ClickHouse/issues/29970): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#30051](https://github.com/ClickHouse/ClickHouse/issues/30051): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29054](https://github.com/ClickHouse/ClickHouse/issues/29054): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28795](https://github.com/ClickHouse/ClickHouse/issues/28795): - Fix the number of arguments required by s2RectAdd and s2RectContains functions. [#28663](https://github.com/ClickHouse/ClickHouse/pull/28663) ([Bharat Nallan](https://github.com/bharatnc)). diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md index d0384d58e23..327e34ca64c 100644 --- a/docs/changelogs/v21.10.3.9-stable.md +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -18,11 +18,11 @@ sidebar_label: 2022 * Backported in [#30620](https://github.com/ClickHouse/ClickHouse/issues/30620): Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#31369](https://github.com/ClickHouse/ClickHouse/issues/31369): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30915](https://github.com/ClickHouse/ClickHouse/issues/30915): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30824](https://github.com/ClickHouse/ClickHouse/issues/30824): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30766](https://github.com/ClickHouse/ClickHouse/issues/30766): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md index 7d1cc93bb98..267f2109f6f 100644 --- a/docs/changelogs/v21.10.4.26-stable.md +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31573](https://github.com/ClickHouse/ClickHouse/issues/31573): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31518](https://github.com/ClickHouse/ClickHouse/issues/31518): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). * Backported in [#31554](https://github.com/ClickHouse/ClickHouse/issues/31554): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md index 88d3d70028e..7c717dfe838 100644 --- a/docs/changelogs/v21.10.5.3-stable.md +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32252](https://github.com/ClickHouse/ClickHouse/issues/32252): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#32346](https://github.com/ClickHouse/ClickHouse/issues/32346): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32151](https://github.com/ClickHouse/ClickHouse/issues/32151): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#32093](https://github.com/ClickHouse/ClickHouse/issues/32093): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md index 74f037b2f8f..0e8e934e2fa 100644 --- a/docs/changelogs/v21.10.6.2-stable.md +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32692](https://github.com/ClickHouse/ClickHouse/issues/32692): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32680](https://github.com/ClickHouse/ClickHouse/issues/32680): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32285](https://github.com/ClickHouse/ClickHouse/issues/32285): Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)). @@ -23,7 +23,7 @@ sidebar_label: 2022 * Backported in [#33182](https://github.com/ClickHouse/ClickHouse/issues/33182): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#33655](https://github.com/ClickHouse/ClickHouse/issues/33655): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md index 407a5196c1d..d6a435dd3ce 100644 --- a/docs/changelogs/v21.11.1.8636-prestable.md +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -124,11 +124,11 @@ sidebar_label: 2022 * Recursive submodules are no longer needed for ClickHouse. [#30315](https://github.com/ClickHouse/ClickHouse/pull/30315) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Added docker image to build docs. [#30499](https://github.com/ClickHouse/ClickHouse/pull/30499) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)). * Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md index 85a8975c6e7..76cd5239cba 100644 --- a/docs/changelogs/v21.11.11.1-stable.md +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33656](https://github.com/ClickHouse/ClickHouse/issues/33656): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md index bf02de235e4..44938addad5 100644 --- a/docs/changelogs/v21.11.2.2-stable.md +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31154](https://github.com/ClickHouse/ClickHouse/issues/31154): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). * Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md index e3886c4efac..f9cc64e2c2b 100644 --- a/docs/changelogs/v21.11.3.6-stable.md +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31246](https://github.com/ClickHouse/ClickHouse/issues/31246): Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31206](https://github.com/ClickHouse/ClickHouse/issues/31206): Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31202](https://github.com/ClickHouse/ClickHouse/issues/31202): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md index f05f43d9de1..8882832d1fe 100644 --- a/docs/changelogs/v21.11.4.14-stable.md +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31370](https://github.com/ClickHouse/ClickHouse/issues/31370): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31282](https://github.com/ClickHouse/ClickHouse/issues/31282): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#31237](https://github.com/ClickHouse/ClickHouse/issues/31237): Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md index 3780d5a2530..11e7f24dbb1 100644 --- a/docs/changelogs/v21.11.5.33-stable.md +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31572](https://github.com/ClickHouse/ClickHouse/issues/31572): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31517](https://github.com/ClickHouse/ClickHouse/issues/31517): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)). * Backported in [#31551](https://github.com/ClickHouse/ClickHouse/issues/31551): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)). diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md index 1f1935d1865..cddd472076a 100644 --- a/docs/changelogs/v21.11.6.7-stable.md +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32254](https://github.com/ClickHouse/ClickHouse/issues/32254): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#32345](https://github.com/ClickHouse/ClickHouse/issues/32345): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32152](https://github.com/ClickHouse/ClickHouse/issues/32152): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#32298](https://github.com/ClickHouse/ClickHouse/issues/32298): Fix recursive user defined functions crash. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md index baa6b0290a5..596d16a28ef 100644 --- a/docs/changelogs/v21.11.7.9-stable.md +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -10,13 +10,13 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32691](https://github.com/ClickHouse/ClickHouse/issues/32691): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32711](https://github.com/ClickHouse/ClickHouse/issues/32711): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#32568](https://github.com/ClickHouse/ClickHouse/issues/32568): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)). * Backported in [#32732](https://github.com/ClickHouse/ClickHouse/issues/32732): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md index bd71374e870..28d413dd2c5 100644 --- a/docs/changelogs/v21.11.8.4-stable.md +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32679](https://github.com/ClickHouse/ClickHouse/issues/32679): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32543](https://github.com/ClickHouse/ClickHouse/issues/32543): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md index 1473a4a152c..ea36479c943 100644 --- a/docs/changelogs/v21.11.9.1-stable.md +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md index e8f2ca283a4..88b8260e312 100644 --- a/docs/changelogs/v21.12.1.9017-prestable.md +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -132,7 +132,7 @@ sidebar_label: 2022 * Build rpm and tgz packages in master and release branches workfolw. [#32048](https://github.com/ClickHouse/ClickHouse/pull/32048) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix broken symlink for sysroot/linux-riscv64/usr/lib. [#32071](https://github.com/ClickHouse/ClickHouse/pull/32071) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)). * Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)). diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md index 94cfc1b88a8..67761ce0e08 100644 --- a/docs/changelogs/v21.12.2.17-stable.md +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#32693](https://github.com/ClickHouse/ClickHouse/issues/32693): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32681](https://github.com/ClickHouse/ClickHouse/issues/32681): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). * Backported in [#32483](https://github.com/ClickHouse/ClickHouse/issues/32483): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). @@ -23,7 +23,7 @@ sidebar_label: 2022 * Backported in [#32733](https://github.com/ClickHouse/ClickHouse/issues/32733): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#32793](https://github.com/ClickHouse/ClickHouse/issues/32793): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md index ea11efa46c5..c8c423a77b9 100644 --- a/docs/changelogs/v21.12.3.32-stable.md +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - ClickHouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32890](https://github.com/ClickHouse/ClickHouse/issues/32890): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#33183](https://github.com/ClickHouse/ClickHouse/issues/33183): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md index bd38dbd0c59..3345f76b317 100644 --- a/docs/changelogs/v21.12.4.1-stable.md +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#33551](https://github.com/ClickHouse/ClickHouse/issues/33551): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33537](https://github.com/ClickHouse/ClickHouse/issues/33537): Fix ORC stripe reading. [#32929](https://github.com/ClickHouse/ClickHouse/pull/32929) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). * Backported in [#33654](https://github.com/ClickHouse/ClickHouse/issues/33654): Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md index 123b27097d4..6aedeff5acb 100644 --- a/docs/changelogs/v21.3.16.5-lts.md +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28075](https://github.com/ClickHouse/ClickHouse/issues/28075): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). * Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md index 6c288b5a0d8..9104ae7aa0a 100644 --- a/docs/changelogs/v21.3.17.2-lts.md +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#28647](https://github.com/ClickHouse/ClickHouse/issues/28647): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28569](https://github.com/ClickHouse/ClickHouse/issues/28569): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). * Backported in [#28857](https://github.com/ClickHouse/ClickHouse/issues/28857): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md index d14dddfb1a5..33f4b86d81c 100644 --- a/docs/changelogs/v21.3.18.4-lts.md +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#30041](https://github.com/ClickHouse/ClickHouse/issues/30041): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29260](https://github.com/ClickHouse/ClickHouse/issues/29260): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29026](https://github.com/ClickHouse/ClickHouse/issues/29026): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md index 4d4404077a5..26c36725610 100644 --- a/docs/changelogs/v21.3.19.1-lts.md +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -14,11 +14,11 @@ sidebar_label: 2022 * Backported in [#31577](https://github.com/ClickHouse/ClickHouse/issues/31577): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). * Backported in [#32347](https://github.com/ClickHouse/ClickHouse/issues/32347): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30913](https://github.com/ClickHouse/ClickHouse/issues/30913): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30750](https://github.com/ClickHouse/ClickHouse/issues/30750): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#31038](https://github.com/ClickHouse/ClickHouse/issues/31038): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md index f9ce3cba78b..1b235556faf 100644 --- a/docs/changelogs/v21.3.20.1-lts.md +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32690](https://github.com/ClickHouse/ClickHouse/issues/32690): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). * Backported in [#33727](https://github.com/ClickHouse/ClickHouse/issues/33727): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md index ca1edeb1722..0a989e4d6b7 100644 --- a/docs/changelogs/v21.6.9.7-stable.md +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -40,7 +40,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28030](https://github.com/ClickHouse/ClickHouse/issues/28030): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28119](https://github.com/ClickHouse/ClickHouse/issues/28119): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#28179](https://github.com/ClickHouse/ClickHouse/issues/28179): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md index daa063ebf5a..238dfd651eb 100644 --- a/docs/changelogs/v21.7.10.4-stable.md +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#27925](https://github.com/ClickHouse/ClickHouse/issues/27925): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28752](https://github.com/ClickHouse/ClickHouse/issues/28752): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28509](https://github.com/ClickHouse/ClickHouse/issues/28509): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#28570](https://github.com/ClickHouse/ClickHouse/issues/28570): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md index b3d1c9a44fd..8ccc31657de 100644 --- a/docs/changelogs/v21.7.11.3-stable.md +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md index ac985f7af37..7aaab54af6b 100644 --- a/docs/changelogs/v21.7.9.7-stable.md +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -23,7 +23,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28032](https://github.com/ClickHouse/ClickHouse/issues/28032): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28116](https://github.com/ClickHouse/ClickHouse/issues/28116): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#28183](https://github.com/ClickHouse/ClickHouse/issues/28183): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md index 56c682ddbd2..5873f2b2ff9 100644 --- a/docs/changelogs/v21.8.10.19-lts.md +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#30452](https://github.com/ClickHouse/ClickHouse/issues/30452): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29724](https://github.com/ClickHouse/ClickHouse/issues/29724): Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30233](https://github.com/ClickHouse/ClickHouse/issues/30233): Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md index d88d191bae2..81bbea4ee21 100644 --- a/docs/changelogs/v21.8.11.4-lts.md +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -16,11 +16,11 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31368](https://github.com/ClickHouse/ClickHouse/issues/31368): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30914](https://github.com/ClickHouse/ClickHouse/issues/30914): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28756](https://github.com/ClickHouse/ClickHouse/issues/28756): Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)). * Backported in [#30825](https://github.com/ClickHouse/ClickHouse/issues/30825): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md index bd1f0c7fe60..8b68a6a3af8 100644 --- a/docs/changelogs/v21.8.12.29-lts.md +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#31575](https://github.com/ClickHouse/ClickHouse/issues/31575): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#31204](https://github.com/ClickHouse/ClickHouse/issues/31204): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31253](https://github.com/ClickHouse/ClickHouse/issues/31253): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md index 63ac956c3d5..205628c6330 100644 --- a/docs/changelogs/v21.8.13.6-lts.md +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#32688](https://github.com/ClickHouse/ClickHouse/issues/32688): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). * Backported in [#32343](https://github.com/ClickHouse/ClickHouse/issues/32343): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#32108](https://github.com/ClickHouse/ClickHouse/issues/32108): Fix crash with empty result on odbc query. Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#32150](https://github.com/ClickHouse/ClickHouse/issues/32150): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)). @@ -26,7 +26,7 @@ sidebar_label: 2022 * Backported in [#33048](https://github.com/ClickHouse/ClickHouse/issues/33048): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#32795](https://github.com/ClickHouse/ClickHouse/issues/32795): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Backported in [#32659](https://github.com/ClickHouse/ClickHouse/issues/32659): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md index 1012d9c5784..75d966ec9e7 100644 --- a/docs/changelogs/v21.8.14.5-lts.md +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33184](https://github.com/ClickHouse/ClickHouse/issues/33184): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#33659](https://github.com/ClickHouse/ClickHouse/issues/33659): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md index eb6bf39d7be..1522c28016a 100644 --- a/docs/changelogs/v21.8.15.7-lts.md +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34121](https://github.com/ClickHouse/ClickHouse/issues/34121): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#34097](https://github.com/ClickHouse/ClickHouse/issues/34097): Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md index 00c6c6e46a7..fa459e093f7 100644 --- a/docs/changelogs/v21.8.5.7-lts.md +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28031](https://github.com/ClickHouse/ClickHouse/issues/28031): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#27974](https://github.com/ClickHouse/ClickHouse/issues/27974): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28117](https://github.com/ClickHouse/ClickHouse/issues/28117): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md index dc8ae4ec9b5..aa51dc1ac1e 100644 --- a/docs/changelogs/v21.8.6.15-lts.md +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2022 * Backported in [#27923](https://github.com/ClickHouse/ClickHouse/issues/27923): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28753](https://github.com/ClickHouse/ClickHouse/issues/28753): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28644](https://github.com/ClickHouse/ClickHouse/issues/28644): Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)). * Backported in [#28508](https://github.com/ClickHouse/ClickHouse/issues/28508): Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md index b6c5b70e096..7a751be4132 100644 --- a/docs/changelogs/v21.8.7.22-lts.md +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29121](https://github.com/ClickHouse/ClickHouse/issues/29121): Better check for connection usability and also catch any exception in RabbitMQ shutdown just in case. [#28797](https://github.com/ClickHouse/ClickHouse/pull/28797) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md index 160d30a6aa9..e988c3c6801 100644 --- a/docs/changelogs/v21.8.8.29-lts.md +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#29128](https://github.com/ClickHouse/ClickHouse/issues/29128): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29262](https://github.com/ClickHouse/ClickHouse/issues/29262): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29106](https://github.com/ClickHouse/ClickHouse/issues/29106): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md index a48ca30080e..71919c48c47 100644 --- a/docs/changelogs/v21.8.9.13-lts.md +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#29817](https://github.com/ClickHouse/ClickHouse/issues/29817): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#29973](https://github.com/ClickHouse/ClickHouse/issues/29973): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29676](https://github.com/ClickHouse/ClickHouse/issues/29676): Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29538](https://github.com/ClickHouse/ClickHouse/issues/29538): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md index 39e3f627f4a..08d208ec97b 100644 --- a/docs/changelogs/v21.9.2.17-stable.md +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -25,7 +25,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#28029](https://github.com/ClickHouse/ClickHouse/issues/28029): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#27973](https://github.com/ClickHouse/ClickHouse/issues/27973): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28118](https://github.com/ClickHouse/ClickHouse/issues/28118): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md index ee2dd24277d..28375c5588e 100644 --- a/docs/changelogs/v21.9.3.30-stable.md +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#28897](https://github.com/ClickHouse/ClickHouse/issues/28897): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#28815](https://github.com/ClickHouse/ClickHouse/issues/28815): Fix possible crash for `SELECT` with partially created aggregate projection in case of exception. [#28700](https://github.com/ClickHouse/ClickHouse/pull/28700) ([Amos Bird](https://github.com/amosbird)). * Backported in [#28789](https://github.com/ClickHouse/ClickHouse/issues/28789): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md index 5a556df1b3a..0b300574559 100644 --- a/docs/changelogs/v21.9.4.35-stable.md +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md index e681322eb45..895e882d257 100644 --- a/docs/changelogs/v21.9.5.16-stable.md +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#29972](https://github.com/ClickHouse/ClickHouse/issues/29972): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#30052](https://github.com/ClickHouse/ClickHouse/issues/30052): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#29055](https://github.com/ClickHouse/ClickHouse/issues/29055): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)). * Backported in [#29107](https://github.com/ClickHouse/ClickHouse/issues/29107): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md index 2407c031873..890dc8d5d02 100644 --- a/docs/changelogs/v21.9.6.24-stable.md +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -21,11 +21,11 @@ sidebar_label: 2022 * Backported in [#31371](https://github.com/ClickHouse/ClickHouse/issues/31371): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#31576](https://github.com/ClickHouse/ClickHouse/issues/31576): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release +#### Bug Fix (user-visible misbehaviour in official stable release * Backported in [#30916](https://github.com/ClickHouse/ClickHouse/issues/30916): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#30823](https://github.com/ClickHouse/ClickHouse/issues/30823): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#30609](https://github.com/ClickHouse/ClickHouse/issues/30609): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index 8d7bb015db6..cacd13c1e12 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -118,7 +118,7 @@ sidebar_label: 2022 * Remove editing /etc/hosts from Dockerfile. [#33635](https://github.com/ClickHouse/ClickHouse/pull/33635) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Properly separate thrift-cmake from arrow-cmake after https://github.com/ClickHouse/ClickHouse/pull/31104 . cc @taiyang-li. [#33661](https://github.com/ClickHouse/ClickHouse/pull/33661) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fixed CAST from String to IPv4 or IPv6 and back. Fixed error message in case of failed conversion. [#29224](https://github.com/ClickHouse/ClickHouse/pull/29224) ([Dmitry Novik](https://github.com/novikd)). * Fix base64Encode adding trailing bytes on small strings. [#31797](https://github.com/ClickHouse/ClickHouse/pull/31797) ([Kevin Michel](https://github.com/kmichel-aiven)). @@ -185,7 +185,7 @@ sidebar_label: 2022 * Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). * Fix wrong database for JOIN w/o explicit database in distributed queries (Fixes: [#10471](https://github.com/ClickHouse/ClickHouse/issues/10471)). [#33611](https://github.com/ClickHouse/ClickHouse/pull/33611) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release): +#### Bug Fix (user-visible misbehaviour in official stable release): * Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index a92a82be290..fd8787f0e75 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -10,6 +10,6 @@ sidebar_label: 2022 #### Improvement * Backported in [#33793](https://github.com/ClickHouse/ClickHouse/issues/33793): Create parent directories in DiskS3::restoreFileOperations method. [#33730](https://github.com/ClickHouse/ClickHouse/pull/33730) ([ianton-ru](https://github.com/ianton-ru)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#33898](https://github.com/ClickHouse/ClickHouse/issues/33898): Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.1.4.30-stable.md b/docs/changelogs/v22.1.4.30-stable.md index c4286d7b64d..b0437382a46 100644 --- a/docs/changelogs/v22.1.4.30-stable.md +++ b/docs/changelogs/v22.1.4.30-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backport CI checks to 22.1 release branch. [#34897](https://github.com/ClickHouse/ClickHouse/pull/34897) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34119](https://github.com/ClickHouse/ClickHouse/issues/34119): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#34124](https://github.com/ClickHouse/ClickHouse/issues/34124): Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.10.1.1877-stable.md b/docs/changelogs/v22.10.1.1877-stable.md index 5b573a3faa4..23c58447c99 100644 --- a/docs/changelogs/v22.10.1.1877-stable.md +++ b/docs/changelogs/v22.10.1.1877-stable.md @@ -105,7 +105,7 @@ sidebar_label: 2022 * Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix power8 support. [#42462](https://github.com/ClickHouse/ClickHouse/pull/42462) ([Boris Kuschel](https://github.com/bkuschel)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Several fixes for DiskWeb. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -147,7 +147,7 @@ sidebar_label: 2022 * This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix function `arrayElement` with type `Map` with `Nullable` values and `Nullable` index. [#42623](https://github.com/ClickHouse/ClickHouse/pull/42623) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v22.10.2.11-stable.md b/docs/changelogs/v22.10.2.11-stable.md index 4e3c382f5a7..196d3fbde80 100644 --- a/docs/changelogs/v22.10.2.11-stable.md +++ b/docs/changelogs/v22.10.2.11-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.10.2.11-stable (d2bfcaba002) FIXME as compared to v22.10.1.1877-stable (98ab5a3c189) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42750](https://github.com/ClickHouse/ClickHouse/issues/42750): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#42793](https://github.com/ClickHouse/ClickHouse/issues/42793): Fix a bug in ParserFunction that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)). diff --git a/docs/changelogs/v22.10.3.27-stable.md b/docs/changelogs/v22.10.3.27-stable.md index 6dc9fd7f3b9..6e0188ad619 100644 --- a/docs/changelogs/v22.10.3.27-stable.md +++ b/docs/changelogs/v22.10.3.27-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42959](https://github.com/ClickHouse/ClickHouse/issues/42959): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43042](https://github.com/ClickHouse/ClickHouse/issues/43042): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42864](https://github.com/ClickHouse/ClickHouse/issues/42864): Fix lowerUTF8()/upperUTF8() in case of symbol was in between 16-byte boundary (very frequent case of you have strings > 16 bytes long). [#42812](https://github.com/ClickHouse/ClickHouse/pull/42812) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#43173](https://github.com/ClickHouse/ClickHouse/issues/43173): Fix rare possible hung on query cancellation. [#42874](https://github.com/ClickHouse/ClickHouse/pull/42874) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.10.4.23-stable.md b/docs/changelogs/v22.10.4.23-stable.md index 04eb8be982f..4438a3470fc 100644 --- a/docs/changelogs/v22.10.4.23-stable.md +++ b/docs/changelogs/v22.10.4.23-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43053](https://github.com/ClickHouse/ClickHouse/issues/43053): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43715](https://github.com/ClickHouse/ClickHouse/issues/43715): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43576](https://github.com/ClickHouse/ClickHouse/issues/43576): Fix possible `Cannot create non-empty column with type Nothing` in functions if/multiIf. Closes [#43356](https://github.com/ClickHouse/ClickHouse/issues/43356). [#43368](https://github.com/ClickHouse/ClickHouse/pull/43368) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.10.5.54-stable.md b/docs/changelogs/v22.10.5.54-stable.md index e372fb30618..73c15cf985d 100644 --- a/docs/changelogs/v22.10.5.54-stable.md +++ b/docs/changelogs/v22.10.5.54-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#44379](https://github.com/ClickHouse/ClickHouse/issues/44379): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44559](https://github.com/ClickHouse/ClickHouse/issues/44559): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44754](https://github.com/ClickHouse/ClickHouse/issues/44754): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43527](https://github.com/ClickHouse/ClickHouse/issues/43527): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -41,4 +41,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.10.6.3-stable.md b/docs/changelogs/v22.10.6.3-stable.md index b0e88c92cb0..c09e9a8670b 100644 --- a/docs/changelogs/v22.10.6.3-stable.md +++ b/docs/changelogs/v22.10.6.3-stable.md @@ -7,7 +7,6 @@ sidebar_label: 2023 ### ClickHouse release v22.10.6.3-stable (645a66d221f) FIXME as compared to v22.10.5.54-stable (dbc7984dc3b) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45084](https://github.com/ClickHouse/ClickHouse/issues/45084): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.10.7.13-stable.md b/docs/changelogs/v22.10.7.13-stable.md index c906e00e524..987bf6cf1e1 100644 --- a/docs/changelogs/v22.10.7.13-stable.md +++ b/docs/changelogs/v22.10.7.13-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.10.7.13-stable (d261d9036cc) FIXME as compared to v22.10.6.3-stable (645a66d221f) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44998](https://github.com/ClickHouse/ClickHouse/issues/44998): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45551](https://github.com/ClickHouse/ClickHouse/issues/45551): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -18,4 +18,3 @@ sidebar_label: 2023 * Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.1.1360-stable.md b/docs/changelogs/v22.11.1.1360-stable.md index 1da53be02b7..4aa110484f8 100644 --- a/docs/changelogs/v22.11.1.1360-stable.md +++ b/docs/changelogs/v22.11.1.1360-stable.md @@ -81,7 +81,7 @@ sidebar_label: 2022 * Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix schema inference in s3Cluster and improve in hdfsCluster. [#41979](https://github.com/ClickHouse/ClickHouse/pull/41979) ([Kruglov Pavel](https://github.com/Avogar)). * Fix retries while reading from http table engines / table function. (retrtiable errors could be retries more times than needed, non-retrialble errors resulted in failed assertion in code). [#42224](https://github.com/ClickHouse/ClickHouse/pull/42224) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.11.2.30-stable.md b/docs/changelogs/v22.11.2.30-stable.md index 7b2febe072a..4759aa4a503 100644 --- a/docs/changelogs/v22.11.2.30-stable.md +++ b/docs/changelogs/v22.11.2.30-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#43511](https://github.com/ClickHouse/ClickHouse/issues/43511): Restrict default access to named collections for user defined in config. It must have explicit `show_named_collections=1` to be able to see them. [#43325](https://github.com/ClickHouse/ClickHouse/pull/43325) ([Kseniia Sumarokova](https://github.com/kssenii)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43716](https://github.com/ClickHouse/ClickHouse/issues/43716): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43431](https://github.com/ClickHouse/ClickHouse/issues/43431): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.11.3.47-stable.md b/docs/changelogs/v22.11.3.47-stable.md index d6451b853f7..a993ff8516f 100644 --- a/docs/changelogs/v22.11.3.47-stable.md +++ b/docs/changelogs/v22.11.3.47-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2023 * Backported in [#44380](https://github.com/ClickHouse/ClickHouse/issues/44380): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44560](https://github.com/ClickHouse/ClickHouse/issues/44560): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44756](https://github.com/ClickHouse/ClickHouse/issues/44756): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43528](https://github.com/ClickHouse/ClickHouse/issues/43528): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -37,4 +37,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.11.4.3-stable.md b/docs/changelogs/v22.11.4.3-stable.md index 33780e848ef..b0e7586277a 100644 --- a/docs/changelogs/v22.11.4.3-stable.md +++ b/docs/changelogs/v22.11.4.3-stable.md @@ -7,7 +7,6 @@ sidebar_label: 2023 ### ClickHouse release v22.11.4.3-stable (7f4cf554f69) FIXME as compared to v22.11.3.47-stable (1c49d124a37) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45085](https://github.com/ClickHouse/ClickHouse/issues/45085): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.11.5.15-stable.md b/docs/changelogs/v22.11.5.15-stable.md index 742a8740514..f11a01c4b1c 100644 --- a/docs/changelogs/v22.11.5.15-stable.md +++ b/docs/changelogs/v22.11.5.15-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.11.5.15-stable (d763e5a9239) FIXME as compared to v22.11.4.3-stable (7f4cf554f69) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44999](https://github.com/ClickHouse/ClickHouse/issues/44999): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45552](https://github.com/ClickHouse/ClickHouse/issues/45552): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -19,4 +19,3 @@ sidebar_label: 2023 * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.6.44-stable.md b/docs/changelogs/v22.11.6.44-stable.md index 6e628b85150..db19e73c666 100644 --- a/docs/changelogs/v22.11.6.44-stable.md +++ b/docs/changelogs/v22.11.6.44-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#46483](https://github.com/ClickHouse/ClickHouse/issues/46483): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46507](https://github.com/ClickHouse/ClickHouse/issues/46507): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45903](https://github.com/ClickHouse/ClickHouse/issues/45903): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#46239](https://github.com/ClickHouse/ClickHouse/issues/46239): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). @@ -34,4 +34,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.1.1752-stable.md b/docs/changelogs/v22.12.1.1752-stable.md index 4f4c4b11150..1549af037f2 100644 --- a/docs/changelogs/v22.12.1.1752-stable.md +++ b/docs/changelogs/v22.12.1.1752-stable.md @@ -91,7 +91,7 @@ sidebar_label: 2022 * Bring sha512 sums back to the building step. [#44017](https://github.com/ClickHouse/ClickHouse/pull/44017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Kill stress tests after 2.5h in case of hanging process. [#44214](https://github.com/ClickHouse/ClickHouse/pull/44214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fixed unable to log in (because of failure to create session_log entry) in rare case of messed up setting profiles. ... [#42641](https://github.com/ClickHouse/ClickHouse/pull/42641) ([Vasily Nemkov](https://github.com/Enmk)). * Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.12.2.25-stable.md b/docs/changelogs/v22.12.2.25-stable.md index 194d0fe3cbc..968854d0428 100644 --- a/docs/changelogs/v22.12.2.25-stable.md +++ b/docs/changelogs/v22.12.2.25-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2023 * Backported in [#44381](https://github.com/ClickHouse/ClickHouse/issues/44381): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44561](https://github.com/ClickHouse/ClickHouse/issues/44561): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44739](https://github.com/ClickHouse/ClickHouse/issues/44739): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#44764](https://github.com/ClickHouse/ClickHouse/issues/44764): Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.12.3.5-stable.md b/docs/changelogs/v22.12.3.5-stable.md index 8cbcbc6a590..5f0cc9cebf3 100644 --- a/docs/changelogs/v22.12.3.5-stable.md +++ b/docs/changelogs/v22.12.3.5-stable.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v22.12.3.5-stable (893de538f02) FIXME as compared to v22.12.2.25-stable (c790cfd4465) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45086](https://github.com/ClickHouse/ClickHouse/issues/45086): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.12.4.76-stable.md b/docs/changelogs/v22.12.4.76-stable.md index 79569ff841e..cdadaae7f7b 100644 --- a/docs/changelogs/v22.12.4.76-stable.md +++ b/docs/changelogs/v22.12.4.76-stable.md @@ -22,7 +22,7 @@ sidebar_label: 2023 * Backported in [#46509](https://github.com/ClickHouse/ClickHouse/issues/46509): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#47058](https://github.com/ClickHouse/ClickHouse/issues/47058): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45904](https://github.com/ClickHouse/ClickHouse/issues/45904): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#45321](https://github.com/ClickHouse/ClickHouse/issues/45321): Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)). @@ -52,4 +52,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.5.34-stable.md b/docs/changelogs/v22.12.5.34-stable.md index 95befaa88ff..61f099462a0 100644 --- a/docs/changelogs/v22.12.5.34-stable.md +++ b/docs/changelogs/v22.12.5.34-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2023 #### Improvement * Backported in [#46983](https://github.com/ClickHouse/ClickHouse/issues/46983): - Apply `ALTER TABLE table_name ON CLUSTER cluster MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name'` to all replicas. Because `ALTER TABLE t MOVE` is not replicated. [#46402](https://github.com/ClickHouse/ClickHouse/pull/46402) ([lizhuoyu5](https://github.com/lzydmxy)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45729](https://github.com/ClickHouse/ClickHouse/issues/45729): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). * Backported in [#46398](https://github.com/ClickHouse/ClickHouse/issues/46398): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -26,4 +26,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index 67db2d9a18d..ca3a84ceaa2 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -141,7 +141,7 @@ sidebar_label: 2022 * - Rework version_helper, make it executable - Reimplement StorageSystemContributors.sh in version_helper - Create a release script. [#34641](https://github.com/ClickHouse/ClickHouse/pull/34641) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * - Fix path in workflows/release.yml - To be backported to branch 22.1. [#34646](https://github.com/ClickHouse/ClickHouse/pull/34646) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix lz4 compression for output. Closes [#31421](https://github.com/ClickHouse/ClickHouse/issues/31421). [#31862](https://github.com/ClickHouse/ClickHouse/pull/31862) ([Kruglov Pavel](https://github.com/Avogar)). * Create a function escapeForLDAPFilter and use it to escape characters '(' and ')' in a final_user_dn variable. [#33401](https://github.com/ClickHouse/ClickHouse/pull/33401) ([IlyaTsoi](https://github.com/IlyaTsoi)). diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index c433669049d..a4368e465aa 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.2.3.5-stable FIXME as compared to v22.2.2.1-stable -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#34848](https://github.com/ClickHouse/ClickHouse/issues/34848): Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index e6203ff18d1..385393cef17 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -95,7 +95,7 @@ sidebar_label: 2022 * Clion has the following problems "The breakpoint will not currently be hit. No executable code is associated with this line". [#35179](https://github.com/ClickHouse/ClickHouse/pull/35179) ([小路](https://github.com/nicelulu)). * Add an ability to build stripped binaries with cmake. [#35196](https://github.com/ClickHouse/ClickHouse/pull/35196) ([alesapin](https://github.com/alesapin)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix distributed subquery max_query_size limitation inconsistency. [#34078](https://github.com/ClickHouse/ClickHouse/pull/34078) ([Chao Ma](https://github.com/godliness)). * Fix incorrect trivial count result when part movement feature is used [#34089](https://github.com/ClickHouse/ClickHouse/issues/34089). [#34385](https://github.com/ClickHouse/ClickHouse/pull/34385) ([nvartolomei](https://github.com/nvartolomei)). diff --git a/docs/changelogs/v22.3.10.22-lts.md b/docs/changelogs/v22.3.10.22-lts.md index a43b8301aad..cc033eb707d 100644 --- a/docs/changelogs/v22.3.10.22-lts.md +++ b/docs/changelogs/v22.3.10.22-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#39761](https://github.com/ClickHouse/ClickHouse/issues/39761): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39206](https://github.com/ClickHouse/ClickHouse/issues/39206): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#39381](https://github.com/ClickHouse/ClickHouse/issues/39381): Fixed error `Not found column Type in block` in selects with `PREWHERE` and read-in-order optimizations. [#39157](https://github.com/ClickHouse/ClickHouse/pull/39157) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#39610](https://github.com/ClickHouse/ClickHouse/issues/39610): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)). * Backported in [#39834](https://github.com/ClickHouse/ClickHouse/issues/39834): Fix `CANNOT_READ_ALL_DATA` exception with `local_filesystem_read_method=pread_threadpool`. This bug affected only Linux kernel version 5.9 and 5.10 according to [man](https://manpages.debian.org/testing/manpages-dev/preadv2.2.en.html#BUGS). [#39800](https://github.com/ClickHouse/ClickHouse/pull/39800) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#39238](https://github.com/ClickHouse/ClickHouse/issues/39238): Fix performance regression of scalar query optimization. [#35986](https://github.com/ClickHouse/ClickHouse/pull/35986) ([Amos Bird](https://github.com/amosbird)). * Backported in [#39531](https://github.com/ClickHouse/ClickHouse/issues/39531): Fix some issues with async reads from remote filesystem which happened when reading low cardinality. [#36763](https://github.com/ClickHouse/ClickHouse/pull/36763) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.3.11.12-lts.md b/docs/changelogs/v22.3.11.12-lts.md index e718493d9a0..58df0c0cadb 100644 --- a/docs/changelogs/v22.3.11.12-lts.md +++ b/docs/changelogs/v22.3.11.12-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39881](https://github.com/ClickHouse/ClickHouse/issues/39881): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39336](https://github.com/ClickHouse/ClickHouse/issues/39336): Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.3.12.19-lts.md b/docs/changelogs/v22.3.12.19-lts.md index 4f6342419f7..6ae342583a9 100644 --- a/docs/changelogs/v22.3.12.19-lts.md +++ b/docs/changelogs/v22.3.12.19-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40695](https://github.com/ClickHouse/ClickHouse/issues/40695): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40160](https://github.com/ClickHouse/ClickHouse/issues/40160): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40122](https://github.com/ClickHouse/ClickHouse/issues/40122): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.3.13.80-lts.md b/docs/changelogs/v22.3.13.80-lts.md index bb84da9a96a..3d0a6a77276 100644 --- a/docs/changelogs/v22.3.13.80-lts.md +++ b/docs/changelogs/v22.3.13.80-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41557](https://github.com/ClickHouse/ClickHouse/issues/41557): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40745](https://github.com/ClickHouse/ClickHouse/issues/40745): * Fix cast lowcard of nullable in JoinSwitcher, close [#37385](https://github.com/ClickHouse/ClickHouse/issues/37385). [#37453](https://github.com/ClickHouse/ClickHouse/pull/37453) ([Vladimir C](https://github.com/vdimir)). * Backported in [#41812](https://github.com/ClickHouse/ClickHouse/issues/41812): Update `simdjson`. This fixes [#38621](https://github.com/ClickHouse/ClickHouse/issues/38621). [#38838](https://github.com/ClickHouse/ClickHouse/pull/38838) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -36,7 +36,7 @@ sidebar_label: 2022 * Backported in [#41639](https://github.com/ClickHouse/ClickHouse/issues/41639): Fix possible `pipeline stuck` exception for queries with `OFFSET`. The error was found with `enable_optimize_predicate_expression = 0` and always false condition in `WHERE`. Fixes [#41383](https://github.com/ClickHouse/ClickHouse/issues/41383). [#41588](https://github.com/ClickHouse/ClickHouse/pull/41588) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#41899](https://github.com/ClickHouse/ClickHouse/issues/41899): Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#41321](https://github.com/ClickHouse/ClickHouse/issues/41321): Fix bug in function `if` when resulting column type differs with resulting data type that led to logical errors like `Logical error: 'Bad cast from type DB::ColumnVector to DB::ColumnVector'.`. Closes [#35367](https://github.com/ClickHouse/ClickHouse/issues/35367). [#35476](https://github.com/ClickHouse/ClickHouse/pull/35476) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.3.14.18-lts.md b/docs/changelogs/v22.3.14.18-lts.md index 88801b268ce..235525bbe81 100644 --- a/docs/changelogs/v22.3.14.18-lts.md +++ b/docs/changelogs/v22.3.14.18-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.3.14.23-lts.md b/docs/changelogs/v22.3.14.23-lts.md index 0a8c645702e..e086e223fb1 100644 --- a/docs/changelogs/v22.3.14.23-lts.md +++ b/docs/changelogs/v22.3.14.23-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42328](https://github.com/ClickHouse/ClickHouse/issues/42328): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42358](https://github.com/ClickHouse/ClickHouse/issues/42358): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42298](https://github.com/ClickHouse/ClickHouse/issues/42298): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#42592](https://github.com/ClickHouse/ClickHouse/issues/42592): This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.3.15.33-lts.md b/docs/changelogs/v22.3.15.33-lts.md index e59bf8bd1cd..3f675bfba1e 100644 --- a/docs/changelogs/v22.3.15.33-lts.md +++ b/docs/changelogs/v22.3.15.33-lts.md @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#42963](https://github.com/ClickHouse/ClickHouse/issues/42963): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43039](https://github.com/ClickHouse/ClickHouse/issues/43039): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43427](https://github.com/ClickHouse/ClickHouse/issues/43427): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). * Backported in [#43720](https://github.com/ClickHouse/ClickHouse/issues/43720): Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). diff --git a/docs/changelogs/v22.3.16.1190-lts.md b/docs/changelogs/v22.3.16.1190-lts.md index a43d34551ca..b65a6484109 100644 --- a/docs/changelogs/v22.3.16.1190-lts.md +++ b/docs/changelogs/v22.3.16.1190-lts.md @@ -18,7 +18,7 @@ sidebar_label: 2023 * Backported in [#44431](https://github.com/ClickHouse/ClickHouse/issues/44431): Kill stress tests after 2.5h in case of hanging process. [#44214](https://github.com/ClickHouse/ClickHouse/pull/44214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44557](https://github.com/ClickHouse/ClickHouse/issues/44557): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43512](https://github.com/ClickHouse/ClickHouse/issues/43512): - Fix several buffer over-reads. [#43159](https://github.com/ClickHouse/ClickHouse/pull/43159) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#43750](https://github.com/ClickHouse/ClickHouse/issues/43750): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). @@ -30,4 +30,3 @@ sidebar_label: 2023 #### NO CL ENTRY * NO CL ENTRY: 'Fix multipart upload for large S3 object, backport to 22.3'. [#44217](https://github.com/ClickHouse/ClickHouse/pull/44217) ([ianton-ru](https://github.com/ianton-ru)). - diff --git a/docs/changelogs/v22.3.18.37-lts.md b/docs/changelogs/v22.3.18.37-lts.md index ff6378f09ad..5a9004e7f98 100644 --- a/docs/changelogs/v22.3.18.37-lts.md +++ b/docs/changelogs/v22.3.18.37-lts.md @@ -14,7 +14,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#45856](https://github.com/ClickHouse/ClickHouse/issues/45856): Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45620](https://github.com/ClickHouse/ClickHouse/issues/45620): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#45549](https://github.com/ClickHouse/ClickHouse/issues/45549): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). @@ -30,4 +30,3 @@ sidebar_label: 2023 * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). - diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md index d5b45f4ce66..dc34ece918d 100644 --- a/docs/changelogs/v22.3.19.6-lts.md +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v22.3.19.6-lts (467e0a7bd77) FIXME as compared to v22.3.18.37-lts (fe512717551) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46440](https://github.com/ClickHouse/ClickHouse/issues/46440): Fix possible `LOGICAL_ERROR` in asynchronous inserts with invalid data sent in format `VALUES`. [#46350](https://github.com/ClickHouse/ClickHouse/pull/46350) ([Anton Popov](https://github.com/CurtizJ)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 3d113d45e68..bf491e46915 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#35928](https://github.com/ClickHouse/ClickHouse/issues/35928): Added settings `input_format_ipv4_default_on_conversion_error`, `input_format_ipv6_default_on_conversion_error` to allow insert of invalid ip address values as default into tables. Closes [#35726](https://github.com/ClickHouse/ClickHouse/issues/35726). [#35733](https://github.com/ClickHouse/ClickHouse/pull/35733) ([Maksim Kita](https://github.com/kitaisreal)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#35415](https://github.com/ClickHouse/ClickHouse/issues/35415): Fix possible deadlock in cache. [#35378](https://github.com/ClickHouse/ClickHouse/pull/35378) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#35563](https://github.com/ClickHouse/ClickHouse/issues/35563): Fix cast into IPv4, IPv6 address in IN section. Fixes [#35528](https://github.com/ClickHouse/ClickHouse/issues/35528). [#35534](https://github.com/ClickHouse/ClickHouse/pull/35534) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.3.4.20-lts.md b/docs/changelogs/v22.3.4.20-lts.md index 72d7b90c743..070ad961beb 100644 --- a/docs/changelogs/v22.3.4.20-lts.md +++ b/docs/changelogs/v22.3.4.20-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * - Add `_le_` method for ClickHouseVersion - Fix auto_version for existing tag - docker_server now support getting version from tags - Add python unit tests to backport workflow. [#36028](https://github.com/ClickHouse/ClickHouse/pull/36028) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36244](https://github.com/ClickHouse/ClickHouse/issues/36244): Fix usage of quota with asynchronous inserts. [#35645](https://github.com/ClickHouse/ClickHouse/pull/35645) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#36240](https://github.com/ClickHouse/ClickHouse/issues/36240): Fix possible loss of subcolumns in type `Object`. [#35682](https://github.com/ClickHouse/ClickHouse/pull/35682) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.5.5-lts.md b/docs/changelogs/v22.3.5.5-lts.md index b4205d5e1a9..4a5dc318b9f 100644 --- a/docs/changelogs/v22.3.5.5-lts.md +++ b/docs/changelogs/v22.3.5.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.5.5-lts (438b4a81f77) FIXME as compared to v22.3.4.20-lts (ecbaf001f49) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36525](https://github.com/ClickHouse/ClickHouse/issues/36525): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36674](https://github.com/ClickHouse/ClickHouse/issues/36674): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 4183332efb0..90e65c7445f 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.6.5-lts (3e44e824cff) FIXME as compared to v22.3.5.5-lts (438b4a81f77) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36795](https://github.com/ClickHouse/ClickHouse/issues/36795): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md index 7347e8e0705..a6a7885abc3 100644 --- a/docs/changelogs/v22.3.7.28-lts.md +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -7,14 +7,14 @@ sidebar_label: 2022 ### ClickHouse release v22.3.7.28-lts (420bdfa2751) FIXME as compared to v22.3.6.5-lts (3e44e824cff) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37715](https://github.com/ClickHouse/ClickHouse/issues/37715): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37511](https://github.com/ClickHouse/ClickHouse/issues/37511): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#37843](https://github.com/ClickHouse/ClickHouse/issues/37843): Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#37940](https://github.com/ClickHouse/ClickHouse/issues/37940): Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#37926](https://github.com/ClickHouse/ClickHouse/issues/37926): Fix check asof join key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37172](https://github.com/ClickHouse/ClickHouse/issues/37172): Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.3.8.39-lts.md b/docs/changelogs/v22.3.8.39-lts.md index 8fff7f00a01..7f9363207d3 100644 --- a/docs/changelogs/v22.3.8.39-lts.md +++ b/docs/changelogs/v22.3.8.39-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38826](https://github.com/ClickHouse/ClickHouse/issues/38826): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38453](https://github.com/ClickHouse/ClickHouse/issues/38453): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#38710](https://github.com/ClickHouse/ClickHouse/issues/38710): Fix incorrect result of distributed queries with `DISTINCT` and `LIMIT`. Fixes [#38282](https://github.com/ClickHouse/ClickHouse/issues/38282). [#38371](https://github.com/ClickHouse/ClickHouse/pull/38371) ([Anton Popov](https://github.com/CurtizJ)). @@ -18,7 +18,7 @@ sidebar_label: 2022 * Backported in [#38776](https://github.com/ClickHouse/ClickHouse/issues/38776): `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#38780](https://github.com/ClickHouse/ClickHouse/issues/38780): Fix use-after-free for Map combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36818](https://github.com/ClickHouse/ClickHouse/issues/36818): Fix projection analysis which might lead to wrong query result when IN subquery is used. This fixes [#35336](https://github.com/ClickHouse/ClickHouse/issues/35336). [#35631](https://github.com/ClickHouse/ClickHouse/pull/35631) ([Amos Bird](https://github.com/amosbird)). * Backported in [#38467](https://github.com/ClickHouse/ClickHouse/issues/38467): - Fix potential error with literals in `WHERE` for join queries. Close [#36279](https://github.com/ClickHouse/ClickHouse/issues/36279). [#36542](https://github.com/ClickHouse/ClickHouse/pull/36542) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v22.3.9.19-lts.md b/docs/changelogs/v22.3.9.19-lts.md index 084793f11cc..e0ebcd9b7b8 100644 --- a/docs/changelogs/v22.3.9.19-lts.md +++ b/docs/changelogs/v22.3.9.19-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.3.9.19-lts (7976930b82e) FIXME as compared to v22.3.8.39-lts (6bcf982f58b) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39097](https://github.com/ClickHouse/ClickHouse/issues/39097): Any allocations inside OvercommitTracker may lead to deadlock. Logging was not very informative so it's easier just to remove logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)). * Backported in [#39080](https://github.com/ClickHouse/ClickHouse/issues/39080): Fix bug in filesystem cache that could happen in some corner case which coincided with cache capacity hitting the limit. Closes [#39066](https://github.com/ClickHouse/ClickHouse/issues/39066). [#39070](https://github.com/ClickHouse/ClickHouse/pull/39070) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index 04ad0bf8955..41f57454d0d 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -161,7 +161,7 @@ sidebar_label: 2022 * call RemoteQueryExecutor with original_query instead of an rewritten query, elimate the AMBIGUOUS_COLUMN_NAME exception. [#35748](https://github.com/ClickHouse/ClickHouse/pull/35748) ([lgbo](https://github.com/lgbo-ustc)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Disallow ALTER TTL for engines that does not support it, to avoid breaking ATTACH TABLE (closes [#33344](https://github.com/ClickHouse/ClickHouse/issues/33344)). [#33391](https://github.com/ClickHouse/ClickHouse/pull/33391) ([zhongyuankai](https://github.com/zhongyuankai)). * Do not delay final part writing by default (fixes possible `Memory limit exceeded` during `INSERT` by adding `max_insert_delayed_streams_for_parallel_write` with default to 1000 for writes to s3 and disabled as before otherwise). [#34780](https://github.com/ClickHouse/ClickHouse/pull/34780) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.4.3.3-stable.md b/docs/changelogs/v22.4.3.3-stable.md index 69f95d8cd27..58d0f7b69a1 100644 --- a/docs/changelogs/v22.4.3.3-stable.md +++ b/docs/changelogs/v22.4.3.3-stable.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v22.4.3.3-stable (def956d6299) FIXME as compared to v22.4.2.1-stable (b34ebdc36ae) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36582](https://github.com/ClickHouse/ClickHouse/issues/36582): Fix nullptr dereference in JOIN and COLUMNS matcher. This fixes [#36416](https://github.com/ClickHouse/ClickHouse/issues/36416) . This is for https://github.com/ClickHouse/ClickHouse/pull/36417. [#36430](https://github.com/ClickHouse/ClickHouse/pull/36430) ([Amos Bird](https://github.com/amosbird)). diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 1dce0c50e0a..af94ecafcf6 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.4.4.7-stable (ba44414f9b3) FIXME as compared to v22.4.3.3-stable (def956d6299) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36524](https://github.com/ClickHouse/ClickHouse/issues/36524): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36673](https://github.com/ClickHouse/ClickHouse/issues/36673): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index 50cec91b12c..524b309f8ec 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.4.5.9-stable (059ef6cadcd) FIXME as compared to v22.4.4.7-stable (ba44414f9b3) -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#36635](https://github.com/ClickHouse/ClickHouse/issues/36635): Fix `Missing column` exception which could happen while using `INTERPOLATE` with `ENGINE = MergeTree` table. [#36549](https://github.com/ClickHouse/ClickHouse/pull/36549) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#36794](https://github.com/ClickHouse/ClickHouse/issues/36794): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). diff --git a/docs/changelogs/v22.4.6.53-stable.md b/docs/changelogs/v22.4.6.53-stable.md index 5dc25697c20..b6380d9c7ba 100644 --- a/docs/changelogs/v22.4.6.53-stable.md +++ b/docs/changelogs/v22.4.6.53-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38828](https://github.com/ClickHouse/ClickHouse/issues/38828): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37717](https://github.com/ClickHouse/ClickHouse/issues/37717): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37512](https://github.com/ClickHouse/ClickHouse/issues/37512): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). @@ -27,7 +27,7 @@ sidebar_label: 2022 * Backported in [#38777](https://github.com/ClickHouse/ClickHouse/issues/38777): `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#38781](https://github.com/ClickHouse/ClickHouse/issues/38781): Fix use-after-free for Map combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Backported in [#37456](https://github.com/ClickHouse/ClickHouse/issues/37456): Server might fail to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#38468](https://github.com/ClickHouse/ClickHouse/issues/38468): - Fix potential error with literals in `WHERE` for join queries. Close [#36279](https://github.com/ClickHouse/ClickHouse/issues/36279). [#36542](https://github.com/ClickHouse/ClickHouse/pull/36542) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index fdd6325fd1f..28dfa0825cf 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -104,7 +104,7 @@ sidebar_label: 2022 * ClickHouse builds for `PowerPC64LE` architecture are now available in universal installation script `curl https://clickhouse.com/ | sh` and by direct link `https://builds.clickhouse.com/master/powerpc64le/clickhouse`. [#37095](https://github.com/ClickHouse/ClickHouse/pull/37095) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * - Make cmake build scripts a bit more robust. [#37169](https://github.com/ClickHouse/ClickHouse/pull/37169) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * The ilike() function on FixedString columns could have returned wrong results (i.e. match less than it should). [#37117](https://github.com/ClickHouse/ClickHouse/pull/37117) ([Robert Schulze](https://github.com/rschu1ze)). * Fix implicit cast for optimize_skip_unused_shards_rewrite_in. [#37153](https://github.com/ClickHouse/ClickHouse/pull/37153) ([Azat Khuzhin](https://github.com/azat)). @@ -120,7 +120,7 @@ sidebar_label: 2022 * Fix system.opentelemetry_span_log attribute.values alias to values instead of keys. [#37275](https://github.com/ClickHouse/ClickHouse/pull/37275) ([Aleksandr Razumov](https://github.com/ernado)). * Fix possible deadlock in OvercommitTracker during logging. cc @alesapin @tavplubix Fixes [#37272](https://github.com/ClickHouse/ClickHouse/issues/37272). [#37299](https://github.com/ClickHouse/ClickHouse/pull/37299) ([Dmitry Novik](https://github.com/novikd)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * - fix substring function range error length when `offset` and `length` is negative constant and `s` is not constant. [#33861](https://github.com/ClickHouse/ClickHouse/pull/33861) ([RogerYK](https://github.com/RogerYK)). * Accidentally ZSTD support for Arrow was not being built. This fixes [#35283](https://github.com/ClickHouse/ClickHouse/issues/35283). [#35486](https://github.com/ClickHouse/ClickHouse/pull/35486) ([Sean Lafferty](https://github.com/seanlaff)). diff --git a/docs/changelogs/v22.5.2.53-stable.md b/docs/changelogs/v22.5.2.53-stable.md index f2fb50b3e14..8af0c7dac45 100644 --- a/docs/changelogs/v22.5.2.53-stable.md +++ b/docs/changelogs/v22.5.2.53-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38827](https://github.com/ClickHouse/ClickHouse/issues/38827): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#37716](https://github.com/ClickHouse/ClickHouse/issues/37716): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). * Backported in [#37408](https://github.com/ClickHouse/ClickHouse/issues/37408): Throw an exception when GROUPING SETS used with ROLLUP or CUBE. [#37367](https://github.com/ClickHouse/ClickHouse/pull/37367) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.5.3.21-stable.md b/docs/changelogs/v22.5.3.21-stable.md index 994bf79f7a8..4915a41a10c 100644 --- a/docs/changelogs/v22.5.3.21-stable.md +++ b/docs/changelogs/v22.5.3.21-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.5.3.21-stable (e03724efec5) FIXME as compared to v22.5.2.53-stable (5fd600fda9e) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38241](https://github.com/ClickHouse/ClickHouse/issues/38241): Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#39098](https://github.com/ClickHouse/ClickHouse/issues/39098): Any allocations inside OvercommitTracker may lead to deadlock. Logging was not very informative so it's easier just to remove logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.5.4.19-stable.md b/docs/changelogs/v22.5.4.19-stable.md index 24903260904..c90f45c770e 100644 --- a/docs/changelogs/v22.5.4.19-stable.md +++ b/docs/changelogs/v22.5.4.19-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39882](https://github.com/ClickHouse/ClickHouse/issues/39882): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39209](https://github.com/ClickHouse/ClickHouse/issues/39209): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#39589](https://github.com/ClickHouse/ClickHouse/issues/39589): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 0437f682789..c915d24fe00 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -119,7 +119,7 @@ sidebar_label: 2022 * Fix overly aggressive stripping which removed the embedded hash required for checking the consistency of the executable. [#37993](https://github.com/ClickHouse/ClickHouse/pull/37993) ([Robert Schulze](https://github.com/rschu1ze)). * fix MacOS build compressor faild. [#38007](https://github.com/ClickHouse/ClickHouse/pull/38007) ([chen](https://github.com/xiedeyantu)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix `GROUP BY` `AggregateFunction` (i.e. you `GROUP BY` by the column that has `AggregateFunction` type). [#37093](https://github.com/ClickHouse/ClickHouse/pull/37093) ([Azat Khuzhin](https://github.com/azat)). * Fix possible heap-use-after-free error when reading system.projection_parts and system.projection_parts_columns . This fixes [#37184](https://github.com/ClickHouse/ClickHouse/issues/37184). [#37185](https://github.com/ClickHouse/ClickHouse/pull/37185) ([Amos Bird](https://github.com/amosbird)). @@ -169,7 +169,7 @@ sidebar_label: 2022 * when open enable_filesystem_query_cache_limit, throw Reserved cache size exceeds the remaining cache size. [#38004](https://github.com/ClickHouse/ClickHouse/pull/38004) ([chen](https://github.com/xiedeyantu)). * Query, containing ORDER BY ... WITH FILL, can generate extra rows when multiple WITH FILL columns are present. [#38074](https://github.com/ClickHouse/ClickHouse/pull/38074) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Fix converting types for UNION queries (may produce LOGICAL_ERROR). [#34775](https://github.com/ClickHouse/ClickHouse/pull/34775) ([Azat Khuzhin](https://github.com/azat)). * TTL merge may not be scheduled again if BackgroundExecutor is busy. --merges_with_ttl_counter is increased in selectPartsToMerge() --merge task will be ignored if BackgroundExecutor is busy --merges_with_ttl_counter will not be decrease. [#36387](https://github.com/ClickHouse/ClickHouse/pull/36387) ([lthaooo](https://github.com/lthaooo)). diff --git a/docs/changelogs/v22.6.2.12-stable.md b/docs/changelogs/v22.6.2.12-stable.md index d8c1cd31936..3c0f2116f1d 100644 --- a/docs/changelogs/v22.6.2.12-stable.md +++ b/docs/changelogs/v22.6.2.12-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#38484](https://github.com/ClickHouse/ClickHouse/issues/38484): Improve the stability for hive storage integration test. Move the data prepare step into test.py. [#38260](https://github.com/ClickHouse/ClickHouse/pull/38260) ([lgbo](https://github.com/lgbo-ustc)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38404](https://github.com/ClickHouse/ClickHouse/issues/38404): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/docs/changelogs/v22.6.3.35-stable.md b/docs/changelogs/v22.6.3.35-stable.md index 0a86c136d35..86b8282e075 100644 --- a/docs/changelogs/v22.6.3.35-stable.md +++ b/docs/changelogs/v22.6.3.35-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38883](https://github.com/ClickHouse/ClickHouse/issues/38883): Add `clickhouse-diagnostics` binary to the packages. [#38647](https://github.com/ClickHouse/ClickHouse/pull/38647) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38690](https://github.com/ClickHouse/ClickHouse/issues/38690): Fix incorrect columns order in subqueries of UNION (in case of duplicated columns in subselects may produce incorrect result). [#37887](https://github.com/ClickHouse/ClickHouse/pull/37887) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#38500](https://github.com/ClickHouse/ClickHouse/issues/38500): Do not allow recursive usage of OvercommitTracker during logging. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794) cc @tavplubix @davenger. [#38246](https://github.com/ClickHouse/ClickHouse/pull/38246) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.6.4.35-stable.md b/docs/changelogs/v22.6.4.35-stable.md index 5c4644f735a..2b5537b5bf9 100644 --- a/docs/changelogs/v22.6.4.35-stable.md +++ b/docs/changelogs/v22.6.4.35-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#38822](https://github.com/ClickHouse/ClickHouse/issues/38822): - Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#38242](https://github.com/ClickHouse/ClickHouse/issues/38242): Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#38865](https://github.com/ClickHouse/ClickHouse/issues/38865): Fix s3 seekable reads with parallel read buffer. (Affected memory usage during query). Closes [#38258](https://github.com/ClickHouse/ClickHouse/issues/38258). [#38802](https://github.com/ClickHouse/ClickHouse/pull/38802) ([Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/docs/changelogs/v22.6.5.22-stable.md b/docs/changelogs/v22.6.5.22-stable.md index 5965f63df14..edb6bdf7c2d 100644 --- a/docs/changelogs/v22.6.5.22-stable.md +++ b/docs/changelogs/v22.6.5.22-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#39883](https://github.com/ClickHouse/ClickHouse/issues/39883): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39207](https://github.com/ClickHouse/ClickHouse/issues/39207): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#38932](https://github.com/ClickHouse/ClickHouse/issues/38932): Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.6.6.16-stable.md b/docs/changelogs/v22.6.6.16-stable.md index 4d44621454b..d4d25771326 100644 --- a/docs/changelogs/v22.6.6.16-stable.md +++ b/docs/changelogs/v22.6.6.16-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.6.6.16-stable (d2a33ebc822) FIXME as compared to v22.6.5.22-stable (47ca5f14a34) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40162](https://github.com/ClickHouse/ClickHouse/issues/40162): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40124](https://github.com/ClickHouse/ClickHouse/issues/40124): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.6.7.7-stable.md b/docs/changelogs/v22.6.7.7-stable.md index f5351cc03ed..0b4cc6836f7 100644 --- a/docs/changelogs/v22.6.7.7-stable.md +++ b/docs/changelogs/v22.6.7.7-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40692](https://github.com/ClickHouse/ClickHouse/issues/40692): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40531](https://github.com/ClickHouse/ClickHouse/issues/40531): Proxy resolver stop on first successful request to endpoint. [#40353](https://github.com/ClickHouse/ClickHouse/pull/40353) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#40623](https://github.com/ClickHouse/ClickHouse/issues/40623): Fix potential dataloss due to a bug in AWS SDK (https://github.com/aws/aws-sdk-cpp/issues/658). Bug can be triggered only when clickhouse is used over S3. [#40506](https://github.com/ClickHouse/ClickHouse/pull/40506) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v22.6.8.35-stable.md b/docs/changelogs/v22.6.8.35-stable.md index e68384d3d9c..b69cabffd4d 100644 --- a/docs/changelogs/v22.6.8.35-stable.md +++ b/docs/changelogs/v22.6.8.35-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#41274](https://github.com/ClickHouse/ClickHouse/issues/41274): Fix memory safety issues with functions `encrypt` and `contingency` if Array of Nullable is used as an argument. This fixes [#41004](https://github.com/ClickHouse/ClickHouse/issues/41004). [#40195](https://github.com/ClickHouse/ClickHouse/pull/40195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41282](https://github.com/ClickHouse/ClickHouse/issues/41282): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#40905](https://github.com/ClickHouse/ClickHouse/issues/40905): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.6.9.11-stable.md b/docs/changelogs/v22.6.9.11-stable.md index 5beb9171d9e..42cf0948ecc 100644 --- a/docs/changelogs/v22.6.9.11-stable.md +++ b/docs/changelogs/v22.6.9.11-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41558](https://github.com/ClickHouse/ClickHouse/issues/41558): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41504](https://github.com/ClickHouse/ClickHouse/issues/41504): Writing data in Apache `ORC` format might lead to a buffer overrun. [#41458](https://github.com/ClickHouse/ClickHouse/pull/41458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.7.1.2484-stable.md b/docs/changelogs/v22.7.1.2484-stable.md index 3f90b3691ea..7464b0449ee 100644 --- a/docs/changelogs/v22.7.1.2484-stable.md +++ b/docs/changelogs/v22.7.1.2484-stable.md @@ -128,7 +128,7 @@ sidebar_label: 2022 * Fix LSan by fixing getauxval(). [#39299](https://github.com/ClickHouse/ClickHouse/pull/39299) ([Azat Khuzhin](https://github.com/azat)). * Adapt universal installation script for FreeBSD. [#39302](https://github.com/ClickHouse/ClickHouse/pull/39302) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fix projection exception when aggregation keys are wrapped inside other functions. This fixes [#37151](https://github.com/ClickHouse/ClickHouse/issues/37151). [#37155](https://github.com/ClickHouse/ClickHouse/pull/37155) ([Amos Bird](https://github.com/amosbird)). * Fix possible logical error `... with argument with type Nothing and default implementation for Nothing is expected to return result with type Nothing, got ...` in some functions. Closes: [#37610](https://github.com/ClickHouse/ClickHouse/issues/37610) Closes: [#37741](https://github.com/ClickHouse/ClickHouse/issues/37741). [#37759](https://github.com/ClickHouse/ClickHouse/pull/37759) ([Kruglov Pavel](https://github.com/Avogar)). @@ -198,7 +198,7 @@ sidebar_label: 2022 * Fix UB (stack-use-after-scope) in extactAll(). [#39397](https://github.com/ClickHouse/ClickHouse/pull/39397) ([Azat Khuzhin](https://github.com/azat)). * Fix incorrect query result when trivial count optimization is in effect with array join. This fixes [#39431](https://github.com/ClickHouse/ClickHouse/issues/39431). [#39444](https://github.com/ClickHouse/ClickHouse/pull/39444) ([Amos Bird](https://github.com/amosbird)). -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) +#### Bug Fix (user-visible misbehaviour in official stable release) * Disable send_logs_level for INSERT into Distributed to avoid possible hung. [#35075](https://github.com/ClickHouse/ClickHouse/pull/35075) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.7.2.15-stable.md b/docs/changelogs/v22.7.2.15-stable.md index a9db8bcf10d..0a3748f90f7 100644 --- a/docs/changelogs/v22.7.2.15-stable.md +++ b/docs/changelogs/v22.7.2.15-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#39750](https://github.com/ClickHouse/ClickHouse/issues/39750): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#39591](https://github.com/ClickHouse/ClickHouse/issues/39591): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#39613](https://github.com/ClickHouse/ClickHouse/issues/39613): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)). diff --git a/docs/changelogs/v22.7.3.5-stable.md b/docs/changelogs/v22.7.3.5-stable.md index 62a5dfee611..b55b16509d4 100644 --- a/docs/changelogs/v22.7.3.5-stable.md +++ b/docs/changelogs/v22.7.3.5-stable.md @@ -11,7 +11,7 @@ sidebar_label: 2022 * Backported in [#39884](https://github.com/ClickHouse/ClickHouse/issues/39884): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#39884](https://github.com/ClickHouse/ClickHouse/issues/39884): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40045](https://github.com/ClickHouse/ClickHouse/issues/40045): Fix big memory usage during fetches. Fixes [#39915](https://github.com/ClickHouse/ClickHouse/issues/39915). [#39990](https://github.com/ClickHouse/ClickHouse/pull/39990) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#40045](https://github.com/ClickHouse/ClickHouse/issues/40045): Fix big memory usage during fetches. Fixes [#39915](https://github.com/ClickHouse/ClickHouse/issues/39915). [#39990](https://github.com/ClickHouse/ClickHouse/pull/39990) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). diff --git a/docs/changelogs/v22.7.4.16-stable.md b/docs/changelogs/v22.7.4.16-stable.md index 52d68283a2f..4847ef8cf64 100644 --- a/docs/changelogs/v22.7.4.16-stable.md +++ b/docs/changelogs/v22.7.4.16-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.7.4.16-stable (0b9272f8fdc) FIXME as compared to v22.7.3.5-stable (e140b8b5f3a) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40163](https://github.com/ClickHouse/ClickHouse/issues/40163): fix HashMethodOneNumber get wrong key value when column is const. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40125](https://github.com/ClickHouse/ClickHouse/issues/40125): Fix bug in collectFilesToSkip() by adding correct file extension(.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.7.5.13-stable.md b/docs/changelogs/v22.7.5.13-stable.md index 0de9e9a26aa..24dbc5c9e7f 100644 --- a/docs/changelogs/v22.7.5.13-stable.md +++ b/docs/changelogs/v22.7.5.13-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40693](https://github.com/ClickHouse/ClickHouse/issues/40693): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40542](https://github.com/ClickHouse/ClickHouse/issues/40542): Fix potential deadlock in WriteBufferFromS3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#40450](https://github.com/ClickHouse/ClickHouse/issues/40450): Fix rare bug with column TTL for MergeTree engines family: In case of repeated vertical merge the error `Cannot unlink file ColumnName.bin ... No such file or directory.` could happen. [#40346](https://github.com/ClickHouse/ClickHouse/pull/40346) ([alesapin](https://github.com/alesapin)). diff --git a/docs/changelogs/v22.7.6.74-stable.md b/docs/changelogs/v22.7.6.74-stable.md index 9060375ce8e..3cf2edfccd1 100644 --- a/docs/changelogs/v22.7.6.74-stable.md +++ b/docs/changelogs/v22.7.6.74-stable.md @@ -16,7 +16,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41559](https://github.com/ClickHouse/ClickHouse/issues/41559): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41283](https://github.com/ClickHouse/ClickHouse/issues/41283): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#40865](https://github.com/ClickHouse/ClickHouse/issues/40865): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). diff --git a/docs/changelogs/v22.7.7.24-stable.md b/docs/changelogs/v22.7.7.24-stable.md index cc3a83c5d4c..16e56156ff6 100644 --- a/docs/changelogs/v22.7.7.24-stable.md +++ b/docs/changelogs/v22.7.7.24-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42329](https://github.com/ClickHouse/ClickHouse/issues/42329): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42359](https://github.com/ClickHouse/ClickHouse/issues/42359): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42268](https://github.com/ClickHouse/ClickHouse/issues/42268): Fix reusing of files > 4GB from base backup. [#42146](https://github.com/ClickHouse/ClickHouse/pull/42146) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#42299](https://github.com/ClickHouse/ClickHouse/issues/42299): Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). diff --git a/docs/changelogs/v22.8.1.2097-lts.md b/docs/changelogs/v22.8.1.2097-lts.md index ef454a7e283..b6b634f4826 100644 --- a/docs/changelogs/v22.8.1.2097-lts.md +++ b/docs/changelogs/v22.8.1.2097-lts.md @@ -106,7 +106,7 @@ sidebar_label: 2022 * Support build with `clang-16` (trunk). This closes [#39949](https://github.com/ClickHouse/ClickHouse/issues/39949). [#40181](https://github.com/ClickHouse/ClickHouse/pull/40181) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Prepare RISC-V 64 build to run in CI. This is for [#40141](https://github.com/ClickHouse/ClickHouse/issues/40141). [#40197](https://github.com/ClickHouse/ClickHouse/pull/40197) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Fixed query hanging for SELECT with ORDER BY WITH FILL with different date/time types. [#37849](https://github.com/ClickHouse/ClickHouse/pull/37849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Fix ORDER BY that matches projections ORDER BY (before it simply returns unsorted result). [#38725](https://github.com/ClickHouse/ClickHouse/pull/38725) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.8.10.29-lts.md b/docs/changelogs/v22.8.10.29-lts.md index ac41d71650c..33ae27b6da8 100644 --- a/docs/changelogs/v22.8.10.29-lts.md +++ b/docs/changelogs/v22.8.10.29-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43051](https://github.com/ClickHouse/ClickHouse/issues/43051): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43513](https://github.com/ClickHouse/ClickHouse/issues/43513): - Fix several buffer over-reads. [#43159](https://github.com/ClickHouse/ClickHouse/pull/43159) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#43428](https://github.com/ClickHouse/ClickHouse/issues/43428): Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.8.11.15-lts.md b/docs/changelogs/v22.8.11.15-lts.md index 337eeba9187..dbe765ca4a2 100644 --- a/docs/changelogs/v22.8.11.15-lts.md +++ b/docs/changelogs/v22.8.11.15-lts.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Bug Fix * Backported in [#43098](https://github.com/ClickHouse/ClickHouse/issues/43098): Updated normaliser to clone the alias ast. resolves [#42452](https://github.com/ClickHouse/ClickHouse/issues/42452) Implementation: * Updated QueryNormalizer to clone alias ast, when its replaced. Previously just assigning the same leads to exception in LogicalExpressinsOptimizer as it would be the same parent being inserted again. * This bug is not seen with new analyser (allow_experimental_analyzer), so no changes for it. I added a test for the same. [#42827](https://github.com/ClickHouse/ClickHouse/pull/42827) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43751](https://github.com/ClickHouse/ClickHouse/issues/43751): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#43617](https://github.com/ClickHouse/ClickHouse/issues/43617): Fix sumMap() for Nullable(Decimal()). [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.8.12.45-lts.md b/docs/changelogs/v22.8.12.45-lts.md index 7412784419c..9fab9daeb95 100644 --- a/docs/changelogs/v22.8.12.45-lts.md +++ b/docs/changelogs/v22.8.12.45-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2023 * Backported in [#44378](https://github.com/ClickHouse/ClickHouse/issues/44378): In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#44558](https://github.com/ClickHouse/ClickHouse/issues/44558): Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#44751](https://github.com/ClickHouse/ClickHouse/issues/44751): [#40651](https://github.com/ClickHouse/ClickHouse/issues/40651) [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43525](https://github.com/ClickHouse/ClickHouse/issues/43525): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). @@ -39,4 +39,3 @@ sidebar_label: 2023 * Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.8.13.20-lts.md b/docs/changelogs/v22.8.13.20-lts.md index d8dd1bd2b1c..0734f40bf3e 100644 --- a/docs/changelogs/v22.8.13.20-lts.md +++ b/docs/changelogs/v22.8.13.20-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v22.8.13.20-lts (e4817946d18) FIXME as compared to v22.8.12.45-lts (86b0ecd5d51) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45565](https://github.com/ClickHouse/ClickHouse/issues/45565): Fix positional arguments exception Positional argument out of bounds. Closes [#40634](https://github.com/ClickHouse/ClickHouse/issues/40634). [#41189](https://github.com/ClickHouse/ClickHouse/pull/41189) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#44997](https://github.com/ClickHouse/ClickHouse/issues/44997): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). @@ -21,4 +21,3 @@ sidebar_label: 2023 * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Merge pull request [#38262](https://github.com/ClickHouse/ClickHouse/issues/38262) from PolyProgrammist/fix-ordinary-system-un… [#45650](https://github.com/ClickHouse/ClickHouse/pull/45650) ([alesapin](https://github.com/alesapin)). - diff --git a/docs/changelogs/v22.8.14.53-lts.md b/docs/changelogs/v22.8.14.53-lts.md index 5978080fa3a..3cceb3475b6 100644 --- a/docs/changelogs/v22.8.14.53-lts.md +++ b/docs/changelogs/v22.8.14.53-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2023 * Backported in [#46482](https://github.com/ClickHouse/ClickHouse/issues/46482): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46505](https://github.com/ClickHouse/ClickHouse/issues/46505): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45908](https://github.com/ClickHouse/ClickHouse/issues/45908): Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Backported in [#46238](https://github.com/ClickHouse/ClickHouse/issues/46238): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). @@ -37,4 +37,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.15.23-lts.md b/docs/changelogs/v22.8.15.23-lts.md index 096a504c9c2..5f49dfb1757 100644 --- a/docs/changelogs/v22.8.15.23-lts.md +++ b/docs/changelogs/v22.8.15.23-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2023 #### Bug Fix * Backported in [#47336](https://github.com/ClickHouse/ClickHouse/issues/47336): Sometimes after changing a role that could be not reflected on the access rights of a user who uses that role. This PR fixes that. [#46772](https://github.com/ClickHouse/ClickHouse/pull/46772) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46901](https://github.com/ClickHouse/ClickHouse/issues/46901): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#47156](https://github.com/ClickHouse/ClickHouse/issues/47156): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). @@ -25,4 +25,3 @@ sidebar_label: 2023 * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.3.13-lts.md b/docs/changelogs/v22.8.3.13-lts.md index 903f5b7a600..5f08bc9ee67 100644 --- a/docs/changelogs/v22.8.3.13-lts.md +++ b/docs/changelogs/v22.8.3.13-lts.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#40694](https://github.com/ClickHouse/ClickHouse/issues/40694): Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40451](https://github.com/ClickHouse/ClickHouse/issues/40451): Fix rare bug with column TTL for MergeTree engines family: In case of repeated vertical merge the error `Cannot unlink file ColumnName.bin ... No such file or directory.` could happen. [#40346](https://github.com/ClickHouse/ClickHouse/pull/40346) ([alesapin](https://github.com/alesapin)). * Backported in [#40533](https://github.com/ClickHouse/ClickHouse/issues/40533): Proxy resolver stop on first successful request to endpoint. [#40353](https://github.com/ClickHouse/ClickHouse/pull/40353) ([Maksim Kita](https://github.com/kitaisreal)). diff --git a/docs/changelogs/v22.8.4.7-lts.md b/docs/changelogs/v22.8.4.7-lts.md index 93d9aa2bf1a..a852120ab8a 100644 --- a/docs/changelogs/v22.8.4.7-lts.md +++ b/docs/changelogs/v22.8.4.7-lts.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.8.4.7-lts (baad27bcd2f) FIXME as compared to v22.8.3.13-lts (6a15b73faea) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40760](https://github.com/ClickHouse/ClickHouse/issues/40760): Fix possible error 'Decimal math overflow' while parsing DateTime64. [#40546](https://github.com/ClickHouse/ClickHouse/pull/40546) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#40811](https://github.com/ClickHouse/ClickHouse/issues/40811): In [#40595](https://github.com/ClickHouse/ClickHouse/issues/40595) it was reported that the `host_regexp` functionality was not working properly with a name to address resolution in `/etc/hosts`. It's fixed. [#40769](https://github.com/ClickHouse/ClickHouse/pull/40769) ([Arthur Passos](https://github.com/arthurpassos)). diff --git a/docs/changelogs/v22.8.5.29-lts.md b/docs/changelogs/v22.8.5.29-lts.md index b7ad3c11a46..1b59501cc28 100644 --- a/docs/changelogs/v22.8.5.29-lts.md +++ b/docs/changelogs/v22.8.5.29-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41157](https://github.com/ClickHouse/ClickHouse/issues/41157): Add macOS binaries to GH release assets, it fixes [#37718](https://github.com/ClickHouse/ClickHouse/issues/37718). [#41088](https://github.com/ClickHouse/ClickHouse/pull/41088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#40866](https://github.com/ClickHouse/ClickHouse/issues/40866): - Fix crash while parsing values of type `Object` that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). * Backported in [#40805](https://github.com/ClickHouse/ClickHouse/issues/40805): During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). diff --git a/docs/changelogs/v22.8.6.71-lts.md b/docs/changelogs/v22.8.6.71-lts.md index 0337c5ba2e2..2ac6ef05bc4 100644 --- a/docs/changelogs/v22.8.6.71-lts.md +++ b/docs/changelogs/v22.8.6.71-lts.md @@ -16,7 +16,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#41560](https://github.com/ClickHouse/ClickHouse/issues/41560): Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41284](https://github.com/ClickHouse/ClickHouse/issues/41284): Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). * Backported in [#41837](https://github.com/ClickHouse/ClickHouse/issues/41837): Fix vertical merge of parts with lightweight deleted rows. [#40559](https://github.com/ClickHouse/ClickHouse/pull/40559) ([Alexander Gololobov](https://github.com/davenger)). diff --git a/docs/changelogs/v22.8.7.34-lts.md b/docs/changelogs/v22.8.7.34-lts.md index ee55f5d9f1f..56f03ecdf3b 100644 --- a/docs/changelogs/v22.8.7.34-lts.md +++ b/docs/changelogs/v22.8.7.34-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42296](https://github.com/ClickHouse/ClickHouse/issues/42296): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42360](https://github.com/ClickHouse/ClickHouse/issues/42360): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42489](https://github.com/ClickHouse/ClickHouse/issues/42489): Removed skipping of mutations in unaffected partitions of `MergeTree` tables, because this feature never worked correctly and might cause resurrection of finished mutations. [#40589](https://github.com/ClickHouse/ClickHouse/pull/40589) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#42121](https://github.com/ClickHouse/ClickHouse/issues/42121): Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v22.8.8.3-lts.md b/docs/changelogs/v22.8.8.3-lts.md index b4673eb955a..d42f333ce3f 100644 --- a/docs/changelogs/v22.8.8.3-lts.md +++ b/docs/changelogs/v22.8.8.3-lts.md @@ -7,6 +7,6 @@ sidebar_label: 2022 ### ClickHouse release v22.8.8.3-lts (ac5a6cababc) FIXME as compared to v22.8.7.34-lts (3c38e5e8ab9) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42677](https://github.com/ClickHouse/ClickHouse/issues/42677): keeper-fix: fix race in accessing logs while snapshot is being installed. [#40627](https://github.com/ClickHouse/ClickHouse/pull/40627) ([Antonio Andelic](https://github.com/antonio2368)). diff --git a/docs/changelogs/v22.8.9.24-lts.md b/docs/changelogs/v22.8.9.24-lts.md index 585327b83a1..15935c4cf4e 100644 --- a/docs/changelogs/v22.8.9.24-lts.md +++ b/docs/changelogs/v22.8.9.24-lts.md @@ -17,7 +17,7 @@ sidebar_label: 2022 * Backported in [#42964](https://github.com/ClickHouse/ClickHouse/issues/42964): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43040](https://github.com/ClickHouse/ClickHouse/issues/43040): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42720](https://github.com/ClickHouse/ClickHouse/issues/42720): Fixed `Unknown identifier (aggregate-function)` exception which appears when a user tries to calculate WINDOW ORDER BY/PARTITION BY expressions over aggregate functions: ``` CREATE TABLE default.tenk1 ( `unique1` Int32, `unique2` Int32, `ten` Int32 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192; SELECT ten, sum(unique1) + sum(unique2) AS res, rank() OVER (ORDER BY sum(unique1) + sum(unique2) ASC) AS rank FROM _complex GROUP BY ten ORDER BY ten ASC; ``` which gives: ``` Code: 47. DB::Exception: Received from localhost:9000. DB::Exception: Unknown identifier: sum(unique1); there are columns: unique1, unique2, ten: While processing sum(unique1) + sum(unique2) ASC. (UNKNOWN_IDENTIFIER) ```. [#39762](https://github.com/ClickHouse/ClickHouse/pull/39762) ([Vladimir Chebotaryov](https://github.com/quickhouse)). * Backported in [#42748](https://github.com/ClickHouse/ClickHouse/issues/42748): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). diff --git a/docs/changelogs/v22.9.2.7-stable.md b/docs/changelogs/v22.9.2.7-stable.md index 5c4db4bfa96..bbd0a6cce32 100644 --- a/docs/changelogs/v22.9.2.7-stable.md +++ b/docs/changelogs/v22.9.2.7-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2022 #### Improvement * Backported in [#41709](https://github.com/ClickHouse/ClickHouse/issues/41709): Check file path for path traversal attacks in errors logger for input formats. [#41694](https://github.com/ClickHouse/ClickHouse/pull/41694) ([Kruglov Pavel](https://github.com/Avogar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41696](https://github.com/ClickHouse/ClickHouse/issues/41696): Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). diff --git a/docs/changelogs/v22.9.3.18-stable.md b/docs/changelogs/v22.9.3.18-stable.md index 656cb1dfc22..a46dba6718c 100644 --- a/docs/changelogs/v22.9.3.18-stable.md +++ b/docs/changelogs/v22.9.3.18-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2022 ### ClickHouse release v22.9.3.18-stable (0cb4b15d2fa) FIXME as compared to v22.9.2.7-stable (362e2cefcef) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#41902](https://github.com/ClickHouse/ClickHouse/issues/41902): Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#41863](https://github.com/ClickHouse/ClickHouse/issues/41863): 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). diff --git a/docs/changelogs/v22.9.4.32-stable.md b/docs/changelogs/v22.9.4.32-stable.md index 658d39af079..92bcc01e408 100644 --- a/docs/changelogs/v22.9.4.32-stable.md +++ b/docs/changelogs/v22.9.4.32-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42297](https://github.com/ClickHouse/ClickHouse/issues/42297): Update cctz to the latest master, update tzdb to 2020e. [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). * Backported in [#42361](https://github.com/ClickHouse/ClickHouse/issues/42361): Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42122](https://github.com/ClickHouse/ClickHouse/issues/42122): Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#41938](https://github.com/ClickHouse/ClickHouse/issues/41938): Don't allow to create or alter merge tree tables with virtual column name _row_exists, which is reserved for lightweight delete. Fixed [#41716](https://github.com/ClickHouse/ClickHouse/issues/41716). [#41763](https://github.com/ClickHouse/ClickHouse/pull/41763) ([Jianmei Zhang](https://github.com/zhangjmruc)). diff --git a/docs/changelogs/v22.9.5.25-stable.md b/docs/changelogs/v22.9.5.25-stable.md index eb46fb893e7..90150726ace 100644 --- a/docs/changelogs/v22.9.5.25-stable.md +++ b/docs/changelogs/v22.9.5.25-stable.md @@ -14,7 +14,7 @@ sidebar_label: 2022 * Backported in [#42965](https://github.com/ClickHouse/ClickHouse/issues/42965): Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#43041](https://github.com/ClickHouse/ClickHouse/issues/43041): Add a CI step to mark commits as ready for release; soft-forbid launching a release script from branches but master. [#43017](https://github.com/ClickHouse/ClickHouse/pull/43017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#42749](https://github.com/ClickHouse/ClickHouse/issues/42749): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#42863](https://github.com/ClickHouse/ClickHouse/issues/42863): Fix lowerUTF8()/upperUTF8() in case of symbol was in between 16-byte boundary (very frequent case of you have strings > 16 bytes long). [#42812](https://github.com/ClickHouse/ClickHouse/pull/42812) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v22.9.6.20-stable.md b/docs/changelogs/v22.9.6.20-stable.md index d450f285848..7abc4adc32e 100644 --- a/docs/changelogs/v22.9.6.20-stable.md +++ b/docs/changelogs/v22.9.6.20-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#43052](https://github.com/ClickHouse/ClickHouse/issues/43052): Wait for all files are in sync before archiving them in integration tests. [#42891](https://github.com/ClickHouse/ClickHouse/pull/42891) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43505](https://github.com/ClickHouse/ClickHouse/issues/43505): Fix a bug when row level filter uses default value of column. [#43387](https://github.com/ClickHouse/ClickHouse/pull/43387) ([Alexander Gololobov](https://github.com/davenger)). * Backported in [#43722](https://github.com/ClickHouse/ClickHouse/issues/43722): Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). diff --git a/docs/changelogs/v22.9.7.34-stable.md b/docs/changelogs/v22.9.7.34-stable.md index 83be449f70d..ed8173eaf50 100644 --- a/docs/changelogs/v22.9.7.34-stable.md +++ b/docs/changelogs/v22.9.7.34-stable.md @@ -13,7 +13,7 @@ sidebar_label: 2022 #### Build/Testing/Packaging Improvement * Backported in [#44111](https://github.com/ClickHouse/ClickHouse/issues/44111): Bring sha512 sums back to the building step. [#44017](https://github.com/ClickHouse/ClickHouse/pull/44017) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#43612](https://github.com/ClickHouse/ClickHouse/issues/43612): Fix bad inefficiency of `remote_filesystem_read_method=read` with filesystem cache. Closes [#42125](https://github.com/ClickHouse/ClickHouse/issues/42125). [#42129](https://github.com/ClickHouse/ClickHouse/pull/42129) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#43526](https://github.com/ClickHouse/ClickHouse/issues/43526): Fix incorrect UserTimeMicroseconds/SystemTimeMicroseconds accounting. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). diff --git a/docs/changelogs/v23.1.1.3077-stable.md b/docs/changelogs/v23.1.1.3077-stable.md index e218be62f09..53ca9e1831c 100644 --- a/docs/changelogs/v23.1.1.3077-stable.md +++ b/docs/changelogs/v23.1.1.3077-stable.md @@ -125,7 +125,7 @@ sidebar_label: 2023 * SQLite library is updated to the latest. It is used for the SQLite database and table integration engines. Also, fixed a false-positive TSan report. This closes [#45027](https://github.com/ClickHouse/ClickHouse/issues/45027). [#45031](https://github.com/ClickHouse/ClickHouse/pull/45031) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix report sending in the case when FastTest failed. [#45588](https://github.com/ClickHouse/ClickHouse/pull/45588) ([Dmitry Novik](https://github.com/novikd)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * #40651 [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). * Fix possible use-of-unitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) CC: @nickitat. [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)). @@ -589,4 +589,3 @@ sidebar_label: 2023 * Resubmit "Fix possible in-use table after DETACH" [#45566](https://github.com/ClickHouse/ClickHouse/pull/45566) ([Alexander Tokmakov](https://github.com/tavplubix)). * Typo: "Granulesis" --> "Granules" [#45598](https://github.com/ClickHouse/ClickHouse/pull/45598) ([Robert Schulze](https://github.com/rschu1ze)). * Fix version in autogenerated_versions.txt [#45624](https://github.com/ClickHouse/ClickHouse/pull/45624) ([Dmitry Novik](https://github.com/novikd)). - diff --git a/docs/changelogs/v23.1.2.9-stable.md b/docs/changelogs/v23.1.2.9-stable.md index 272a2b95a86..7d34a6b9ec2 100644 --- a/docs/changelogs/v23.1.2.9-stable.md +++ b/docs/changelogs/v23.1.2.9-stable.md @@ -13,11 +13,10 @@ sidebar_label: 2023 #### Bug Fix * Backported in [#45673](https://github.com/ClickHouse/ClickHouse/issues/45673): Fix wiping sensitive info in logs. [#45603](https://github.com/ClickHouse/ClickHouse/pull/45603) ([Vitaly Baranov](https://github.com/vitlibar)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45730](https://github.com/ClickHouse/ClickHouse/issues/45730): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.3.5-stable.md b/docs/changelogs/v23.1.3.5-stable.md index d4f39894bec..9f8ef928138 100644 --- a/docs/changelogs/v23.1.3.5-stable.md +++ b/docs/changelogs/v23.1.3.5-stable.md @@ -7,11 +7,10 @@ sidebar_label: 2023 ### ClickHouse release v23.1.3.5-stable (548b494bcce) FIXME as compared to v23.1.2.9-stable (8dfb1700858) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#45896](https://github.com/ClickHouse/ClickHouse/issues/45896): Bugfix IPv6 parser for mixed ip4 address with missed first octet (like `::.1.2.3`). [#45871](https://github.com/ClickHouse/ClickHouse/pull/45871) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.4.58-stable.md b/docs/changelogs/v23.1.4.58-stable.md index d1ffe87f58e..9081d700308 100644 --- a/docs/changelogs/v23.1.4.58-stable.md +++ b/docs/changelogs/v23.1.4.58-stable.md @@ -20,7 +20,7 @@ sidebar_label: 2023 * Backported in [#46477](https://github.com/ClickHouse/ClickHouse/issues/46477): Get rid of unnecessary build for standalone clickhouse-keeper. [#46367](https://github.com/ClickHouse/ClickHouse/pull/46367) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Backported in [#46511](https://github.com/ClickHouse/ClickHouse/issues/46511): Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46228](https://github.com/ClickHouse/ClickHouse/issues/46228): A couple of seg faults have been reported around `c-ares`. All of the recent stack traces observed fail on inserting into `std::unodered_set<>`. I believe I have found the root cause of this, it seems to be unprocessed queries. Prior to this PR, CH calls `poll` to wait on the file descriptors in the `c-ares` channel. According to the [poll docs](https://man7.org/linux/man-pages/man2/poll.2.html), a negative return value means an error has ocurred. Because of this, we would abort the execution and return failure. The problem is that `poll` will also return a negative value if a system interrupt occurs. A system interrupt does not mean the processing has failed or ended, but we would abort it anyways because we were checking for negative values. Once the execution is aborted, the whole stack is destroyed, which includes the `std::unordered_set` passed to the `void *` parameter of the c-ares callback. Once c-ares completed the request, the callback would be invoked and would access an invalid memory address causing a segfault. [#45629](https://github.com/ClickHouse/ClickHouse/pull/45629) ([Arthur Passos](https://github.com/arthurpassos)). * Backported in [#46967](https://github.com/ClickHouse/ClickHouse/issues/46967): Backward compatibility - allow implicit narrowing conversion from UInt64 to IPv4 - required for "INSERT ... VALUES ..." expression. [#45865](https://github.com/ClickHouse/ClickHouse/pull/45865) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). @@ -44,4 +44,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.5.24-stable.md b/docs/changelogs/v23.1.5.24-stable.md index 1c2c127a8c3..934e97312c0 100644 --- a/docs/changelogs/v23.1.5.24-stable.md +++ b/docs/changelogs/v23.1.5.24-stable.md @@ -10,7 +10,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#47060](https://github.com/ClickHouse/ClickHouse/issues/47060): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46401](https://github.com/ClickHouse/ClickHouse/issues/46401): Fix `SYSTEM UNFREEZE` queries failing with the exception `CANNOT_PARSE_INPUT_ASSERTION_FAILED`. [#46325](https://github.com/ClickHouse/ClickHouse/pull/46325) ([Aleksei Filatov](https://github.com/aalexfvk)). * Backported in [#46905](https://github.com/ClickHouse/ClickHouse/issues/46905): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). @@ -25,4 +25,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.1.2537-stable.md b/docs/changelogs/v23.2.1.2537-stable.md index 3fdcf6d6571..9da81c039ea 100644 --- a/docs/changelogs/v23.2.1.2537-stable.md +++ b/docs/changelogs/v23.2.1.2537-stable.md @@ -161,7 +161,7 @@ sidebar_label: 2023 * Some time ago the ccache compression was changed to `zst`, but `gz` archives are downloaded by default. It fixes it by prioritizing zst archive. [#46490](https://github.com/ClickHouse/ClickHouse/pull/46490) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Raised the minimum Clang version needed to build ClickHouse from 12 to 15. [#46710](https://github.com/ClickHouse/ClickHouse/pull/46710) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Flush data exactly by `rabbitmq_flush_interval_ms` or by `rabbitmq_max_block_size` in `StorageRabbitMQ`. Closes [#42389](https://github.com/ClickHouse/ClickHouse/issues/42389). Closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#44404](https://github.com/ClickHouse/ClickHouse/pull/44404) ([Kseniia Sumarokova](https://github.com/kssenii)). * - Use PODArray to render in sparkBar function, so we can control the memory usage. Close [#44467](https://github.com/ClickHouse/ClickHouse/issues/44467). [#44489](https://github.com/ClickHouse/ClickHouse/pull/44489) ([Duc Canh Le](https://github.com/canhld94)). @@ -470,4 +470,3 @@ sidebar_label: 2023 #### Testing Improvement * Fixed functional test 00304_http_external_data for s390x. [#45807](https://github.com/ClickHouse/ClickHouse/pull/45807) ([Harry Lee](https://github.com/HarryLeeIBM)). - diff --git a/docs/changelogs/v23.2.2.20-stable.md b/docs/changelogs/v23.2.2.20-stable.md index 60aeaa66cbf..b92fbdebe33 100644 --- a/docs/changelogs/v23.2.2.20-stable.md +++ b/docs/changelogs/v23.2.2.20-stable.md @@ -17,7 +17,7 @@ sidebar_label: 2023 #### Build/Testing/Packaging Improvement * Backported in [#47062](https://github.com/ClickHouse/ClickHouse/issues/47062): Fix error during server startup on old distros (e.g. Amazon Linux 2) and on ARM that glibc 2.28 symbols are not found. [#47008](https://github.com/ClickHouse/ClickHouse/pull/47008) ([Robert Schulze](https://github.com/rschu1ze)). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46895](https://github.com/ClickHouse/ClickHouse/issues/46895): Fixed a bug in automatic retries of `DROP TABLE` query with `ReplicatedMergeTree` tables and `Atomic` databases. In rare cases it could lead to `Can't get data for node /zk_path/log_pointer` and `The specified key does not exist` errors if ZooKeeper session expired during DROP and a new replicated table with the same path in ZooKeeper was created in parallel. [#46384](https://github.com/ClickHouse/ClickHouse/pull/46384) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#46865](https://github.com/ClickHouse/ClickHouse/issues/46865): Fix a bug in the `Map` data type. This closes [#46855](https://github.com/ClickHouse/ClickHouse/issues/46855). [#46856](https://github.com/ClickHouse/ClickHouse/pull/46856) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -27,4 +27,3 @@ sidebar_label: 2023 * More concise logging at trace level for PREWHERE steps [#46771](https://github.com/ClickHouse/ClickHouse/pull/46771) ([Alexander Gololobov](https://github.com/davenger)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.3.17-stable.md b/docs/changelogs/v23.2.3.17-stable.md index fb2c4e394dc..75b7f8b2b20 100644 --- a/docs/changelogs/v23.2.3.17-stable.md +++ b/docs/changelogs/v23.2.3.17-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v23.2.3.17-stable (dec18bf7281) FIXME as compared to v23.2.2.20-stable (f6c269c8df2) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#46907](https://github.com/ClickHouse/ClickHouse/issues/46907): - Fix incorrect alias recursion in QueryNormalizer. [#46609](https://github.com/ClickHouse/ClickHouse/pull/46609) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#47091](https://github.com/ClickHouse/ClickHouse/issues/47091): - Fix arithmetic operations in aggregate optimization with `min` and `max`. [#46705](https://github.com/ClickHouse/ClickHouse/pull/46705) ([Duc Canh Le](https://github.com/canhld94)). @@ -20,4 +20,3 @@ sidebar_label: 2023 * Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * do flushUntrackedMemory when context switches [#47102](https://github.com/ClickHouse/ClickHouse/pull/47102) ([Sema Checherinda](https://github.com/CheSema)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.4.12-stable.md b/docs/changelogs/v23.2.4.12-stable.md index 2b6a689aee5..1542e3257ce 100644 --- a/docs/changelogs/v23.2.4.12-stable.md +++ b/docs/changelogs/v23.2.4.12-stable.md @@ -7,7 +7,7 @@ sidebar_label: 2023 ### ClickHouse release v23.2.4.12-stable (8fe866cb035) FIXME as compared to v23.2.3.17-stable (dec18bf7281) -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix (user-visible misbehavior in official stable release) * Backported in [#47277](https://github.com/ClickHouse/ClickHouse/issues/47277): Fix IPv4/IPv6 serialization/deserialization in binary formats that was broken in https://github.com/ClickHouse/ClickHouse/pull/43221. Closes [#46522](https://github.com/ClickHouse/ClickHouse/issues/46522). [#46616](https://github.com/ClickHouse/ClickHouse/pull/46616) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#47212](https://github.com/ClickHouse/ClickHouse/issues/47212): `INSERT` queries through native TCP protocol and HTTP protocol were not canceled correctly in some cases. It could lead to a partially applied query if a client canceled the query, or if a client died or, in rare cases, on network errors. As a result, it could lead to not working deduplication. Fixes [#27667](https://github.com/ClickHouse/ClickHouse/issues/27667) and [#45377](https://github.com/ClickHouse/ClickHouse/issues/45377). [#46681](https://github.com/ClickHouse/ClickHouse/pull/46681) ([Alexander Tokmakov](https://github.com/tavplubix)). @@ -17,4 +17,3 @@ sidebar_label: 2023 * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 41574907492..8dc136e01f4 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -35,8 +35,8 @@ LABELS = { "pr-backward-incompatible": ["Backward Incompatible Change"], "pr-bugfix": [ "Bug Fix", - "Bug Fix (user-visible misbehaviour in official stable or prestable release)", - "Bug Fix (user-visible misbehavior in official stable or prestable release)", + "Bug Fix (user-visible misbehaviour in official stable release)", + "Bug Fix (user-visible misbehavior in official stable release)", ], "pr-build": [ "Build/Testing/Packaging Improvement", diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 3b66b68193b..9972ce8a627 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -256,6 +256,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri category = "NOT FOR CHANGELOG / INSIGNIFICANT" return Description(item.number, item.user, item.html_url, item.title, category) + # Normalize bug fixes + if re.match( + r"(?i)bug\Wfix", + category, + ): + category = "Bug Fix (user-visible misbehavior in an official stable release)" + return Description(item.number, item.user, item.html_url, item.title, category) + # Filter out documentations changelog if re.match( r"(?i)doc", From a71454f8ec53d56ca4332220843ea0540dcafca4 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Mon, 27 Mar 2023 12:53:31 +0200 Subject: [PATCH 34/49] Fix automatic indentation in the built-in UI SQL editor --- programs/server/play.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index a7fc951ec37..323ba7d0b9e 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -687,7 +687,7 @@ e.preventDefault(); return false; - } else if (e.key === 'Enter') { + } else if (e.key === 'Enter' && !(event.metaKey || event.ctrlKey)) { // If the user presses Enter, and the previous line starts with spaces, // then we will insert the same number of spaces. const elem = e.target; From b01f08ccb8958d1526cc53f6797bb9e3117a208d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 27 Mar 2023 13:23:56 +0200 Subject: [PATCH 35/49] rename marked_dropped_tables to dropped_tables --- ...roppedTables.cpp => StorageSystemDroppedTables.cpp} | 6 +++--- ...kedDroppedTables.h => StorageSystemDroppedTables.h} | 2 +- src/Storages/System/attachSystemTables.cpp | 4 ++-- .../0_stateless/25400_marked_dropped_tables.reference | 2 +- .../0_stateless/25400_marked_dropped_tables.sql | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) rename src/Storages/System/{StorageSystemMarkedDroppedTables.cpp => StorageSystemDroppedTables.cpp} (91%) rename src/Storages/System/{StorageSystemMarkedDroppedTables.h => StorageSystemDroppedTables.h} (78%) diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.cpp b/src/Storages/System/StorageSystemDroppedTables.cpp similarity index 91% rename from src/Storages/System/StorageSystemMarkedDroppedTables.cpp rename to src/Storages/System/StorageSystemDroppedTables.cpp index fcdd6e1edcf..1d6c8824c76 100644 --- a/src/Storages/System/StorageSystemMarkedDroppedTables.cpp +++ b/src/Storages/System/StorageSystemDroppedTables.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -13,7 +13,7 @@ namespace DB { -NamesAndTypesList StorageSystemMarkedDroppedTables::getNamesAndTypes() +NamesAndTypesList StorageSystemDroppedTables::getNamesAndTypes() { NamesAndTypesList names_and_types{ {"index", std::make_shared()}, @@ -28,7 +28,7 @@ NamesAndTypesList StorageSystemMarkedDroppedTables::getNamesAndTypes() } -void StorageSystemMarkedDroppedTables::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const +void StorageSystemDroppedTables::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const { auto tables_mark_dropped = DatabaseCatalog::instance().getTablesMarkedDropped(); diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.h b/src/Storages/System/StorageSystemDroppedTables.h similarity index 78% rename from src/Storages/System/StorageSystemMarkedDroppedTables.h rename to src/Storages/System/StorageSystemDroppedTables.h index ea2a864311c..44cc8122603 100644 --- a/src/Storages/System/StorageSystemMarkedDroppedTables.h +++ b/src/Storages/System/StorageSystemDroppedTables.h @@ -6,7 +6,7 @@ namespace DB { -class StorageSystemMarkedDroppedTables final : public IStorageSystemOneBlock +class StorageSystemDroppedTables final : public IStorageSystemOneBlock { public: std::string getName() const override { return "SystemMarkedDroppedTables"; } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index fd1cf2f1623..d6982ba30d5 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -79,7 +79,7 @@ #include #include #include -#include +#include #ifdef OS_LINUX #include @@ -141,7 +141,7 @@ void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database) attach(context, system_database, "time_zones"); attach(context, system_database, "backups"); attach(context, system_database, "schema_inference_cache"); - attach(context, system_database, "marked_dropped_tables"); + attach(context, system_database, "dropped_tables"); #ifdef OS_LINUX attach(context, system_database, "stack_trace"); #endif diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.reference b/tests/queries/0_stateless/25400_marked_dropped_tables.reference index 6fc5caff0cb..44906da9527 100644 --- a/tests/queries/0_stateless/25400_marked_dropped_tables.reference +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.reference @@ -1,4 +1,4 @@ -25400_marked_dropped_tables MergeTree +25400_dropped_tables MergeTree index UInt32 database String table String diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.sql b/tests/queries/0_stateless/25400_marked_dropped_tables.sql index 101642fa779..9bf6579b583 100644 --- a/tests/queries/0_stateless/25400_marked_dropped_tables.sql +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.sql @@ -1,11 +1,11 @@ -- Tags: no-ordinary-database SET database_atomic_wait_for_drop_and_detach_synchronously = 0; -DROP TABLE IF EXISTS 25400_marked_dropped_tables; +DROP TABLE IF EXISTS 25400_dropped_tables; -CREATE TABLE 25400_marked_dropped_tables (id Int32) Engine=MergeTree() ORDER BY id; -DROP TABLE 25400_marked_dropped_tables; +CREATE TABLE 25400_dropped_tables (id Int32) Engine=MergeTree() ORDER BY id; +DROP TABLE 25400_dropped_tables; -SELECT table, engine FROM system.marked_dropped_tables WHERE database = currentDatabase() LIMIT 1; -DESCRIBE TABLE system.marked_dropped_tables; +SELECT table, engine FROM system.dropped_tables WHERE database = currentDatabase() LIMIT 1; +DESCRIBE TABLE system.dropped_tables; From 2773e5c35241d9f1cb53cbe46e90d4124725b6cb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 27 Mar 2023 13:50:09 +0200 Subject: [PATCH 36/49] Fix test 02050_client_profile_events (#47951) --- src/Client/ClientBase.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index d3bac40d709..d3ba1d7e84c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1131,6 +1131,8 @@ void ClientBase::onProfileEvents(Block & block) { if (profile_events.watch.elapsedMilliseconds() >= profile_events.delay_ms) { + /// We need to restart the watch each time we flushed these events + profile_events.watch.restart(); initLogsOutputStream(); if (need_render_progress && tty_buf) progress_indication.clearProgressOutput(*tty_buf); @@ -1144,7 +1146,6 @@ void ClientBase::onProfileEvents(Block & block) incrementProfileEventsBlock(profile_events.last_block, block); } } - profile_events.watch.restart(); } } From 4234c38a64a0ba7a7c2c152592b7f857813b30f1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Mar 2023 14:45:49 +0200 Subject: [PATCH 37/49] Automatically correct some mistakes in the changelog --- docs/changelogs/v22.10.5.54-stable.md | 1 - docs/changelogs/v22.10.6.3-stable.md | 1 - docs/changelogs/v22.10.7.13-stable.md | 1 - docs/changelogs/v22.11.3.47-stable.md | 1 - docs/changelogs/v22.11.4.3-stable.md | 1 - docs/changelogs/v22.11.5.15-stable.md | 1 - docs/changelogs/v22.11.6.44-stable.md | 1 - docs/changelogs/v22.12.3.5-stable.md | 1 - docs/changelogs/v22.12.4.76-stable.md | 1 - docs/changelogs/v22.12.5.34-stable.md | 1 - docs/changelogs/v22.3.16.1190-lts.md | 1 - docs/changelogs/v22.3.18.37-lts.md | 1 - docs/changelogs/v22.3.19.6-lts.md | 1 - docs/changelogs/v22.8.12.45-lts.md | 1 - docs/changelogs/v22.8.13.20-lts.md | 1 - docs/changelogs/v22.8.14.53-lts.md | 1 - docs/changelogs/v22.8.15.23-lts.md | 1 - docs/changelogs/v23.1.1.3077-stable.md | 1 - docs/changelogs/v23.1.2.9-stable.md | 1 - docs/changelogs/v23.1.3.5-stable.md | 1 - docs/changelogs/v23.1.4.58-stable.md | 1 - docs/changelogs/v23.1.5.24-stable.md | 1 - docs/changelogs/v23.2.1.2537-stable.md | 1 - docs/changelogs/v23.2.2.20-stable.md | 1 - docs/changelogs/v23.2.3.17-stable.md | 1 - docs/changelogs/v23.2.4.12-stable.md | 1 - utils/changelog/changelog.py | 20 ++++++++++++++++++-- 27 files changed, 18 insertions(+), 28 deletions(-) diff --git a/docs/changelogs/v22.10.5.54-stable.md b/docs/changelogs/v22.10.5.54-stable.md index e372fb30618..aa1173dc671 100644 --- a/docs/changelogs/v22.10.5.54-stable.md +++ b/docs/changelogs/v22.10.5.54-stable.md @@ -41,4 +41,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.10.6.3-stable.md b/docs/changelogs/v22.10.6.3-stable.md index b0e88c92cb0..43ca032c774 100644 --- a/docs/changelogs/v22.10.6.3-stable.md +++ b/docs/changelogs/v22.10.6.3-stable.md @@ -10,4 +10,3 @@ sidebar_label: 2023 #### Bug Fix (user-visible misbehavior in official stable or prestable release) * Backported in [#45084](https://github.com/ClickHouse/ClickHouse/issues/45084): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.10.7.13-stable.md b/docs/changelogs/v22.10.7.13-stable.md index c906e00e524..46e876f36b4 100644 --- a/docs/changelogs/v22.10.7.13-stable.md +++ b/docs/changelogs/v22.10.7.13-stable.md @@ -18,4 +18,3 @@ sidebar_label: 2023 * Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.3.47-stable.md b/docs/changelogs/v22.11.3.47-stable.md index d6451b853f7..832591e68f3 100644 --- a/docs/changelogs/v22.11.3.47-stable.md +++ b/docs/changelogs/v22.11.3.47-stable.md @@ -37,4 +37,3 @@ sidebar_label: 2023 * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.11.4.3-stable.md b/docs/changelogs/v22.11.4.3-stable.md index 33780e848ef..fbf2baa9f7b 100644 --- a/docs/changelogs/v22.11.4.3-stable.md +++ b/docs/changelogs/v22.11.4.3-stable.md @@ -10,4 +10,3 @@ sidebar_label: 2023 #### Bug Fix (user-visible misbehavior in official stable or prestable release) * Backported in [#45085](https://github.com/ClickHouse/ClickHouse/issues/45085): fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). - diff --git a/docs/changelogs/v22.11.5.15-stable.md b/docs/changelogs/v22.11.5.15-stable.md index 742a8740514..6927dccf7b7 100644 --- a/docs/changelogs/v22.11.5.15-stable.md +++ b/docs/changelogs/v22.11.5.15-stable.md @@ -19,4 +19,3 @@ sidebar_label: 2023 * Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.11.6.44-stable.md b/docs/changelogs/v22.11.6.44-stable.md index 6e628b85150..cb7b2667667 100644 --- a/docs/changelogs/v22.11.6.44-stable.md +++ b/docs/changelogs/v22.11.6.44-stable.md @@ -34,4 +34,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.3.5-stable.md b/docs/changelogs/v22.12.3.5-stable.md index 8cbcbc6a590..2770de71f0c 100644 --- a/docs/changelogs/v22.12.3.5-stable.md +++ b/docs/changelogs/v22.12.3.5-stable.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.12.4.76-stable.md b/docs/changelogs/v22.12.4.76-stable.md index 79569ff841e..7f4334c565b 100644 --- a/docs/changelogs/v22.12.4.76-stable.md +++ b/docs/changelogs/v22.12.4.76-stable.md @@ -52,4 +52,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.12.5.34-stable.md b/docs/changelogs/v22.12.5.34-stable.md index 95befaa88ff..11613bd838b 100644 --- a/docs/changelogs/v22.12.5.34-stable.md +++ b/docs/changelogs/v22.12.5.34-stable.md @@ -26,4 +26,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.3.16.1190-lts.md b/docs/changelogs/v22.3.16.1190-lts.md index a43d34551ca..5f38dc32c16 100644 --- a/docs/changelogs/v22.3.16.1190-lts.md +++ b/docs/changelogs/v22.3.16.1190-lts.md @@ -30,4 +30,3 @@ sidebar_label: 2023 #### NO CL ENTRY * NO CL ENTRY: 'Fix multipart upload for large S3 object, backport to 22.3'. [#44217](https://github.com/ClickHouse/ClickHouse/pull/44217) ([ianton-ru](https://github.com/ianton-ru)). - diff --git a/docs/changelogs/v22.3.18.37-lts.md b/docs/changelogs/v22.3.18.37-lts.md index ff6378f09ad..65c05b35149 100644 --- a/docs/changelogs/v22.3.18.37-lts.md +++ b/docs/changelogs/v22.3.18.37-lts.md @@ -30,4 +30,3 @@ sidebar_label: 2023 * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). - diff --git a/docs/changelogs/v22.3.19.6-lts.md b/docs/changelogs/v22.3.19.6-lts.md index d5b45f4ce66..1cc4b9cc013 100644 --- a/docs/changelogs/v22.3.19.6-lts.md +++ b/docs/changelogs/v22.3.19.6-lts.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.12.45-lts.md b/docs/changelogs/v22.8.12.45-lts.md index 7412784419c..7fde9de07f2 100644 --- a/docs/changelogs/v22.8.12.45-lts.md +++ b/docs/changelogs/v22.8.12.45-lts.md @@ -39,4 +39,3 @@ sidebar_label: 2023 * Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). - diff --git a/docs/changelogs/v22.8.13.20-lts.md b/docs/changelogs/v22.8.13.20-lts.md index d8dd1bd2b1c..a2ed41fc97d 100644 --- a/docs/changelogs/v22.8.13.20-lts.md +++ b/docs/changelogs/v22.8.13.20-lts.md @@ -21,4 +21,3 @@ sidebar_label: 2023 * Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Merge pull request [#38262](https://github.com/ClickHouse/ClickHouse/issues/38262) from PolyProgrammist/fix-ordinary-system-un… [#45650](https://github.com/ClickHouse/ClickHouse/pull/45650) ([alesapin](https://github.com/alesapin)). - diff --git a/docs/changelogs/v22.8.14.53-lts.md b/docs/changelogs/v22.8.14.53-lts.md index 5978080fa3a..0b5170953eb 100644 --- a/docs/changelogs/v22.8.14.53-lts.md +++ b/docs/changelogs/v22.8.14.53-lts.md @@ -37,4 +37,3 @@ sidebar_label: 2023 * Add helping logging to auto-merge script [#46080](https://github.com/ClickHouse/ClickHouse/pull/46080) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix write buffer destruction order for vertical merge. [#46205](https://github.com/ClickHouse/ClickHouse/pull/46205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v22.8.15.23-lts.md b/docs/changelogs/v22.8.15.23-lts.md index 096a504c9c2..faa5dcd9860 100644 --- a/docs/changelogs/v22.8.15.23-lts.md +++ b/docs/changelogs/v22.8.15.23-lts.md @@ -25,4 +25,3 @@ sidebar_label: 2023 * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.1.3077-stable.md b/docs/changelogs/v23.1.1.3077-stable.md index e218be62f09..025d1242b11 100644 --- a/docs/changelogs/v23.1.1.3077-stable.md +++ b/docs/changelogs/v23.1.1.3077-stable.md @@ -589,4 +589,3 @@ sidebar_label: 2023 * Resubmit "Fix possible in-use table after DETACH" [#45566](https://github.com/ClickHouse/ClickHouse/pull/45566) ([Alexander Tokmakov](https://github.com/tavplubix)). * Typo: "Granulesis" --> "Granules" [#45598](https://github.com/ClickHouse/ClickHouse/pull/45598) ([Robert Schulze](https://github.com/rschu1ze)). * Fix version in autogenerated_versions.txt [#45624](https://github.com/ClickHouse/ClickHouse/pull/45624) ([Dmitry Novik](https://github.com/novikd)). - diff --git a/docs/changelogs/v23.1.2.9-stable.md b/docs/changelogs/v23.1.2.9-stable.md index 272a2b95a86..825481d8382 100644 --- a/docs/changelogs/v23.1.2.9-stable.md +++ b/docs/changelogs/v23.1.2.9-stable.md @@ -20,4 +20,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.3.5-stable.md b/docs/changelogs/v23.1.3.5-stable.md index d4f39894bec..bd4a59ac512 100644 --- a/docs/changelogs/v23.1.3.5-stable.md +++ b/docs/changelogs/v23.1.3.5-stable.md @@ -14,4 +14,3 @@ sidebar_label: 2023 #### NOT FOR CHANGELOG / INSIGNIFICANT * Get rid of progress timestamps in release publishing [#45818](https://github.com/ClickHouse/ClickHouse/pull/45818) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.4.58-stable.md b/docs/changelogs/v23.1.4.58-stable.md index d1ffe87f58e..035c55863a0 100644 --- a/docs/changelogs/v23.1.4.58-stable.md +++ b/docs/changelogs/v23.1.4.58-stable.md @@ -44,4 +44,3 @@ sidebar_label: 2023 * Fix dependencies for InstallPackagesTestAarch64 [#46597](https://github.com/ClickHouse/ClickHouse/pull/46597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Get rid of legacy DocsReleaseChecks [#46665](https://github.com/ClickHouse/ClickHouse/pull/46665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.1.5.24-stable.md b/docs/changelogs/v23.1.5.24-stable.md index 1c2c127a8c3..f490b92e725 100644 --- a/docs/changelogs/v23.1.5.24-stable.md +++ b/docs/changelogs/v23.1.5.24-stable.md @@ -25,4 +25,3 @@ sidebar_label: 2023 * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.1.2537-stable.md b/docs/changelogs/v23.2.1.2537-stable.md index 3fdcf6d6571..7feafe7af65 100644 --- a/docs/changelogs/v23.2.1.2537-stable.md +++ b/docs/changelogs/v23.2.1.2537-stable.md @@ -470,4 +470,3 @@ sidebar_label: 2023 #### Testing Improvement * Fixed functional test 00304_http_external_data for s390x. [#45807](https://github.com/ClickHouse/ClickHouse/pull/45807) ([Harry Lee](https://github.com/HarryLeeIBM)). - diff --git a/docs/changelogs/v23.2.2.20-stable.md b/docs/changelogs/v23.2.2.20-stable.md index 60aeaa66cbf..ae41fb24100 100644 --- a/docs/changelogs/v23.2.2.20-stable.md +++ b/docs/changelogs/v23.2.2.20-stable.md @@ -27,4 +27,3 @@ sidebar_label: 2023 * More concise logging at trace level for PREWHERE steps [#46771](https://github.com/ClickHouse/ClickHouse/pull/46771) ([Alexander Gololobov](https://github.com/davenger)). * Reduce updates of Mergeable Check [#46781](https://github.com/ClickHouse/ClickHouse/pull/46781) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.3.17-stable.md b/docs/changelogs/v23.2.3.17-stable.md index fb2c4e394dc..1735c0e5925 100644 --- a/docs/changelogs/v23.2.3.17-stable.md +++ b/docs/changelogs/v23.2.3.17-stable.md @@ -20,4 +20,3 @@ sidebar_label: 2023 * Use /etc/default/clickhouse in systemd too [#47003](https://github.com/ClickHouse/ClickHouse/pull/47003) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * do flushUntrackedMemory when context switches [#47102](https://github.com/ClickHouse/ClickHouse/pull/47102) ([Sema Checherinda](https://github.com/CheSema)). * Update typing for a new PyGithub version [#47123](https://github.com/ClickHouse/ClickHouse/pull/47123) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/docs/changelogs/v23.2.4.12-stable.md b/docs/changelogs/v23.2.4.12-stable.md index 2b6a689aee5..071fde68298 100644 --- a/docs/changelogs/v23.2.4.12-stable.md +++ b/docs/changelogs/v23.2.4.12-stable.md @@ -17,4 +17,3 @@ sidebar_label: 2023 * Follow-up to [#46681](https://github.com/ClickHouse/ClickHouse/issues/46681) [#47284](https://github.com/ClickHouse/ClickHouse/pull/47284) ([Alexander Tokmakov](https://github.com/tavplubix)). * Add a manual trigger for release workflow [#47302](https://github.com/ClickHouse/ClickHouse/pull/47302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 3b66b68193b..6b70952eced 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -126,8 +126,8 @@ def get_descriptions(prs: PullRequests) -> Dict[str, List[Description]]: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Generate a changelog in MD format between given tags. " - "It fetches all tags and unshallow the git repositore automatically", + description="Generate a changelog in Markdown format between given tags. " + "It fetches all tags and unshallow the git repository automatically", ) parser.add_argument( "-v", @@ -243,6 +243,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri else: i += 1 + # Remove excessive bullets from the entry. + if re.match(r"^[\-\*] ", entry): + entry = entry[2:] + + # Better style. + if re.match(r"^[a-z]", entry): + entry = entry.capitalize() + if not category: # Shouldn't happen, because description check in CI should catch such PRs. # Fall through, so that it shows up in output and the user can fix it. @@ -256,6 +264,14 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri category = "NOT FOR CHANGELOG / INSIGNIFICANT" return Description(item.number, item.user, item.html_url, item.title, category) + # Normalize bug fixes + if re.match( + r"(?i)bug\Wfix", + category, + ): + category = "Bug Fix (user-visible misbehavior in an official stable release)" + return Description(item.number, item.user, item.html_url, item.title, category) + # Filter out documentations changelog if re.match( r"(?i)doc", From 64db4c478b76e179d067806795a3353d10372410 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 27 Mar 2023 13:19:45 +0000 Subject: [PATCH 38/49] Use one map --- src/Interpreters/ProcessList.cpp | 19 ++++++++++--------- src/Interpreters/ProcessList.h | 5 ++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index a86377a37d5..d66d4bdea64 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -362,7 +362,7 @@ QueryStatus::~QueryStatus() { #if !defined(NDEBUG) /// Check that all executors were invalidated. - for (const auto & e : executors) + for (const auto & [_, e] : executors) assert(!e->executor); #endif @@ -400,7 +400,9 @@ CancellationCode QueryStatus::cancelQuery(bool) { /// Create a snapshot of executors under a mutex. std::lock_guard lock(executors_mutex); - executors_snapshot = executors; + executors_snapshot.reserve(executors.size()); + for (const auto & [_, e] : executors) + executors_snapshot.push_back(e); } /// We should call cancel() for each executor with unlocked executors_mutex, because @@ -428,9 +430,8 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(!executor_indexes.contains(e)); - executors.push_back(std::make_shared(e)); - executor_indexes[e] = executors.size() - 1; + assert(!executors.contains(e)); + executors[e] = std::make_shared(e); } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) @@ -439,12 +440,12 @@ void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { std::lock_guard lock(executors_mutex); - assert(executor_indexes.contains(e)); - executor_holder = executors[executor_indexes[e]]; - executor_indexes.erase(e); + assert(executors.contains(e)); + executor_holder = executors[e]; + executors.erase(e); } - /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) + /// Invalidate executor pointer inside holder. /// We should do it with released executors_mutex to avoid possible lock order inversion. executor_holder->remove(); } diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 9e5c91bf2ed..a04beac4901 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -133,9 +133,8 @@ protected: using ExecutorHolderPtr = std::shared_ptr; - /// Array of PipelineExecutors to be cancelled when a cancelQuery is received - std::vector executors; - std::unordered_map executor_indexes; + /// Container of PipelineExecutors to be cancelled when a cancelQuery is received + std::unordered_map executors; enum QueryStreamsStatus { From 1845bfc6446f033475ae18799e20af4daface821 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 14:13:32 +0200 Subject: [PATCH 39/49] Revert change to ld64.lld linker on Mac This reverts PRs #42470, #47673 and #47744. The problem was that after the switch to ld64.lld, server binaries build in Debug mode no longer came up, see the discussion after (*). My attempt to fix it with `-no_compact_unwind` didn't help. I also tried - `-keep_dwarf_unwind`, - `-unwindlib` (to use the OS unwinder), - the unwinder in contrib (CMake variable EXCEPTION_HANDLING_LIBRARY) but w/o success. Just tons of wasted time. Rolling back to lld as linker on Mac. This will bring back many linker warnings (#42282) but I rather accept that (temporarily, maybe someone can figure out how to fix them) than have a broken Debug binary. (*) https://github.com/ClickHouse/ClickHouse/pull/42470#issuecomment-1312344068 --- CMakeLists.txt | 14 ++------------ cmake/tools.cmake | 6 +----- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5550a19b699..bd7de46474b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ endif () if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. - if (LINKER_NAME MATCHES "lld" AND OS_LINUX) + if (LINKER_NAME MATCHES "lld") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") message (STATUS "Adding .gdb-index via --gdb-index linker option.") endif () @@ -212,7 +212,7 @@ endif () # Create BuildID when using lld. For other linkers it is created by default. # (NOTE: LINKER_NAME can be either path or name, and in different variants) -if (LINKER_NAME MATCHES "lld" AND OS_LINUX) +if (LINKER_NAME MATCHES "lld") # SHA1 is not cryptographically secure but it is the best what lld is offering. set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") endif () @@ -349,17 +349,7 @@ set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_I if (COMPILER_CLANG) if (OS_DARWIN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main") - - # The LLVM MachO linker (ld64.lld, used in native builds) generates by default unwind info in 'compact' format which the internal - # unwinder doesn't support and the server will not come up ('invalid compact unwind encoding'). Disable it. You will see warning - # during the build "ld64.lld: warning: Option `-no_compact_unwind' is undocumented. Should lld implement it?". Yes, ld64.lld does - # not document the option, likely for compat with Apple's system ld after which ld64.lld is modeled after and which also does not - # document it. - if (NOT CMAKE_CROSSCOMPILING) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_compact_unwind") - endif () endif() # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 974b0bd1d3d..3572134d89f 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -61,17 +61,13 @@ if (NOT LINKER_NAME) if (COMPILER_GCC) find_program (LLD_PATH NAMES "ld.lld") elseif (COMPILER_CLANG) - # llvm lld is a generic driver. - # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead if (OS_LINUX) if (NOT ARCH_S390X) # s390x doesnt support lld find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld") endif () - elseif (OS_DARWIN) - find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld") endif () endif () - if (OS_LINUX OR OS_DARWIN) + if (OS_LINUX) if (LLD_PATH) if (COMPILER_GCC) # GCC driver requires one of supported linker names like "lld". From f1258a932fcaf34e5a6241501ffc9d5519f1fe72 Mon Sep 17 00:00:00 2001 From: nellicus Date: Mon, 27 Mar 2023 16:58:20 +0200 Subject: [PATCH 40/49] fix typo --- docs/en/operations/external-authenticators/ldap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index fa44e6e2978..ee2336e9378 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -120,7 +120,7 @@ Query: CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server'; ``` -## LDAP Exernal User Directory {#ldap-external-user-directory} +## LDAP External User Directory {#ldap-external-user-directory} In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. To achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. From eac71397c4bb68b20bb071582ba33b71de84c60d Mon Sep 17 00:00:00 2001 From: rfraposa Date: Mon, 27 Mar 2023 09:19:50 -0600 Subject: [PATCH 41/49] Create marked_dropped_tables.md --- .../system-tables/marked_dropped_tables.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/en/operations/system-tables/marked_dropped_tables.md diff --git a/docs/en/operations/system-tables/marked_dropped_tables.md b/docs/en/operations/system-tables/marked_dropped_tables.md new file mode 100644 index 00000000000..a260aea1c5e --- /dev/null +++ b/docs/en/operations/system-tables/marked_dropped_tables.md @@ -0,0 +1,37 @@ +--- +slug: /en/operations/system-tables/marked_dropped_tables +--- +# marked_dropped_tables + +Contains information about tables that drop table has been executed but data cleanup has not been actually performed. + +Columns: + +- `index` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Index in marked_dropped_tables queue. +- `database` ([String](../../sql-reference/data-types/string.md)) — Database. +- `table` ([String](../../sql-reference/data-types/string.md)) — Table name. +- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid. +- `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name. +- `metadata_dropped_path` ([String](../../sql-reference/data-types/string.md)) — Path of table's metadata file in metadate_dropped directory. +- `table_dropped_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — The time when the next attempt to remove table's data is scheduled on. Usually it's the table when the table was dropped plus `database_atomic_delay_before_drop_table_sec` + +**Example** + +The following example shows how to get information about marked_dropped_tables. + +``` sql +SELECT * +FROM system.marked_dropped_tables\G +``` + +``` text +Row 1: +────── +index: 0 +database: default +table: test +uuid: 03141bb2-e97a-4d7c-a172-95cc066bb3bd +engine: MergeTree +metadata_dropped_path: /data/ClickHouse/build/programs/data/metadata_dropped/default.test.03141bb2-e97a-4d7c-a172-95cc066bb3bd.sql +table_dropped_time: 2023-03-16 23:43:31 +``` \ No newline at end of file From 246316dd49c5eca803e67b6018eea03756f3b4e1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 15:23:30 +0000 Subject: [PATCH 42/49] Docs: Document [FULL] keyword in SHOW TABLES Follow-up to #43910 --- docs/en/sql-reference/statements/show.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index a9f0aedccdf..7d5e2845c4a 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -117,7 +117,7 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" Displays a list of tables. ```sql -SHOW [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` If the `FROM` clause is not specified, the query returns the list of tables from the current database. @@ -521,4 +521,4 @@ Outputs the content of the [system.table_engines](../../operations/system-tables **See Also** -- [system.table_engines](../../operations/system-tables/table_engines.md) table \ No newline at end of file +- [system.table_engines](../../operations/system-tables/table_engines.md) table From 15039d33271acfb93c393c3165a72cdf3beac259 Mon Sep 17 00:00:00 2001 From: rfraposa Date: Mon, 27 Mar 2023 09:36:56 -0600 Subject: [PATCH 43/49] Rename file --- .../system-tables/{marked_dropped_tables.md => dropped_tables.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/en/operations/system-tables/{marked_dropped_tables.md => dropped_tables.md} (100%) diff --git a/docs/en/operations/system-tables/marked_dropped_tables.md b/docs/en/operations/system-tables/dropped_tables.md similarity index 100% rename from docs/en/operations/system-tables/marked_dropped_tables.md rename to docs/en/operations/system-tables/dropped_tables.md From 1b63b0b043b0cd53569329d5089d29836f2200f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 27 Mar 2023 17:51:22 +0200 Subject: [PATCH 44/49] Fix stateless tests numbers --- ..._bug.reference => 02695_logical_optimizer_alias_bug.reference} | 0 ...imizer_alias_bug.sql => 02695_logical_optimizer_alias_bug.sql} | 0 ...erence => 02695_storage_join_insert_select_deadlock.reference} | 0 ...deadlock.sql => 02695_storage_join_insert_select_deadlock.sql} | 0 ...checksums.reference => 02696_inverted_idx_checksums.reference} | 0 ...nverted_idx_checksums.sql => 02696_inverted_idx_checksums.sql} | 0 ...cel.reference => 02697_stop_reading_on_first_cancel.reference} | 0 ...g_on_first_cancel.sh => 02697_stop_reading_on_first_cancel.sh} | 0 ...ped_tables.reference => 02698_marked_dropped_tables.reference} | 0 ..._marked_dropped_tables.sql => 02698_marked_dropped_tables.sql} | 0 ...p.reference => 02699_polygons_sym_difference_rollup.reference} | 0 ...erence_rollup.sql => 02699_polygons_sym_difference_rollup.sql} | 0 ..._regexp_operator.reference => 02700_regexp_operator.reference} | 0 .../{25401_regexp_operator.sql => 02700_regexp_operator.sql} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{25340_logical_optimizer_alias_bug.reference => 02695_logical_optimizer_alias_bug.reference} (100%) rename tests/queries/0_stateless/{25340_logical_optimizer_alias_bug.sql => 02695_logical_optimizer_alias_bug.sql} (100%) rename tests/queries/0_stateless/{25340_storage_join_insert_select_deadlock.reference => 02695_storage_join_insert_select_deadlock.reference} (100%) rename tests/queries/0_stateless/{25340_storage_join_insert_select_deadlock.sql => 02695_storage_join_insert_select_deadlock.sql} (100%) rename tests/queries/0_stateless/{25341_inverted_idx_checksums.reference => 02696_inverted_idx_checksums.reference} (100%) rename tests/queries/0_stateless/{25341_inverted_idx_checksums.sql => 02696_inverted_idx_checksums.sql} (100%) rename tests/queries/0_stateless/{25341_stop_reading_on_first_cancel.reference => 02697_stop_reading_on_first_cancel.reference} (100%) rename tests/queries/0_stateless/{25341_stop_reading_on_first_cancel.sh => 02697_stop_reading_on_first_cancel.sh} (100%) rename tests/queries/0_stateless/{25400_marked_dropped_tables.reference => 02698_marked_dropped_tables.reference} (100%) rename tests/queries/0_stateless/{25400_marked_dropped_tables.sql => 02698_marked_dropped_tables.sql} (100%) rename tests/queries/0_stateless/{25401_polygons_sym_difference_rollup.reference => 02699_polygons_sym_difference_rollup.reference} (100%) rename tests/queries/0_stateless/{25401_polygons_sym_difference_rollup.sql => 02699_polygons_sym_difference_rollup.sql} (100%) rename tests/queries/0_stateless/{25401_regexp_operator.reference => 02700_regexp_operator.reference} (100%) rename tests/queries/0_stateless/{25401_regexp_operator.sql => 02700_regexp_operator.sql} (100%) diff --git a/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference b/tests/queries/0_stateless/02695_logical_optimizer_alias_bug.reference similarity index 100% rename from tests/queries/0_stateless/25340_logical_optimizer_alias_bug.reference rename to tests/queries/0_stateless/02695_logical_optimizer_alias_bug.reference diff --git a/tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql b/tests/queries/0_stateless/02695_logical_optimizer_alias_bug.sql similarity index 100% rename from tests/queries/0_stateless/25340_logical_optimizer_alias_bug.sql rename to tests/queries/0_stateless/02695_logical_optimizer_alias_bug.sql diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference b/tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.reference similarity index 100% rename from tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference rename to tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.reference diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql b/tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.sql similarity index 100% rename from tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql rename to tests/queries/0_stateless/02695_storage_join_insert_select_deadlock.sql diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.reference b/tests/queries/0_stateless/02696_inverted_idx_checksums.reference similarity index 100% rename from tests/queries/0_stateless/25341_inverted_idx_checksums.reference rename to tests/queries/0_stateless/02696_inverted_idx_checksums.reference diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.sql b/tests/queries/0_stateless/02696_inverted_idx_checksums.sql similarity index 100% rename from tests/queries/0_stateless/25341_inverted_idx_checksums.sql rename to tests/queries/0_stateless/02696_inverted_idx_checksums.sql diff --git a/tests/queries/0_stateless/25341_stop_reading_on_first_cancel.reference b/tests/queries/0_stateless/02697_stop_reading_on_first_cancel.reference similarity index 100% rename from tests/queries/0_stateless/25341_stop_reading_on_first_cancel.reference rename to tests/queries/0_stateless/02697_stop_reading_on_first_cancel.reference diff --git a/tests/queries/0_stateless/25341_stop_reading_on_first_cancel.sh b/tests/queries/0_stateless/02697_stop_reading_on_first_cancel.sh similarity index 100% rename from tests/queries/0_stateless/25341_stop_reading_on_first_cancel.sh rename to tests/queries/0_stateless/02697_stop_reading_on_first_cancel.sh diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.reference b/tests/queries/0_stateless/02698_marked_dropped_tables.reference similarity index 100% rename from tests/queries/0_stateless/25400_marked_dropped_tables.reference rename to tests/queries/0_stateless/02698_marked_dropped_tables.reference diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.sql b/tests/queries/0_stateless/02698_marked_dropped_tables.sql similarity index 100% rename from tests/queries/0_stateless/25400_marked_dropped_tables.sql rename to tests/queries/0_stateless/02698_marked_dropped_tables.sql diff --git a/tests/queries/0_stateless/25401_polygons_sym_difference_rollup.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference similarity index 100% rename from tests/queries/0_stateless/25401_polygons_sym_difference_rollup.reference rename to tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference diff --git a/tests/queries/0_stateless/25401_polygons_sym_difference_rollup.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql similarity index 100% rename from tests/queries/0_stateless/25401_polygons_sym_difference_rollup.sql rename to tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql diff --git a/tests/queries/0_stateless/25401_regexp_operator.reference b/tests/queries/0_stateless/02700_regexp_operator.reference similarity index 100% rename from tests/queries/0_stateless/25401_regexp_operator.reference rename to tests/queries/0_stateless/02700_regexp_operator.reference diff --git a/tests/queries/0_stateless/25401_regexp_operator.sql b/tests/queries/0_stateless/02700_regexp_operator.sql similarity index 100% rename from tests/queries/0_stateless/25401_regexp_operator.sql rename to tests/queries/0_stateless/02700_regexp_operator.sql From 86f682855687cda2494acf33fa9cc63183da0883 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 27 Mar 2023 15:56:31 +0000 Subject: [PATCH 45/49] Docs: Update syntax of some SHOW queries --- docs/en/sql-reference/statements/show.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 7d5e2845c4a..544c556d4b3 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -21,13 +21,13 @@ Note that if you use this statement to get `CREATE` query of system tables, you Prints a list of all databases. ```sql -SHOW DATABASES [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] +SHOW DATABASES [[NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] ``` This statement is identical to the query: ```sql -SELECT name FROM system.databases [WHERE name LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] +SELECT name FROM system.databases [WHERE name [NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE filename] [FORMAT format] ``` ### Examples @@ -117,7 +117,7 @@ $ watch -n1 "clickhouse-client --query='SHOW PROCESSLIST'" Displays a list of tables. ```sql -SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SHOW [FULL] [TEMPORARY] TABLES [{FROM | IN} ] [[NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` If the `FROM` clause is not specified, the query returns the list of tables from the current database. @@ -125,7 +125,7 @@ If the `FROM` clause is not specified, the query returns the list of tables from This statement is identical to the query: ```sql -SELECT name FROM system.tables [WHERE name LIKE | ILIKE | NOT LIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] +SELECT name FROM system.tables [WHERE name [NOT] LIKE | ILIKE ''] [LIMIT ] [INTO OUTFILE ] [FORMAT ] ``` ### Examples @@ -370,7 +370,7 @@ Returns a list of clusters. All available clusters are listed in the [system.clu ``` sql SHOW CLUSTER '' -SHOW CLUSTERS [LIKE|NOT LIKE ''] [LIMIT ] +SHOW CLUSTERS [[NOT] LIKE|ILIKE ''] [LIMIT ] ``` ### Examples From feac46eac4b06ff21addd4985559dc7b53e28c2e Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Mon, 27 Mar 2023 11:16:28 -0600 Subject: [PATCH 46/49] Update dropped_tables.md (#48067) * Update dropped_tables.md * Update dropped_tables.md * Update docs/en/operations/system-tables/dropped_tables.md --------- Co-authored-by: Alexander Tokmakov --- docs/en/operations/system-tables/dropped_tables.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/system-tables/dropped_tables.md b/docs/en/operations/system-tables/dropped_tables.md index a260aea1c5e..cb6cec0035a 100644 --- a/docs/en/operations/system-tables/dropped_tables.md +++ b/docs/en/operations/system-tables/dropped_tables.md @@ -1,7 +1,7 @@ --- -slug: /en/operations/system-tables/marked_dropped_tables +slug: /en/operations/system-tables/dropped_tables --- -# marked_dropped_tables +# dropped_tables Contains information about tables that drop table has been executed but data cleanup has not been actually performed. @@ -17,11 +17,11 @@ Columns: **Example** -The following example shows how to get information about marked_dropped_tables. +The following example shows how to get information about dropped_tables. ``` sql SELECT * -FROM system.marked_dropped_tables\G +FROM system.dropped_tables\G ``` ``` text @@ -34,4 +34,4 @@ uuid: 03141bb2-e97a-4d7c-a172-95cc066bb3bd engine: MergeTree metadata_dropped_path: /data/ClickHouse/build/programs/data/metadata_dropped/default.test.03141bb2-e97a-4d7c-a172-95cc066bb3bd.sql table_dropped_time: 2023-03-16 23:43:31 -``` \ No newline at end of file +``` From 98488f30ab25062e789acf40b6a8d13cc6b1e0cb Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 27 Mar 2023 14:40:45 -0300 Subject: [PATCH 47/49] Update settings.md --- docs/en/operations/settings/settings.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f3c0f20f3a6..eb94836b3ba 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -988,6 +988,17 @@ The interval in microseconds for checking whether request execution has been can Default value: 100,000 (checks for cancelling and sends the progress ten times per second). +## idle_connection_timeout {#idle_connection_timeout} + +Timeout to close idle TCP connections after specified number of seconds. + +Possible values: + +- Positive integer. +- 0 — Never. + +Default value: 3600. + ## connect_timeout, receive_timeout, send_timeout {#connect-timeout-receive-timeout-send-timeout} Timeouts in seconds on the socket used for communicating with the client. From 295d3b751013f0f946e7d189d9054bfdef6d3ad4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 27 Mar 2023 14:45:43 -0300 Subject: [PATCH 48/49] Update settings.md --- docs/en/operations/settings/settings.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index eb94836b3ba..4e5ae6d63cf 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -994,8 +994,7 @@ Timeout to close idle TCP connections after specified number of seconds. Possible values: -- Positive integer. -- 0 — Never. +- Positive integer (0 - close immediatly, after 0 seconds). Default value: 3600. From 6d8a2bbd48d7d753e52b9cfcc75b03bdb9b63dae Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Mon, 27 Mar 2023 14:54:05 -0400 Subject: [PATCH 49/49] standardize admonitions --- .../database-engines/materialized-mysql.md | 4 +-- .../materialized-postgresql.md | 2 +- docs/en/engines/database-engines/mysql.md | 2 +- .../table-engines/integrations/hdfs.md | 4 +-- .../table-engines/integrations/kafka.md | 2 +- .../integrations/materialized-postgresql.md | 2 +- .../table-engines/integrations/postgresql.md | 2 +- .../engines/table-engines/integrations/s3.md | 4 +-- .../mergetree-family/aggregatingmergetree.md | 2 +- .../mergetree-family/collapsingmergetree.md | 2 +- .../custom-partitioning-key.md | 4 +-- .../mergetree-family/graphitemergetree.md | 6 ++-- .../mergetree-family/invertedindexes.md | 2 +- .../mergetree-family/mergetree.md | 4 +-- .../mergetree-family/replacingmergetree.md | 4 +-- .../mergetree-family/replication.md | 2 +- .../mergetree-family/summingmergetree.md | 2 +- .../versionedcollapsingmergetree.md | 2 +- .../engines/table-engines/special/buffer.md | 2 +- docs/en/engines/table-engines/special/file.md | 2 +- .../example-datasets/star-schema.md | 2 +- docs/en/interfaces/formats.md | 28 +++++++++---------- docs/en/interfaces/http.md | 4 +-- docs/en/interfaces/postgresql.md | 2 +- .../third-party/client-libraries.md | 2 +- .../en/interfaces/third-party/integrations.md | 2 +- .../external-authenticators/kerberos.md | 6 ++-- docs/en/operations/opentelemetry.md | 2 +- docs/en/operations/query-cache.md | 2 +- .../settings.md | 4 +-- .../settings/merge-tree-settings.md | 2 +- .../operations/settings/settings-formats.md | 2 +- docs/en/operations/settings/settings-users.md | 2 +- docs/en/operations/settings/settings.md | 20 ++++++------- docs/en/operations/storing-data.md | 2 +- docs/en/operations/system-tables/parts.md | 2 +- .../operations/utilities/clickhouse-copier.md | 2 +- .../parametric-functions.md | 4 +-- docs/en/sql-reference/data-types/float.md | 2 +- docs/en/sql-reference/data-types/json.md | 4 +-- .../data-types/special-data-types/interval.md | 2 +- docs/en/sql-reference/dictionaries/index.md | 6 ++-- docs/en/sql-reference/distributed-ddl.md | 4 +-- .../sql-reference/functions/hash-functions.md | 4 +-- .../sql-reference/functions/introspection.md | 2 +- .../sql-reference/functions/nlp-functions.md | 2 +- .../functions/other-functions.md | 8 +++--- docs/en/sql-reference/operators/exists.md | 2 +- docs/en/sql-reference/operators/index.md | 2 +- .../sql-reference/statements/alter/column.md | 2 +- .../statements/alter/constraint.md | 2 +- .../statements/create/row-policy.md | 2 +- .../sql-reference/statements/create/table.md | 8 +++--- .../sql-reference/statements/create/user.md | 6 ++-- docs/en/sql-reference/statements/optimize.md | 2 +- docs/en/sql-reference/statements/system.md | 2 +- docs/en/sql-reference/statements/watch.md | 4 +-- docs/en/sql-reference/table-functions/file.md | 2 +- docs/en/sql-reference/table-functions/hdfs.md | 2 +- .../table-functions/hdfsCluster.md | 2 +- .../sql-reference/table-functions/iceberg.md | 2 +- .../en/sql-reference/table-functions/index.md | 2 +- docs/en/sql-reference/table-functions/s3.md | 2 +- .../table-functions/s3Cluster.md | 2 +- 64 files changed, 113 insertions(+), 113 deletions(-) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index 899c8d024f1..45719b1340e 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -6,7 +6,7 @@ sidebar_position: 70 # [experimental] MaterializedMySQL -:::warning +:::note This is an experimental feature that should not be used in production. ::: @@ -245,7 +245,7 @@ extra care needs to be taken. You may specify overrides for tables that do not exist yet. -:::warning +:::important It is easy to break replication with table overrides if not used with care. For example: * If an ALIAS column is added with a table override, and a column with the same name is later added to the source diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index b43f71a7576..32e3435afa2 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -145,7 +145,7 @@ FROM pg_class WHERE oid = 'postgres_table'::regclass; ``` -:::warning +:::note Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used. ::: diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index e4ff734d55f..7c8c3459ec5 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -60,7 +60,7 @@ These variables are supported: - `version` - `max_allowed_packet` -:::warning +:::note By now these variables are stubs and don't correspond to anything. ::: diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index 7c04a6594a6..1497ea47eca 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -64,7 +64,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2 - Indexes. - [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is possible, but not recommended. - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: @@ -110,7 +110,7 @@ Table consists of all the files in both directories (all files should satisfy fo CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV') ``` -:::warning +:::note If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 255ba06f056..e2a7304dc59 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -102,7 +102,7 @@ Examples: Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/integrations/materialized-postgresql.md b/docs/en/engines/table-engines/integrations/materialized-postgresql.md index 11e7928c3ed..3920b402a49 100644 --- a/docs/en/engines/table-engines/integrations/materialized-postgresql.md +++ b/docs/en/engines/table-engines/integrations/materialized-postgresql.md @@ -52,6 +52,6 @@ PRIMARY KEY key; SELECT key, value, _version FROM postgresql_db.postgresql_replica; ``` -:::warning +:::note Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used. ::: diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 18e884f3bcc..2222d1fc016 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -74,7 +74,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp PostgreSQL `Array` types are converted into ClickHouse arrays. -:::warning +:::note Be careful - in PostgreSQL an array data, created like a `type_name[]`, may contain multi-dimensional arrays of different dimensions in different table rows in same column. But in ClickHouse it is only allowed to have multidimensional arrays of the same count of dimensions in all table rows in same column. ::: diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index dd843945e10..2901edb9f35 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -63,7 +63,7 @@ For more information about virtual columns see [here](../../../engines/table-eng - Indexes. - [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is possible, but not supported. - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: @@ -78,7 +78,7 @@ For more information about virtual columns see [here](../../../engines/table-eng Constructions with `{}` are similar to the [remote](../../../sql-reference/table-functions/remote.md) table function. -:::warning +:::note If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md index 9677f75a358..6591f666244 100644 --- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -43,7 +43,7 @@ When creating an `AggregatingMergeTree` table the same [clauses](../../../engine Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch the old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md index 0bd665116f0..7e16f4926db 100644 --- a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -45,7 +45,7 @@ When creating a `CollapsingMergeTree` table, the same [query clauses](../../../e Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index b1e79c4c3fd..f5b8436fdfe 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -6,7 +6,7 @@ sidebar_label: Custom Partitioning Key # Custom Partitioning Key -:::warning +:::note In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. You should never use too granular of partitioning. Don't partition your data by client identifiers or names. Instead, make a client identifier or name the first column in the ORDER BY expression. @@ -159,7 +159,7 @@ FROM session_log GROUP BY UserID; ``` -:::warning +:::note Performance of such a query heavily depends on the table layout. Because of that the optimisation is not enabled by default. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 104ec049ec4..9577c8dc936 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -55,7 +55,7 @@ When creating a `GraphiteMergeTree` table, the same [clauses](../../../engines/t Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: @@ -129,7 +129,7 @@ default ... ``` -:::warning +:::important Patterns must be strictly ordered: 1. Patterns without `function` or `retention`. @@ -263,6 +263,6 @@ Valid values: ``` -:::warning +:::note Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). ::: diff --git a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md index aa11258dc4a..701615495de 100644 --- a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md @@ -15,7 +15,7 @@ tokenized cells of the string column. For example, the string cell "I will be a " wi", "wil", "ill", "ll ", "l b", " be" etc. The more fine-granular the input strings are tokenized, the bigger but also the more useful the resulting inverted index will be. -:::warning +:::note Inverted indexes are experimental and should not be used in production environments yet. They may change in the future in backward-incompatible ways, for example with respect to their DDL/DQL syntax or performance/compression characteristics. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index f1b7a40094d..fa97579c33e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -192,7 +192,7 @@ The `index_granularity` setting can be omitted because 8192 is the default value Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: @@ -1092,7 +1092,7 @@ Other parameters: Examples of working configurations can be found in integration tests directory (see e.g. [test_merge_tree_azure_blob_storage](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml) or [test_azure_blob_storage_zero_copy_replication](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_azure_blob_storage_zero_copy_replication/configs/config.d/storage_conf.xml)). - :::warning Zero-copy replication is not ready for production + :::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index f5d81182898..8351a31db55 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -30,7 +30,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of request parameters, see [statement description](../../../sql-reference/statements/create/table.md). -:::warning +:::note Uniqueness of rows is determined by the `ORDER BY` table section, not `PRIMARY KEY`. ::: @@ -96,7 +96,7 @@ When creating a `ReplacingMergeTree` table the same [clauses](../../../engines/t Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index c50433f2aeb..b2b967c685a 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -43,7 +43,7 @@ ClickHouse uses [ClickHouse Keeper](/docs/en/guides/sre/keeper/index.md) for sto To use replication, set parameters in the [zookeeper](/docs/en/operations/server-configuration-parameters/settings.md/#server-settings_zookeeper) server configuration section. -:::warning +:::note Don’t neglect the security setting. ClickHouse supports the `digest` [ACL scheme](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) of the ZooKeeper security subsystem. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index b2b6272c58e..d0078656b5d 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -44,7 +44,7 @@ When creating a `SummingMergeTree` table the same [clauses](../../../engines/tab Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects and, if possible, switch the old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 2891907f79a..74ac9c97fc0 100644 --- a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -58,7 +58,7 @@ When creating a `VersionedCollapsingMergeTree` table, the same [clauses](../../. Deprecated Method for Creating a Table -:::warning +:::note Do not use this method in new projects. If possible, switch old projects to the method described above. ::: diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 5f28fafbc26..f7d84b9b452 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -86,7 +86,7 @@ If the set of columns in the Buffer table does not match the set of columns in a If the types do not match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log, and the buffer is cleared. The same happens if the subordinate table does not exist when the buffer is flushed. -:::warning +:::note Running ALTER on the Buffer table in releases made before 26 Oct 2021 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117) and [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565)), so deleting the Buffer table and then recreating is the only option. Check that this error is fixed in your release before trying to run ALTER on the Buffer table. ::: diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md index 8314c511236..e99b1c83cbc 100644 --- a/docs/en/engines/table-engines/special/file.md +++ b/docs/en/engines/table-engines/special/file.md @@ -31,7 +31,7 @@ When creating table using `File(Format)` it creates empty subdirectory in that f You may manually create this subfolder and file in server filesystem and then [ATTACH](../../../sql-reference/statements/attach.md) it to table information with matching name, so you can query data from that file. -:::warning +:::note Be careful with this functionality, because ClickHouse does not keep track of external changes to such files. The result of simultaneous writes via ClickHouse and outside of ClickHouse is undefined. ::: diff --git a/docs/en/getting-started/example-datasets/star-schema.md b/docs/en/getting-started/example-datasets/star-schema.md index 1702be70410..72ced87ef55 100644 --- a/docs/en/getting-started/example-datasets/star-schema.md +++ b/docs/en/getting-started/example-datasets/star-schema.md @@ -18,7 +18,7 @@ $ make Generating data: -:::warning +:::note With `-s 100` dbgen generates 600 million rows (67 GB), while while `-s 1000` it generates 6 billion rows (which takes a lot of time) ::: diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ae3756d5d41..8430946a6c6 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -205,7 +205,7 @@ Differs from the `TabSeparated` format in that the column names are written in t During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness. -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from the input data will be mapped to the columns of the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -217,7 +217,7 @@ This format is also available under the name `TSVWithNames`. Differs from the `TabSeparated` format in that the column names are written to the first row, while the column types are in the second row. -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from the input data will be mapped to the columns in the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -470,7 +470,7 @@ The CSV format supports the output of totals and extremes the same way as `TabSe Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -480,7 +480,7 @@ Otherwise, the first row will be skipped. Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -500,7 +500,7 @@ There is also `CustomSeparatedIgnoreSpaces` format, which is similar to [Templat Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -510,7 +510,7 @@ Otherwise, the first row will be skipped. Also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -969,7 +969,7 @@ Differs from `JSONEachRow`/`JSONStringsEachRow` in that ClickHouse will also yie Differs from `JSONCompactEachRow` format in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -979,7 +979,7 @@ Otherwise, the first row will be skipped. Differs from `JSONCompactEachRow` format in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -991,7 +991,7 @@ the types from input data will be compared with the types of the corresponding c Differs from `JSONCompactStringsEachRow` in that in that it also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1001,7 +1001,7 @@ Otherwise, the first row will be skipped. Differs from `JSONCompactStringsEachRow` in that it also prints two header rows with column names and types, similar to [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes). -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1120,7 +1120,7 @@ CREATE TABLE IF NOT EXISTS example_table - If `input_format_defaults_for_omitted_fields = 0`, then the default value for `x` and `a` equals `0` (as the default value for the `UInt32` data type). - If `input_format_defaults_for_omitted_fields = 1`, then the default value for `x` equals `0`, but the default value of `a` equals `x * 2`. -:::warning +:::note When inserting data with `input_format_defaults_for_omitted_fields = 1`, ClickHouse consumes more computational resources, compared to insertion with `input_format_defaults_for_omitted_fields = 0`. ::: @@ -1450,7 +1450,7 @@ Similar to [RowBinary](#rowbinary), but with added header: - [LEB128](https://en.wikipedia.org/wiki/LEB128)-encoded number of columns (N) - N `String`s specifying column names -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1464,7 +1464,7 @@ Similar to [RowBinary](#rowbinary), but with added header: - N `String`s specifying column names - N `String`s specifying column types -:::warning +:::note If setting [input_format_with_names_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_names_use_header) is set to 1, the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](/docs/en/operations/settings/settings-formats.md/#input_format_skip_unknown_fields) is set to 1. Otherwise, the first row will be skipped. @@ -1915,7 +1915,7 @@ SET format_avro_schema_registry_url = 'http://schema-registry'; SELECT * FROM topic1_stream; ``` -:::warning +:::note Setting `format_avro_schema_registry_url` needs to be configured in `users.xml` to maintain it’s value after a restart. Also you can use the `format_avro_schema_registry_url` setting of the `Kafka` table engine. ::: diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 32f323a63d5..4bc108cac7c 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -446,7 +446,7 @@ Next are the configuration methods for different `type`. The following example defines the values of [max_threads](../operations/settings/settings.md#settings-max_threads) and `max_final_threads` settings, then queries the system table to check whether these settings were set successfully. -:::warning +:::note To keep the default `handlers` such as` query`, `play`,` ping`, add the `` rule. ::: @@ -477,7 +477,7 @@ $ curl -H 'XXX:TEST_HEADER_VALUE' -H 'PARAMS_XXX:max_threads' 'http://localhost: max_final_threads 2 ``` -:::warning +:::note In one `predefined_query_handler` only supports one `query` of an insert type. ::: diff --git a/docs/en/interfaces/postgresql.md b/docs/en/interfaces/postgresql.md index f7a619ca620..1146274b012 100644 --- a/docs/en/interfaces/postgresql.md +++ b/docs/en/interfaces/postgresql.md @@ -54,7 +54,7 @@ default=> And that's it! You now have a PostgreSQL client connected to ClickHouse, and all commands and queries are executed on ClickHouse. -:::caution +:::note The PostgreSQL protocol currently only supports plain-text passwords. ::: diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index 0e065cb7179..1069a04391f 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -6,7 +6,7 @@ sidebar_label: Client Libraries # Client Libraries from Third-party Developers -:::warning +:::note ClickHouse Inc does **not** maintain the libraries listed below and hasn’t done any extensive testing to ensure their quality. ::: diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 90a4f088be7..a089b3eff17 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -6,7 +6,7 @@ sidebar_label: Integrations # Integration Libraries from Third-party Developers -:::warning Disclaimer +:::note Disclaimer ClickHouse, Inc. does **not** maintain the tools and libraries listed below and haven’t done extensive testing to ensure their quality. ::: diff --git a/docs/en/operations/external-authenticators/kerberos.md b/docs/en/operations/external-authenticators/kerberos.md index b7a11d7445b..27dc23c9792 100644 --- a/docs/en/operations/external-authenticators/kerberos.md +++ b/docs/en/operations/external-authenticators/kerberos.md @@ -59,11 +59,11 @@ With filtering by realm: ``` -:::warning +:::note You can define only one `kerberos` section. The presence of multiple `kerberos` sections will force ClickHouse to disable Kerberos authentication. ::: -:::warning +:::note `principal` and `realm` sections cannot be specified at the same time. The presence of both `principal` and `realm` sections will force ClickHouse to disable Kerberos authentication. ::: @@ -103,7 +103,7 @@ Example (goes into `users.xml`): ``` -:::warning +:::note Note that Kerberos authentication cannot be used alongside with any other authentication mechanism. The presence of any other sections like `password` alongside `kerberos` will force ClickHouse to shutdown. ::: diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md index 1de5a09db0c..15185f7ae6b 100644 --- a/docs/en/operations/opentelemetry.md +++ b/docs/en/operations/opentelemetry.md @@ -7,7 +7,7 @@ title: "[experimental] Tracing ClickHouse with OpenTelemetry" [OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry. -:::warning +:::note This is an experimental feature that will change in backwards-incompatible ways in future releases. ::: diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md index a5afee63e6e..ff4236ab8c3 100644 --- a/docs/en/operations/query-cache.md +++ b/docs/en/operations/query-cache.md @@ -29,7 +29,7 @@ Transactionally inconsistent caching is traditionally provided by client tools o the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side. This reduces maintenance effort and avoids redundancy. -:::warning +:::note The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query processing) where wrong results are returned. ::: diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index bd0fb03bad0..c2c577d3d79 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -25,7 +25,7 @@ Default value: 3600. Data compression settings for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)-engine tables. -:::warning +:::note Don’t use it if you have just started using ClickHouse. ::: @@ -1368,7 +1368,7 @@ The following settings are available: Changed settings take effect immediately. -:::warning +:::note Data for the query cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `max_size` or disable the query cache altogether. ::: diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 5bc174727ad..7e77394729a 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -289,7 +289,7 @@ Default value: 0 (seconds) When this setting has a value greater than than zero only a single replica starts the merge immediately if merged part on shared storage and `allow_remote_fs_zero_copy_replication` is enabled. -:::warning Zero-copy replication is not ready for production +:::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index aa991cd9f15..40ca914b738 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -142,7 +142,7 @@ y Nullable(String) z IPv4 ``` -:::warning +:::note If the `schema_inference_hints` is not formated properly, or if there is a typo or a wrong datatype, etc... the whole schema_inference_hints will be ignored. ::: diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index 9b27af61851..bafac853377 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -118,7 +118,7 @@ To open access for user from any network, specify: ::/0 ``` -:::warning +:::note It’s insecure to open access from any network unless you have a firewall properly configured or the server is not directly connected to Internet. ::: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f3c0f20f3a6..239931b6757 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -460,7 +460,7 @@ Possible values: Changes the behaviour of join operations with `ANY` strictness. -:::warning +:::note This setting applies only for `JOIN` operations with [Join](../../engines/table-engines/special/join.md) engine tables. ::: @@ -550,7 +550,7 @@ Default value: 64. Enables legacy ClickHouse server behaviour in `ANY INNER|LEFT JOIN` operations. -:::warning +:::note Use this setting only for backward compatibility if your use cases depend on legacy `JOIN` behaviour. ::: @@ -942,7 +942,7 @@ Higher values will lead to higher memory usage. The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying a smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. -:::warning +:::note This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. ::: @@ -960,7 +960,7 @@ We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won’t be decompressed. -:::warning +:::note This is an expert-level setting, and you shouldn't change it if you're just getting started with ClickHouse. ::: @@ -1247,7 +1247,7 @@ Possible values: Default value: 1. -:::warning +:::note Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas) without [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key). If [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) is set, disable this setting only if it's used on a cluster with multiple shards containing multiple replicas. If it's used on a cluster with a single shard and multiple replicas, disabling this setting will have negative effects. @@ -1277,7 +1277,7 @@ Default value: `1`. This options will produce different results depending on the settings used. -:::warning +:::note This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. ::: @@ -2186,7 +2186,7 @@ Default value: 0. This setting also affects broken batches (that may appears because of abnormal server (machine) termination and no `fsync_after_insert`/`fsync_directories` for [Distributed](../../engines/table-engines/special/distributed.md) table engine). ::: -:::warning +:::note You should not rely on automatic batch splitting, since this may hurt performance. ::: @@ -2194,7 +2194,7 @@ You should not rely on automatic batch splitting, since this may hurt performanc Sets the priority ([nice](https://en.wikipedia.org/wiki/Nice_(Unix))) for threads that execute queries. The OS scheduler considers this priority when choosing the next thread to run on each available CPU core. -:::warning +:::note To use this setting, you need to set the `CAP_SYS_NICE` capability. The `clickhouse-server` package sets it up during installation. Some virtual environments do not allow you to set the `CAP_SYS_NICE` capability. In this case, `clickhouse-server` shows a message about it at the start. ::: @@ -2858,11 +2858,11 @@ Possible values: Default value: `0`. -:::warning +:::note Nullable primary key usually indicates bad design. It is forbidden in almost all main stream DBMS. The feature is mainly for [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) and is not heavily tested. Use with care. ::: -:::warning +:::note Do not enable this feature in version `<= 21.8`. It's not properly implemented and may lead to server crash. ::: diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index e019a3741cf..af7c526e29b 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -471,6 +471,6 @@ Use [http_max_single_read_retries](/docs/en/operations/settings/settings.md/#htt Zero-copy replication is possible, but not recommended, with `S3` and `HDFS` disks. Zero-copy replication means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself. -:::warning Zero-copy replication is not ready for production +:::note Zero-copy replication is not ready for production Zero-copy replication is disabled by default in ClickHouse version 22.8 and higher. This feature is not recommended for production use. ::: diff --git a/docs/en/operations/system-tables/parts.md b/docs/en/operations/system-tables/parts.md index 106d3c59dea..e7700562e35 100644 --- a/docs/en/operations/system-tables/parts.md +++ b/docs/en/operations/system-tables/parts.md @@ -99,7 +99,7 @@ Columns: - `move_ttl_info.expression` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Array of expressions. Each expression defines a [TTL MOVE rule](../../engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-ttl). -:::warning +:::note The `move_ttl_info.expression` array is kept mostly for backward compatibility, now the simpliest way to check `TTL MOVE` rule is to use the `move_ttl_info.min` and `move_ttl_info.max` fields. ::: diff --git a/docs/en/operations/utilities/clickhouse-copier.md b/docs/en/operations/utilities/clickhouse-copier.md index 87280bc3ba8..ccce7ea1b79 100644 --- a/docs/en/operations/utilities/clickhouse-copier.md +++ b/docs/en/operations/utilities/clickhouse-copier.md @@ -8,7 +8,7 @@ sidebar_label: clickhouse-copier Copies data from the tables in one cluster to tables in another (or the same) cluster. -:::warning +:::note To get a consistent copy, the data in the source tables and partitions should not change during the entire process. ::: diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 40184c0aa02..3b874dbe7cd 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -90,7 +90,7 @@ Checks whether the sequence contains an event chain that matches the pattern. sequenceMatch(pattern)(timestamp, cond1, cond2, ...) ``` -:::warning +:::note Events that occur at the same second may lay in the sequence in an undefined order affecting the result. ::: @@ -176,7 +176,7 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM Counts the number of event chains that matched the pattern. The function searches event chains that do not overlap. It starts to search for the next chain after the current chain is matched. -:::warning +:::note Events that occur at the same second may lay in the sequence in an undefined order affecting the result. ::: diff --git a/docs/en/sql-reference/data-types/float.md b/docs/en/sql-reference/data-types/float.md index 38c414fa8cd..c89b24ad235 100644 --- a/docs/en/sql-reference/data-types/float.md +++ b/docs/en/sql-reference/data-types/float.md @@ -6,7 +6,7 @@ sidebar_label: Float32, Float64 # Float32, Float64 -:::warning +:::note If you need accurate calculations, in particular if you work with financial or business data requiring a high precision you should consider using Decimal instead. Floats might lead to inaccurate results as illustrated below: ``` diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md index a21898de9a2..f727f0d75f7 100644 --- a/docs/en/sql-reference/data-types/json.md +++ b/docs/en/sql-reference/data-types/json.md @@ -6,7 +6,7 @@ sidebar_label: JSON # JSON -:::warning +:::note This feature is experimental and is not production ready. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json.md) instead. ::: @@ -14,7 +14,7 @@ Stores JavaScript Object Notation (JSON) documents in a single column. `JSON` is an alias for `Object('json')`. -:::warning +:::note The JSON data type is an experimental feature. To use it, set `allow_experimental_object_type = 1`. ::: diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 5169bc646c9..0ac5248e36c 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -8,7 +8,7 @@ sidebar_label: Interval The family of data types representing time and date intervals. The resulting types of the [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator. -:::warning +:::note `Interval` data type values can’t be stored in tables. ::: diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index 2185e2b31c1..f697b1ecdcf 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -417,7 +417,7 @@ Example: The table contains discounts for each advertiser in the format: To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](#dictionary-key-and-fields). These elements must contain elements `name` and `type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others). -:::warning +:::note Values of `range_min` and `range_max` should fit in `Int64` type. ::: @@ -706,7 +706,7 @@ Set a large enough cache size. You need to experiment to select the number of ce 3. Assess memory consumption using the `system.dictionaries` table. 4. Increase or decrease the number of cells until the required memory consumption is reached. -:::warning +:::note Do not use ClickHouse as a source, because it is slow to process queries with random reads. ::: @@ -1888,7 +1888,7 @@ ClickHouse supports the following types of keys: An xml structure can contain either `` or ``. DDL-query must contain single `PRIMARY KEY`. -:::warning +:::note You must not describe key as an attribute. ::: diff --git a/docs/en/sql-reference/distributed-ddl.md b/docs/en/sql-reference/distributed-ddl.md index ff5155391be..d170f3765c2 100644 --- a/docs/en/sql-reference/distributed-ddl.md +++ b/docs/en/sql-reference/distributed-ddl.md @@ -18,6 +18,6 @@ In order to run these queries correctly, each host must have the same cluster de The local version of the query will eventually be executed on each host in the cluster, even if some hosts are currently not available. -:::warning +:::important The order for executing queries within a single host is guaranteed. -::: \ No newline at end of file +::: diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 69dc73e2fb0..2943ba13861 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -125,7 +125,7 @@ SELECT sipHash64Keyed((506097522914230528, 1084818905618843912), array('e','x',' Like [sipHash64](#hash_functions-siphash64) but produces a 128-bit hash value, i.e. the final xor-folding state is done up to 128 bits. -:::warning +:::note This 128-bit variant differs from the reference implementation and it's weaker. This version exists because, when it was written, there was no official 128-bit extension for SipHash. New projects should probably use [sipHash128Reference](#hash_functions-siphash128reference). @@ -165,7 +165,7 @@ Result: Same as [sipHash128](#hash_functions-siphash128) but additionally takes an explicit key argument instead of using a fixed key. -:::warning +:::note This 128-bit variant differs from the reference implementation and it's weaker. This version exists because, when it was written, there was no official 128-bit extension for SipHash. New projects should probably use [sipHash128ReferenceKeyed](#hash_functions-siphash128referencekeyed). diff --git a/docs/en/sql-reference/functions/introspection.md b/docs/en/sql-reference/functions/introspection.md index 9357f75b8e6..c01abd6ed89 100644 --- a/docs/en/sql-reference/functions/introspection.md +++ b/docs/en/sql-reference/functions/introspection.md @@ -8,7 +8,7 @@ sidebar_label: Introspection You can use functions described in this chapter to introspect [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) and [DWARF](https://en.wikipedia.org/wiki/DWARF) for query profiling. -:::warning +:::note These functions are slow and may impose security considerations. ::: diff --git a/docs/en/sql-reference/functions/nlp-functions.md b/docs/en/sql-reference/functions/nlp-functions.md index f68448af2be..132a126edee 100644 --- a/docs/en/sql-reference/functions/nlp-functions.md +++ b/docs/en/sql-reference/functions/nlp-functions.md @@ -5,7 +5,7 @@ sidebar_label: NLP title: "[experimental] Natural Language Processing functions" --- -:::warning +:::note This is an experimental feature that is currently in development and is not ready for general use. It will change in unpredictable backwards-incompatible ways in future releases. Set `allow_experimental_nlp_functions = 1` to enable it. ::: diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 011b73405c5..2e44fa5e9f6 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -792,7 +792,7 @@ neighbor(column, offset[, default_value]) The result of the function depends on the affected data blocks and the order of data in the block. -:::warning +:::tip It can reach the neighbor rows only inside the currently processed data block. ::: @@ -902,7 +902,7 @@ Result: Calculates the difference between successive row values ​​in the data block. Returns 0 for the first row and the difference from the previous row for each subsequent row. -:::warning +:::tip It can reach the previous row only inside the currently processed data block. ::: @@ -986,7 +986,7 @@ Each event has a start time and an end time. The start time is included in the e The function calculates the total number of active (concurrent) events for each event start time. -:::warning +:::tip Events must be ordered by the start time in ascending order. If this requirement is violated the function raises an exception. Every data block is processed separately. If events from different data blocks overlap then they can not be processed correctly. ::: @@ -1674,7 +1674,7 @@ Result: Accumulates states of an aggregate function for each row of a data block. -:::warning +:::tip The state is reset for each new data block. ::: diff --git a/docs/en/sql-reference/operators/exists.md b/docs/en/sql-reference/operators/exists.md index 4bc29389c9c..5e96e11b924 100644 --- a/docs/en/sql-reference/operators/exists.md +++ b/docs/en/sql-reference/operators/exists.md @@ -7,7 +7,7 @@ The `EXISTS` operator checks how many records are in the result of a subquery. I `EXISTS` can be used in a [WHERE](../../sql-reference/statements/select/where.md) clause. -:::warning +:::tip References to main query tables and columns are not supported in a subquery. ::: diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 0fe7ebbf4b6..ade2d601f80 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -229,7 +229,7 @@ Types of intervals: You can also use a string literal when setting the `INTERVAL` value. For example, `INTERVAL 1 HOUR` is identical to the `INTERVAL '1 hour'` or `INTERVAL '1' hour`. -:::warning +:::tip Intervals with different types can’t be combined. You can’t use expressions like `INTERVAL 4 DAY 1 HOUR`. Specify intervals in units that are smaller or equal to the smallest unit of the interval, for example, `INTERVAL 25 HOUR`. You can use consecutive operations, like in the example below. ::: diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index d580efa4992..921785102a8 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -75,7 +75,7 @@ Deletes the column with the name `name`. If the `IF EXISTS` clause is specified, Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly. -:::warning +:::tip You can’t delete a column if it is referenced by [materialized view](/docs/en/sql-reference/statements/create/view.md/#materialized). Otherwise, it returns an error. ::: diff --git a/docs/en/sql-reference/statements/alter/constraint.md b/docs/en/sql-reference/statements/alter/constraint.md index 844b24d7374..7a8f5809320 100644 --- a/docs/en/sql-reference/statements/alter/constraint.md +++ b/docs/en/sql-reference/statements/alter/constraint.md @@ -17,7 +17,7 @@ See more on [constraints](../../../sql-reference/statements/create/table.md#cons Queries will add or remove metadata about constraints from table so they are processed immediately. -:::warning +:::tip Constraint check **will not be executed** on existing data if it was added. ::: diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index 56a57534234..aa0a07747f2 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -7,7 +7,7 @@ title: "CREATE ROW POLICY" Creates a [row policy](../../../guides/sre/user-management/index.md#row-policy-management), i.e. a filter used to determine which rows a user can read from a table. -:::warning +:::tip Row policies makes sense only for users with readonly access. If user can modify table or copy partitions between tables, it defeats the restrictions of row policies. ::: diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index b29bf31e26c..3432066864f 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -285,7 +285,7 @@ ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); ``` -:::warning +:::tip You can't combine both ways in one query. ::: @@ -343,7 +343,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -:::warning +:::tip You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. ::: @@ -438,11 +438,11 @@ Encryption codecs: These codecs use a fixed nonce and encryption is therefore deterministic. This makes it compatible with deduplicating engines such as [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md) but has a weakness: when the same data block is encrypted twice, the resulting ciphertext will be exactly the same so an adversary who can read the disk can see this equivalence (although only the equivalence, without getting its content). -:::warning +:::note Most engines including the "\*MergeTree" family create index files on disk without applying codecs. This means plaintext will appear on disk if an encrypted column is indexed. ::: -:::warning +:::note If you perform a SELECT query mentioning a specific value in an encrypted column (such as in its WHERE clause), the value may appear in [system.query_log](../../../operations/system-tables/query_log.md). You may want to disable the logging. ::: diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index a38523ee37b..b9bd2c1c507 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -50,7 +50,7 @@ There are multiple ways of user identification: CREATE USER name2 IDENTIFIED WITH plaintext_password BY 'my_password' ``` - :::warning + :::tip The password is stored in a SQL text file in `/var/lib/clickhouse/access`, so it's not a good idea to use `plaintext_password`. Try `sha256_password` instead, as demonstrated next... ::: @@ -76,7 +76,7 @@ There are multiple ways of user identification: ATTACH USER name3 IDENTIFIED WITH sha256_hash BY '0C268556C1680BEF0640AAC1E7187566704208398DA31F03D18C74F5C5BE5053' SALT '4FB16307F5E10048196966DD7E6876AE53DE6A1D1F625488482C75F14A5097C7'; ``` - :::note + :::tip If you have already created a hash value and corresponding salt value for a username, then you can use `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'`. For identification with `sha256_hash` using `SALT` - hash must be calculated from concatenation of 'password' and 'salt'. ::: @@ -109,7 +109,7 @@ Another way of specifying host is to use `@` syntax following the username. Exam - `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` syntax. - `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` syntax. -:::warning +:::tip ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we do not recommend to do so. ::: diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 78615a2f9ad..427ee75cd5f 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -7,7 +7,7 @@ title: "OPTIMIZE Statement" This query tries to initialize an unscheduled merge of data parts for tables. -:::warning +:::note `OPTIMIZE` can’t fix the `Too many parts` error. ::: diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index d069ae8413a..b0877d94e8a 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -312,7 +312,7 @@ One may execute query after: Replica attaches locally found parts and sends info about them to Zookeeper. Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). -:::warning +:::note Parts in all states are moved to `detached/` folder. Parts active before data loss (committed) are attached. ::: diff --git a/docs/en/sql-reference/statements/watch.md b/docs/en/sql-reference/statements/watch.md index 90d19e6be0e..5230479cbd2 100644 --- a/docs/en/sql-reference/statements/watch.md +++ b/docs/en/sql-reference/statements/watch.md @@ -6,7 +6,7 @@ sidebar_label: WATCH # WATCH Statement (Experimental) -:::warning +:::note This is an experimental feature that may change in backwards-incompatible ways in the future releases. Enable live views and `WATCH` query using `set allow_experimental_live_view = 1`. ::: @@ -107,4 +107,4 @@ The `FORMAT` clause works the same way as for the [SELECT](../../sql-reference/s :::note The [JSONEachRowWithProgress](../../interfaces/formats.md#jsoneachrowwithprogress) format should be used when watching [LIVE VIEW](./create/view.md#live-view) tables over the HTTP interface. The progress messages will be added to the output to keep the long-lived HTTP connection alive until the query result changes. The interval between progress messages is controlled using the [live_view_heartbeat_interval](./create/view.md#live-view-settings) setting. -::: \ No newline at end of file +::: diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index 594c328c3ff..48c2cadc62c 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -109,7 +109,7 @@ Query the number of rows in all files of these two directories: SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32'); ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md index 97a253a5356..1b4588a9b55 100644 --- a/docs/en/sql-reference/table-functions/hdfs.md +++ b/docs/en/sql-reference/table-functions/hdfs.md @@ -79,7 +79,7 @@ SELECT count(*) FROM hdfs('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/hdfsCluster.md b/docs/en/sql-reference/table-functions/hdfsCluster.md index 231c552610f..546c6a3d1b1 100644 --- a/docs/en/sql-reference/table-functions/hdfsCluster.md +++ b/docs/en/sql-reference/table-functions/hdfsCluster.md @@ -50,7 +50,7 @@ SELECT count(*) FROM hdfsCluster('cluster_simple', 'hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32') ``` -:::warning +:::note If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/iceberg.md b/docs/en/sql-reference/table-functions/iceberg.md index 7c2648d3dcf..713b0f9bbf5 100644 --- a/docs/en/sql-reference/table-functions/iceberg.md +++ b/docs/en/sql-reference/table-functions/iceberg.md @@ -32,7 +32,7 @@ A table with the specified structure for reading data in the specified Iceberg t SELECT * FROM iceberg('http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') ``` -:::warning +:::important ClickHouse currently supports reading v1 (v2 support is coming soon!) of the Iceberg format via the `iceberg` table function and `Iceberg` table engine. ::: diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index b49c2f8da20..1010d53e86d 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -20,6 +20,6 @@ You can use table functions in: - [INSERT INTO TABLE FUNCTION](../../sql-reference/statements/insert-into.md#inserting-into-table-function) query. -:::warning +:::note You can’t use table functions if the [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) setting is disabled. ::: diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 99b7832394d..44d1efdee7a 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -118,7 +118,7 @@ FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/ └─────────┘ ``` -:::warning +:::tip If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. ::: diff --git a/docs/en/sql-reference/table-functions/s3Cluster.md b/docs/en/sql-reference/table-functions/s3Cluster.md index f420a69596c..504f92b4dc0 100644 --- a/docs/en/sql-reference/table-functions/s3Cluster.md +++ b/docs/en/sql-reference/table-functions/s3Cluster.md @@ -42,7 +42,7 @@ SELECT * FROM s3Cluster( Count the total amount of rows in all files in the cluster `cluster_simple`: -:::warning +:::tip If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. :::