From fa2afb292c1a2bbf08cb186edf2017f71922418a Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 16 May 2022 22:48:37 +0800 Subject: [PATCH 001/408] Improve performance for column replicate for uint32 type. In our benchmark unit test there has about 2x performance boost Co-authored-by: vesslanjin --- src/Columns/ColumnVector.cpp | 104 +++++++++++++++++++++++++++++++++++ src/Columns/ColumnVector.h | 4 +- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index eca10049a0b..01f5c2d3782 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -401,6 +401,12 @@ ColumnPtr ColumnVector::index(const IColumn & indexes, size_t limit) const template ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { + #ifdef __SSE4_2__ + if constexpr (std::is_same_v) + { + return replicateSSE2(offsets); + } + #endif const size_t size = data.size(); if (size != offsets.size()) throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); @@ -421,6 +427,104 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const return res; } +#ifdef __SSE4_2__ +template +ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const +{ + const size_t size = data.size(); + if (size != offsets.size()) + throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); + + if (0 == size) + return this->create(); + + auto res = this->create(offsets.back()); + + auto it = res->getData().begin(); // NOLINT + ///column use paddedpodarray.Don't worry about the 4 conitnues op will out of range + if constexpr (std::is_same_v) + { + size_t prev_offset = 0; + int cp_begin = -1; + for (size_t i = 0; i < size; ++i) + { + size_t span = offsets[i] - prev_offset; + prev_offset = offsets[i]; + if (span == 1) + { + if (cp_begin == -1) + cp_begin = i; + continue; + } + ///data : 11 22 33 44 55 + ///offsets: 0 1 2 3 3 + ///res: 22 33 44 + size_t cpsz = (!(cp_begin == -1)) * (i - cp_begin); + bool remain = (cpsz & 3); + size_t sse_cp_counter = (cpsz >> 2); + sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); + auto it_tmp = it; + size_t data_start = cp_begin; + cp_begin = -1; + constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); + while (sse_cp_counter--) + { + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto cres = _mm_shuffle_epi32(cdata, msk_cp); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + data_start += 4; + } + it += cpsz; + if (span == 0) + { + continue; + } + ///data : 11 22 33 + ///offsets: 0 0 4 + ///res: 33 33 33 33 + size_t shuffle_sz = span; + bool shuffle_remain = (shuffle_sz & 3); + size_t sse_shuffle_counter = (shuffle_sz >> 2); + sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); + it_tmp = it; + constexpr const int msk_shuffle = (_MM_SHUFFLE(0, 0, 0, 0)); + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[i])); + while (sse_shuffle_counter--) + { + auto cres = _mm_shuffle_epi32(cdata, msk_shuffle); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + } + it += shuffle_sz; + } + ///data : 11 22 33 44 55 + ///offsets: 1 2 3 4 5 + ///res: 11 22 33 44 55 + if (cp_begin != -1) + { + size_t cpsz = (size - cp_begin); + bool remain = (cpsz & 3); + size_t sse_cp_counter = (cpsz >> 2); + sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); + auto it_tmp = it; + size_t data_start = cp_begin; + constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); + while (sse_cp_counter--) + { + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto cres = _mm_shuffle_epi32(cdata, msk_cp); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + data_start += 4; + } + it += cpsz; + } + } + return res; +} +#endif + template void ColumnVector::gather(ColumnGathererStream & gatherer) { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 62a0e3a1190..30cbff0570e 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -132,7 +132,9 @@ private: /// Sugar constructor. ColumnVector(std::initializer_list il) : data{il} {} - + #ifdef __SSE4_2__ + ColumnPtr replicateSSE2(const IColumn::Offsets & offsets) const; + #endif public: bool isNumeric() const override { return is_arithmetic_v; } From 4b2a24b5ec27eb1728d2b009eedf9dedea04266a Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 16 May 2022 23:31:54 +0800 Subject: [PATCH 002/408] change comment format --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 01f5c2d3782..1f7158e64d6 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -441,7 +441,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///column use paddedpodarray.Don't worry about the 4 conitnues op will out of range + ///column using padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From dbea0c122e9e65124e74b210d4c896c43d5111dd Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Wed, 18 May 2022 21:10:01 +0800 Subject: [PATCH 003/408] change macro from sse4.2 to sse2 --- src/Columns/ColumnVector.cpp | 4 ++-- src/Columns/ColumnVector.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 1f7158e64d6..97d184a5d61 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -401,7 +401,7 @@ ColumnPtr ColumnVector::index(const IColumn & indexes, size_t limit) const template ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { - #ifdef __SSE4_2__ + #ifdef __SSE2__ if constexpr (std::is_same_v) { return replicateSSE2(offsets); @@ -427,7 +427,7 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const return res; } -#ifdef __SSE4_2__ +#ifdef __SSE2__ template ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 30cbff0570e..6dcc0647781 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -132,7 +132,7 @@ private: /// Sugar constructor. ColumnVector(std::initializer_list il) : data{il} {} - #ifdef __SSE4_2__ + #ifdef __SSE2__ ColumnPtr replicateSSE2(const IColumn::Offsets & offsets) const; #endif public: From 44f2e3a06d09de37e0a0634b473e3e47a21b97e6 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Thu, 19 May 2022 02:49:26 +0800 Subject: [PATCH 004/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index d478d6ce8ee..b0a21b471f9 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///column using padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column using padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From 58518047fe1f30c97bbce60c9ef70deb867d7359 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Thu, 19 May 2022 19:13:12 +0800 Subject: [PATCH 005/408] iterator not always pointer type. so use the auto* substitue auto that clang tidy suggest is not alwyas ok. --- src/Columns/ColumnVector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index b0a21b471f9..c702b20105d 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -590,7 +590,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const bool remain = (cpsz & 3); size_t sse_cp_counter = (cpsz >> 2); sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); - auto it_tmp = it; + auto it_tmp = it; // NOLINT size_t data_start = cp_begin; cp_begin = -1; constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); @@ -634,7 +634,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const bool remain = (cpsz & 3); size_t sse_cp_counter = (cpsz >> 2); sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); - auto it_tmp = it; + auto it_tmp = it; // NOLINT size_t data_start = cp_begin; constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); while (sse_cp_counter--) From ad1f0c025e5cad2b7dc9017c43da45e5563b69fc Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 05:52:21 +0800 Subject: [PATCH 006/408] change some comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index c702b20105d..21d84b85e7d 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column using padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From 69d50bf232e821971e46e02692a1a018b8d0cf6a Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 06:33:47 +0800 Subject: [PATCH 007/408] change some comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 21d84b85e7d..f312aa7e59a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column use padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From ce9305939bddeef4353e98793b16230957c1ad66 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 06:42:22 +0800 Subject: [PATCH 008/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index f312aa7e59a..21d84b85e7d 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column use padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From 114125dde2a7a405b81a42d35fb05c726057ad48 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 07:32:26 +0800 Subject: [PATCH 009/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 21d84b85e7d..f312aa7e59a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column use padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From 59ecd3fa82f4e052e1b7549f0df5538451d84ae6 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 18:20:36 +0800 Subject: [PATCH 010/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index f312aa7e59a..21d84b85e7d 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column use padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From b5ddbf10d88dc541e3e424af60039029824ea22a Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Fri, 20 May 2022 22:41:47 +0800 Subject: [PATCH 011/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 21d84b85e7d..483f0db19ed 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column uses padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column used padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From dc58f82788b3bdc5b4c3597ace5861f56b471f37 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Sat, 21 May 2022 03:19:02 +0800 Subject: [PATCH 012/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 483f0db19ed..c702b20105d 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column used padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column using padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From a876435f4774d83a36a756ca351054bf9ec4c892 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Sat, 21 May 2022 13:51:34 +0800 Subject: [PATCH 013/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index c702b20105d..483f0db19ed 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column using padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column used padded pod array. Don't worry about the 4 conitnues op will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From ae739ca9baea2d4dbd97512578eba49143977389 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Sat, 21 May 2022 20:12:43 +0800 Subject: [PATCH 014/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 483f0db19ed..53695ffb167 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -568,7 +568,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column used padded pod array. Don't worry about the 4 conitnues op will out of range + ///Column used padded pod array. Don't worry about the 4 conitnue ops will out of range if constexpr (std::is_same_v) { size_t prev_offset = 0; From 50c02af959423905cca83e6e19a9d88c5970d183 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 21 May 2022 23:35:04 +0300 Subject: [PATCH 015/408] Make the code slightly more humane. --- src/Columns/ColumnVector.cpp | 184 ++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 91 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 53695ffb167..54f4745bb5e 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -437,10 +437,10 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s const T * data_pos = data.data(); /** A slightly more optimized version. - * Based on the assumption that often pieces of consecutive values - * completely pass or do not pass the filter. - * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. - */ + * Based on the assumption that often pieces of consecutive values + * completely pass or do not pass the filter. + * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. + */ static constexpr size_t SIMD_BYTES = 64; const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; @@ -528,12 +528,6 @@ ColumnPtr ColumnVector::index(const IColumn & indexes, size_t limit) const template ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { - #ifdef __SSE2__ - if constexpr (std::is_same_v) - { - return replicateSSE2(offsets); - } - #endif const size_t size = data.size(); if (size != offsets.size()) throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); @@ -541,6 +535,11 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const if (0 == size) return this->create(); +#ifdef __SSE2__ + if constexpr (std::is_same_v) + return replicateSSE2(offsets); +#endif + auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT @@ -554,104 +553,107 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const return res; } + #ifdef __SSE2__ + template ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const { - const size_t size = data.size(); - if (size != offsets.size()) - throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); - - if (0 == size) - return this->create(); - auto res = this->create(offsets.back()); auto it = res->getData().begin(); // NOLINT - ///Column used padded pod array. Don't worry about the 4 conitnue ops will out of range - if constexpr (std::is_same_v) + + /// Column is using PaddedPODArray, so we don't have to worry about the 4 out of range elements. + + size_t prev_offset = 0; + int copy_begin = -1; + + for (size_t i = 0; i < size; ++i) { - size_t prev_offset = 0; - int cp_begin = -1; - for (size_t i = 0; i < size; ++i) + size_t span = offsets[i] - prev_offset; + prev_offset = offsets[i]; + if (span == 1) { - size_t span = offsets[i] - prev_offset; - prev_offset = offsets[i]; - if (span == 1) - { - if (cp_begin == -1) - cp_begin = i; - continue; - } - ///data : 11 22 33 44 55 - ///offsets: 0 1 2 3 3 - ///res: 22 33 44 - size_t cpsz = (!(cp_begin == -1)) * (i - cp_begin); - bool remain = (cpsz & 3); - size_t sse_cp_counter = (cpsz >> 2); - sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); - auto it_tmp = it; // NOLINT - size_t data_start = cp_begin; - cp_begin = -1; - constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); - while (sse_cp_counter--) - { - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto cres = _mm_shuffle_epi32(cdata, msk_cp); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); - it_tmp += 4; - data_start += 4; - } - it += cpsz; - if (span == 0) - { - continue; - } - ///data : 11 22 33 - ///offsets: 0 0 4 - ///res: 33 33 33 33 - size_t shuffle_sz = span; - bool shuffle_remain = (shuffle_sz & 3); - size_t sse_shuffle_counter = (shuffle_sz >> 2); - sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); - it_tmp = it; - constexpr const int msk_shuffle = (_MM_SHUFFLE(0, 0, 0, 0)); - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[i])); - while (sse_shuffle_counter--) - { - auto cres = _mm_shuffle_epi32(cdata, msk_shuffle); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); - it_tmp += 4; - } - it += shuffle_sz; + if (copy_begin == -1) + copy_begin = i; + continue; } - ///data : 11 22 33 44 55 - ///offsets: 1 2 3 4 5 - ///res: 11 22 33 44 55 - if (cp_begin != -1) + + /// data : 11 22 33 44 55 + /// offsets: 0 1 2 3 3 + /// res: 22 33 44 + + size_t copy_size = (!(copy_begin == -1)) * (i - copy_begin); + bool remain = (copy_size & 3); + size_t sse_copy_counter = (copy_size >> 2); + sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); + auto it_tmp = it; // NOLINT + size_t data_start = copy_begin; + copy_begin = -1; + constexpr const int msk_cp = _MM_SHUFFLE(3, 2, 1, 0); + while (sse_copy_counter) { - size_t cpsz = (size - cp_begin); - bool remain = (cpsz & 3); - size_t sse_cp_counter = (cpsz >> 2); - sse_cp_counter = remain * (sse_cp_counter + 1) + (!remain) * (sse_cp_counter); - auto it_tmp = it; // NOLINT - size_t data_start = cp_begin; - constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); - while (sse_cp_counter--) - { - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto cres = _mm_shuffle_epi32(cdata, msk_cp); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); - it_tmp += 4; - data_start += 4; - } - it += cpsz; + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto cres = _mm_shuffle_epi32(cdata, msk_cp); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + data_start += 4; + --sse_copy_counter; } + + it += copy_size; + if (span == 0) + continue; + + /// data : 11 22 33 + /// offsets: 0 0 4 + /// res: 33 33 33 33 + size_t shuffle_sz = span; + bool shuffle_remain = (shuffle_sz & 3); + size_t sse_shuffle_counter = (shuffle_sz >> 2); + sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); + it_tmp = it; + constexpr const int msk_shuffle = (_MM_SHUFFLE(0, 0, 0, 0)); + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[i])); + while (sse_shuffle_counter) + { + auto cres = _mm_shuffle_epi32(cdata, msk_shuffle); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + --sse_shuffle_counter; + } + it += shuffle_sz; } + + /// data : 11 22 33 44 55 + /// offsets: 1 2 3 4 5 + /// res: 11 22 33 44 55 + if (copy_begin != -1) + { + size_t copy_size = (size - copy_begin); + bool remain = (copy_size & 3); + size_t sse_copy_counter = (copy_size >> 2); + sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); + auto it_tmp = it; // NOLINT + size_t data_start = copy_begin; + constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); + while (sse_copy_counter) + { + __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto cres = _mm_shuffle_epi32(cdata, msk_cp); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + it_tmp += 4; + data_start += 4; + --sse_copy_counter; + } + it += copy_size; + } + return res; } #endif + template void ColumnVector::gather(ColumnGathererStream & gatherer) { From fa032a76b61562b2b51a73ff2d0d78e4faa694de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 21 May 2022 23:38:51 +0300 Subject: [PATCH 016/408] Make variable names less disgusting. --- src/Columns/ColumnVector.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 54f4745bb5e..d49b3e7f948 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -590,12 +590,12 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto it_tmp = it; // NOLINT size_t data_start = copy_begin; copy_begin = -1; - constexpr const int msk_cp = _MM_SHUFFLE(3, 2, 1, 0); + constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); while (sse_copy_counter) { - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto cres = _mm_shuffle_epi32(cdata, msk_cp); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); it_tmp += 4; data_start += 4; --sse_copy_counter; @@ -613,12 +613,12 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const size_t sse_shuffle_counter = (shuffle_sz >> 2); sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); it_tmp = it; - constexpr const int msk_shuffle = (_MM_SHUFFLE(0, 0, 0, 0)); - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[i])); + constexpr const int shuffle_mask = (_MM_SHUFFLE(0, 0, 0, 0)); + __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[i])); while (sse_shuffle_counter) { - auto cres = _mm_shuffle_epi32(cdata, msk_shuffle); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + auto copy_result = _mm_shuffle_epi32(data_to_copy, shuffle_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); it_tmp += 4; --sse_shuffle_counter; } @@ -636,12 +636,12 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); auto it_tmp = it; // NOLINT size_t data_start = copy_begin; - constexpr const int msk_cp = (_MM_SHUFFLE(3, 2, 1, 0)); + constexpr const int copy_mask = (_MM_SHUFFLE(3, 2, 1, 0)); while (sse_copy_counter) { - __m128i cdata = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto cres = _mm_shuffle_epi32(cdata, msk_cp); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), cres); + __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); it_tmp += 4; data_start += 4; --sse_copy_counter; From d36ec5e34d272203edb10b57b1fe6e6be5645e0d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 21 May 2022 23:39:34 +0300 Subject: [PATCH 017/408] There are so many disgusting names. --- src/Columns/ColumnVector.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index d49b3e7f948..2f52fa6aa85 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -608,9 +608,9 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// data : 11 22 33 /// offsets: 0 0 4 /// res: 33 33 33 33 - size_t shuffle_sz = span; - bool shuffle_remain = (shuffle_sz & 3); - size_t sse_shuffle_counter = (shuffle_sz >> 2); + size_t shuffle_size = span; + bool shuffle_remain = (shuffle_size & 3); + size_t sse_shuffle_counter = (shuffle_size >> 2); sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); it_tmp = it; constexpr const int shuffle_mask = (_MM_SHUFFLE(0, 0, 0, 0)); @@ -622,7 +622,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const it_tmp += 4; --sse_shuffle_counter; } - it += shuffle_sz; + it += shuffle_size; } /// data : 11 22 33 44 55 From a1a0d5b07565f9b1d0ff57d1dbf2b973b53f72ca Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Sun, 22 May 2022 14:22:54 +0800 Subject: [PATCH 018/408] fix compile error --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 2f52fa6aa85..7e2748c472c 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -567,7 +567,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const size_t prev_offset = 0; int copy_begin = -1; - + size_t size = offsets.size(); for (size_t i = 0; i < size; ++i) { size_t span = offsets[i] - prev_offset; From eff6adbaa4aa17f47ac23b680ec4550dfc5ab0f9 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Sun, 22 May 2022 15:50:47 +0800 Subject: [PATCH 019/408] fix offset type issue --- src/Columns/ColumnVector.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 7e2748c472c..3a9ccafb57f 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -565,8 +565,8 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// Column is using PaddedPODArray, so we don't have to worry about the 4 out of range elements. - size_t prev_offset = 0; - int copy_begin = -1; + IColumn::Offset prev_offset = 0; + std::optional copy_begin; size_t size = offsets.size(); for (size_t i = 0; i < size; ++i) { @@ -574,7 +574,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const prev_offset = offsets[i]; if (span == 1) { - if (copy_begin == -1) + if (!copy_begin) copy_begin = i; continue; } @@ -583,13 +583,13 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// offsets: 0 1 2 3 3 /// res: 22 33 44 - size_t copy_size = (!(copy_begin == -1)) * (i - copy_begin); + size_t copy_size = (static_cast(copy_begin)) * (i - (*copy_begin)); bool remain = (copy_size & 3); size_t sse_copy_counter = (copy_size >> 2); sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); auto it_tmp = it; // NOLINT - size_t data_start = copy_begin; - copy_begin = -1; + size_t data_start = (static_cast(copy_begin))*(*copy_begin); + copy_begin.reset(); constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); while (sse_copy_counter) { @@ -628,14 +628,14 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// data : 11 22 33 44 55 /// offsets: 1 2 3 4 5 /// res: 11 22 33 44 55 - if (copy_begin != -1) + if (copy_begin) { - size_t copy_size = (size - copy_begin); + size_t copy_size = (size - (*copy_begin)); bool remain = (copy_size & 3); size_t sse_copy_counter = (copy_size >> 2); sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); auto it_tmp = it; // NOLINT - size_t data_start = copy_begin; + size_t data_start = *copy_begin; constexpr const int copy_mask = (_MM_SHUFFLE(3, 2, 1, 0)); while (sse_copy_counter) { From a6347dcea626fa21f26cdfd38055efa25acaba14 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 23 May 2022 01:11:01 +0800 Subject: [PATCH 020/408] fix deference null optional value issue. Dereference NUll optioanl value in debug mode will trigger about signal. --- src/Columns/ColumnVector.cpp | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 3a9ccafb57f..5d84cbfb782 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -574,34 +574,37 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const prev_offset = offsets[i]; if (span == 1) { - if (!copy_begin) - copy_begin = i; - continue; + if (!copy_begin) + copy_begin = i; + continue; } /// data : 11 22 33 44 55 /// offsets: 0 1 2 3 3 /// res: 22 33 44 + if(copy_begin) + { + size_t copy_size = i - (*copy_begin); + bool remain = (copy_size & 3); + size_t sse_copy_counter = (copy_size >> 2); + sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); + auto it_tmp = it; // NOLINT + size_t data_start = *copy_begin; + copy_begin.reset(); + constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); + while (sse_copy_counter) + { + __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); + it_tmp += 4; + data_start += 4; + --sse_copy_counter; + } - size_t copy_size = (static_cast(copy_begin)) * (i - (*copy_begin)); - bool remain = (copy_size & 3); - size_t sse_copy_counter = (copy_size >> 2); - sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); - auto it_tmp = it; // NOLINT - size_t data_start = (static_cast(copy_begin))*(*copy_begin); - copy_begin.reset(); - constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); - while (sse_copy_counter) - { - __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); - it_tmp += 4; - data_start += 4; - --sse_copy_counter; - } + it += copy_size; + } - it += copy_size; if (span == 0) continue; @@ -612,13 +615,13 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const bool shuffle_remain = (shuffle_size & 3); size_t sse_shuffle_counter = (shuffle_size >> 2); sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); - it_tmp = it; + auto it_tmp = it; // NOLINT constexpr const int shuffle_mask = (_MM_SHUFFLE(0, 0, 0, 0)); - __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[i])); + __m128i data_to_shuffle = _mm_loadu_si128(reinterpret_cast(&data[i])); + auto shuffle_result = _mm_shuffle_epi32(data_to_shuffle, shuffle_mask); while (sse_shuffle_counter) { - auto copy_result = _mm_shuffle_epi32(data_to_copy, shuffle_mask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), shuffle_result); it_tmp += 4; --sse_shuffle_counter; } From a3d6cb2e58ed30f83fdcf7a09b490768d541fd25 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Mon, 23 May 2022 01:36:59 +0800 Subject: [PATCH 021/408] fix style issue --- src/Columns/ColumnVector.cpp | 52 ++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 5d84cbfb782..7920124d4f2 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -565,7 +565,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// Column is using PaddedPODArray, so we don't have to worry about the 4 out of range elements. - IColumn::Offset prev_offset = 0; + IColumn::Offset prev_offset = 0; std::optional copy_begin; size_t size = offsets.size(); for (size_t i = 0; i < size; ++i) @@ -574,36 +574,36 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const prev_offset = offsets[i]; if (span == 1) { - if (!copy_begin) - copy_begin = i; - continue; + if (!copy_begin) + copy_begin = i; + continue; } /// data : 11 22 33 44 55 /// offsets: 0 1 2 3 3 /// res: 22 33 44 - if(copy_begin) - { - size_t copy_size = i - (*copy_begin); - bool remain = (copy_size & 3); - size_t sse_copy_counter = (copy_size >> 2); - sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); - auto it_tmp = it; // NOLINT - size_t data_start = *copy_begin; - copy_begin.reset(); - constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); - while (sse_copy_counter) - { - __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); - it_tmp += 4; - data_start += 4; - --sse_copy_counter; - } + if (copy_begin) + { + size_t copy_size = i - (*copy_begin); + bool remain = (copy_size & 3); + size_t sse_copy_counter = (copy_size >> 2); + sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); + auto it_tmp = it; // NOLINT + size_t data_start = (*copy_begin); + copy_begin.reset(); + constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); + while (sse_copy_counter) + { + __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); + auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); + _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); + it_tmp += 4; + data_start += 4; + --sse_copy_counter; + } - it += copy_size; - } + it += copy_size; + } if (span == 0) continue; @@ -618,7 +618,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const auto it_tmp = it; // NOLINT constexpr const int shuffle_mask = (_MM_SHUFFLE(0, 0, 0, 0)); __m128i data_to_shuffle = _mm_loadu_si128(reinterpret_cast(&data[i])); - auto shuffle_result = _mm_shuffle_epi32(data_to_shuffle, shuffle_mask); + auto shuffle_result = _mm_shuffle_epi32(data_to_shuffle, shuffle_mask); while (sse_shuffle_counter) { _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), shuffle_result); From 5162e87dd2005e6d75159d0ab773d35c8c192d13 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Tue, 24 May 2022 10:22:29 +0800 Subject: [PATCH 022/408] remove parentheses --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 7920124d4f2..920fb12124c 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -589,7 +589,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const size_t sse_copy_counter = (copy_size >> 2); sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); auto it_tmp = it; // NOLINT - size_t data_start = (*copy_begin); + size_t data_start = *copy_begin; copy_begin.reset(); constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); while (sse_copy_counter) From 2ff2c5547969fd058d3634de931134741da1ef6c Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Thu, 26 May 2022 21:11:56 +0800 Subject: [PATCH 023/408] change some comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 920fb12124c..b659e50426f 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -581,7 +581,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const /// data : 11 22 33 44 55 /// offsets: 0 1 2 3 3 - /// res: 22 33 44 + /// res: 22 33 44 if (copy_begin) { size_t copy_size = i - (*copy_begin); From 781701b223ece9ab16e1a09a089545e64f471af6 Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Thu, 26 May 2022 22:23:22 +0800 Subject: [PATCH 024/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index b659e50426f..7ab2a7e387a 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -580,7 +580,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const } /// data : 11 22 33 44 55 - /// offsets: 0 1 2 3 3 + /// offsets: 0 1 2 3 3 /// res: 22 33 44 if (copy_begin) { From bef9971890522dde57756938fd79cc4196cab53b Mon Sep 17 00:00:00 2001 From: zhao zhou Date: Tue, 31 May 2022 21:13:02 +0800 Subject: [PATCH 025/408] change comment --- src/Columns/ColumnVector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 7ab2a7e387a..b659e50426f 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -580,7 +580,7 @@ ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const } /// data : 11 22 33 44 55 - /// offsets: 0 1 2 3 3 + /// offsets: 0 1 2 3 3 /// res: 22 33 44 if (copy_begin) { From 82ce2d76c31547539cdb6ed4bec6da065e13e7cb Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 3 Jun 2022 12:06:31 +0300 Subject: [PATCH 026/408] Add KerberosInit class; add kerberos_init console example; modify HDFSCommon.cpp --- src/Access/CMakeLists.txt | 3 + src/Access/KerberosInit.cpp | 162 ++++++++++++++++++++++++++ src/Access/KerberosInit.h | 37 ++++++ src/Access/examples/CMakeLists.txt | 4 + src/Access/examples/kerberos_init.cpp | 36 ++++++ src/Storages/HDFS/HDFSCommon.cpp | 15 ++- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 src/Access/KerberosInit.cpp create mode 100644 src/Access/KerberosInit.h create mode 100644 src/Access/examples/CMakeLists.txt create mode 100644 src/Access/examples/kerberos_init.cpp diff --git a/src/Access/CMakeLists.txt b/src/Access/CMakeLists.txt index e69de29bb2d..83bbe418246 100644 --- a/src/Access/CMakeLists.txt +++ b/src/Access/CMakeLists.txt @@ -0,0 +1,3 @@ +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp new file mode 100644 index 00000000000..8954eccf136 --- /dev/null +++ b/src/Access/KerberosInit.cpp @@ -0,0 +1,162 @@ +#include +#include + +#include +#include +#include + + +int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) +{ + auto adqm_log = &Poco::Logger::get("ADQM"); + LOG_DEBUG(adqm_log,"KerberosInit: begin"); + + krb5_error_code ret; + + // todo: use deftype + //const char *deftype = nullptr; + int flags = 0; + principal_name = new char[256]; + std::copy(principal.begin(), principal.end(), principal_name); + principal_name[principal.size()] = '\0'; + + //memset(&k5, 0, sizeof(k5)); + memset(&k5d, 0, sizeof(k5d)); + k5 = &k5d; + //memset(k5, 0, sizeof(k5_data)); + // begin + ret = krb5_init_context(&k5->ctx); + if (ret) + throw DB::Exception(0, "Error while initializing Kerberos 5 library"); + + if (!cache_name.empty()) + { + ret = krb5_cc_resolve(k5->ctx, cache_name.c_str(), &k5->out_cc); + // todo: analyze return code + LOG_DEBUG(adqm_log,"Resolved cache"); + } + else + { + // Resolve the default ccache and get its type and default principal (if it is initialized). + ret = krb5_cc_default(k5->ctx, &defcache); + if (ret) + throw DB::Exception(0, "Error while getting default ccache"); + LOG_DEBUG(adqm_log,"Resolved default cache"); + // todo: deftype + /*deftype = */krb5_cc_get_type(k5->ctx, defcache); + if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) + defcache_princ = nullptr; + } + + // Use the specified principal name. + ret = krb5_parse_name_flags(k5->ctx, principal_name, flags, &k5->me); + if (ret) + throw DB::Exception(0, "Error when parsing principal name " + String(principal_name)); + + + // to-do: add more cache init commands + + ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); + if (ret) + throw DB::Exception(0, "Error when unparsing name"); + + LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); + + principal_name = k5->name; + + // init: + memset(&my_creds, 0, sizeof(my_creds)); + + ret = krb5_get_init_creds_opt_alloc(k5->ctx, &options); + if (ret) + throw DB::Exception(0, "Error in options allocation"); + + // todo +/* +#ifndef _WIN32 + if (strncmp(opts->keytab_name, "KDB:", 4) == 0) { + ret = kinit_kdb_init(&k5->ctx, k5->me->realm.data); + errctx = k5->ctx; + if (ret) { + com_err(progname, ret, + _("while setting up KDB keytab for realm %s"), + k5->me->realm.data); + goto cleanup; + } + } +#endif +*/ + // Resolve keytab + ret = krb5_kt_resolve(k5->ctx, keytab_file.c_str(), &keytab); + if (ret) + throw DB::Exception(0, "Error resolving keytab "+keytab_file); + + LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); + + // todo: num_pa_opts + + + // todo: in_cc / ccache + + // action: init or renew + // todo: doing only init action: + ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + if (ret) + LOG_DEBUG(adqm_log,"Getting initial credentials"); + + // todo: implement renew action + + + LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); + LOG_DEBUG(adqm_log,"KerberosInit: end"); + return 0; +} + +KerberosInit::~KerberosInit() +{ + if (k5->ctx) + { + //begin. cleanup: + if (defcache) + krb5_cc_close(k5->ctx, defcache); + //todo + krb5_free_principal(k5->ctx, defcache_princ); + + // init. cleanup: + //todo: + /* + #ifndef _WIN32 + kinit_kdb_fini(); + #endif + */ + if (options) + krb5_get_init_creds_opt_free(k5->ctx, options); + if (my_creds.client == k5->me) + my_creds.client = nullptr; + /* + if (opts->pa_opts) { + free(opts->pa_opts); + opts->pa_opts = NULL; + opts->num_pa_opts = 0; + } + */ + krb5_free_cred_contents(k5->ctx, &my_creds); + if (keytab) + krb5_kt_close(k5->ctx, keytab); + + + // end: + krb5_free_unparsed_name(k5->ctx, k5->name); + krb5_free_principal(k5->ctx, k5->me); + /* + if (k5->in_cc != NULL) + krb5_cc_close(k5->ctx, k5->in_cc); + if (k5->out_cc != NULL) + krb5_cc_close(k5->ctx, k5->out_cc); + */ + krb5_free_context(k5->ctx); + } + memset(k5, 0, sizeof(*k5)); + + delete[] principal_name; +} diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h new file mode 100644 index 00000000000..ed64ba3b4a8 --- /dev/null +++ b/src/Access/KerberosInit.h @@ -0,0 +1,37 @@ +#pragma once + +#include "config_core.h" + +#include + +//#include +//#include "k5-platform.h" +#include +//#include + +struct k5_data +{ + krb5_context ctx; + krb5_ccache in_cc, out_cc; + krb5_principal me; + char *name; + krb5_boolean switch_to_cache; +}; + +class KerberosInit +{ +public: + int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + ~KerberosInit(); +private: + struct k5_data * k5 = nullptr; + //struct k5_data k5; + struct k5_data k5d; + krb5_ccache defcache = nullptr; + krb5_get_init_creds_opt *options = nullptr; + krb5_creds my_creds; + krb5_keytab keytab = nullptr; + char * principal_name; + krb5_principal defcache_princ = nullptr; +}; + diff --git a/src/Access/examples/CMakeLists.txt b/src/Access/examples/CMakeLists.txt new file mode 100644 index 00000000000..07f75ca0b47 --- /dev/null +++ b/src/Access/examples/CMakeLists.txt @@ -0,0 +1,4 @@ +if (TARGET ch_contrib::krb5) + add_executable (kerberos_init kerberos_init.cpp) + target_link_libraries (kerberos_init PRIVATE dbms ch_contrib::krb5) +endif() diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp new file mode 100644 index 00000000000..fb2575dc581 --- /dev/null +++ b/src/Access/examples/kerberos_init.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include + +using namespace DB; + +int main(int argc, char ** argv) +{ + std::cout << "Kerberos Init" << "\n"; + + if (argc < 3) + { + std::cout << "Usage:" << "\n" << " kerberos_init keytab principal [cache]" << "\n"; + return 0; + } + + String cache_name = ""; + if (argc == 4) + cache_name = argv[3]; + + Poco::AutoPtr app_channel(new Poco::ConsoleChannel(std::cerr)); + Poco::Logger::root().setChannel(app_channel); + Poco::Logger::root().setLevel("trace"); + + KerberosInit k_init; + try { + k_init.init(argv[1], argv[2], cache_name); + } catch (const Exception & e) { + std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + } + std::cout << "Done" << "\n"; + return 0; +} diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 2f7b03790ee..71989bfd549 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -101,7 +102,7 @@ String HDFSBuilderWrapper::getKinitCmd() } void HDFSBuilderWrapper::runKinit() -{ +{ /* String cmd = getKinitCmd(); LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "running kinit: {}", cmd); @@ -113,6 +114,18 @@ void HDFSBuilderWrapper::runKinit() { throw Exception("kinit failure: " + cmd, ErrorCodes::BAD_ARGUMENTS); } + */ + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); + std::unique_lock lck(kinit_mtx); + KerberosInit k_init; + try { + k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal); + } catch (const DB::Exception & e) { + String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); + throw Exception(0, msg); + } + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) From 8b5bf0292748c5cfb0875b66c1d3a02118fea39d Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 3 Jun 2022 18:07:18 +0300 Subject: [PATCH 027/408] Add support of cache commands in KerberosInit --- src/Access/KerberosInit.cpp | 88 +++++++++++++++------------ src/Access/KerberosInit.h | 1 - src/Access/examples/kerberos_init.cpp | 3 +- src/Storages/HDFS/HDFSCommon.cpp | 2 +- 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 8954eccf136..0c643e27bd2 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -1,10 +1,9 @@ #include #include - #include #include #include - +#include int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { @@ -14,16 +13,14 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; // todo: use deftype - //const char *deftype = nullptr; + const char *deftype = nullptr; int flags = 0; - principal_name = new char[256]; - std::copy(principal.begin(), principal.end(), principal_name); - principal_name[principal.size()] = '\0'; - //memset(&k5, 0, sizeof(k5)); + if (!std::filesystem::exists(keytab_file)) + throw DB::Exception(0, "Error keytab file does not exist"); + memset(&k5d, 0, sizeof(k5d)); k5 = &k5d; - //memset(k5, 0, sizeof(k5_data)); // begin ret = krb5_init_context(&k5->ctx); if (ret) @@ -42,19 +39,46 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (ret) throw DB::Exception(0, "Error while getting default ccache"); LOG_DEBUG(adqm_log,"Resolved default cache"); - // todo: deftype - /*deftype = */krb5_cc_get_type(k5->ctx, defcache); + deftype = krb5_cc_get_type(k5->ctx, defcache); if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; } // Use the specified principal name. - ret = krb5_parse_name_flags(k5->ctx, principal_name, flags, &k5->me); + ret = krb5_parse_name_flags(k5->ctx, principal.c_str(), flags, &k5->me); if (ret) - throw DB::Exception(0, "Error when parsing principal name " + String(principal_name)); + throw DB::Exception(0, "Error when parsing principal name " + principal); + // Cache related commands + if (k5->out_cc == nullptr && krb5_cc_support_switch(k5->ctx, deftype)) + { + // Use an existing cache for the client principal if we can. + ret = krb5_cc_cache_match(k5->ctx, k5->me, &k5->out_cc); + if (ret && ret != KRB5_CC_NOTFOUND) + throw DB::Exception(0, "Error while searching for ccache for " + principal); + if (!ret) + { + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + k5->switch_to_cache = 1; + } + else if (defcache_princ != nullptr) + { + // Create a new cache to avoid overwriting the initialized default cache. + ret = krb5_cc_new_unique(k5->ctx, deftype, nullptr, &k5->out_cc); + if (ret) + throw DB::Exception(0, "Error while generating new ccache"); + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + k5->switch_to_cache = 1; + } + } - // to-do: add more cache init commands + // Use the default cache if we haven't picked one yet. + if (k5->out_cc == nullptr) + { + k5->out_cc = defcache; + defcache = nullptr; + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + } ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); if (ret) @@ -62,8 +86,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); - principal_name = k5->name; - // init: memset(&my_creds, 0, sizeof(my_creds)); @@ -93,20 +115,23 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); - // todo: num_pa_opts - - - // todo: in_cc / ccache + if (k5->in_cc) + { + ret = krb5_get_init_creds_opt_set_in_ccache(k5->ctx, options, k5->in_cc); + if (ret) + throw DB::Exception(0, "Error in setting input credential cache"); + } + ret = krb5_get_init_creds_opt_set_out_ccache(k5->ctx, options, k5->out_cc); + if (ret) + throw DB::Exception(0, "Error in setting output credential cache"); // action: init or renew + // todo: implement renew action // todo: doing only init action: ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); if (ret) LOG_DEBUG(adqm_log,"Getting initial credentials"); - // todo: implement renew action - - LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); LOG_DEBUG(adqm_log,"KerberosInit: end"); return 0; @@ -119,7 +144,6 @@ KerberosInit::~KerberosInit() //begin. cleanup: if (defcache) krb5_cc_close(k5->ctx, defcache); - //todo krb5_free_principal(k5->ctx, defcache_princ); // init. cleanup: @@ -133,30 +157,18 @@ KerberosInit::~KerberosInit() krb5_get_init_creds_opt_free(k5->ctx, options); if (my_creds.client == k5->me) my_creds.client = nullptr; - /* - if (opts->pa_opts) { - free(opts->pa_opts); - opts->pa_opts = NULL; - opts->num_pa_opts = 0; - } - */ krb5_free_cred_contents(k5->ctx, &my_creds); if (keytab) krb5_kt_close(k5->ctx, keytab); - - // end: + // end. cleanup: krb5_free_unparsed_name(k5->ctx, k5->name); krb5_free_principal(k5->ctx, k5->me); - /* - if (k5->in_cc != NULL) + if (k5->in_cc != nullptr) krb5_cc_close(k5->ctx, k5->in_cc); - if (k5->out_cc != NULL) + if (k5->out_cc != nullptr) krb5_cc_close(k5->ctx, k5->out_cc); - */ krb5_free_context(k5->ctx); } memset(k5, 0, sizeof(*k5)); - - delete[] principal_name; } diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index ed64ba3b4a8..07f619aaa0b 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -31,7 +31,6 @@ private: krb5_get_init_creds_opt *options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; - char * principal_name; krb5_principal defcache_princ = nullptr; }; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index fb2575dc581..554d57fb8b2 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -26,7 +26,8 @@ int main(int argc, char ** argv) Poco::Logger::root().setLevel("trace"); KerberosInit k_init; - try { + try + { k_init.init(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 71989bfd549..f2ec7af368a 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -119,7 +119,7 @@ void HDFSBuilderWrapper::runKinit() std::unique_lock lck(kinit_mtx); KerberosInit k_init; try { - k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal); + k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); From 6de7590af40e7b7a7d5e5ecab7b8207591b0e0bf Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Sat, 4 Jun 2022 23:30:38 +0800 Subject: [PATCH 028/408] compile the new libgsasl with supportting three modes(privacy/integrity/authentication) --- contrib/libgsasl-cmake/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/libgsasl-cmake/CMakeLists.txt b/contrib/libgsasl-cmake/CMakeLists.txt index 3cf087c2f4c..f0c2395a831 100644 --- a/contrib/libgsasl-cmake/CMakeLists.txt +++ b/contrib/libgsasl-cmake/CMakeLists.txt @@ -113,4 +113,8 @@ if (TARGET ch_contrib::krb5) target_compile_definitions(_gsasl PRIVATE HAVE_GSSAPI_H=1 USE_GSSAPI=1) endif() +if (TARGET OpenSSL::SSL) + target_link_libraries(_gsasl PRIVATE OpenSSL::Crypto OpenSSL::SSL) +endif() + add_library(ch_contrib::gsasl ALIAS _gsasl) From 323835f51d600f4b406492d5122304095b4b7cbb Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Mon, 6 Jun 2022 11:34:10 +0300 Subject: [PATCH 029/408] Add renew/init logic in KerberosInit --- src/Access/KerberosInit.cpp | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 0c643e27bd2..0142708c699 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -126,11 +126,35 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw DB::Exception(0, "Error in setting output credential cache"); // action: init or renew - // todo: implement renew action - // todo: doing only init action: - ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + LOG_DEBUG(adqm_log,"Trying to renew credentials"); + ret = krb5_get_renewed_creds(k5->ctx, &my_creds, k5->me, k5->out_cc, nullptr); if (ret) - LOG_DEBUG(adqm_log,"Getting initial credentials"); + { + LOG_DEBUG(adqm_log,"Renew failed, making init credentials"); + ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + if (ret) + throw DB::Exception(0, "Error in init"); + else + LOG_DEBUG(adqm_log,"Getting initial credentials"); + } + else + { + LOG_DEBUG(adqm_log,"Successfull reviewal"); + ret = krb5_cc_initialize(k5->ctx, k5->out_cc, k5->me); + if (ret) + throw DB::Exception(0, "Error when initializing cache"); + LOG_DEBUG(adqm_log,"Initialized cache"); + ret = krb5_cc_store_cred(k5->ctx, k5->out_cc, &my_creds); + if (ret) + LOG_DEBUG(adqm_log,"Error while storing credentials"); + LOG_DEBUG(adqm_log,"Stored credentials"); + } + + if (k5->switch_to_cache) { + ret = krb5_cc_switch(k5->ctx, k5->out_cc); + if (ret) + throw DB::Exception(0, "Error while switching to new ccache"); + } LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); LOG_DEBUG(adqm_log,"KerberosInit: end"); From cb53aa15ec0a670eb98fd07738d95b74cbd1f73c Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 7 Jun 2022 12:06:22 +0300 Subject: [PATCH 030/408] Fix HDFSCommon and test_storage_kerberized_hdfs to make running integration tests --- src/Storages/HDFS/HDFSCommon.cpp | 4 +--- tests/integration/test_storage_kerberized_hdfs/test.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index f2ec7af368a..b13866d7a7b 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -121,9 +121,7 @@ void HDFSBuilderWrapper::runKinit() try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { - String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); - throw Exception(0, msg); + throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index fb00403b952..5ac8b4670f9 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -113,7 +113,7 @@ def test_read_table_expired(started_cluster): ) assert False, "Exception have to be thrown" except Exception as ex: - assert "DB::Exception: kinit failure:" in str(ex) + assert "DB::Exception: KerberosInit failure:" in str(ex) started_cluster.unpause_container("hdfskerberos") From a156a77890eeab8423f86a93e7a48e2784eeb0b0 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 7 Jun 2022 14:59:46 +0300 Subject: [PATCH 031/408] Add KerberosInit into StorageKafka --- src/Access/KerberosInit.cpp | 1 + src/Storages/HDFS/HDFSCommon.cpp | 3 ++- src/Storages/Kafka/StorageKafka.cpp | 21 +++++++++++++++++++ .../test_storage_kerberized_kafka/test.py | 4 +++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 0142708c699..724aa223c2b 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -9,6 +9,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con { auto adqm_log = &Poco::Logger::get("ADQM"); LOG_DEBUG(adqm_log,"KerberosInit: begin"); + //LOG_DEBUG(adqm_log,"KerberosInit: do nothing"); return 0; krb5_error_code ret; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index b13866d7a7b..35319ce1d83 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -118,7 +118,8 @@ void HDFSBuilderWrapper::runKinit() LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); std::unique_lock lck(kinit_mtx); KerberosInit k_init; - try { + try + { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 2409f8dcb6e..f6850f02511 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -44,6 +44,8 @@ #include #include +#include + namespace CurrentMetrics { @@ -515,6 +517,25 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) + { + LOG_DEBUG(log, "ADQM: preparing KerberosInit"); + String keytab = conf.get("sasl.kerberos.keytab"); + String principal = conf.get("sasl.kerberos.principal"); + LOG_DEBUG(log, "ADQM: keytab: {}, principal: {}", keytab, principal); + LOG_DEBUG(log, "ADQM: running KerberosInit"); + KerberosInit k_init; + try + { + k_init.init(keytab,principal); + } catch (const DB::Exception & e) { + LOG_ERROR(log, "ADQM: KerberosInit failure: {}", DB::getExceptionMessage(e, false)); + } + LOG_DEBUG(log, "ADQM: finished KerberosInit"); + conf.set("sasl.kerberos.kinit.cmd",""); + conf.set("sasl.kerberos.min.time.before.relogin","0"); + } + // Update consumer topic-specific configuration for (const auto & topic : topics) { diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 6347ba89c16..98d39dd5d4f 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -122,6 +122,7 @@ def test_kafka_json_as_string(kafka_cluster): {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} """ + logging.debug("ADQM: logs: %s", instance.grep_in_log("ADQM")) assert TSV(result) == TSV(expected) assert instance.contains_in_log( "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" @@ -182,7 +183,8 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") - assert instance.contains_in_log("Kerberos ticket refresh failed") + #~ assert instance.contains_in_log("Kerberos ticket refresh failed") + assert instance.contains_in_log("KerberosInit failure:") if __name__ == "__main__": From 2b76d0c6a992631b7727c5b051c98d4e077f4803 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 8 Jun 2022 12:26:35 +0300 Subject: [PATCH 032/408] Add new integration test for kerberized Kafka; remove old kinit code from HDFSCommon --- src/Access/KerberosInit.cpp | 1 - src/Storages/HDFS/HDFSCommon.cpp | 38 +---------------- src/Storages/HDFS/HDFSCommon.h | 2 - .../test_storage_kerberized_kafka/test.py | 41 ++++++++++++++++++- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 724aa223c2b..1fa550a4ec1 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -13,7 +13,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; - // todo: use deftype const char *deftype = nullptr; int flags = 0; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 35319ce1d83..a3554bafeff 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -77,44 +77,8 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration } } -String HDFSBuilderWrapper::getKinitCmd() -{ - - if (hadoop_kerberos_keytab.empty() || hadoop_kerberos_principal.empty()) - { - throw Exception("Not enough parameters to run kinit", - ErrorCodes::NO_ELEMENTS_IN_CONFIG); - } - - WriteBufferFromOwnString ss; - - String cache_name = hadoop_security_kerberos_ticket_cache_path.empty() ? - String() : - (String(" -c \"") + hadoop_security_kerberos_ticket_cache_path + "\""); - - // command to run looks like - // kinit -R -t /keytab_dir/clickhouse.keytab -k somebody@TEST.CLICKHOUSE.TECH || .. - ss << hadoop_kerberos_kinit_command << cache_name << - " -R -t \"" << hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal << - "|| " << hadoop_kerberos_kinit_command << cache_name << " -t \"" << - hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal; - return ss.str(); -} - void HDFSBuilderWrapper::runKinit() -{ /* - String cmd = getKinitCmd(); - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "running kinit: {}", cmd); - - std::unique_lock lck(kinit_mtx); - - auto command = ShellCommand::execute(cmd); - auto status = command->tryWait(); - if (status) - { - throw Exception("kinit failure: " + cmd, ErrorCodes::BAD_ARGUMENTS); - } - */ +{ LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); std::unique_lock lck(kinit_mtx); KerberosInit k_init; diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 0523849abe5..773661b0745 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -69,8 +69,6 @@ public: private: void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false); - String getKinitCmd(); - void runKinit(); // hdfs builder relies on an external config data storage diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 98d39dd5d4f..091307a2b01 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -122,12 +122,51 @@ def test_kafka_json_as_string(kafka_cluster): {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} """ - logging.debug("ADQM: logs: %s", instance.grep_in_log("ADQM")) assert TSV(result) == TSV(expected) assert instance.contains_in_log( "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) +def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster): + # Ticket should be expired after the wait time + # On run of SELECT query new ticket should be requested and SELECT query should run fine. + + kafka_produce( + kafka_cluster, + "kafka_json_as_string", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) + + instance.query( + """ + CREATE TABLE test.kafka (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string', + kafka_commit_on_select = 1, + kafka_group_name = 'kafka_json_as_string', + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + """ + ) + + time.sleep(45) # wait for ticket expiration + + result = instance.query("SELECT * FROM test.kafka;") + expected = """\ +{"t": 123, "e": {"x": "woof"} } +{"t": 124, "e": {"x": "test"} } +{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} +""" + assert TSV(result) == TSV(expected) + assert instance.contains_in_log( + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + ) def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), From 3cfea6e76f88e26b24cf4e6a371f033216dfa37f Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 8 Jun 2022 17:57:45 +0300 Subject: [PATCH 033/408] Cleanup code in KerberosInit, HDFSCommon and StorageKafka; update English and Russian documentation. --- .../table-engines/integrations/hdfs.md | 4 +- .../table-engines/integrations/kafka.md | 2 +- .../table-engines/integrations/hdfs.md | 3 +- .../table-engines/integrations/kafka.md | 2 +- src/Access/KerberosInit.cpp | 188 ++++++++---------- src/Access/KerberosInit.h | 19 +- src/Access/examples/kerberos_init.cpp | 2 + src/Storages/HDFS/HDFSCommon.cpp | 11 +- src/Storages/HDFS/HDFSCommon.h | 3 - src/Storages/Kafka/StorageKafka.cpp | 4 +- 10 files changed, 104 insertions(+), 134 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index 503bd779abf..146d1fcb3a6 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -186,7 +186,6 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | |libhdfs3\_conf | "" | ### Limitations {#limitations} @@ -200,8 +199,7 @@ Note that due to libhdfs3 limitations only old-fashioned approach is supported, datanode communications are not secured by SASL (`HADOOP_SECURE_DN_USER` is a reliable indicator of such security approach). Use `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` for reference. -If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_kerberos_kinit_command` is specified, `kinit` will be invoked. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. `kinit` tool and krb5 configuration files are required. - +If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_security_kerberos_ticket_cache_path` are specified, Kerberos authentication will be used. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. ## HDFS Namenode HA support {#namenode-ha} libhdfs3 support HDFS namenode HA. diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index a9d13194a59..94fcbab51ad 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -168,7 +168,7 @@ For a list of possible configuration options, see the [librdkafka configuration ### Kerberos support {#kafka-kerberos-support} To deal with Kerberos-aware Kafka, add `security_protocol` child element with `sasl_plaintext` value. It is enough if Kerberos ticket-granting ticket is obtained and cached by OS facilities. -ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` and `sasl.kerberos.kinit.cmd` child elements. +ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab` and `sasl_kerberos_principal` child elements. Example: diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index 0857359e987..84f31c0afcc 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -183,7 +183,6 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | ### Ограничения {#limitations} * `hadoop_security_kerberos_ticket_cache_path` и `libhdfs3_conf` могут быть определены только на глобальном, а не на пользовательском уровне @@ -196,7 +195,7 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 коммуникация с узлами данных не защищена SASL (`HADOOP_SECURE_DN_USER` надежный показатель такого подхода к безопасности). Используйте `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` для примера настроек. -Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_kerberos_kinit_command` указаны в настройках, `kinit` будет вызван. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. Необходимо также будет установить `kinit` и файлы конфигурации krb5. +Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_security_kerberos_ticket_cache_path` указаны в настройках, будет использоваться аутентификация с помощью Kerberos. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. ## Виртуальные столбцы {#virtual-columns} diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index b24a096015d..b51a0113302 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -167,7 +167,7 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ### Поддержка Kerberos {#kafka-kerberos-support} Чтобы начать работу с Kafka с поддержкой Kerberos, добавьте дочерний элемент `security_protocol` со значением `sasl_plaintext`. Этого будет достаточно, если получен тикет на получение тикета (ticket-granting ticket) Kerberos и он кэшируется средствами ОС. -ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` и `sasl.kerberos.kinit.cmd`. +ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab` и `sasl_kerberos_principal`. Пример: diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 1fa550a4ec1..1130447048b 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -5,11 +5,17 @@ #include #include +using namespace DB; + +std::mutex KerberosInit::kinit_mtx; + int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - auto adqm_log = &Poco::Logger::get("ADQM"); - LOG_DEBUG(adqm_log,"KerberosInit: begin"); - //LOG_DEBUG(adqm_log,"KerberosInit: do nothing"); return 0; + // Using mutex to prevent cache file corruptions + std::unique_lock lck(kinit_mtx); + + auto log = &Poco::Logger::get("ADQM"); + LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -17,182 +23,152 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con int flags = 0; if (!std::filesystem::exists(keytab_file)) - throw DB::Exception(0, "Error keytab file does not exist"); + throw Exception("Error keytab file does not exist", ErrorCodes::KERBEROS_ERROR); - memset(&k5d, 0, sizeof(k5d)); - k5 = &k5d; - // begin - ret = krb5_init_context(&k5->ctx); + memset(&k5, 0, sizeof(k5)); + ret = krb5_init_context(&k5.ctx); if (ret) - throw DB::Exception(0, "Error while initializing Kerberos 5 library"); + throw Exception("Error while initializing Kerberos 5 library", ErrorCodes::KERBEROS_ERROR); if (!cache_name.empty()) { - ret = krb5_cc_resolve(k5->ctx, cache_name.c_str(), &k5->out_cc); - // todo: analyze return code - LOG_DEBUG(adqm_log,"Resolved cache"); + ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); + if (ret) + throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Resolved cache"); } else { // Resolve the default ccache and get its type and default principal (if it is initialized). - ret = krb5_cc_default(k5->ctx, &defcache); + ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw DB::Exception(0, "Error while getting default ccache"); - LOG_DEBUG(adqm_log,"Resolved default cache"); - deftype = krb5_cc_get_type(k5->ctx, defcache); - if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) + throw Exception("Error while getting default ccache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Resolved default cache"); + deftype = krb5_cc_get_type(k5.ctx, defcache); + if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; } // Use the specified principal name. - ret = krb5_parse_name_flags(k5->ctx, principal.c_str(), flags, &k5->me); + ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), flags, &k5.me); if (ret) - throw DB::Exception(0, "Error when parsing principal name " + principal); + throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); // Cache related commands - if (k5->out_cc == nullptr && krb5_cc_support_switch(k5->ctx, deftype)) + if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) { // Use an existing cache for the client principal if we can. - ret = krb5_cc_cache_match(k5->ctx, k5->me, &k5->out_cc); + ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); if (ret && ret != KRB5_CC_NOTFOUND) - throw DB::Exception(0, "Error while searching for ccache for " + principal); + throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); if (!ret) { - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); - k5->switch_to_cache = 1; + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; } else if (defcache_princ != nullptr) { // Create a new cache to avoid overwriting the initialized default cache. - ret = krb5_cc_new_unique(k5->ctx, deftype, nullptr, &k5->out_cc); + ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) - throw DB::Exception(0, "Error while generating new ccache"); - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); - k5->switch_to_cache = 1; + throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; } } // Use the default cache if we haven't picked one yet. - if (k5->out_cc == nullptr) + if (k5.out_cc == nullptr) { - k5->out_cc = defcache; + k5.out_cc = defcache; defcache = nullptr; - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); } - ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); + ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) - throw DB::Exception(0, "Error when unparsing name"); + throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using principal: {}", k5.name); - LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); - - // init: memset(&my_creds, 0, sizeof(my_creds)); - - ret = krb5_get_init_creds_opt_alloc(k5->ctx, &options); + ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) - throw DB::Exception(0, "Error in options allocation"); + throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); - // todo -/* -#ifndef _WIN32 - if (strncmp(opts->keytab_name, "KDB:", 4) == 0) { - ret = kinit_kdb_init(&k5->ctx, k5->me->realm.data); - errctx = k5->ctx; - if (ret) { - com_err(progname, ret, - _("while setting up KDB keytab for realm %s"), - k5->me->realm.data); - goto cleanup; - } - } -#endif -*/ // Resolve keytab - ret = krb5_kt_resolve(k5->ctx, keytab_file.c_str(), &keytab); + ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) - throw DB::Exception(0, "Error resolving keytab "+keytab_file); + throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using keytab: {}", keytab_file); - LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); - - if (k5->in_cc) + if (k5.in_cc) { - ret = krb5_get_init_creds_opt_set_in_ccache(k5->ctx, options, k5->in_cc); + ret = krb5_get_init_creds_opt_set_in_ccache(k5.ctx, options, k5.in_cc); if (ret) - throw DB::Exception(0, "Error in setting input credential cache"); + throw Exception("Error in setting input credential cache", ErrorCodes::KERBEROS_ERROR); } - ret = krb5_get_init_creds_opt_set_out_ccache(k5->ctx, options, k5->out_cc); + ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) - throw DB::Exception(0, "Error in setting output credential cache"); + throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); - // action: init or renew - LOG_DEBUG(adqm_log,"Trying to renew credentials"); - ret = krb5_get_renewed_creds(k5->ctx, &my_creds, k5->me, k5->out_cc, nullptr); + // Action: init or renew + LOG_DEBUG(log,"Trying to renew credentials"); + ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(adqm_log,"Renew failed, making init credentials"); - ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + LOG_DEBUG(log,"Renew failed, trying to get initial credentials"); + ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) - throw DB::Exception(0, "Error in init"); + throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); else - LOG_DEBUG(adqm_log,"Getting initial credentials"); + LOG_DEBUG(log,"Got initial credentials"); } else { - LOG_DEBUG(adqm_log,"Successfull reviewal"); - ret = krb5_cc_initialize(k5->ctx, k5->out_cc, k5->me); + LOG_DEBUG(log,"Successfull renewal"); + ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) - throw DB::Exception(0, "Error when initializing cache"); - LOG_DEBUG(adqm_log,"Initialized cache"); - ret = krb5_cc_store_cred(k5->ctx, k5->out_cc, &my_creds); + throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Initialized cache"); + ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) - LOG_DEBUG(adqm_log,"Error while storing credentials"); - LOG_DEBUG(adqm_log,"Stored credentials"); + LOG_DEBUG(log,"Error while storing credentials"); + LOG_DEBUG(log,"Stored credentials"); } - if (k5->switch_to_cache) { - ret = krb5_cc_switch(k5->ctx, k5->out_cc); + if (k5.switch_to_cache) { + ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw DB::Exception(0, "Error while switching to new ccache"); + throw Exception("Error while switching to new ccache", ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); - LOG_DEBUG(adqm_log,"KerberosInit: end"); + LOG_DEBUG(log,"Authenticated to Kerberos v5"); return 0; } KerberosInit::~KerberosInit() { - if (k5->ctx) + std::unique_lock lck(kinit_mtx); + if (k5.ctx) { - //begin. cleanup: if (defcache) - krb5_cc_close(k5->ctx, defcache); - krb5_free_principal(k5->ctx, defcache_princ); + krb5_cc_close(k5.ctx, defcache); + krb5_free_principal(k5.ctx, defcache_princ); - // init. cleanup: - //todo: - /* - #ifndef _WIN32 - kinit_kdb_fini(); - #endif - */ if (options) - krb5_get_init_creds_opt_free(k5->ctx, options); - if (my_creds.client == k5->me) + krb5_get_init_creds_opt_free(k5.ctx, options); + if (my_creds.client == k5.me) my_creds.client = nullptr; - krb5_free_cred_contents(k5->ctx, &my_creds); + krb5_free_cred_contents(k5.ctx, &my_creds); if (keytab) - krb5_kt_close(k5->ctx, keytab); + krb5_kt_close(k5.ctx, keytab); - // end. cleanup: - krb5_free_unparsed_name(k5->ctx, k5->name); - krb5_free_principal(k5->ctx, k5->me); - if (k5->in_cc != nullptr) - krb5_cc_close(k5->ctx, k5->in_cc); - if (k5->out_cc != nullptr) - krb5_cc_close(k5->ctx, k5->out_cc); - krb5_free_context(k5->ctx); + krb5_free_unparsed_name(k5.ctx, k5.name); + krb5_free_principal(k5.ctx, k5.me); + if (k5.in_cc != nullptr) + krb5_cc_close(k5.ctx, k5.in_cc); + if (k5.out_cc != nullptr) + krb5_cc_close(k5.ctx, k5.out_cc); + krb5_free_context(k5.ctx); } - memset(k5, 0, sizeof(*k5)); } diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 07f619aaa0b..48401cbac1b 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,10 +4,16 @@ #include -//#include -//#include "k5-platform.h" #include -//#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int KERBEROS_ERROR; +} +} struct k5_data { @@ -24,13 +30,14 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct k5_data * k5 = nullptr; - //struct k5_data k5; - struct k5_data k5d; + //struct k5_data * k5 = nullptr; + struct k5_data k5; + //struct k5_data k5d; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt *options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; + static std::mutex kinit_mtx; }; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 554d57fb8b2..49923e81fd7 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -13,6 +13,7 @@ int main(int argc, char ** argv) if (argc < 3) { + std::cout << "kerberos_init obtains and caches an initial ticket-granting ticket for principal." << "\n\n"; std::cout << "Usage:" << "\n" << " kerberos_init keytab principal [cache]" << "\n"; return 0; } @@ -31,6 +32,7 @@ int main(int argc, char ** argv) k_init.init(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + return -1; } std::cout << "Done" << "\n"; return 0; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index a3554bafeff..e3592a7e53b 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -52,12 +52,6 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); continue; } - else if (key == "hadoop_kerberos_kinit_command") - { - need_kinit = true; - hadoop_kerberos_kinit_command = config.getString(key_path); - continue; - } else if (key == "hadoop_security_kerberos_ticket_cache_path") { if (isUser) @@ -80,13 +74,12 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); - std::unique_lock lck(kinit_mtx); KerberosInit k_init; try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { - throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); + throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } @@ -168,8 +161,6 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A return builder; } -std::mutex HDFSBuilderWrapper::kinit_mtx; - HDFSFSPtr createHDFSFS(hdfsBuilder * builder) { HDFSFSPtr fs(hdfsBuilderConnect(builder)); diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 773661b0745..cacb986a091 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -80,10 +79,8 @@ private: hdfsBuilder * hdfs_builder; String hadoop_kerberos_keytab; String hadoop_kerberos_principal; - String hadoop_kerberos_kinit_command = "kinit"; String hadoop_security_kerberos_ticket_cache_path; - static std::mutex kinit_mtx; std::vector> config_stor; bool need_kinit{false}; }; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index f6850f02511..d8018aebbcc 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -528,8 +528,8 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) try { k_init.init(keytab,principal); - } catch (const DB::Exception & e) { - LOG_ERROR(log, "ADQM: KerberosInit failure: {}", DB::getExceptionMessage(e, false)); + } catch (const Exception & e) { + LOG_ERROR(log, "ADQM: KerberosInit failure: {}", getExceptionMessage(e, false)); } LOG_DEBUG(log, "ADQM: finished KerberosInit"); conf.set("sasl.kerberos.kinit.cmd",""); From d1d6d874322a012fc5f16975d668dfd0dd554cad Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 9 Jun 2022 11:51:15 +0300 Subject: [PATCH 034/408] Cleanup code in KerberosInit --- src/Access/KerberosInit.cpp | 10 +++++----- src/Storages/HDFS/HDFSCommon.cpp | 4 ++-- src/Storages/Kafka/StorageKafka.cpp | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 1130447048b..549520e63ea 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -14,7 +14,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Using mutex to prevent cache file corruptions std::unique_lock lck(kinit_mtx); - auto log = &Poco::Logger::get("ADQM"); + auto log = &Poco::Logger::get("KerberosInit"); LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -39,10 +39,10 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } else { - // Resolve the default ccache and get its type and default principal (if it is initialized). + // Resolve the default cache and get its type and default principal (if it is initialized). ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw Exception("Error while getting default ccache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); LOG_DEBUG(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) @@ -116,7 +116,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(log,"Renew failed, trying to get initial credentials"); + LOG_DEBUG(log,"Renew failed ({}). Trying to get initial credentials", ret); ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); @@ -139,7 +139,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (k5.switch_to_cache) { ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw Exception("Error while switching to new ccache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(log,"Authenticated to Kerberos v5"); diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index e3592a7e53b..160c2279d17 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -73,7 +73,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); KerberosInit k_init; try { @@ -81,7 +81,7 @@ void HDFSBuilderWrapper::runKinit() } catch (const DB::Exception & e) { throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index d8018aebbcc..b473b9fee56 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -519,19 +519,17 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) { - LOG_DEBUG(log, "ADQM: preparing KerberosInit"); String keytab = conf.get("sasl.kerberos.keytab"); String principal = conf.get("sasl.kerberos.principal"); - LOG_DEBUG(log, "ADQM: keytab: {}, principal: {}", keytab, principal); - LOG_DEBUG(log, "ADQM: running KerberosInit"); + LOG_DEBUG(log, "Running KerberosInit"); KerberosInit k_init; try { k_init.init(keytab,principal); } catch (const Exception & e) { - LOG_ERROR(log, "ADQM: KerberosInit failure: {}", getExceptionMessage(e, false)); + LOG_ERROR(log, "KerberosInit failure: {}", getExceptionMessage(e, false)); } - LOG_DEBUG(log, "ADQM: finished KerberosInit"); + LOG_DEBUG(log, "Finished KerberosInit"); conf.set("sasl.kerberos.kinit.cmd",""); conf.set("sasl.kerberos.min.time.before.relogin","0"); } From 4c560584c700cdffbd66beab89a7e06e7546f646 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 10 Jun 2022 12:38:39 +0300 Subject: [PATCH 035/408] Code cleanup in KerberosInit and kafka integration tests --- src/Access/KerberosInit.h | 6 ++---- tests/integration/test_storage_kerberized_kafka/test.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 48401cbac1b..f5269793c97 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -20,7 +20,7 @@ struct k5_data krb5_context ctx; krb5_ccache in_cc, out_cc; krb5_principal me; - char *name; + char * name; krb5_boolean switch_to_cache; }; @@ -30,11 +30,9 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - //struct k5_data * k5 = nullptr; struct k5_data k5; - //struct k5_data k5d; krb5_ccache defcache = nullptr; - krb5_get_init_creds_opt *options = nullptr; + krb5_get_init_creds_opt * options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 091307a2b01..7856361deda 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -127,6 +127,7 @@ def test_kafka_json_as_string(kafka_cluster): "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) + def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster): # Ticket should be expired after the wait time # On run of SELECT query new ticket should be requested and SELECT query should run fine. @@ -155,7 +156,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) """ ) - time.sleep(45) # wait for ticket expiration + time.sleep(45) # wait for ticket expiration result = instance.query("SELECT * FROM test.kafka;") expected = """\ @@ -168,6 +169,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) + def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), # we need a ticket to @@ -222,7 +224,6 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") - #~ assert instance.contains_in_log("Kerberos ticket refresh failed") assert instance.contains_in_log("KerberosInit failure:") From 9555153f9532fca25145834e5e3f056f0e3f90b4 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 13 Jun 2022 13:29:08 +0000 Subject: [PATCH 036/408] Throw exception when xml user profile does not exist Closes #26086 --- src/Access/SettingsProfileElement.h | 4 ++++ src/Access/SettingsProfilesCache.cpp | 11 +++++++++-- src/Access/UsersConfigAccessStorage.cpp | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index c9f6936c832..e4be0821407 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -19,6 +19,10 @@ class AccessControl; struct SettingsProfileElement { std::optional parent_profile; + /// parent_profile_xml_name is set only when users are configured via XML + /// and is used for user-friendly error messages. + std::optional parent_profile_xml_name; + String setting_name; Field value; Field min_value; diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 2a3dedbbd7a..37a544cf051 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -170,8 +170,15 @@ void SettingsProfilesCache::substituteProfiles( continue; auto profile_it = all_profiles.find(profile_id); - if (profile_it == all_profiles.end()) - continue; + if (profile_it == all_profiles.end()) { + /// If the textual profile name is set, then users are configured via XML. + /// For these users we want to throw an exception when their profile can't + /// be found. Otherwise, these users are super admins. + if (element.parent_profile_xml_name) + throw Exception(ErrorCodes::THERE_IS_NO_PROFILE, "There is no profile '{}' in configuration file", *element.parent_profile_xml_name); + else + continue; + } const auto & profile = profile_it->second; const auto & profile_elements = profile->elements; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index a6c4388fef8..6a2995f2579 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -142,6 +142,7 @@ namespace auto profile_name = config.getString(profile_name_config); SettingsProfileElement profile_element; profile_element.parent_profile = generateID(AccessEntityType::SETTINGS_PROFILE, profile_name); + profile_element.parent_profile_xml_name = profile_name; user->settings.push_back(std::move(profile_element)); } @@ -473,6 +474,7 @@ namespace String parent_profile_name = config.getString(profile_config + "." + key); SettingsProfileElement profile_element; profile_element.parent_profile = generateID(AccessEntityType::SETTINGS_PROFILE, parent_profile_name); + profile_element.parent_profile_xml_name = parent_profile_name; profile->elements.emplace_back(std::move(profile_element)); continue; } From fc626b2897d366599239c20751ca51bacc4f138e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 14 Jun 2022 00:16:28 +0200 Subject: [PATCH 037/408] Update SettingsProfilesCache.cpp --- src/Access/SettingsProfilesCache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 37a544cf051..959cbb01ea2 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -170,7 +170,8 @@ void SettingsProfilesCache::substituteProfiles( continue; auto profile_it = all_profiles.find(profile_id); - if (profile_it == all_profiles.end()) { + if (profile_it == all_profiles.end()) + { /// If the textual profile name is set, then users are configured via XML. /// For these users we want to throw an exception when their profile can't /// be found. Otherwise, these users are super admins. From 9bf6b9d491fa5e8bba23a61fcca0a004b6c66451 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 11:37:02 +0300 Subject: [PATCH 038/408] Add kinit presence handling in StorageKafka; Cleanup code in HDFSCommon --- src/Storages/HDFS/HDFSCommon.cpp | 4 +++- src/Storages/Kafka/StorageKafka.cpp | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 160c2279d17..3ade1b1cfdf 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -78,7 +78,9 @@ void HDFSBuilderWrapper::runKinit() try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); - } catch (const DB::Exception & e) { + } + catch (const DB::Exception & e) + { throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index b473b9fee56..39b8adfbec4 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -517,6 +517,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + if (conf.has_property("sasl.kerberos.kinit.cmd")) + LOG_WARNING(log, "kinit executable is not allowed."); + + conf.set("sasl.kerberos.kinit.cmd",""); + conf.set("sasl.kerberos.min.time.before.relogin","0"); + if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) { String keytab = conf.get("sasl.kerberos.keytab"); @@ -526,12 +532,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) try { k_init.init(keytab,principal); - } catch (const Exception & e) { + } + catch (const Exception & e) + { LOG_ERROR(log, "KerberosInit failure: {}", getExceptionMessage(e, false)); } LOG_DEBUG(log, "Finished KerberosInit"); - conf.set("sasl.kerberos.kinit.cmd",""); - conf.set("sasl.kerberos.min.time.before.relogin","0"); } // Update consumer topic-specific configuration From dd5b0ee0656c6b10f25920f8557c2ef9f2b08260 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 17:02:53 +0300 Subject: [PATCH 039/408] Add kerberosInit() function to call KeberosInit --- src/Access/KerberosInit.cpp | 36 +++++++++++++++++++++++---- src/Access/KerberosInit.h | 25 +------------------ src/Access/examples/kerberos_init.cpp | 3 +-- src/Storages/HDFS/HDFSCommon.cpp | 3 +-- src/Storages/Kafka/StorageKafka.cpp | 3 +-- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 549520e63ea..95633dea077 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -7,13 +7,31 @@ using namespace DB; -std::mutex KerberosInit::kinit_mtx; +struct k5_data +{ + krb5_context ctx; + krb5_ccache in_cc, out_cc; + krb5_principal me; + char * name; + krb5_boolean switch_to_cache; +}; + +class KerberosInit +{ +public: + int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + ~KerberosInit(); +private: + struct k5_data k5; + krb5_ccache defcache = nullptr; + krb5_get_init_creds_opt * options = nullptr; + krb5_creds my_creds; + krb5_keytab keytab = nullptr; + krb5_principal defcache_princ = nullptr; +}; int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - // Using mutex to prevent cache file corruptions - std::unique_lock lck(kinit_mtx); - auto log = &Poco::Logger::get("KerberosInit"); LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); @@ -148,7 +166,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con KerberosInit::~KerberosInit() { - std::unique_lock lck(kinit_mtx); if (k5.ctx) { if (defcache) @@ -172,3 +189,12 @@ KerberosInit::~KerberosInit() krb5_free_context(k5.ctx); } } + +int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) +{ + // Using mutex to prevent cache file corruptions + static std::mutex kinit_mtx; + std::unique_lock lck(kinit_mtx); + KerberosInit k_init; + return k_init.init(keytab_file, principal, cache_name); +} diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index f5269793c97..94910661056 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -15,27 +15,4 @@ namespace ErrorCodes } } -struct k5_data -{ - krb5_context ctx; - krb5_ccache in_cc, out_cc; - krb5_principal me; - char * name; - krb5_boolean switch_to_cache; -}; - -class KerberosInit -{ -public: - int init(const String & keytab_file, const String & principal, const String & cache_name = ""); - ~KerberosInit(); -private: - struct k5_data k5; - krb5_ccache defcache = nullptr; - krb5_get_init_creds_opt * options = nullptr; - krb5_creds my_creds; - krb5_keytab keytab = nullptr; - krb5_principal defcache_princ = nullptr; - static std::mutex kinit_mtx; -}; - +int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 49923e81fd7..9a42a9e00e5 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -26,10 +26,9 @@ int main(int argc, char ** argv) Poco::Logger::root().setChannel(app_channel); Poco::Logger::root().setLevel("trace"); - KerberosInit k_init; try { - k_init.init(argv[1], argv[2], cache_name); + kerberosInit(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; return -1; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 3ade1b1cfdf..9d3cb2353b6 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -74,10 +74,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); - KerberosInit k_init; try { - k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); + kerberosInit(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 39b8adfbec4..789df36fcf0 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -528,10 +528,9 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) String keytab = conf.get("sasl.kerberos.keytab"); String principal = conf.get("sasl.kerberos.principal"); LOG_DEBUG(log, "Running KerberosInit"); - KerberosInit k_init; try { - k_init.init(keytab,principal); + kerberosInit(keytab,principal); } catch (const Exception & e) { From 1c26424371adc53f8eefd8d0e3cc604954be089e Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 19:35:21 +0300 Subject: [PATCH 040/408] Change message in StorageKafka; Code style correction --- src/Access/KerberosInit.cpp | 3 ++- src/Storages/Kafka/StorageKafka.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 95633dea077..6c4927dd4fa 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -154,7 +154,8 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(log,"Stored credentials"); } - if (k5.switch_to_cache) { + if (k5.switch_to_cache) + { ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 789df36fcf0..b4072cd6c0e 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -518,7 +518,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) loadFromConfig(conf, config, config_prefix); if (conf.has_property("sasl.kerberos.kinit.cmd")) - LOG_WARNING(log, "kinit executable is not allowed."); + LOG_WARNING(log, "sasl.kerberos.kinit.cmd configuration parameter is ignored."); conf.set("sasl.kerberos.kinit.cmd",""); conf.set("sasl.kerberos.min.time.before.relogin","0"); From 89a659e738a6a7236300f918b508ad0273745304 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 20:08:16 +0300 Subject: [PATCH 041/408] Move krb header files from KerberosInit.h to KerberosInit.cpp --- src/Access/KerberosInit.cpp | 2 ++ src/Access/KerberosInit.h | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 6c4927dd4fa..4fcb280f033 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include using namespace DB; diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 94910661056..60f959ab4ba 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,9 +4,6 @@ #include -#include -#include - namespace DB { namespace ErrorCodes From 344fbe8de4038ec6f93d67eb3455eda1ad82ce87 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 20:26:42 +0300 Subject: [PATCH 042/408] Fix code style --- src/Access/KerberosInit.cpp | 10 +++++++++- src/Access/KerberosInit.h | 8 -------- src/Access/examples/kerberos_init.cpp | 6 ++++-- src/Storages/HDFS/HDFSCommon.cpp | 1 + 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 4fcb280f033..304bb69d424 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -9,6 +9,14 @@ using namespace DB; +namespace DB +{ +namespace ErrorCodes +{ + extern const int KERBEROS_ERROR; +} +} + struct k5_data { krb5_context ctx; @@ -145,7 +153,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } else { - LOG_DEBUG(log,"Successfull renewal"); + LOG_DEBUG(log,"Successful renewal"); ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 60f959ab4ba..cfbbfc2e7d8 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,12 +4,4 @@ #include -namespace DB -{ -namespace ErrorCodes -{ - extern const int KERBEROS_ERROR; -} -} - int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 9a42a9e00e5..4868182fba2 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -29,8 +29,10 @@ int main(int argc, char ** argv) try { kerberosInit(argv[1], argv[2], cache_name); - } catch (const Exception & e) { - std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + } + catch (const Exception & e) + { + std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; return -1; } std::cout << "Done" << "\n"; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 9d3cb2353b6..8134aaf4981 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int NO_ELEMENTS_IN_CONFIG; + extern const int KERBEROS_ERROR; } const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs"; From 4f13521aa6de2fc6813214307da5e4c83ded9ea5 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Wed, 15 Jun 2022 14:50:47 +0800 Subject: [PATCH 043/408] struct type support for storage hive --- .../hive_server/prepare_hive_data.sh | 5 +++ src/DataTypes/NestedUtils.cpp | 22 +++++++++++-- src/Storages/Hive/StorageHive.cpp | 21 +++++++++--- src/Storages/Hive/StorageHive.h | 2 ++ tests/integration/test_hive_query/test.py | 33 +++++++++++++++++++ 5 files changed, 75 insertions(+), 8 deletions(-) diff --git a/docker/test/integration/hive_server/prepare_hive_data.sh b/docker/test/integration/hive_server/prepare_hive_data.sh index 8126b975612..cee5581e2db 100755 --- a/docker/test/integration/hive_server/prepare_hive_data.sh +++ b/docker/test/integration/hive_server/prepare_hive_data.sh @@ -8,3 +8,8 @@ hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo pa hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" + +hive -e "CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" + +hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3)); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3));" + diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 20aae08f363..db66d4d979b 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -17,7 +17,6 @@ #include - namespace DB { @@ -76,8 +75,7 @@ Block flatten(const Block & block) for (const auto & elem : block) { - const DataTypeArray * type_arr = typeid_cast(elem.type.get()); - if (type_arr) + if (const DataTypeArray * type_arr = typeid_cast(elem.type.get())) { const DataTypeTuple * type_tuple = typeid_cast(type_arr->getNestedType().get()); if (type_tuple && type_tuple->haveExplicitNames()) @@ -114,6 +112,24 @@ Block flatten(const Block & block) else res.insert(elem); } + else if (const DataTypeTuple * type_tuple = typeid_cast(elem.type.get())) + { + if (type_tuple->haveExplicitNames()) + { + const DataTypes & element_types = type_tuple->getElements(); + const Strings & names = type_tuple->getElementNames(); + const ColumnTuple * column_tuple = typeid_cast(elem.column.get()); + size_t tuple_size = column_tuple->tupleSize(); + for (size_t i = 0; i < tuple_size; ++i) + { + const auto & element_column = column_tuple->getColumn(i); + String nested_name = concatenateName(elem.name, names[i]); + res.insert(ColumnWithTypeAndName(element_column.getPtr(), element_types[i], nested_name)); + } + } + else + res.insert(elem); + } else res.insert(elem); } diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 6046dd58677..bc65ab2fd97 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -37,6 +37,8 @@ #include #include #include +#include +#include namespace DB { @@ -563,8 +565,8 @@ HiveFiles StorageHive::collectHiveFilesFromPartition( const ContextPtr & context_, PruneLevel prune_level) const { - LOG_DEBUG( - log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level)); + //LOG_DEBUG( + // log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level)); /// Skip partition "__HIVE_DEFAULT_PARTITION__" bool has_default_partition = false; @@ -766,8 +768,13 @@ Pipe StorageHive::read( sources_info->hive_metastore_client = hive_metastore_client; sources_info->partition_name_types = partition_name_types; - Block sample_block; const auto header_block = storage_snapshot->metadata->getSampleBlock(); + bool support_subset_columns = supportsSubsetOfColumns(); + Block flatten_block; + if (support_subset_columns) + flatten_block = Nested::flatten(header_block); + + Block sample_block; for (const auto & column : column_names) { if (header_block.has(column)) @@ -775,13 +782,17 @@ Pipe StorageHive::read( sample_block.insert(header_block.getByName(column)); continue; } - + else if (support_subset_columns && flatten_block.has(column)) + { + sample_block.insert(flatten_block.getByName(column)); + continue; + } if (column == "_path") sources_info->need_path_column = true; if (column == "_file") sources_info->need_file_column = true; } - + LOG_TRACE(&Poco::Logger::get("StorageHive"), "sample_block={}", sample_block.dumpNames()); if (num_streams > sources_info->hive_files.size()) num_streams = sources_info->hive_files.size(); diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h index d92d2dbd745..2f69ae00b6e 100644 --- a/src/Storages/Hive/StorageHive.h +++ b/src/Storages/Hive/StorageHive.h @@ -42,6 +42,8 @@ public: String getName() const override { return "Hive"; } bool supportsIndexForIn() const override { return true; } + + bool supportsSubcolumns() const override { return true; } bool mayBenefitFromIndexForIn( const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index fd4d91d6f78..00aaaec7afe 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -400,3 +400,36 @@ def test_cache_dir_use(started_cluster): ["bash", "-c", "ls /tmp/clickhouse_local_cache1 | wc -l"] ) assert result0 != "0" and result1 != "0" + +def test_cache_dir_use(started_cluster): + node = started_cluster.instances["h0_0_0"] + result0 = node.exec_in_container( + ["bash", "-c", "ls /tmp/clickhouse_local_cache | wc -l"] + ) + result1 = node.exec_in_container( + ["bash", "-c", "ls /tmp/clickhouse_local_cache1 | wc -l"] + ) + assert result0 != "0" and result1 != "0" + +def test_hive_struct_type(started_cluster): + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + CREATE TABLE IF NOT EXISTS default.test_hive_types (`f_tinyint` Int8, `f_smallint` Int16, `f_int` Int32, `f_integer` Int32, `f_bigint` Int64, `f_float` Float32, `f_double` Float64, `f_decimal` Float64, `f_timestamp` DateTime, `f_date` Date, `f_string` String, `f_varchar` String, `f_char` String, `f_bool` Boolean, `f_array_int` Array(Int32), `f_array_string` Array(String), `f_array_float` Array(Float32), `f_map_int` Map(String, Int32), `f_map_string` Map(String, String), `f_map_float` Map(String, Float32), `f_struct` Tuple(a String, b Int32, c Float32), `day` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'test_hive_types') PARTITION BY (day) + """ + ) + result = node.query( + """ + SELECT * FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 + """ + ) + expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3) 2022-02-20""" + assert result.strip() == expected_result + + result = node.query( + """ + SELECT day, f_struct.a FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 + """ + ) + expected_result = """2022-02-20 aaa""" + assert result.strip() == expected_result From 655e42c9bcd9c4fe409f0d72f30b6697f2a57eeb Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 16 Jun 2022 09:44:41 +0800 Subject: [PATCH 044/408] remove trace logs --- src/Storages/Hive/StorageHive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index bc65ab2fd97..0135da97445 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -792,7 +792,7 @@ Pipe StorageHive::read( if (column == "_file") sources_info->need_file_column = true; } - LOG_TRACE(&Poco::Logger::get("StorageHive"), "sample_block={}", sample_block.dumpNames()); + if (num_streams > sources_info->hive_files.size()) num_streams = sources_info->hive_files.size(); From e115e3f7311400d0e3c1af2b4dfb13bd3ad617f5 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 16 Jun 2022 09:53:04 +0800 Subject: [PATCH 045/408] remove unused header --- src/Storages/Hive/StorageHive.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 0135da97445..ab2b1ef7a09 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -38,7 +38,6 @@ #include #include #include -#include namespace DB { From 35d534c213faf5abde296dc041737d8970d0d2f3 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 16 Jun 2022 16:16:34 +0800 Subject: [PATCH 046/408] nested struct in struct --- .../hive_server/prepare_hive_data.sh | 4 +- src/DataTypes/NestedUtils.cpp | 68 +++++++++++++++- src/DataTypes/NestedUtils.h | 13 +++ .../Formats/Impl/ArrowColumnToCHColumn.cpp | 81 ++++++++++--------- src/Storages/Hive/StorageHive.cpp | 24 ++++-- tests/integration/test_hive_query/test.py | 6 +- 6 files changed, 145 insertions(+), 51 deletions(-) diff --git a/docker/test/integration/hive_server/prepare_hive_data.sh b/docker/test/integration/hive_server/prepare_hive_data.sh index cee5581e2db..39d435eb05a 100755 --- a/docker/test/integration/hive_server/prepare_hive_data.sh +++ b/docker/test/integration/hive_server/prepare_hive_data.sh @@ -9,7 +9,7 @@ hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo pa hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" -hive -e "CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" +hive -e "CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" -hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3)); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3));" +hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 10, 'y', 'xyz')); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 11, 'y', 'abc'));" diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index db66d4d979b..0df664ad408 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -259,7 +260,72 @@ std::unordered_set getAllTableNames(const Block & block, bool to_lower_c } return nested_table_names; } - } +NestedColumnExtractHelper::NestedColumnExtractHelper(const Block & block_, bool case_insentive_) + : block(block_) + , case_insentive(case_insentive_) +{} + +std::optional NestedColumnExtractHelper::extractColumn(const String & column_name) +{ + if (block.has(column_name, case_insentive)) + return {block.getByName(column_name, case_insentive)}; + + auto nested_names = Nested::splitName(column_name); + if (case_insentive) + { + boost::to_lower(nested_names.first); + boost::to_lower(nested_names.second); + } + if (!block.has(nested_names.first, case_insentive)) + return {}; + + if (!nested_tables.contains(nested_names.first)) + { + ColumnsWithTypeAndName columns = {block.getByName(nested_names.first, case_insentive)}; + nested_tables[nested_names.first] = std::make_shared(Nested::flatten(columns)); + } + + return extractColumn(column_name, nested_names.first, nested_names.second); +} + +std::optional NestedColumnExtractHelper::extractColumn( + const String & original_column_name, const String & column_name_prefix, const String & column_name_suffix) +{ + auto table_iter = nested_tables.find(column_name_prefix); + if (table_iter == nested_tables.end()) + { + return {}; + } + + auto & nested_table = table_iter->second; + auto nested_names = Nested::splitName(column_name_suffix); + auto new_column_name_prefix = Nested::concatenateName(column_name_prefix, nested_names.first); + if (nested_names.second.empty()) + { + if (nested_table->has(new_column_name_prefix, case_insentive)) + { + ColumnWithTypeAndName column = nested_table->getByName(new_column_name_prefix, case_insentive); + if (case_insentive) + column.name = original_column_name; + return {column}; + } + else + { + return {}; + } + } + + if (!nested_table->has(new_column_name_prefix, case_insentive)) + { + return {}; + } + + ColumnsWithTypeAndName columns = {nested_table->getByName(new_column_name_prefix, case_insentive)}; + Block sub_block(columns); + nested_tables[new_column_name_prefix] = std::make_shared(Nested::flatten(sub_block)); + return extractColumn(original_column_name, new_column_name_prefix, nested_names.second); + +} } diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index f6dc42d5c58..39f73b65100 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -35,4 +35,17 @@ namespace Nested std::unordered_set getAllTableNames(const Block & block, bool to_lower_case = false); } +class NestedColumnExtractHelper +{ +public: + explicit NestedColumnExtractHelper(const Block & block_, bool case_insentive_); + std::optional extractColumn(const String & column_name); +private: + std::optional + extractColumn(const String & original_column_name, const String & column_name_prefix, const String & column_name_suffix); + const Block & block; + bool case_insentive; + std::map nested_tables; +}; + } diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index c792d828e44..da89d6f7321 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,4 +1,6 @@ #include "ArrowColumnToCHColumn.h" +#include +#include #if USE_ARROW || USE_ORC || USE_PARQUET @@ -602,8 +604,8 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & Columns columns_list; UInt64 num_rows = name_to_column_ptr.begin()->second->length(); - columns_list.reserve(header.rows()); - std::unordered_map nested_tables; + columns_list.reserve(header.columns()); + std::unordered_map>> nested_tables; bool skipped = false; for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i) { @@ -613,55 +615,57 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & if (case_insensitive_matching) boost::to_lower(search_column_name); - bool read_from_nested = false; - String nested_table_name = Nested::extractTableName(header_column.name); - String search_nested_table_name = nested_table_name; - if (case_insensitive_matching) - boost::to_lower(search_nested_table_name); - + ColumnWithTypeAndName column; if (!name_to_column_ptr.contains(search_column_name)) { + bool read_from_nested = false; /// Check if it's a column from nested table. - if (import_nested && name_to_column_ptr.contains(search_nested_table_name)) + if (import_nested) { - if (!nested_tables.contains(search_nested_table_name)) + String nested_table_name = Nested::extractTableName(header_column.name); + String search_nested_table_name = nested_table_name; + if (case_insensitive_matching) + boost::to_lower(search_nested_table_name); + if (name_to_column_ptr.contains(search_nested_table_name)) { - std::shared_ptr arrow_column = name_to_column_ptr[search_nested_table_name]; - ColumnsWithTypeAndName cols - = {readColumnFromArrowColumn(arrow_column, nested_table_name, format_name, false, dictionary_values, true, true, false, skipped)}; - Block block(cols); - nested_tables[search_nested_table_name] = std::make_shared(Nested::flatten(block)); + if (!nested_tables.contains(search_nested_table_name)) + { + std::shared_ptr arrow_column = name_to_column_ptr[search_nested_table_name]; + ColumnsWithTypeAndName cols = {readColumnFromArrowColumn( + arrow_column, nested_table_name, format_name, false, dictionary_values, true, true, false, skipped)}; + BlockPtr block_ptr = std::make_shared(cols); + auto column_extractor = std::make_shared(*block_ptr, case_insensitive_matching); + nested_tables[search_nested_table_name] = {block_ptr, column_extractor}; + } + auto nested_column = nested_tables[search_nested_table_name].second->extractColumn(search_column_name); + if (nested_column) + { + column = *nested_column; + if (case_insensitive_matching) + column.name = header_column.name; + read_from_nested = true; + } } - - read_from_nested = nested_tables[search_nested_table_name]->has(header_column.name, case_insensitive_matching); } - if (!read_from_nested) { if (!allow_missing_columns) throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - - ColumnWithTypeAndName column; - column.name = header_column.name; - column.type = header_column.type; - column.column = header_column.column->cloneResized(num_rows); - columns_list.push_back(std::move(column.column)); - continue; + else + { + column.name = header_column.name; + column.type = header_column.type; + column.column = header_column.column->cloneResized(num_rows); + columns_list.push_back(std::move(column.column)); + continue; + } } } - - - ColumnWithTypeAndName column; - if (read_from_nested) - { - column = nested_tables[search_nested_table_name]->getByName(header_column.name, case_insensitive_matching); - if (case_insensitive_matching) - column.name = header_column.name; - } else { auto arrow_column = name_to_column_ptr[search_column_name]; - column = readColumnFromArrowColumn(arrow_column, header_column.name, format_name, false, dictionary_values, true, true, false, skipped); + column = readColumnFromArrowColumn( + arrow_column, header_column.name, format_name, false, dictionary_values, true, true, false, skipped); } try @@ -689,17 +693,16 @@ std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema { std::vector missing_columns; auto block_from_arrow = arrowSchemaToCHHeader(schema, format_name, false, &header, case_insensitive_matching); - auto flatten_block_from_arrow = Nested::flatten(block_from_arrow); + NestedColumnExtractHelper nested_columns_extractor(block_from_arrow, case_insensitive_matching); for (size_t i = 0, columns = header.columns(); i < columns; ++i) { const auto & header_column = header.getByPosition(i); bool read_from_nested = false; - String nested_table_name = Nested::extractTableName(header_column.name); if (!block_from_arrow.has(header_column.name, case_insensitive_matching)) { - if (import_nested && block_from_arrow.has(nested_table_name, case_insensitive_matching)) - read_from_nested = flatten_block_from_arrow.has(header_column.name, case_insensitive_matching); + if (import_nested && nested_columns_extractor.extractColumn(header_column.name)) + read_from_nested = true; if (!read_from_nested) { diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index ab2b1ef7a09..9dbfd1119dc 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -1,3 +1,4 @@ +#include #include #if USE_HIVE @@ -769,11 +770,18 @@ Pipe StorageHive::read( const auto header_block = storage_snapshot->metadata->getSampleBlock(); bool support_subset_columns = supportsSubsetOfColumns(); - Block flatten_block; - if (support_subset_columns) - flatten_block = Nested::flatten(header_block); + auto settings = context_->getSettingsRef(); + auto case_insensitive_matching = [&]() -> bool + { + if (format_name == "Parquet") + return settings.input_format_parquet_case_insensitive_column_matching; + else if (format_name == "ORC") + return settings.input_format_orc_case_insensitive_column_matching; + return false; + }; Block sample_block; + NestedColumnExtractHelper nested_columns_extractor(header_block, case_insensitive_matching()); for (const auto & column : column_names) { if (header_block.has(column)) @@ -781,10 +789,14 @@ Pipe StorageHive::read( sample_block.insert(header_block.getByName(column)); continue; } - else if (support_subset_columns && flatten_block.has(column)) + else if (support_subset_columns) { - sample_block.insert(flatten_block.getByName(column)); - continue; + auto subset_column = nested_columns_extractor.extractColumn(column); + if (subset_column) + { + sample_block.insert(*subset_column); + continue; + } } if (column == "_path") sources_info->need_path_column = true; diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index 00aaaec7afe..4bac09320ce 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -415,7 +415,7 @@ def test_hive_struct_type(started_cluster): node = started_cluster.instances["h0_0_0"] result = node.query( """ - CREATE TABLE IF NOT EXISTS default.test_hive_types (`f_tinyint` Int8, `f_smallint` Int16, `f_int` Int32, `f_integer` Int32, `f_bigint` Int64, `f_float` Float32, `f_double` Float64, `f_decimal` Float64, `f_timestamp` DateTime, `f_date` Date, `f_string` String, `f_varchar` String, `f_char` String, `f_bool` Boolean, `f_array_int` Array(Int32), `f_array_string` Array(String), `f_array_float` Array(Float32), `f_map_int` Map(String, Int32), `f_map_string` Map(String, String), `f_map_float` Map(String, Float32), `f_struct` Tuple(a String, b Int32, c Float32), `day` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'test_hive_types') PARTITION BY (day) + CREATE TABLE IF NOT EXISTS default.test_hive_types (`f_tinyint` Int8, `f_smallint` Int16, `f_int` Int32, `f_integer` Int32, `f_bigint` Int64, `f_float` Float32, `f_double` Float64, `f_decimal` Float64, `f_timestamp` DateTime, `f_date` Date, `f_string` String, `f_varchar` String, `f_char` String, `f_bool` Boolean, `f_array_int` Array(Int32), `f_array_string` Array(String), `f_array_float` Array(Float32), `f_map_int` Map(String, Int32), `f_map_string` Map(String, String), `f_map_float` Map(String, Float32), `f_struct` Tuple(a String, b Int32, c Float32, d Tuple(x Int32, y String)), `day` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'test_hive_types') PARTITION BY (day) """ ) result = node.query( @@ -423,7 +423,7 @@ def test_hive_struct_type(started_cluster): SELECT * FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 """ ) - expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3) 2022-02-20""" + expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3,(10,'xyz')) 2022-02-20""" assert result.strip() == expected_result result = node.query( @@ -431,5 +431,5 @@ def test_hive_struct_type(started_cluster): SELECT day, f_struct.a FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 """ ) - expected_result = """2022-02-20 aaa""" + expected_result = """2022-02-20 aaa 10""" assert result.strip() == expected_result From a8b17fec848a698dc8d0f6886b041089c991a923 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 16 Jun 2022 16:53:11 +0800 Subject: [PATCH 047/408] fixed a bug --- src/Storages/Hive/StorageHive.cpp | 2 +- src/Storages/Hive/StorageHive.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 9dbfd1119dc..60936f6a3f4 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -769,7 +769,7 @@ Pipe StorageHive::read( sources_info->partition_name_types = partition_name_types; const auto header_block = storage_snapshot->metadata->getSampleBlock(); - bool support_subset_columns = supportsSubsetOfColumns(); + bool support_subset_columns = supportsSubcolumns(); auto settings = context_->getSettingsRef(); auto case_insensitive_matching = [&]() -> bool diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h index 2f69ae00b6e..fd806553a86 100644 --- a/src/Storages/Hive/StorageHive.h +++ b/src/Storages/Hive/StorageHive.h @@ -44,6 +44,7 @@ public: bool supportsIndexForIn() const override { return true; } bool supportsSubcolumns() const override { return true; } + bool mayBenefitFromIndexForIn( const ASTPtr & /* left_in_operand */, ContextPtr /* query_context */, From d93fd3bd2dcdfec6d1eba790508e393d91858999 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 09:30:40 +0000 Subject: [PATCH 048/408] Add complilation support for case when krb5 is not used --- src/Access/KerberosInit.cpp | 2 ++ src/Access/KerberosInit.h | 4 ++++ src/Storages/HDFS/HDFSCommon.cpp | 5 ++++- src/Storages/Kafka/StorageKafka.cpp | 6 ++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 304bb69d424..8d4e2339bbf 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,7 @@ #include #include #include +#if USE_KRB5 #include #include @@ -209,3 +210,4 @@ int kerberosInit(const String & keytab_file, const String & principal, const Str KerberosInit k_init; return k_init.init(keytab_file, principal, cache_name); } +#endif // USE_KRB5 diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index cfbbfc2e7d8..4731baf706f 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,4 +4,8 @@ #include +#if USE_KRB5 + int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); + +#endif // USE_KRB5 diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 8134aaf4981..bee8ac21acb 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -10,8 +10,9 @@ #include #include #include +#if USE_KRB5 #include - +#endif // USE_KRB5 namespace DB { @@ -74,6 +75,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { + #if USE_KRB5 LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); try { @@ -84,6 +86,7 @@ void HDFSBuilderWrapper::runKinit() throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); + #endif // USE_KRB5 } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index b4072cd6c0e..a24630fa3b1 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -43,9 +43,9 @@ #include #include - +#if USE_KRB5 #include - +#endif // USE_KRB5 namespace CurrentMetrics { @@ -517,6 +517,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + #if USE_KRB5 if (conf.has_property("sasl.kerberos.kinit.cmd")) LOG_WARNING(log, "sasl.kerberos.kinit.cmd configuration parameter is ignored."); @@ -538,6 +539,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) } LOG_DEBUG(log, "Finished KerberosInit"); } + #endif // USE_KRB5 // Update consumer topic-specific configuration for (const auto & topic : topics) From 6e28275569d9ede757ab490f110e578524775458 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:21:04 +0300 Subject: [PATCH 049/408] Add warnings about using krb5 parameters --- src/Storages/HDFS/HDFSCommon.cpp | 19 ++++++++++++++++--- src/Storages/HDFS/HDFSCommon.h | 9 +++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index bee8ac21acb..87b2d02ada0 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -20,9 +20,10 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NETWORK_ERROR; + #if USE_KRB5 extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int NO_ELEMENTS_IN_CONFIG; extern const int KERBEROS_ERROR; + #endif // USE_KRB5 } const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs"; @@ -43,19 +44,28 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration String key_name; if (key == "hadoop_kerberos_keytab") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_keytab = config.getString(key_path); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_keytab parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_kerberos_principal") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_principal = config.getString(key_path); hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_principal parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_security_kerberos_ticket_cache_path") { + #if USE_KRB5 if (isUser) { throw Exception("hadoop.security.kerberos.ticket.cache.path cannot be set per user", @@ -64,6 +74,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration hadoop_security_kerberos_ticket_cache_path = config.getString(key_path); // standard param - pass further + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop.security.kerberos.ticket.cache.path parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 } key_name = boost::replace_all_copy(key, "_", "."); @@ -73,9 +86,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration } } +#if USE_KRB5 void HDFSBuilderWrapper::runKinit() { - #if USE_KRB5 LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); try { @@ -86,8 +99,8 @@ void HDFSBuilderWrapper::runKinit() throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); - #endif // USE_KRB5 } +#endif // USE_KRB5 HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) { diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index cacb986a091..9eb2dfd3e46 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -68,8 +68,6 @@ public: private: void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false); - void runKinit(); - // hdfs builder relies on an external config data storage std::pair& keep(const String & k, const String & v) { @@ -77,12 +75,15 @@ private: } hdfsBuilder * hdfs_builder; + std::vector> config_stor; + + #if USE_KRB5 + void runKinit(); String hadoop_kerberos_keytab; String hadoop_kerberos_principal; String hadoop_security_kerberos_ticket_cache_path; - - std::vector> config_stor; bool need_kinit{false}; + #endif // USE_KRB5 }; using HDFSFSPtr = std::unique_ptr, detail::HDFSFsDeleter>; From 0ab6bfd5d805b358cb61ae053909527a9253e47b Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:25:37 +0300 Subject: [PATCH 050/408] Add warning about using krb5 parameter in StorageKafka.cpp --- src/Storages/Kafka/StorageKafka.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index a24630fa3b1..e65ac27b096 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -539,6 +539,9 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) } LOG_DEBUG(log, "Finished KerberosInit"); } + #else // USE_KRB5 + if (conf.has_property("sasl.kerberos.keytab") || conf.has_property("sasl.kerberos.principal")) + LOG_WARNING(log, "Kerberos-related parameters are ignored because ClickHouse was built without support of krb5 library."); #endif // USE_KRB5 // Update consumer topic-specific configuration From 9753ea91575e306ee8729ec533ed93abc0adc261 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 16 Jun 2022 19:27:30 +0800 Subject: [PATCH 051/408] fixed a bug fixed a bug in TranslateQualifiedNamesMatcher to handle nested tuple field with the table qulified name --- src/Interpreters/IdentifierSemantic.cpp | 7 +++ src/Interpreters/IdentifierSemantic.h | 1 + .../TranslateQualifiedNamesVisitor.cpp | 46 +++++++++---------- ...ranslate_qualified_names_matcher.reference | 2 + ...heck_sranslate_qualified_names_matcher.sql | 10 ++++ 5 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference create mode 100644 tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index dbfdba77619..14cb43e2a2e 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -188,6 +188,13 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const return canReferColumnToTable(identifier, table_with_columns.table); } +std::optional IdentifierSemantic::getColumnNamePart(const ASTIdentifier & node, size_t pos) +{ + if (pos >= node.name_parts.size()) + return {}; + return node.name_parts[pos]; +} + /// Strip qualifications from left side of column name. /// Example: 'database.table.name' -> 'name'. void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table) diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 5dc828c36ba..622813a8abe 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -40,6 +40,7 @@ struct IdentifierSemantic /// @returns name for column identifiers static std::optional getColumnName(const ASTIdentifier & node); static std::optional getColumnName(const ASTPtr & ast); + static std::optional getColumnNamePart(const ASTIdentifier & node, size_t pos); /// @returns name for 'not a column' identifiers static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 3129f9d7fe2..91d9ca082d5 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -33,35 +33,33 @@ namespace ErrorCodes bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const { const auto & table = tables[table_pos].table; - auto nested1 = IdentifierSemantic::extractNestedName(identifier, table.table); - auto nested2 = IdentifierSemantic::extractNestedName(identifier, table.alias); - - const String & short_name = identifier.shortName(); const auto & columns = tables[table_pos].columns; + if (columns.empty()) + return false; + auto match = IdentifierSemantic::canReferColumnToTable(identifier, table); + size_t to_strip = 0; + switch (match) + { + case IdentifierSemantic::ColumnMatch::TableName: + case IdentifierSemantic::ColumnMatch::AliasedTableName: + case IdentifierSemantic::ColumnMatch::TableAlias: + to_strip = 1; + break; + case IdentifierSemantic::ColumnMatch::DBAndTable: + to_strip = 2; + break; + default: + break; + } + const auto & column_name = IdentifierSemantic::getColumnNamePart(identifier, to_strip); + if (!column_name) + return true; for (const auto & column : columns) { - const String & known_name = column.name; - if (short_name == known_name) - return false; - if (nested1 && *nested1 == known_name) - return false; - if (nested2 && *nested2 == known_name) + if (*column_name == column.name) return false; } - - const auto & hidden_columns = tables[table_pos].hidden_columns; - for (const auto & column : hidden_columns) - { - const String & known_name = column.name; - if (short_name == known_name) - return false; - if (nested1 && *nested1 == known_name) - return false; - if (nested2 && *nested2 == known_name) - return false; - } - - return !columns.empty(); + return true; } bool TranslateQualifiedNamesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child) diff --git a/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference b/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference new file mode 100644 index 00000000000..c094c553f81 --- /dev/null +++ b/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference @@ -0,0 +1,2 @@ +12 +12 diff --git a/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql b/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql new file mode 100644 index 00000000000..f2b5c372ba7 --- /dev/null +++ b/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql @@ -0,0 +1,10 @@ +CREATE TABLE nested_name_tuples +( + `a` Tuple(x String, y Tuple(i Int32, j String)) +) +ENGINE = Memory; + +INSERT INTO nested_name_tuples VALUS(('asd', (12, 'ddd'))); + +SELECT t.a.y.i FROM nested_name_tuples as t; +SELECT nested_name_tuples.a.y.i FROM nested_name_tuples as t; From ed3fe84b63d14cb05898cf12eac8024ae3921110 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:45:27 +0300 Subject: [PATCH 052/408] Fix runKinit() is called only for USE_KRB5 --- src/Storages/HDFS/HDFSCommon.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 87b2d02ada0..3ac9f50231d 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -171,10 +171,12 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A } } + #if USE_KRB5 if (builder.need_kinit) { builder.runKinit(); } + #endif // USE_KRB5 return builder; } From 285bb44b2542a16a36e6bcd1496a13682066b7fb Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 09:33:16 +0800 Subject: [PATCH 053/408] nothing --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 91d9ca082d5..a1432b10b3c 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -36,6 +36,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const const auto & columns = tables[table_pos].columns; if (columns.empty()) return false; + auto match = IdentifierSemantic::canReferColumnToTable(identifier, table); size_t to_strip = 0; switch (match) From 8c629085e485f15a221f2e443f3b89e95bc0d9e0 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 09:36:59 +0800 Subject: [PATCH 054/408] simplified code --- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index da89d6f7321..305009a070f 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -698,18 +698,13 @@ std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema for (size_t i = 0, columns = header.columns(); i < columns; ++i) { const auto & header_column = header.getByPosition(i); - bool read_from_nested = false; if (!block_from_arrow.has(header_column.name, case_insensitive_matching)) { - if (import_nested && nested_columns_extractor.extractColumn(header_column.name)) - read_from_nested = true; - - if (!read_from_nested) + if (!import_nested || !nested_columns_extractor.extractColumn(header_column.name)) { if (!allow_missing_columns) throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - - missing_columns.push_back(i); + missing_columns.push_back(i); } } } From 68c588585187f90e5c920abafb1ddfa0f2f132aa Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 12:19:25 +0800 Subject: [PATCH 055/408] fixed bugs --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index a1432b10b3c..9a24aed9814 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -19,7 +19,6 @@ #include #include - namespace DB { @@ -55,11 +54,19 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const const auto & column_name = IdentifierSemantic::getColumnNamePart(identifier, to_strip); if (!column_name) return true; + for (const auto & column : columns) { if (*column_name == column.name) return false; } + const auto & hidden_columns = tables[table_pos].hidden_columns; + for (const auto & column : hidden_columns) + { + const String & known_name = column.name; + if (*column_name == known_name) + return false; + } return true; } From 55757dfb34bb0c27fd7d960912925cca0aae42e5 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 15:13:31 +0800 Subject: [PATCH 056/408] fixed bugs --- src/Interpreters/IdentifierSemantic.cpp | 34 ++++++++--- src/Interpreters/IdentifierSemantic.h | 3 +- .../TranslateQualifiedNamesVisitor.cpp | 59 ++++++++++++------- .../TranslateQualifiedNamesVisitor.h | 2 + 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 14cb43e2a2e..f0658cb7c9b 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -142,6 +142,33 @@ std::optional IdentifierSemantic::extractNestedName(const ASTIdentifier return {}; } +String IdentifierSemantic::extractNestedName(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & table) +{ + auto match = IdentifierSemantic::canReferColumnToTable(identifier, table); + size_t to_strip = 0; + switch (match) + { + case IdentifierSemantic::ColumnMatch::TableName: + case IdentifierSemantic::ColumnMatch::AliasedTableName: + case IdentifierSemantic::ColumnMatch::TableAlias: + to_strip = 1; + break; + case IdentifierSemantic::ColumnMatch::DBAndTable: + to_strip = 2; + break; + default: + break; + } + String res; + for (size_t i = to_strip, sz = identifier.name_parts.size(); i < sz; ++i) + { + if (!res.empty()) + res += "."; + res += identifier.name_parts[i]; + } + return res; +} + bool IdentifierSemantic::doesIdentifierBelongTo(const ASTIdentifier & identifier, const String & database, const String & table) { size_t num_components = identifier.name_parts.size(); @@ -188,13 +215,6 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const return canReferColumnToTable(identifier, table_with_columns.table); } -std::optional IdentifierSemantic::getColumnNamePart(const ASTIdentifier & node, size_t pos) -{ - if (pos >= node.name_parts.size()) - return {}; - return node.name_parts[pos]; -} - /// Strip qualifications from left side of column name. /// Example: 'database.table.name' -> 'name'. void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table) diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 622813a8abe..c082e83b75c 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -40,11 +40,12 @@ struct IdentifierSemantic /// @returns name for column identifiers static std::optional getColumnName(const ASTIdentifier & node); static std::optional getColumnName(const ASTPtr & ast); - static std::optional getColumnNamePart(const ASTIdentifier & node, size_t pos); /// @returns name for 'not a column' identifiers static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); + static String extractNestedName(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & table); + static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & table_with_columns); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 9a24aed9814..ac060a48270 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -6,6 +7,7 @@ #include #include #include +#include #include #include @@ -19,6 +21,7 @@ #include #include + namespace DB { @@ -28,43 +31,55 @@ namespace ErrorCodes extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; } +bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const String & name, const String & column_name, DataTypePtr column_type) +{ + if (name.size() < column_name.size()) + return false; + + if (std::strncmp(name.data(), column_name.data(), column_name.size()) != 0) + return false; + + if (name.size() == column_name.size()) + return true; + /// In case the type is named tuple, check the name recursively. + if (const DataTypeTuple * type_tuple = typeid_cast(column_type.get())) + { + if (type_tuple->haveExplicitNames() && name.at(column_name.size()) == '.') + { + const Strings & names = type_tuple->getElementNames(); + const DataTypes & element_types = type_tuple->getElements(); + for (size_t i = 0; i < names.size(); ++i) + { + if (matchColumnName(name.substr(column_name.size() + 1, name.size() - column_name.size()), names[i], element_types[i])) + { + return true; + } + } + } + } + + return false; +} bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const { const auto & table = tables[table_pos].table; const auto & columns = tables[table_pos].columns; if (columns.empty()) return false; - - auto match = IdentifierSemantic::canReferColumnToTable(identifier, table); - size_t to_strip = 0; - switch (match) - { - case IdentifierSemantic::ColumnMatch::TableName: - case IdentifierSemantic::ColumnMatch::AliasedTableName: - case IdentifierSemantic::ColumnMatch::TableAlias: - to_strip = 1; - break; - case IdentifierSemantic::ColumnMatch::DBAndTable: - to_strip = 2; - break; - default: - break; - } - const auto & column_name = IdentifierSemantic::getColumnNamePart(identifier, to_strip); - if (!column_name) - return true; + + // Remove database and table name from the identifier'name + auto full_name = IdentifierSemantic::extractNestedName(identifier, table); for (const auto & column : columns) { - if (*column_name == column.name) + if (matchColumnName(full_name, column.name, column.type)) return false; } const auto & hidden_columns = tables[table_pos].hidden_columns; for (const auto & column : hidden_columns) { - const String & known_name = column.name; - if (*column_name == known_name) + if (matchColumnName(full_name, column.name, column.type)) return false; } return true; diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index 9c46d926eca..b1d4d94d01c 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -39,6 +39,7 @@ public: bool hasTable() const { return !tables.empty(); } bool processAsterisks() const { return hasTable() && has_columns; } bool unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const; + static bool matchColumnName(const String & name, const String & column_name, DataTypePtr column_type); }; static void visit(ASTPtr & ast, Data & data); @@ -53,6 +54,7 @@ private: static void visit(ASTFunction &, const ASTPtr &, Data &); static void extractJoinUsingColumns(ASTPtr ast, Data & data); + }; /// Visits AST for names qualification. From 14609e043ceaf84c88fa733896ef8b851436077b Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 15:15:09 +0800 Subject: [PATCH 057/408] fixed style --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index ac060a48270..60753dcc1f5 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -35,10 +35,10 @@ bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const String & name, { if (name.size() < column_name.size()) return false; - + if (std::strncmp(name.data(), column_name.data(), column_name.size()) != 0) return false; - + if (name.size() == column_name.size()) return true; @@ -67,7 +67,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const const auto & columns = tables[table_pos].columns; if (columns.empty()) return false; - + // Remove database and table name from the identifier'name auto full_name = IdentifierSemantic::extractNestedName(identifier, table); From 4a0a6e8cde7efb460810c17abb5c2cd9caa154bc Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 15:17:56 +0800 Subject: [PATCH 058/408] small optimization --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 60753dcc1f5..9292ef1353e 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -49,9 +49,10 @@ bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const String & name, { const Strings & names = type_tuple->getElementNames(); const DataTypes & element_types = type_tuple->getElements(); + String sub_name = name.substr(column_name.size() + 1, name.size() - column_name.size()); for (size_t i = 0; i < names.size(); ++i) { - if (matchColumnName(name.substr(column_name.size() + 1, name.size() - column_name.size()), names[i], element_types[i])) + if (matchColumnName(sub_name, names[i], element_types[i])) { return true; } From 308ca625181a4c8cc044250c977939a434c5265b Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 15:37:54 +0800 Subject: [PATCH 059/408] fixed bugs --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 9292ef1353e..fd71dc01595 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -66,8 +66,6 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const { const auto & table = tables[table_pos].table; const auto & columns = tables[table_pos].columns; - if (columns.empty()) - return false; // Remove database and table name from the identifier'name auto full_name = IdentifierSemantic::extractNestedName(identifier, table); @@ -83,7 +81,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const if (matchColumnName(full_name, column.name, column.type)) return false; } - return true; + return !columns.empty(); } bool TranslateQualifiedNamesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child) From 3d3c044d572be17197caaf007e05efd7818fff3b Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 17 Jun 2022 16:39:28 +0800 Subject: [PATCH 060/408] rename files --- ... => 02337_check_translate_qualified_names_matcher.reference} | 0 ...er.sql => 02337_check_translate_qualified_names_matcher.sql} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/queries/0_stateless/{02337_check_sranslate_qualified_names_matcher.reference => 02337_check_translate_qualified_names_matcher.reference} (100%) rename tests/queries/0_stateless/{02337_check_sranslate_qualified_names_matcher.sql => 02337_check_translate_qualified_names_matcher.sql} (77%) diff --git a/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference b/tests/queries/0_stateless/02337_check_translate_qualified_names_matcher.reference similarity index 100% rename from tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.reference rename to tests/queries/0_stateless/02337_check_translate_qualified_names_matcher.reference diff --git a/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql b/tests/queries/0_stateless/02337_check_translate_qualified_names_matcher.sql similarity index 77% rename from tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql rename to tests/queries/0_stateless/02337_check_translate_qualified_names_matcher.sql index f2b5c372ba7..09ab591f98a 100644 --- a/tests/queries/0_stateless/02337_check_sranslate_qualified_names_matcher.sql +++ b/tests/queries/0_stateless/02337_check_translate_qualified_names_matcher.sql @@ -4,7 +4,7 @@ CREATE TABLE nested_name_tuples ) ENGINE = Memory; -INSERT INTO nested_name_tuples VALUS(('asd', (12, 'ddd'))); +INSERT INTO nested_name_tuples VALUES(('asd', (12, 'ddd'))); SELECT t.a.y.i FROM nested_name_tuples as t; SELECT nested_name_tuples.a.y.i FROM nested_name_tuples as t; From c13bf03fe066950a48d85625d55d7d0552fad636 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Mon, 20 Jun 2022 09:56:21 +0800 Subject: [PATCH 061/408] fixed code style --- src/DataTypes/NestedUtils.cpp | 4 ++-- tests/integration/test_hive_query/test.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 0df664ad408..2e429bcff10 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -311,7 +311,7 @@ std::optional NestedColumnExtractHelper::extractColumn( column.name = original_column_name; return {column}; } - else + else { return {}; } @@ -325,7 +325,7 @@ std::optional NestedColumnExtractHelper::extractColumn( ColumnsWithTypeAndName columns = {nested_table->getByName(new_column_name_prefix, case_insentive)}; Block sub_block(columns); nested_tables[new_column_name_prefix] = std::make_shared(Nested::flatten(sub_block)); - return extractColumn(original_column_name, new_column_name_prefix, nested_names.second); + return extractColumn(original_column_name, new_column_name_prefix, nested_names.second); } } diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index 4bac09320ce..538c99d1a65 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -401,6 +401,7 @@ def test_cache_dir_use(started_cluster): ) assert result0 != "0" and result1 != "0" + def test_cache_dir_use(started_cluster): node = started_cluster.instances["h0_0_0"] result0 = node.exec_in_container( @@ -411,6 +412,7 @@ def test_cache_dir_use(started_cluster): ) assert result0 != "0" and result1 != "0" + def test_hive_struct_type(started_cluster): node = started_cluster.instances["h0_0_0"] result = node.query( @@ -419,17 +421,17 @@ def test_hive_struct_type(started_cluster): """ ) result = node.query( - """ + """ SELECT * FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 - """ + """ ) - expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3,(10,'xyz')) 2022-02-20""" + expected_result = """1 2 3 4 5 6.11 7.22 8 2022-02-20 14:47:04 2022-02-20 hello world hello world hello world true [1,2,3] ['hello world','hello world'] [1.1,1.2] {'a':100,'b':200,'c':300} {'a':'aa','b':'bb','c':'cc'} {'a':111.1,'b':222.2,'c':333.3} ('aaa',200,333.3,(10,'xyz')) 2022-02-20""" assert result.strip() == expected_result result = node.query( - """ + """ SELECT day, f_struct.a FROM default.test_hive_types WHERE day = '2022-02-20' SETTINGS input_format_parquet_import_nested=1 - """ + """ ) expected_result = """2022-02-20 aaa 10""" assert result.strip() == expected_result From 6ac68e8303c0208979089ba6d8083c329d72f8f1 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 30 May 2022 22:17:46 +0000 Subject: [PATCH 062/408] DISTINCT in order optimization + optimization for DISTINCT containing primary key columns --- src/Core/Settings.h | 1 + src/Interpreters/ExpressionAnalyzer.cpp | 2 + src/Interpreters/ExpressionAnalyzer.h | 1 + src/Interpreters/InterpreterSelectQuery.cpp | 24 +- src/Interpreters/InterpreterSelectQuery.h | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 2 +- src/Processors/QueryPlan/DistinctStep.cpp | 31 ++- src/Processors/QueryPlan/DistinctStep.h | 13 +- .../DistinctPrimaryKeyTransform.cpp | 214 ++++++++++++++++++ .../Transforms/DistinctPrimaryKeyTransform.h | 63 ++++++ src/Storages/ReadInOrderOptimizer.cpp | 33 ++- src/Storages/ReadInOrderOptimizer.h | 10 + src/Storages/SelectQueryInfo.h | 12 +- ...7_distinct_in_order_optimization.reference | 83 +++++++ .../02317_distinct_in_order_optimization.sql | 47 ++++ 15 files changed, 512 insertions(+), 26 deletions(-) create mode 100644 src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp create mode 100644 src/Processors/Transforms/DistinctPrimaryKeyTransform.h create mode 100644 tests/queries/0_stateless/02317_distinct_in_order_optimization.reference create mode 100644 tests/queries/0_stateless/02317_distinct_in_order_optimization.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 756e6eb651e..27902b8f2df 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -603,6 +603,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ + M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization for primary key (prefix) columns in MergeTree family table engines.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 00333503db1..f05cada7929 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1859,6 +1859,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( && !query.final() && join_allow_read_in_order; + optimize_distinct_in_order = settings.optimize_distinct_in_order && storage && query.distinct; + /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 80c664832e5..79d2c16efff 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -232,6 +232,7 @@ struct ExpressionAnalysisResult bool remove_where_filter = false; bool optimize_read_in_order = false; bool optimize_aggregation_in_order = false; + bool optimize_distinct_in_order = false; bool join_has_delayed_stream = false; bool use_grouping_set_key = false; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ec7c3878b06..ed6fd7ec7fd 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1243,7 +1243,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

getInputOrder(metadata_snapshot, context, limit); } + if (!analysis_result.optimize_aggregation_in_order && analysis_result.optimize_distinct_in_order) + { + query_info.distinct_optimizer = std::make_shared(analysis_result.selected_columns); + query_info.distinct_order_info = query_info.distinct_optimizer->getInputOrder(metadata_snapshot); + } query_info.storage_limits = std::make_shared(storage_limits); @@ -2554,7 +2559,7 @@ void InterpreterSelectQuery::executeProjection(QueryPlan & query_plan, const Act } -void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct) +void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct, InputOrderInfoPtr distinct_info) { auto & query = getSelectQuery(); if (query.distinct) @@ -2571,8 +2576,11 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - auto distinct_step - = std::make_unique(query_plan.getCurrentDataStream(), limits, limit_for_distinct, columns, pre_distinct); + if (distinct_info && !settings.optimize_distinct_in_order) + distinct_info = nullptr; + + auto distinct_step = std::make_unique( + query_plan.getCurrentDataStream(), limits, limit_for_distinct, columns, pre_distinct, distinct_info); if (pre_distinct) distinct_step->setStepDescription("Preliminary DISTINCT"); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 40afaaaeed0..f441c3774a4 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -180,7 +180,7 @@ private: void executeLimit(QueryPlan & query_plan); void executeOffset(QueryPlan & query_plan); static void executeProjection(QueryPlan & query_plan, const ActionsDAGPtr & expression); - void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct); + void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct, InputOrderInfoPtr distinct_info); void executeExtremes(QueryPlan & query_plan); void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan, std::unordered_map & subqueries_for_sets); void diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index f0150fe663f..a1d8431d626 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -319,7 +319,7 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); auto distinct_step - = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false); + = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, nullptr); query_plan.addStep(std::move(distinct_step)); } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 16670c53fd5..6d8f1cc4f92 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -43,7 +44,8 @@ DistinctStep::DistinctStep( const SizeLimits & set_size_limits_, UInt64 limit_hint_, const Names & columns_, - bool pre_distinct_) + bool pre_distinct_, + const InputOrderInfoPtr & distinct_info_) : ITransformingStep( input_stream_, input_stream_.header, @@ -51,6 +53,7 @@ DistinctStep::DistinctStep( , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) + , distinct_info(distinct_info_) , pre_distinct(pre_distinct_) { if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing @@ -71,13 +74,29 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil if (!pre_distinct) pipeline.resize(1); - pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + if (pre_distinct && distinct_info) { - if (stream_type != QueryPipelineBuilder::StreamType::Main) - return nullptr; + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; - return std::make_shared(header, set_size_limits, limit_hint, columns); - }); + return std::make_shared( + header, set_size_limits, limit_hint, distinct_info->order_key_prefix_descr, columns); + }); + } + else + { + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; + + return std::make_shared(header, set_size_limits, limit_hint, columns); + }); + } } void DistinctStep::describeActions(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index a48a779425d..c64b9f6ccb5 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include namespace DB { @@ -10,11 +11,12 @@ class DistinctStep : public ITransformingStep { public: DistinctStep( - const DataStream & input_stream_, - const SizeLimits & set_size_limits_, - UInt64 limit_hint_, - const Names & columns_, - bool pre_distinct_); /// If is enabled, execute distinct for separate streams. Otherwise, merge streams. + const DataStream & input_stream_, + const SizeLimits & set_size_limits_, + UInt64 limit_hint_, + const Names & columns_, + bool pre_distinct_, /// If is enabled, execute distinct for separate streams. Otherwise, merge streams. + const InputOrderInfoPtr & distinct_info_); String getName() const override { return "Distinct"; } @@ -27,6 +29,7 @@ private: SizeLimits set_size_limits; UInt64 limit_hint; Names columns; + InputOrderInfoPtr distinct_info; bool pre_distinct; }; diff --git a/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp b/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp new file mode 100644 index 00000000000..ba3a72f1285 --- /dev/null +++ b/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp @@ -0,0 +1,214 @@ +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SET_SIZE_LIMIT_EXCEEDED; +} + +DistinctPrimaryKeyTransform::DistinctPrimaryKeyTransform( + const Block & header_, + const SizeLimits & output_size_limits_, + UInt64 limit_hint_, + const SortDescription & sorted_columns_descr_, + const Names & source_columns) + : ISimpleTransform(header_, header_, true) + , limit_hint(limit_hint_) + , output_size_limits(output_size_limits_) + , sorted_columns_descr(sorted_columns_descr_) +{ + /// calculate sorted columns positions + sorted_columns_pos.reserve(sorted_columns_descr.size()); + for (auto const & descr : sorted_columns_descr) + { + size_t pos = header_.getPositionByName(descr.column_name); + sorted_columns_pos.emplace_back(pos); + } + + /// calculate not-sorted columns positions + other_columns_pos.reserve(source_columns.size()); + for (const auto & source_column : source_columns) + { + size_t pos = header_.getPositionByName(source_column); + if (std::find(sorted_columns_pos.begin(), sorted_columns_pos.end(), pos) != sorted_columns_pos.end()) + continue; + + const auto & col = header_.getByPosition(pos).column; + if (col && !isColumnConst(*col)) + other_columns_pos.emplace_back(pos); + } + + /// reserve space in auxiliary column vectors for processing + sorted_columns.reserve(sorted_columns_pos.size()); + other_columns.reserve(other_columns_pos.size()); + current_key.reserve(sorted_columns.size()); +} + +void DistinctPrimaryKeyTransform::initChunkProcessing(const Columns & input_columns) +{ + sorted_columns.clear(); + for (size_t pos : sorted_columns_pos) + sorted_columns.emplace_back(input_columns[pos].get()); + + other_columns.clear(); + for (size_t pos : other_columns_pos) + other_columns.emplace_back(input_columns[pos].get()); + + if (!other_columns.empty() && data.type == ClearableSetVariants::Type::EMPTY) + data.init(ClearableSetVariants::chooseMethod(other_columns, other_columns_sizes)); +} + +size_t DistinctPrimaryKeyTransform::ordinaryDistinctOnRange(IColumn::Filter & filter, size_t range_begin, size_t range_end) +{ + size_t count = 0; + switch (data.type) + { + case ClearableSetVariants::Type::EMPTY: + break; + // clang-format off +#define M(NAME) \ + case ClearableSetVariants::Type::NAME: \ + count = buildFilterForRange(*data.NAME, filter, range_begin, range_end, data); \ + break; + + APPLY_FOR_SET_VARIANTS(M) +#undef M + // clang-format on + } + return count; +} + +template +size_t DistinctPrimaryKeyTransform::buildFilterForRange( + Method & method, IColumn::Filter & filter, size_t range_begin, size_t range_end, ClearableSetVariants & variants) +{ + typename Method::State state(other_columns, other_columns_sizes, nullptr); + method.data.clear(); + + size_t count = 0; + for (size_t i = range_begin; i < range_end; ++i) + { + auto emplace_result = state.emplaceKey(method.data, i, variants.string_pool); + + /// emit the record if there is no such key in the current set, skip otherwise + filter[i] = emplace_result.isInserted(); + if (filter[i]) + ++count; + } + return count; +} + +void DistinctPrimaryKeyTransform::setCurrentKey(const size_t row_pos) +{ + current_key.clear(); + for (auto const & col : sorted_columns) + { + current_key.emplace_back(col->cloneEmpty()); + current_key.back()->insertFrom(*col, row_pos); + } +} + +bool DistinctPrimaryKeyTransform::isCurrentKey(const size_t row_pos) +{ + for (size_t i = 0; i < sorted_columns.size(); ++i) + { + const auto & sort_col_desc = sorted_columns_descr[i]; + int res = sort_col_desc.direction * current_key[i]->compareAt(0, row_pos, *sorted_columns[i], sort_col_desc.nulls_direction); + if (res != 0) + return false; + } + return true; +} + +size_t DistinctPrimaryKeyTransform::getRangeEnd(size_t range_begin, size_t range_end) +{ + size_t low = range_begin; + size_t high = range_end-1; + while (low <= high) + { + size_t mid = low + (high - low) / 2; + if (isCurrentKey(mid)) + low = mid + 1; + else + { + high = mid - 1; + range_end = mid; + } + } + return range_end; +} + +size_t DistinctPrimaryKeyTransform::getStartPosition(const size_t chunk_rows) +{ + if (!current_key.empty()) // current_key is empty on very first transform() call + { + if (other_columns.empty() && isCurrentKey(0)) + return getRangeEnd(0, chunk_rows); + } + return 0; +} + +void DistinctPrimaryKeyTransform::transform(Chunk & chunk) +{ + const size_t chunk_rows = chunk.getNumRows(); + size_t output_rows = 0; + Columns input_columns = chunk.detachColumns(); + + /// split input columns into sorted and other("non-sorted") columns + initChunkProcessing(input_columns); + + /// build filter: + /// (1) find range with the same values in sorted columns -> [range_begin, range_end) + /// (2) for found range + /// if there is no "non-sorted" columns: filter out all rows in range except first one + /// otherwise: apply ordinary distinct + /// (3) repeat until chunk is processed + IColumn::Filter filter(chunk_rows); + size_t range_begin = getStartPosition(chunk_rows); + if (range_begin > 0) + std::fill(filter.begin(), filter.begin() + range_begin, 0); /// skip rows already included in distinct on previous transform() + + size_t range_end = range_begin; + while (range_end != chunk_rows) + { + // set current key to find range + setCurrentKey(range_begin); + + // find new range [range_begin, range_end) + range_end = getRangeEnd(range_begin, chunk_rows); + + // update filter for range + if (other_columns.empty()) + { + filter[range_begin] = 1; + std::fill(filter.begin() + range_begin + 1, filter.begin() + range_end, 0); + ++output_rows; + } + else + { + // ordinary distinct in range if there are "non-sorted" columns + output_rows += ordinaryDistinctOnRange(filter, range_begin, range_end); + } + + // set where next range start + range_begin = range_end; + } + + /// apply the built filter + for (auto & input_column : input_columns) + input_column = input_column->filter(filter, output_rows); + + chunk.setColumns(std::move(input_columns), output_rows); + + /// Update total output rows and check limits + total_output_rows += output_rows; + if ((limit_hint && total_output_rows >= limit_hint) + || !output_size_limits.check(total_output_rows, data.getTotalByteCount(), "DISTINCT", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) + { + stopReading(); + } +} + +} diff --git a/src/Processors/Transforms/DistinctPrimaryKeyTransform.h b/src/Processors/Transforms/DistinctPrimaryKeyTransform.h new file mode 100644 index 00000000000..d862e4a28ce --- /dev/null +++ b/src/Processors/Transforms/DistinctPrimaryKeyTransform.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +/// +/// DISTINCT optimization for MergeTree family engines +/// Applied in case of DISTINCT is done over primary key(prefix) columns +/// It leverages their sorting property +/// +class DistinctPrimaryKeyTransform : public ISimpleTransform +{ +public: + DistinctPrimaryKeyTransform( + const Block & header_, + const SizeLimits & output_size_limits_, + UInt64 limit_hint_, + const SortDescription & sorted_columns_descr_, + const Names & source_columns_); + + String getName() const override { return "DistinctPrimaryKeyTransform"; } + +protected: + void transform(Chunk & chunk) override; + +private: + void initChunkProcessing(const Columns & input_columns); + size_t getStartPosition(size_t chunk_rows); + size_t ordinaryDistinctOnRange(IColumn::Filter & filter, size_t range_begin, size_t range_end); + inline void setCurrentKey(size_t row_pos); + inline bool isCurrentKey(size_t row_pos); + inline size_t getRangeEnd(size_t range_begin, size_t range_end); + + template + size_t + buildFilterForRange(Method & method, IColumn::Filter & filter, size_t range_begin, size_t range_end, ClearableSetVariants & variants); + + + ClearableSetVariants data; + const size_t limit_hint; + size_t total_output_rows = 0; + + /// Restrictions on the maximum size of the output data. + const SizeLimits output_size_limits; + + const SortDescription sorted_columns_descr; + ColumnNumbers sorted_columns_pos; + ColumnRawPtrs sorted_columns; // used during processing + + ColumnNumbers other_columns_pos; + Sizes other_columns_sizes; + ColumnRawPtrs other_columns; // used during processing + + MutableColumns current_key; +}; + +} diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 3ff4baa0b11..36c4c39c6b1 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -1,13 +1,15 @@ #include +#include +#include #include #include +#include #include #include -#include -#include -#include #include +#include +#include namespace DB { @@ -260,4 +262,29 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( return getInputOrderImpl(metadata_snapshot, required_sort_description, elements_actions, limit); } + +ReadInOrderOptimizerForDistinct::ReadInOrderOptimizerForDistinct(const Names & source_columns_) + : source_columns(source_columns_.begin(), source_columns_.end()) +{ +} + +InputOrderInfoPtr ReadInOrderOptimizerForDistinct::getInputOrder(const StorageMetadataPtr & metadata_snapshot) const +{ + Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); + if (sorting_key_columns.empty()) + return {}; + + SortDescription order_key_prefix_descr; + for (const auto & sorting_key_column : sorting_key_columns) + { + if (source_columns.find(sorting_key_column) == source_columns.end()) + break; + order_key_prefix_descr.emplace_back(sorting_key_column, 1, 1); + } + if (order_key_prefix_descr.empty()) + return {}; + + // todo: InputOrderInfo contains more info then needed for distinct, probably it makes sense to replace it + return std::make_shared(SortDescription{}, std::move(order_key_prefix_descr), 1, 0); +} } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index fd8c9187ddb..fe24541f750 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -39,4 +39,14 @@ private: SortDescription required_sort_description; const ASTSelectQuery & query; }; + +class ReadInOrderOptimizerForDistinct +{ +public: + explicit ReadInOrderOptimizerForDistinct(const Names & source_columns_); + InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot) const; + +private: + NameSet source_columns; +}; } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index bdb4c392c48..f0523415e79 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -38,6 +38,9 @@ using TreeRewriterResultPtr = std::shared_ptr; class ReadInOrderOptimizer; using ReadInOrderOptimizerPtr = std::shared_ptr; +class ReadInOrderOptimizerForDistinct; +using ReadInOrderOptimizerForDistinctPtr = std::shared_ptr; + class Cluster; using ClusterPtr = std::shared_ptr; @@ -95,10 +98,12 @@ struct InputOrderInfo InputOrderInfo( const SortDescription & order_key_fixed_prefix_descr_, const SortDescription & order_key_prefix_descr_, - int direction_, UInt64 limit_) + int direction_, + UInt64 limit_) : order_key_fixed_prefix_descr(order_key_fixed_prefix_descr_) , order_key_prefix_descr(order_key_prefix_descr_) - , direction(direction_), limit(limit_) + , direction(direction_) + , limit(limit_) { } @@ -160,6 +165,9 @@ struct SelectQueryInfoBase /// Can be modified while reading from storage InputOrderInfoPtr input_order_info; + ReadInOrderOptimizerForDistinctPtr distinct_optimizer; + InputOrderInfoPtr distinct_order_info; + /// Prepared sets are used for indices by storage engine. /// Example: x IN (1, 2, 3) PreparedSets sets; diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference new file mode 100644 index 00000000000..87063502bb8 --- /dev/null +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -0,0 +1,83 @@ +disable optimize_distinct_in_order +pipeline does _not_ contain the optimization +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Distinct) + DistinctTransform + (Expression) + ExpressionTransform + (ReadFromPreparedSource) + NullSource 0 → 1 +enable optimize_distinct_in_order +distinct with primary key prefix -> pipeline contains the optimization +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Distinct) + DistinctPrimaryKeyTransform + (Expression) + ExpressionTransform + (ReadFromPreparedSource) + NullSource 0 → 1 +distinct with non-primary key prefix -> pipeline does _not_ contain the optimization +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Distinct) + DistinctTransform + (Expression) + ExpressionTransform + (ReadFromPreparedSource) + NullSource 0 → 1 +the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one +single-threaded distinct +0 +multi-threaded distinct +0 +skip part of chunk since it contians values from previous one +single-threaded distinct +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +multi-threaded distinct +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +table with not only primary key columns +distinct with key-prefix only +0 +distinct with full key, 2 columns +0 0 +0 1 +0 2 +0 3 +0 4 +0 5 +0 6 +0 7 +0 8 +0 9 +distinct with key prefix and non-sorted column +0 0 +0 1 +0 2 +0 3 +0 4 diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql new file mode 100644 index 00000000000..93bf520b3f4 --- /dev/null +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -0,0 +1,47 @@ + +drop table if exists distinct_in_order sync; +create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by (a, b, c); + +select 'disable optimize_distinct_in_order'; +set optimize_distinct_in_order=0; +select 'pipeline does _not_ contain the optimization'; +explain pipeline select distinct * from distinct_in_order settings max_threads=1; + +select 'enable optimize_distinct_in_order'; +set optimize_distinct_in_order=1; +select 'distinct with primary key prefix -> pipeline contains the optimization'; +explain pipeline select distinct a, c from distinct_in_order settings max_threads=1; +select 'distinct with non-primary key prefix -> pipeline does _not_ contain the optimization'; +explain pipeline select distinct b, c from distinct_in_order settings max_threads=1; + +select 'the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one'; +drop table if exists distinct_in_order sync; +create table distinct_in_order (a int) engine=MergeTree() order by a settings index_granularity=10; +insert into distinct_in_order (a) select * from zeros(30); +select 'single-threaded distinct'; +select distinct * from distinct_in_order settings max_block_size=10, max_threads=1; +select 'multi-threaded distinct'; +select distinct * from distinct_in_order settings max_block_size=10; + +select 'skip part of chunk since it contians values from previous one'; +drop table if exists distinct_in_order sync; +create table distinct_in_order (a int) engine=MergeTree() order by a settings index_granularity=10; +insert into distinct_in_order (a) select * from zeros(10); +insert into distinct_in_order select * from numbers(10); +select 'single-threaded distinct'; +select distinct a from distinct_in_order settings max_block_size=10, max_threads=1; +select 'multi-threaded distinct'; +select distinct a from distinct_in_order settings max_block_size=10; + +select 'table with not only primary key columns'; +drop table if exists distinct_in_order sync; +create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by (a, b); +insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,1000000); +select 'distinct with key-prefix only'; +select distinct a from distinct_in_order; +select 'distinct with full key, 2 columns'; +select distinct a,b from distinct_in_order order by b; +select 'distinct with key prefix and non-sorted column'; +select distinct a,c from distinct_in_order order by c; + +-- drop table if exists distinct_in_order sync; From 53d656d89f9f1dafd89c21d32e3f73ed741f72db Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Jun 2022 17:35:24 +0000 Subject: [PATCH 063/408] Fix deadlock in OvercommitTracker logging --- src/Common/MemoryTracker.cpp | 7 +-- src/Common/OvercommitTracker.cpp | 31 ++++++++---- src/Interpreters/InternalTextLogsQueue.cpp | 1 - src/Interpreters/InternalTextLogsQueue.h | 56 ---------------------- 4 files changed, 23 insertions(+), 72 deletions(-) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 51f4c83dc23..a9aef7d8465 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -338,11 +338,8 @@ void MemoryTracker::free(Int64 size) accounted_size += new_amount; } } - if (!OvercommitTrackerBlockerInThread::isBlocked()) - { - if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr) - overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); - } + if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed)) + overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 3cef72eb8b4..097b6637569 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -23,8 +23,16 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} +#define LOG_DEBUG_SAFE(...) \ + { \ + OvercommitTrackerBlockerInThread blocker; \ + LOG_DEBUG(__VA_ARGS__); \ + } + OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { + if (OvercommitTrackerBlockerInThread::isBlocked()) + return OvercommitResult::NONE; // NOTE: Do not change the order of locks // // global_mutex must be acquired before overcommit_m, because @@ -73,7 +81,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); - LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); + LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")) required_memory -= amount; Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 @@ -98,6 +106,9 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) { + if (OvercommitTrackerBlockerInThread::isBlocked()) + return; + std::lock_guard guard(overcommit_m); if (cancellation_state != QueryCancellationState::NONE) { @@ -112,7 +123,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { - LOG_DEBUG(getLogger(), "Picked query stopped"); + LOG_DEBUG_SAFE(getLogger(), "Picked query stopped") reset(); cv.notify_all(); @@ -140,7 +151,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. auto & queries = user_process_list->queries; - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()) for (auto const & query : queries) { if (query.second->isKilled()) @@ -151,15 +162,15 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() continue; auto ratio = memory_tracker->getOvercommitRatio(); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) if (ratio.soft_limit != 0 && current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit) picked_tracker = query_tracker; } @@ -174,7 +185,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() OvercommitRatio current_ratio{0, 0}; // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", process_list->size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()) for (auto const & query : process_list->processes) { if (query.isKilled()) @@ -190,15 +201,15 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() if (!memory_tracker) continue; auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) if (current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit) picked_tracker = query_tracker; } diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index 6176e3cc865..2172a6f4261 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -38,7 +38,6 @@ MutableColumns InternalTextLogsQueue::getSampleColumns() void InternalTextLogsQueue::pushBlock(Block && log_block) { - OvercommitTrackerBlockerInThread blocker; static Block sample_block = getSampleBlock(); if (blocksHaveEqualStructure(sample_block, log_block)) diff --git a/src/Interpreters/InternalTextLogsQueue.h b/src/Interpreters/InternalTextLogsQueue.h index a7193a55178..53710fa3bd2 100644 --- a/src/Interpreters/InternalTextLogsQueue.h +++ b/src/Interpreters/InternalTextLogsQueue.h @@ -18,62 +18,6 @@ public: static Block getSampleBlock(); static MutableColumns getSampleColumns(); - template - bool push(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::push(std::forward(args)...); - } - - template - bool emplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::emplace(std::forward(args)...); - } - - template - bool pop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::pop(std::forward(args)...); - } - - template - bool tryPush(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPush(std::forward(args)...); - } - - template - bool tryEmplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryEmplace(std::forward(args)...); - } - - template - bool tryPop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPop(std::forward(args)...); - } - - template - void clear(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clear(std::forward(args)...); - } - - template - void clearAndFinish(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clearAndFinish(std::forward(args)...); - } - /// Is used to pass block from remote server to the client void pushBlock(Block && log_block); From 9ca368ac20a35e6e49f3162f00f06a6206b9e6dd Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Jun 2022 23:10:40 +0000 Subject: [PATCH 064/408] Use do while(false) in macro --- src/Common/OvercommitTracker.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 097b6637569..1e36c99b41c 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -24,10 +24,10 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) {} #define LOG_DEBUG_SAFE(...) \ - { \ + do { \ OvercommitTrackerBlockerInThread blocker; \ LOG_DEBUG(__VA_ARGS__); \ - } + } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { @@ -81,7 +81,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); - LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")) + LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 @@ -123,7 +123,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { - LOG_DEBUG_SAFE(getLogger(), "Picked query stopped") + LOG_DEBUG_SAFE(getLogger(), "Picked query stopped"); reset(); cv.notify_all(); @@ -151,7 +151,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. auto & queries = user_process_list->queries; - LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()) + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()); for (auto const & query : queries) { if (query.second->isKilled()) @@ -162,7 +162,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() continue; auto ratio = memory_tracker->getOvercommitRatio(); - LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (ratio.soft_limit != 0 && current_ratio < ratio) { query_tracker = memory_tracker; @@ -170,7 +170,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() } } LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit) + current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } @@ -185,7 +185,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() OvercommitRatio current_ratio{0, 0}; // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. - LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()) + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()); for (auto const & query : process_list->processes) { if (query.isKilled()) @@ -201,7 +201,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() if (!memory_tracker) continue; auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); - LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (current_ratio < ratio) { query_tracker = memory_tracker; @@ -209,7 +209,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() } } LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit) + current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } From fc641d9ce49bbd0e256ddb45d01cee02405f2f2d Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Tue, 21 Jun 2022 11:29:55 +0800 Subject: [PATCH 065/408] some changes --- src/DataTypes/NestedUtils.cpp | 9 +++++++-- src/DataTypes/NestedUtils.h | 8 ++++++-- src/Storages/Hive/StorageHive.cpp | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 2e429bcff10..9cab49a509d 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -4,6 +4,7 @@ #include #include #include +#include "Columns/IColumn.h" #include #include @@ -119,7 +120,11 @@ Block flatten(const Block & block) { const DataTypes & element_types = type_tuple->getElements(); const Strings & names = type_tuple->getElementNames(); - const ColumnTuple * column_tuple = typeid_cast(elem.column.get()); + const ColumnTuple * column_tuple; + if(isColumnConst(*elem.column)) + column_tuple = typeid_cast(&assert_cast(*elem.column).getDataColumn()); + else + column_tuple = typeid_cast(elem.column.get()); size_t tuple_size = column_tuple->tupleSize(); for (size_t i = 0; i < tuple_size; ++i) { @@ -306,7 +311,7 @@ std::optional NestedColumnExtractHelper::extractColumn( { if (nested_table->has(new_column_name_prefix, case_insentive)) { - ColumnWithTypeAndName column = nested_table->getByName(new_column_name_prefix, case_insentive); + ColumnWithTypeAndName column = *nested_table->findByName(new_column_name_prefix, case_insentive); if (case_insentive) column.name = original_column_name; return {column}; diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index 39f73b65100..9473d30497a 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -18,8 +18,9 @@ namespace Nested /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. std::string extractTableName(const std::string & nested_name); - /// Replace Array(Tuple(...)) columns to a multiple of Array columns in a form of `column_name.element_name`. - /// only for named tuples that actually represent Nested structures. + /// Flat a column of nested type into columns + /// 1) For named tuples，t Tuple(x .., y ..., ...), replace it with t.x ..., t.y ... , ... + /// 2) For an Array with named Tuple element column, a Array(Tuple(x ..., y ..., ...)), replace it with multiple Array Columns, a.x ..., a.y ..., ... Block flatten(const Block & block); /// Collect Array columns in a form of `column_name.element_name` to single Array(Tuple(...)) column. @@ -35,6 +36,9 @@ namespace Nested std::unordered_set getAllTableNames(const Block & block, bool to_lower_case = false); } +/// Use this class to extract element columns from columns of nested type in a block, e.g. named Tuple. +/// It can extract a column from a multiple nested type column, e.g. named Tuple in named Tuple +/// Keeps some intermediate datas to avoid rebuild them multi-times. class NestedColumnExtractHelper { public: diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 60936f6a3f4..6d298c0033c 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -565,8 +565,8 @@ HiveFiles StorageHive::collectHiveFilesFromPartition( const ContextPtr & context_, PruneLevel prune_level) const { - //LOG_DEBUG( - // log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level)); + LOG_DEBUG( + log, "Collect hive files from partition {}, prune_level:{}", boost::join(partition.values, ","), pruneLevelToString(prune_level)); /// Skip partition "__HIVE_DEFAULT_PARTITION__" bool has_default_partition = false; @@ -794,7 +794,7 @@ Pipe StorageHive::read( auto subset_column = nested_columns_extractor.extractColumn(column); if (subset_column) { - sample_block.insert(*subset_column); + sample_block.insert(std::move(*subset_column)); continue; } } From e10f079bd35c377170abb2a1b984e83f1ce7c0a3 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Jun 2022 10:15:33 +0000 Subject: [PATCH 066/408] Get rid of allocations in OvercommitTracker --- src/Common/OvercommitTracker.cpp | 19 ++++++++++--------- src/Common/OvercommitTracker.h | 10 ++++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 1e36c99b41c..6f688ca28ff 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -20,6 +20,8 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , global_mutex(global_mutex_) , freed_memory(0) , required_memory(0) + , next_id(0) + , id_to_release(0) , allow_release(true) {} @@ -42,6 +44,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int std::unique_lock global_lock(global_mutex); std::unique_lock lk(overcommit_m); + size_t id = next_id++; + auto max_wait_time = tracker->getOvercommitWaitingTime(); if (max_wait_time == ZERO_MICROSEC) @@ -73,23 +77,21 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int allow_release = true; required_memory += amount; - required_per_thread[tracker] = amount; auto wait_start_time = std::chrono::system_clock::now(); - bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]() + bool timeout = !cv.wait_for(lk, max_wait_time, [this, id]() { - return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE; + return id < id_to_release || cancellation_state == QueryCancellationState::NONE; }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; - Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 - required_per_thread.erase(tracker); + bool still_need = !(id < id_to_release); // True if thread wasn't released // If threads where not released since last call of this method, // we can release them now. - if (allow_release && required_memory <= freed_memory && still_need != 0) + if (allow_release && required_memory <= freed_memory && still_need) releaseThreads(); // All required amount of memory is free now and selected query to stop doesn't know about it. @@ -98,7 +100,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int reset(); if (timeout) return OvercommitResult::TIMEOUTED; - if (still_need != 0) + if (still_need) return OvercommitResult::NOT_ENOUGH_FREED; else return OvercommitResult::MEMORY_FREED; @@ -132,8 +134,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) void OvercommitTracker::releaseThreads() { - for (auto & required : required_per_thread) - required.second = 0; + id_to_release = next_id; freed_memory = 0; allow_release = false; // To avoid repeating call of this method in OvercommitTracker::needToStopQuery cv.notify_all(); diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 80aaed68e37..6a03c679422 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -104,6 +105,10 @@ private: picked_tracker = nullptr; cancellation_state = QueryCancellationState::NONE; freed_memory = 0; + + next_id = 0; + id_to_release = 0; + allow_release = true; } @@ -111,8 +116,6 @@ private: QueryCancellationState cancellation_state; - std::unordered_map required_per_thread; - // Global mutex which is used in ProcessList to synchronize // insertion and deletion of queries. // OvercommitTracker::pickQueryToExcludeImpl() implementations @@ -122,6 +125,9 @@ private: Int64 freed_memory; Int64 required_memory; + size_t next_id; // Id provided to the next thread to come in OvercommitTracker + size_t id_to_release; // We can release all threads with id smaller than this + bool allow_release; }; From ba0fcec993953fc129f4305f890f1cdca4e4b8e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 12:35:47 +0200 Subject: [PATCH 067/408] add background cleanup of store/ subdirs --- src/Access/DiskAccessStorage.cpp | 10 +- src/Common/filesystemHelpers.cpp | 2 - src/Databases/DatabaseOrdinary.cpp | 6 +- src/IO/ReadHelpers.h | 2 + src/Interpreters/DatabaseCatalog.cpp | 149 +++++++++++++- src/Interpreters/DatabaseCatalog.h | 15 ++ src/Interpreters/InterpreterCreateQuery.cpp | 8 + tests/config/config.d/database_atomic.xml | 5 + .../configs/store_cleanup.xml | 11 ++ .../test.py | 183 +++++++++++++++++- 10 files changed, 371 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index a9b7a6a265b..231e325196d 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -153,15 +153,7 @@ namespace bool tryParseUUID(const String & str, UUID & id) { - try - { - id = parseFromString(str); - return true; - } - catch (...) - { - return false; - } + return tryParse(id, str); } } diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 7c10b24eb9c..32ae3219d9f 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -1,12 +1,10 @@ #include "filesystemHelpers.h" #if defined(OS_LINUX) -# include # include # include #endif #include -#include #include #include #include diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index ada9030905d..2e4ee785a94 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -172,9 +172,9 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables if (fs::exists(full_path.string() + detached_suffix)) { - /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. - /// if (create_query->uuid != UUIDHelpers::Nil) - /// DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + /// Even if we don't load the table we can still mark the uuid of it as taken. + if (create_query->uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 8fe8e2aa23e..4cd07dddf25 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -1065,6 +1065,8 @@ inline bool tryReadText(is_integer auto & x, ReadBuffer & buf) return tryReadIntText(x, buf); } +inline bool tryReadText(UUID & x, ReadBuffer & buf) { return tryReadUUIDText(x, buf); } + inline void readText(is_floating_point auto & x, ReadBuffer & buf) { readFloatText(x, buf); } inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index fa9444a7e66..623e4ed0845 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -16,9 +16,11 @@ #include #include #include -#include #include +#include +#include + #include "config_core.h" #if USE_MYSQL @@ -27,12 +29,11 @@ #endif #if USE_LIBPQXX -# include +# include # include +# include #endif -namespace fs = std::filesystem; - namespace CurrentMetrics { extern const Metric TablesToDropQueueSize; @@ -143,6 +144,9 @@ StoragePtr TemporaryTableHolder::getTable() const void DatabaseCatalog::initializeAndLoadTemporaryDatabase() { drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec); + unused_dir_hide_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_hide_timeout_sec", unused_dir_hide_timeout_sec); + unused_dir_rm_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_rm_timeout_sec", unused_dir_rm_timeout_sec); + unused_dir_cleanup_period_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_cleanup_period_sec", unused_dir_cleanup_period_sec); auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE, getContext()); attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); @@ -150,6 +154,12 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { + auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->cleanupStoreDirectoryTask(); }); + cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + (*cleanup_task)->activate(); + /// Do not start task immediately on server startup, it's not urgent. + (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); drop_task = std::make_unique(std::move(task_holder)); (*drop_task)->activate(); @@ -166,6 +176,9 @@ void DatabaseCatalog::shutdownImpl() { TemporaryLiveViewCleaner::shutdown(); + if (cleanup_task) + (*cleanup_task)->deactivate(); + if (drop_task) (*drop_task)->deactivate(); @@ -372,6 +385,10 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (drop) { + UUID db_uuid = db->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMappingFinally(db_uuid); + /// Delete the database. db->drop(local_context); @@ -560,6 +577,15 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, prev_table = std::move(table); } +bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + return it != map_part.map.end(); +} + std::unique_ptr DatabaseCatalog::database_catalog; DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) @@ -1072,6 +1098,121 @@ void DatabaseCatalog::updateLoadingDependencies(const StorageID & table_id, Tabl old_dependencies = std::move(new_dependencies); } +void DatabaseCatalog::cleanupStoreDirectoryTask() +{ + fs::path store_path = fs::path(getContext()->getPath()) / "store"; + size_t affected_dirs = 0; + for (const auto & prefix_dir : fs::directory_iterator{store_path}) + { + String prefix = prefix_dir.path().filename(); + bool expected_prefix_dir = prefix_dir.is_directory() && + prefix.size() == 3 && + isHexDigit(prefix[0]) && + isHexDigit(prefix[1]) && + isHexDigit(prefix[2]); + + if (!expected_prefix_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", prefix_dir.path().string()); + maybeRemoveDirectory(prefix_dir.path()); + continue; + } + + for (const auto & uuid_dir : fs::directory_iterator{prefix_dir.path()}) + { + String uuid_str = uuid_dir.path().filename(); + UUID uuid; + bool parsed = tryParse(uuid, uuid_str); + + bool expected_dir = uuid_dir.is_directory() && + parsed && + uuid != UUIDHelpers::Nil && + uuid_str.starts_with(prefix); + + if (!expected_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", uuid_dir.path().string()); + maybeRemoveDirectory(uuid_dir.path()); + continue; + } + + if (!hasUUIDMapping(uuid)) + { + /// We load uuids even for detached and permanently detached tables, + /// so it looks safe enough to remove directory if we don't have uuid mapping for it. + /// No table or database using this directory should concurrently appear, + /// because creation of new table would fail with "directory already exists". + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); + } + } + } + + if (affected_dirs) + LOG_INFO(log, "Cleaned up {} directories from store/", affected_dirs); + + (*cleanup_task)->scheduleAfter(unused_dir_cleanup_period_sec * 1000); +} + +bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) +{ + /// "Safe" automatic removal of some directory. + /// At first we do not remove anything and only revoke all access right. + /// And remove only if nobody noticed it after, for example, one month. + + struct stat st; + if (stat(unused_dir.string().c_str(), &st)) + { + LOG_ERROR(log, "Failed to stat {}, errno: {}", unused_dir.string(), errno); + return false; + } + + if (st.st_uid != geteuid()) + { + /// Directory is not owned by clickhouse, it's weird, let's ignore it (chmod will likely fail anyway). + LOG_WARNING(log, "Found directory {} with unexpected owner (uid={})", unused_dir.string(), st.st_uid); + return false; + } + + time_t max_modification_time = std::max(st.st_atime, std::max(st.st_mtime, st.st_ctime)); + time_t current_time = time(nullptr); + if (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) + { + if (current_time <= max_modification_time + unused_dir_hide_timeout_sec) + return false; + + LOG_INFO(log, "Removing access rights for unused directory {} (will remove it when timeout exceed)", unused_dir.string()); + + /// Explicitly update modification time just in case + + struct utimbuf tb; + tb.actime = current_time; + tb.modtime = current_time; + if (utime(unused_dir.string().c_str(), &tb) != 0) + LOG_ERROR(log, "Failed to utime {}, errno: {}", unused_dir.string(), errno); + + /// Remove all access right + if (chmod(unused_dir.string().c_str(), 0)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + return true; + } + else + { + if (current_time <= max_modification_time + unused_dir_rm_timeout_sec) + return false; + + LOG_INFO(log, "Removing unused directory {}", unused_dir.string()); + + /// We have to set these access rights to make recursive removal work + if (chmod(unused_dir.string().c_str(), S_IRWXU)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + fs::remove_all(unused_dir); + + return true; + } +} + DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 79ba4052038..9ca0bb276d6 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -20,7 +20,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -203,6 +205,8 @@ public: /// this method will throw an exception. void addUUIDMapping(const UUID & uuid); + bool hasUUIDMapping(const UUID & uuid); + static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -261,6 +265,9 @@ private: void dropTableDataTask(); void dropTableFinally(const TableMarkedAsDropped & table); + void cleanupStoreDirectoryTask(); + bool maybeRemoveDirectory(const fs::path & unused_dir); + static constexpr size_t reschedule_time_ms = 100; static constexpr time_t drop_error_cooldown_sec = 5; @@ -298,6 +305,14 @@ private: static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; std::condition_variable wait_table_finally_dropped; + + std::unique_ptr cleanup_task; + static constexpr time_t default_unused_dir_hide_timeout_sec = 60 * 60; /// 1 hour + time_t unused_dir_hide_timeout_sec = default_unused_dir_hide_timeout_sec; + static constexpr time_t default_unused_dir_rm_timeout_sec = 30 * 24 * 60 * 60; /// 30 days + time_t unused_dir_rm_timeout_sec = default_unused_dir_rm_timeout_sec; + static constexpr time_t default_unused_dir_cleanup_period_sec = 24 * 60 * 60; /// 1 day + time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 304cfa2f3f4..9e745ffcabe 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -255,6 +255,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); + bool need_lock_uuid = internal || !create.attach; if (need_write_metadata) { @@ -279,6 +280,13 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// We attach database before loading it's tables, so do not allow concurrent DDL queries auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); + if (create.uuid != UUIDHelpers::Nil && need_lock_uuid) + { + /// Lock uuid, so we will known it's already in use. + /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); + DatabaseCatalog::instance().addUUIDMapping(create.uuid); + } + bool added = false; bool renamed = false; try diff --git a/tests/config/config.d/database_atomic.xml b/tests/config/config.d/database_atomic.xml index b3f51d51a79..a551e710ca3 100644 --- a/tests/config/config.d/database_atomic.xml +++ b/tests/config/config.d/database_atomic.xml @@ -1,3 +1,8 @@ 60 + + + 0 + 5 + 10 diff --git a/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml new file mode 100644 index 00000000000..3b0260dd07a --- /dev/null +++ b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml @@ -0,0 +1,11 @@ + + 0 + 15 + 1 + + + + testkeeper + + \ No newline at end of file diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index 3d9134bdc54..dd6744b4228 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -1,14 +1,15 @@ import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk from helpers.corrupt_part_data_on_disk import break_part import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", stay_alive=True, with_zookeeper=True) +node1 = cluster.add_instance( + "node1", stay_alive=True, main_configs=["configs/store_cleanup.xml"] +) path_to_data = "/var/lib/clickhouse/" @@ -147,3 +148,181 @@ def test_remove_broken_detached_part_replicated_merge_tree(started_cluster): ) remove_broken_detached_part_impl("replicated_mt", node1, "broken") + + +def test_store_cleanup(started_cluster): + node1.query("CREATE DATABASE db UUID '10000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db.log UUID '10000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mt UUID '10000000-1000-4000-8000-000000000003' ENGINE=MergeTree ORDER BY tuple() AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mem UUID '10000000-1000-4000-8000-000000000004' ENGINE=Memory AS SELECT 1" + ) + + node1.query("CREATE DATABASE db2 UUID '20000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db2.log UUID '20000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH DATABASE db2") + + node1.query("CREATE DATABASE db3 UUID '30000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db3.log UUID '30000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db3.log2 UUID '30000000-1000-4000-8000-000000000003' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH TABLE db3.log") + node1.query("DETACH TABLE db3.log2 PERMANENTLY") + + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + node1.stop_clickhouse(kill=True) + # All dirs related to `db` will be removed + node1.exec_in_container(["rm", f"{path_to_data}/metadata/db.sql"]) + + node1.exec_in_container(["mkdir", f"{path_to_data}/store/kek"]) + node1.exec_in_container(["touch", f"{path_to_data}/store/12"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456/testgarbage"]) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/30000000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["touch", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000004"] + ) + + node1.start_clickhouse() + node1.query("DETACH DATABASE db2") + node1.query("DETACH TABLE db3.log") + + node1.wait_for_log_line("Removing access rights for unused directory") + time.sleep(1) + node1.wait_for_log_line("testgarbage") + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" in store + assert "12" in store + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) + assert "10000000-1000-4000-8000-000000000001" in store100 + assert "10000000-1000-4000-8000-000000000002" in store100 + assert "10000000-1000-4000-8000-000000000003" in store100 + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000004" in store456 + assert "testgarbage" in store456 + assert "----------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + node1.wait_for_log_line("Removing unused directory") + time.sleep(1) + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" not in store # changed + assert "12" not in store # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) # changed + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) # changed + assert "10000000-1000-4000-8000-000000000001" not in store100 # changed + assert "10000000-1000-4000-8000-000000000002" not in store100 # changed + assert "10000000-1000-4000-8000-000000000003" not in store100 # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) # changed + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000004" not in store456 # changed + assert "testgarbage" not in store456 # changed + assert "---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) # changed + + node1.query("ATTACH TABLE db3.log2") + node1.query("ATTACH DATABASE db2") + node1.query("ATTACH TABLE db3.log") + + assert "1\n" == node1.query("SELECT * FROM db3.log") + assert "1\n" == node1.query("SELECT * FROM db3.log2") + assert "1\n" == node1.query("SELECT * FROM db2.log") From 099055c1833224bf9212897a7a283a7cf529c803 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Jun 2022 12:26:13 +0000 Subject: [PATCH 068/408] Explicitly forbid allocations in OvercommitTracker --- src/Common/OvercommitTracker.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 6f688ca28ff..74235748345 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -25,14 +25,24 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} -#define LOG_DEBUG_SAFE(...) \ - do { \ - OvercommitTrackerBlockerInThread blocker; \ - LOG_DEBUG(__VA_ARGS__); \ +#define LOG_DEBUG_SAFE(...) \ + do { \ + OvercommitTrackerBlockerInThread blocker; \ + try \ + { \ + ALLOW_ALLOCATIONS_IN_SCOPE; \ + LOG_DEBUG(__VA_ARGS__); \ + } \ + catch (std::bad_alloc const &) \ + { \ + fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n"); \ + } \ } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + if (OvercommitTrackerBlockerInThread::isBlocked()) return OvercommitResult::NONE; // NOTE: Do not change the order of locks @@ -108,6 +118,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + if (OvercommitTrackerBlockerInThread::isBlocked()) return; @@ -122,6 +134,8 @@ void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) void OvercommitTracker::onQueryStop(MemoryTracker * tracker) { + DENY_ALLOCATIONS_IN_SCOPE; + std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { From 13e3d40db43bf0a257f608ddab344ee22d0fa3f1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 15:11:03 +0200 Subject: [PATCH 069/408] fix --- src/Interpreters/DatabaseCatalog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 623e4ed0845..dc0d346be92 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "config_core.h" From 1d6eea6cfa8068284ab06f2c7d5f8dfc8e091783 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 21 Jun 2022 18:55:17 +0300 Subject: [PATCH 070/408] Replace LOG_DEBUG by LOG_TRACE in KerberosInit; Correct exception message; Prohibit making a copy of KerberosInit --- src/Access/KerberosInit.cpp | 40 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 8d4e2339bbf..e78830bdaa5 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #if USE_KRB5 #include #include @@ -27,7 +28,10 @@ struct k5_data krb5_boolean switch_to_cache; }; -class KerberosInit +/** + * This class implements programmatic implementation of kinit. + */ +class KerberosInit : boost::noncopyable { public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); @@ -44,7 +48,7 @@ private: int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { auto log = &Poco::Logger::get("KerberosInit"); - LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); + LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -52,7 +56,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con int flags = 0; if (!std::filesystem::exists(keytab_file)) - throw Exception("Error keytab file does not exist", ErrorCodes::KERBEROS_ERROR); + throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); memset(&k5, 0, sizeof(k5)); ret = krb5_init_context(&k5.ctx); @@ -64,7 +68,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); if (ret) throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Resolved cache"); + LOG_TRACE(log,"Resolved cache"); } else { @@ -72,7 +76,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_default(k5.ctx, &defcache); if (ret) throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Resolved default cache"); + LOG_TRACE(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; @@ -92,7 +96,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); if (!ret) { - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } else if (defcache_princ != nullptr) @@ -101,7 +105,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } } @@ -111,13 +115,13 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con { k5.out_cc = defcache; defcache = nullptr; - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); } ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using principal: {}", k5.name); + LOG_TRACE(log,"Using principal: {}", k5.name); memset(&my_creds, 0, sizeof(my_creds)); ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); @@ -128,7 +132,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using keytab: {}", keytab_file); + LOG_TRACE(log,"Using keytab: {}", keytab_file); if (k5.in_cc) { @@ -141,28 +145,28 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); // Action: init or renew - LOG_DEBUG(log,"Trying to renew credentials"); + LOG_TRACE(log,"Trying to renew credentials"); ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(log,"Renew failed ({}). Trying to get initial credentials", ret); + LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); else - LOG_DEBUG(log,"Got initial credentials"); + LOG_TRACE(log,"Got initial credentials"); } else { - LOG_DEBUG(log,"Successful renewal"); + LOG_TRACE(log,"Successful renewal"); ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Initialized cache"); + LOG_TRACE(log,"Initialized cache"); ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) - LOG_DEBUG(log,"Error while storing credentials"); - LOG_DEBUG(log,"Stored credentials"); + LOG_TRACE(log,"Error while storing credentials"); + LOG_TRACE(log,"Stored credentials"); } if (k5.switch_to_cache) @@ -172,7 +176,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(log,"Authenticated to Kerberos v5"); + LOG_TRACE(log,"Authenticated to Kerberos v5"); return 0; } From 8288b9fe609485804c5c41f387d8ec3322e5c9be Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 18:47:55 +0200 Subject: [PATCH 071/408] fixes --- src/Databases/DatabaseOrdinary.cpp | 15 ++++++++++++++- tests/queries/0_stateless/replication.lib | 4 ++-- tests/queries/shell_config.sh | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 2e4ee785a94..ff5628ecf0f 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -174,7 +174,20 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables { /// Even if we don't load the table we can still mark the uuid of it as taken. if (create_query->uuid != UUIDHelpers::Nil) - DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + { + /// A bit tricky way to distinguish ATTACH DATABASE and server startup. + if (getContext()->isServerCompletelyStarted()) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } + else + { + /// Server is starting up. Lock UUID used by permanently detached table. + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + } + } const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib index 6bf3c35f344..fd32fa28ba0 100755 --- a/tests/queries/0_stateless/replication.lib +++ b/tests/queries/0_stateless/replication.lib @@ -45,8 +45,8 @@ function check_replication_consistency() while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do sleep 0.5; num_tries=$((num_tries+1)) - if [ $num_tries -eq 100 ]; then - $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%' FORMAT Vertical" + if [ $num_tries -eq 200 ]; then + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%' FORMAT Vertical" break fi done diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 8e9b9c2ac20..e3ab454f356 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -138,7 +138,7 @@ function wait_for_queries_to_finish() sleep 0.5; num_tries=$((num_tries+1)) if [ $num_tries -eq 20 ]; then - $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%' FORMAT Vertical" + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%' FORMAT Vertical" break fi done From b0a98bd875b99827380d4381008c57147d19ec28 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 21 Jun 2022 15:06:47 +0000 Subject: [PATCH 072/408] DISTINCT in order optimization + use SortDescription from input data stream in DistinctStep to decide if the optimization is applicable --- src/Core/Settings.h | 2 +- src/Interpreters/ExpressionAnalyzer.cpp | 2 - src/Interpreters/ExpressionAnalyzer.h | 1 - src/Interpreters/InterpreterSelectQuery.cpp | 22 +++---- src/Interpreters/InterpreterSelectQuery.h | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 4 +- src/Processors/QueryPlan/DistinctStep.cpp | 58 ++++++++++++------- src/Processors/QueryPlan/DistinctStep.h | 4 +- .../QueryPlan/ReadFromMergeTree.cpp | 9 +++ src/Processors/QueryPlan/ReadFromMergeTree.h | 2 +- .../DistinctPrimaryKeyTransform.cpp | 5 +- src/Storages/ReadInOrderOptimizer.cpp | 25 -------- src/Storages/ReadInOrderOptimizer.h | 10 ---- src/Storages/SelectQueryInfo.h | 6 -- ...7_distinct_in_order_optimization.reference | 45 +++++++++++--- .../02317_distinct_in_order_optimization.sql | 20 +++++-- 16 files changed, 113 insertions(+), 104 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 27902b8f2df..0c7bee9d57d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -603,7 +603,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ - M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization for primary key (prefix) columns in MergeTree family table engines.", 0) \ + M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if columns in DISTINCT are sorted", 1) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index f05cada7929..00333503db1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1859,8 +1859,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( && !query.final() && join_allow_read_in_order; - optimize_distinct_in_order = settings.optimize_distinct_in_order && storage && query.distinct; - /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 79d2c16efff..80c664832e5 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -232,7 +232,6 @@ struct ExpressionAnalysisResult bool remove_where_filter = false; bool optimize_read_in_order = false; bool optimize_aggregation_in_order = false; - bool optimize_distinct_in_order = false; bool join_has_delayed_stream = false; bool use_grouping_set_key = false; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ed6fd7ec7fd..ec93dfcde17 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1243,7 +1243,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

getInputOrder(metadata_snapshot, context, limit); } - if (!analysis_result.optimize_aggregation_in_order && analysis_result.optimize_distinct_in_order) - { - query_info.distinct_optimizer = std::make_shared(analysis_result.selected_columns); - query_info.distinct_order_info = query_info.distinct_optimizer->getInputOrder(metadata_snapshot); - } query_info.storage_limits = std::make_shared(storage_limits); @@ -2559,7 +2554,7 @@ void InterpreterSelectQuery::executeProjection(QueryPlan & query_plan, const Act } -void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct, InputOrderInfoPtr distinct_info) +void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct) { auto & query = getSelectQuery(); if (query.distinct) @@ -2576,11 +2571,8 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - if (distinct_info && !settings.optimize_distinct_in_order) - distinct_info = nullptr; - auto distinct_step = std::make_unique( - query_plan.getCurrentDataStream(), limits, limit_for_distinct, columns, pre_distinct, distinct_info); + query_plan.getCurrentDataStream(), limits, limit_for_distinct, columns, pre_distinct, settings.optimize_distinct_in_order); if (pre_distinct) distinct_step->setStepDescription("Preliminary DISTINCT"); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index f441c3774a4..40afaaaeed0 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -180,7 +180,7 @@ private: void executeLimit(QueryPlan & query_plan); void executeOffset(QueryPlan & query_plan); static void executeProjection(QueryPlan & query_plan, const ActionsDAGPtr & expression); - void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct, InputOrderInfoPtr distinct_info); + void executeDistinct(QueryPlan & query_plan, bool before_order, Names columns, bool pre_distinct); void executeExtremes(QueryPlan & query_plan); void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan, std::unordered_map & subqueries_for_sets); void diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index a1d8431d626..c9ba5bcd653 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -318,8 +318,8 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) /// Add distinct transform SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); - auto distinct_step - = std::make_unique(query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, nullptr); + auto distinct_step = std::make_unique( + query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, settings.optimize_distinct_in_order); query_plan.addStep(std::move(distinct_step)); } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 6d8f1cc4f92..9e8eabc53c8 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -38,6 +39,18 @@ static ITransformingStep::Traits getTraits(bool pre_distinct, bool already_disti }; } +static SortDescription getSortDescription(const SortDescription & input_sort_desc, const Names& columns) +{ + SortDescription distinct_sort_desc; + for (const auto & sort_column_desc : input_sort_desc) + { + if (std::find(begin(columns), end(columns), sort_column_desc.column_name) == columns.end()) + break; + distinct_sort_desc.emplace_back(sort_column_desc); + } + return distinct_sort_desc; +} + DistinctStep::DistinctStep( const DataStream & input_stream_, @@ -45,7 +58,7 @@ DistinctStep::DistinctStep( UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, - const InputOrderInfoPtr & distinct_info_) + bool optimize_distinct_in_order_) : ITransformingStep( input_stream_, input_stream_.header, @@ -53,8 +66,8 @@ DistinctStep::DistinctStep( , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) - , distinct_info(distinct_info_) , pre_distinct(pre_distinct_) + , optimize_distinct_in_order(optimize_distinct_in_order_) { if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing && (!pre_distinct /// Main distinct @@ -74,29 +87,32 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil if (!pre_distinct) pipeline.resize(1); - if (pre_distinct && distinct_info) + if (pre_distinct && optimize_distinct_in_order) { - pipeline.addSimpleTransform( - [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr - { - if (stream_type != QueryPipelineBuilder::StreamType::Main) - return nullptr; + if (SortDescription distinct_sort_desc = getSortDescription(input_streams.front().sort_description, columns); + !distinct_sort_desc.empty()) + { + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; - return std::make_shared( - header, set_size_limits, limit_hint, distinct_info->order_key_prefix_descr, columns); - }); + return std::make_shared( + header, set_size_limits, limit_hint, distinct_sort_desc, columns); + }); + return; + } } - else - { - pipeline.addSimpleTransform( - [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr - { - if (stream_type != QueryPipelineBuilder::StreamType::Main) - return nullptr; - return std::make_shared(header, set_size_limits, limit_hint, columns); - }); - } + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; + + return std::make_shared(header, set_size_limits, limit_hint, columns); + }); } void DistinctStep::describeActions(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index c64b9f6ccb5..7009739e3ad 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -16,7 +16,7 @@ public: UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, /// If is enabled, execute distinct for separate streams. Otherwise, merge streams. - const InputOrderInfoPtr & distinct_info_); + bool optimize_distinct_in_order_); String getName() const override { return "Distinct"; } @@ -29,8 +29,8 @@ private: SizeLimits set_size_limits; UInt64 limit_hint; Names columns; - InputOrderInfoPtr distinct_info; bool pre_distinct; + bool optimize_distinct_in_order; }; } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 8adaf2f1027..05a311ee268 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -117,6 +117,15 @@ ReadFromMergeTree::ReadFromMergeTree( /// Add explicit description. setStepDescription(data.getStorageID().getFullNameNotQuoted()); + + Names sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); + + SortDescription sort_description; + for (const auto & column : sorting_key_columns) + { + sort_description.emplace_back(column, 1); + } + output_stream->sort_description = std::move(sort_description); } Pipe ReadFromMergeTree::readFromPool( diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 6846506f260..a5f212a8c8a 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -111,7 +111,7 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; void describeIndexes(JSONBuilder::JSONMap & map) const override; - const StorageID getStorageID() const { return data.getStorageID(); } + StorageID getStorageID() const { return data.getStorageID(); } UInt64 getSelectedParts() const { return selected_parts; } UInt64 getSelectedRows() const { return selected_rows; } UInt64 getSelectedMarks() const { return selected_marks; } diff --git a/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp b/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp index ba3a72f1285..b3e3e15d963 100644 --- a/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp +++ b/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp @@ -114,8 +114,7 @@ bool DistinctPrimaryKeyTransform::isCurrentKey(const size_t row_pos) { for (size_t i = 0; i < sorted_columns.size(); ++i) { - const auto & sort_col_desc = sorted_columns_descr[i]; - int res = sort_col_desc.direction * current_key[i]->compareAt(0, row_pos, *sorted_columns[i], sort_col_desc.nulls_direction); + int res = current_key[i]->compareAt(0, row_pos, *sorted_columns[i], sorted_columns_descr[i].nulls_direction); if (res != 0) return false; } @@ -125,7 +124,7 @@ bool DistinctPrimaryKeyTransform::isCurrentKey(const size_t row_pos) size_t DistinctPrimaryKeyTransform::getRangeEnd(size_t range_begin, size_t range_end) { size_t low = range_begin; - size_t high = range_end-1; + size_t high = range_end - 1; while (low <= high) { size_t mid = low + (high - low) / 2; diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 36c4c39c6b1..e3f1c79719f 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -262,29 +262,4 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( return getInputOrderImpl(metadata_snapshot, required_sort_description, elements_actions, limit); } - -ReadInOrderOptimizerForDistinct::ReadInOrderOptimizerForDistinct(const Names & source_columns_) - : source_columns(source_columns_.begin(), source_columns_.end()) -{ -} - -InputOrderInfoPtr ReadInOrderOptimizerForDistinct::getInputOrder(const StorageMetadataPtr & metadata_snapshot) const -{ - Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); - if (sorting_key_columns.empty()) - return {}; - - SortDescription order_key_prefix_descr; - for (const auto & sorting_key_column : sorting_key_columns) - { - if (source_columns.find(sorting_key_column) == source_columns.end()) - break; - order_key_prefix_descr.emplace_back(sorting_key_column, 1, 1); - } - if (order_key_prefix_descr.empty()) - return {}; - - // todo: InputOrderInfo contains more info then needed for distinct, probably it makes sense to replace it - return std::make_shared(SortDescription{}, std::move(order_key_prefix_descr), 1, 0); -} } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index fe24541f750..fd8c9187ddb 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -39,14 +39,4 @@ private: SortDescription required_sort_description; const ASTSelectQuery & query; }; - -class ReadInOrderOptimizerForDistinct -{ -public: - explicit ReadInOrderOptimizerForDistinct(const Names & source_columns_); - InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot) const; - -private: - NameSet source_columns; -}; } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index f0523415e79..339e7b609d8 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -38,9 +38,6 @@ using TreeRewriterResultPtr = std::shared_ptr; class ReadInOrderOptimizer; using ReadInOrderOptimizerPtr = std::shared_ptr; -class ReadInOrderOptimizerForDistinct; -using ReadInOrderOptimizerForDistinctPtr = std::shared_ptr; - class Cluster; using ClusterPtr = std::shared_ptr; @@ -165,9 +162,6 @@ struct SelectQueryInfoBase /// Can be modified while reading from storage InputOrderInfoPtr input_order_info; - ReadInOrderOptimizerForDistinctPtr distinct_optimizer; - InputOrderInfoPtr distinct_order_info; - /// Prepared sets are used for indices by storage engine. /// Example: x IN (1, 2, 3) PreparedSets sets; diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index 87063502bb8..24c2edca708 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -1,3 +1,16 @@ +enable optimize_distinct_in_order +distinct pipeline on empty table -> no optimization, source is ReadFromPreparedSource instead of ReadFromMergeTree +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Distinct) + DistinctTransform + (Expression) + ExpressionTransform + (ReadFromPreparedSource) + NullSource 0 → 1 +insert into table to use ReadFromMergeTree source disable optimize_distinct_in_order pipeline does _not_ contain the optimization (Expression) @@ -8,10 +21,10 @@ ExpressionTransform DistinctTransform (Expression) ExpressionTransform - (ReadFromPreparedSource) - NullSource 0 → 1 + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 enable optimize_distinct_in_order -distinct with primary key prefix -> pipeline contains the optimization +distinct with all primary key columns -> optimization applied (Expression) ExpressionTransform (Distinct) @@ -20,9 +33,22 @@ ExpressionTransform DistinctPrimaryKeyTransform (Expression) ExpressionTransform - (ReadFromPreparedSource) - NullSource 0 → 1 -distinct with non-primary key prefix -> pipeline does _not_ contain the optimization + (ReadFromMergeTree) + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 +distinct with primary key prefix -> optimization applied +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Distinct) + DistinctPrimaryKeyTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 +distinct with non-primary key prefix -> no optimization (Expression) ExpressionTransform (Distinct) @@ -31,8 +57,9 @@ ExpressionTransform DistinctTransform (Expression) ExpressionTransform - (ReadFromPreparedSource) - NullSource 0 → 1 + (ReadFromMergeTree) + Concat 2 → 1 + MergeTreeInOrder × 2 0 → 1 the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one single-threaded distinct 0 @@ -64,7 +91,7 @@ multi-threaded distinct table with not only primary key columns distinct with key-prefix only 0 -distinct with full key, 2 columns +distinct with full key 0 0 0 1 0 2 diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index 93bf520b3f4..f718986ae70 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -1,7 +1,14 @@ - drop table if exists distinct_in_order sync; create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by (a, b, c); +select 'enable optimize_distinct_in_order'; +set optimize_distinct_in_order=1; +select 'distinct pipeline on empty table -> no optimization, source is ReadFromPreparedSource instead of ReadFromMergeTree'; +explain pipeline select distinct * from distinct_in_order settings max_threads=1; + +select 'insert into table to use ReadFromMergeTree source'; +insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,10); + select 'disable optimize_distinct_in_order'; set optimize_distinct_in_order=0; select 'pipeline does _not_ contain the optimization'; @@ -9,9 +16,12 @@ explain pipeline select distinct * from distinct_in_order settings max_threads=1 select 'enable optimize_distinct_in_order'; set optimize_distinct_in_order=1; -select 'distinct with primary key prefix -> pipeline contains the optimization'; +select 'distinct with all primary key columns -> optimization applied'; +insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,10); +explain pipeline select distinct * from distinct_in_order settings max_threads=1; +select 'distinct with primary key prefix -> optimization applied'; explain pipeline select distinct a, c from distinct_in_order settings max_threads=1; -select 'distinct with non-primary key prefix -> pipeline does _not_ contain the optimization'; +select 'distinct with non-primary key prefix -> no optimization'; explain pipeline select distinct b, c from distinct_in_order settings max_threads=1; select 'the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one'; @@ -39,9 +49,9 @@ create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,1000000); select 'distinct with key-prefix only'; select distinct a from distinct_in_order; -select 'distinct with full key, 2 columns'; +select 'distinct with full key'; select distinct a,b from distinct_in_order order by b; select 'distinct with key prefix and non-sorted column'; select distinct a,c from distinct_in_order order by c; --- drop table if exists distinct_in_order sync; +drop table if exists distinct_in_order sync; From dc1f5963265d08f027592ae96fd3976e4c112c6b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Jun 2022 00:50:16 +0200 Subject: [PATCH 073/408] fix --- src/Databases/DatabaseOrdinary.cpp | 25 +++++---- src/Databases/DatabaseOrdinary.h | 2 +- src/Databases/IDatabase.h | 2 +- src/Databases/TablesLoader.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 57 ++++++++++++++++----- src/Interpreters/DatabaseCatalog.h | 16 ++++++ src/Interpreters/InterpreterCreateQuery.cpp | 2 + 7 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index ff5628ecf0f..e44f5e5e628 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -29,6 +29,12 @@ namespace fs = std::filesystem; namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; namespace @@ -83,7 +89,7 @@ void DatabaseOrdinary::loadStoredObjects( */ ParsedTablesMetadata metadata; - loadTablesMetadata(local_context, metadata); + loadTablesMetadata(local_context, metadata, force_attach); size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries; @@ -151,12 +157,12 @@ void DatabaseOrdinary::loadStoredObjects( } } -void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata) +void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata, bool is_startup) { size_t prev_tables_count = metadata.parsed_tables.size(); size_t prev_total_dictionaries = metadata.total_dictionaries; - auto process_metadata = [&metadata, this](const String & file_name) + auto process_metadata = [&metadata, is_startup, this](const String & file_name) { fs::path path(getMetadataPath()); fs::path file_path(file_name); @@ -176,17 +182,16 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables if (create_query->uuid != UUIDHelpers::Nil) { /// A bit tricky way to distinguish ATTACH DATABASE and server startup. - if (getContext()->isServerCompletelyStarted()) - { - /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. - if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); - } - else + if (is_startup) { /// Server is starting up. Lock UUID used by permanently detached table. DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); } + else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } } const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 982be2024ce..6e524ae18b0 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -25,7 +25,7 @@ public: bool supportsLoadingInTopologicalOrder() const override { return true; } - void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata) override; + void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override; void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 38c85cf3d05..a81df0a389a 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -148,7 +148,7 @@ public: { } - virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/) + virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/, bool /*is_startup*/) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 898376f8e0f..e973c9211be 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -93,7 +93,7 @@ void TablesLoader::loadTables() for (auto & database_name : databases_to_load) { databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach); - databases[database_name]->loadTablesMetadata(global_context, metadata); + databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach); } LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec", diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index dc0d346be92..116cc8aa359 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -155,11 +155,15 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { - auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->cleanupStoreDirectoryTask(); }); - cleanup_task = std::make_unique(std::move(cleanup_task_holder)); - (*cleanup_task)->activate(); - /// Do not start task immediately on server startup, it's not urgent. - (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER) + { + auto cleanup_task_holder + = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this]() { this->cleanupStoreDirectoryTask(); }); + cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + (*cleanup_task)->activate(); + /// Do not start task immediately on server startup, it's not urgent. + (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + } auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); drop_task = std::make_unique(std::move(task_holder)); @@ -387,8 +391,6 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (drop) { UUID db_uuid = db->getUUID(); - if (db_uuid != UUIDHelpers::Nil) - removeUUIDMappingFinally(db_uuid); /// Delete the database. db->drop(local_context); @@ -399,6 +401,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri fs::remove(database_metadata_dir); fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); fs::remove(database_metadata_file); + + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMappingFinally(db_uuid); } return db; @@ -583,8 +588,7 @@ bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; - auto it = map_part.map.find(uuid); - return it != map_part.map.end(); + return map_part.map.contains(uuid); } std::unique_ptr DatabaseCatalog::database_catalog; @@ -1115,7 +1119,7 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() if (!expected_prefix_dir) { LOG_WARNING(log, "Found invalid directory {}, will try to remove it", prefix_dir.path().string()); - maybeRemoveDirectory(prefix_dir.path()); + affected_dirs += maybeRemoveDirectory(prefix_dir.path()); continue; } @@ -1133,11 +1137,12 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() if (!expected_dir) { LOG_WARNING(log, "Found invalid directory {}, will try to remove it", uuid_dir.path().string()); - maybeRemoveDirectory(uuid_dir.path()); + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); continue; } - if (!hasUUIDMapping(uuid)) + /// Order is important + if (!isProtectedUUIDDir(uuid) && !hasUUIDMapping(uuid)) { /// We load uuids even for detached and permanently detached tables, /// so it looks safe enough to remove directory if we don't have uuid mapping for it. @@ -1214,6 +1219,34 @@ bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) } } +void DatabaseCatalog::addProtectedUUIDDir(const UUID & uuid) +{ + if (uuid == UUIDHelpers::Nil) + return; + std::lock_guard lock{protected_uuid_dirs_mutex}; + bool inserted = protected_uuid_dirs.insert(uuid).second; + if (inserted) + return; + + throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " + "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); +} + +void DatabaseCatalog::removeProtectedUUIDDir(const UUID & uuid) +{ + if (uuid == UUIDHelpers::Nil) + return; + std::lock_guard lock{protected_uuid_dirs_mutex}; + chassert(protected_uuid_dirs.contains(uuid)); + protected_uuid_dirs.erase(uuid); +} + +bool DatabaseCatalog::isProtectedUUIDDir(const UUID & uuid) +{ + std::lock_guard lock{protected_uuid_dirs_mutex}; + return protected_uuid_dirs.contains(uuid); +} + DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 9ca0bb276d6..7dde0643dd9 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -207,6 +207,10 @@ public: bool hasUUIDMapping(const UUID & uuid); + void addProtectedUUIDDir(const UUID & uuid); + void removeProtectedUUIDDir(const UUID & uuid); + bool isProtectedUUIDDir(const UUID & uuid); + static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -301,6 +305,9 @@ private: std::unordered_set tables_marked_dropped_ids; mutable std::mutex tables_marked_dropped_mutex; + std::unordered_set protected_uuid_dirs; + mutable std::mutex protected_uuid_dirs_mutex; + std::unique_ptr drop_task; static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; @@ -315,4 +322,13 @@ private: time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; + +class UUIDDirectoryProtector : private boost::noncopyable +{ + UUID uuid; +public: + UUIDDirectoryProtector(UUID uuid_) : uuid(uuid_) { DatabaseCatalog::instance().addProtectedUUIDDir(uuid); } + ~UUIDDirectoryProtector() { DatabaseCatalog::instance().removeProtectedUUIDDir(uuid); } +}; + } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9e745ffcabe..20ebf96e617 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -249,6 +249,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } + UUIDDirectoryProtector uuid_lock{create.uuid}; DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); if (create.uuid != UUIDHelpers::Nil) @@ -1269,6 +1270,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database->checkDetachedTableNotInUse(create.uuid); } + UUIDDirectoryProtector uuid_lock{create.uuid}; StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) From f2811995887893e800bea3d0fe87cbd89d187d27 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 22 Jun 2022 11:28:00 +0300 Subject: [PATCH 074/408] Fix code style in KerberosInit; Add anonymous namespace; Add comment about using kerberos_init --- src/Access/KerberosInit.cpp | 9 ++++++--- src/Access/examples/kerberos_init.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index e78830bdaa5..435a8126b05 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -19,7 +19,9 @@ namespace ErrorCodes } } -struct k5_data +namespace +{ +struct K5Data { krb5_context ctx; krb5_ccache in_cc, out_cc; @@ -37,17 +39,18 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct k5_data k5; + struct K5Data k5; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; }; +} int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - auto log = &Poco::Logger::get("KerberosInit"); + auto * log = &Poco::Logger::get("KerberosInit"); LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 4868182fba2..5dbe92a5b57 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -5,6 +5,13 @@ #include #include +/** The example demonstrates using of kerberosInit function to obtain and cache Kerberos ticket-granting ticket. + * The first argument specifies keytab file. The second argument specifies principal name. + * The third argument is optional. It specifies credentials cache location. + * After successful run of kerberos_init it is useful to call klist command to list cached Kerberos tickets. + * It is also useful to run kdestroy to destroy Kerberos tickets if needed. + */ + using namespace DB; int main(int argc, char ** argv) From 7bd65c8c24b444d23514758a27269b614560a96c Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 22 Jun 2022 16:31:48 +0300 Subject: [PATCH 075/408] Add comments to KerberosInit; Remove input cache and flags from KerberosInit --- src/Access/KerberosInit.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 435a8126b05..d3be559cba7 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -24,7 +24,7 @@ namespace struct K5Data { krb5_context ctx; - krb5_ccache in_cc, out_cc; + krb5_ccache out_cc; krb5_principal me; char * name; krb5_boolean switch_to_cache; @@ -42,6 +42,7 @@ private: struct K5Data k5; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; + // Credentials structure including ticket, session key, and lifetime info. krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; @@ -56,7 +57,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; const char *deftype = nullptr; - int flags = 0; if (!std::filesystem::exists(keytab_file)) throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); @@ -86,7 +86,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } // Use the specified principal name. - ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), flags, &k5.me); + ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); if (ret) throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); @@ -126,7 +126,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using principal: {}", k5.name); - memset(&my_creds, 0, sizeof(my_creds)); + // Allocate a new initial credential options structure. ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); @@ -137,22 +137,20 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using keytab: {}", keytab_file); - if (k5.in_cc) - { - ret = krb5_get_init_creds_opt_set_in_ccache(k5.ctx, options, k5.in_cc); - if (ret) - throw Exception("Error in setting input credential cache", ErrorCodes::KERBEROS_ERROR); - } + // Set an output credential cache in initial credential options. ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); // Action: init or renew LOG_TRACE(log,"Trying to renew credentials"); + memset(&my_creds, 0, sizeof(my_creds)); + // Get renewed credential from KDC using an existing credential from output cache. ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); + // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); @@ -162,10 +160,12 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con else { LOG_TRACE(log,"Successful renewal"); + // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Initialized cache"); + // Store credentials in a credential cache. ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) LOG_TRACE(log,"Error while storing credentials"); @@ -174,6 +174,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (k5.switch_to_cache) { + // Make a credential cache the primary cache for its collection. ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); @@ -201,8 +202,6 @@ KerberosInit::~KerberosInit() krb5_free_unparsed_name(k5.ctx, k5.name); krb5_free_principal(k5.ctx, k5.me); - if (k5.in_cc != nullptr) - krb5_cc_close(k5.ctx, k5.in_cc); if (k5.out_cc != nullptr) krb5_cc_close(k5.ctx, k5.out_cc); krb5_free_context(k5.ctx); From 3544fbe5c49b13f81e5a1870b75b3950ab5f2647 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 22 Jun 2022 14:05:20 +0000 Subject: [PATCH 076/408] cleanup --- src/Common/OvercommitTracker.cpp | 25 +++++++++++++------------ src/Common/OvercommitTracker.h | 1 - 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 74235748345..4faed833428 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -25,18 +25,19 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} -#define LOG_DEBUG_SAFE(...) \ - do { \ - OvercommitTrackerBlockerInThread blocker; \ - try \ - { \ - ALLOW_ALLOCATIONS_IN_SCOPE; \ - LOG_DEBUG(__VA_ARGS__); \ - } \ - catch (std::bad_alloc const &) \ - { \ - fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n"); \ - } \ +#define LOG_DEBUG_SAFE(...) \ + do { \ + OvercommitTrackerBlockerInThread blocker; \ + try \ + { \ + ALLOW_ALLOCATIONS_IN_SCOPE; \ + LOG_DEBUG(__VA_ARGS__); \ + } \ + catch (...) \ + { \ + if (fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n") != -1) \ + ; \ + } \ } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 6a03c679422..6f6788493a9 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include From d553b30f808d2adb0ec58242769d1ac3e79d6128 Mon Sep 17 00:00:00 2001 From: Laurie Li Date: Wed, 22 Jun 2022 23:28:25 +0800 Subject: [PATCH 077/408] Update replication.md of zh docs docs/zh/engines/table-engines/mergetree-family/replication.md Signed-off-by: Laurie Li --- .../mergetree-family/replication.md | 100 ++++++++++++++++-- 1 file changed, 89 insertions(+), 11 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index c3be3a382cb..568484c6256 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -18,11 +18,19 @@ 而 `CREATE`，`DROP`，`ATTACH`，`DETACH` 和 `RENAME` 语句只会在单个服务器上执行，不会被复制。 -- `The CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 -- `The DROP TABLE` 删除运行此查询的服务器上的副本。 -- `The RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 +- `CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 +- `DROP TABLE` 删除运行此查询的服务器上的副本。 +- `RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 -要使用副本，需在配置文件中设置 ZooKeeper 集群的地址。例如： +ClickHouse 使用 [Apache ZooKeeper](https://zookeeper.apache.org) 存储副本的元信息。请使用 ZooKeeper 3.4.5 或更高版本。 + +要使用副本，需在 [Zookeeper](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) 服务器的配置部分设置相应参数。 + +:::warning +不要忽视安全设置。 ClickHouse 支持 ZooKeeper 安全子系统中的 `digest` [ACL 方案](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) 。 +::: + +Zookeeper 集群地址的设置样例如下： ``` xml @@ -41,7 +49,41 @@ ``` -需要 ZooKeeper 3.4.5 或更高版本。 +通过以引擎参数的形式提供 ZooKeeper 集群的名称和路径，ClickHouse 同样支持将副本的元信息存储在备用 ZooKeeper 集群上。也就是说，支持将不同数据表的元数据存储在不同的 ZooKeeper 集群上。 + +设置备用 ZooKeeper 集群地址的样例如下： + +``` xml + + + + example_2_1 + 2181 + + + example_2_2 + 2181 + + + example_2_3 + 2181 + + + + + example_3_1 + 2181 + + + +``` + +为了将数据表的元数据存储到备用 ZooKeeper 集群而非默认 ZooKeeper 集群，我们可以通过如下 SQL 的方式创建使用 +ReplicatedMergeTree 引擎的数据表： + +``` +CREATE TABLE table_name ( ... ) ENGINE = ReplicatedMergeTree('zookeeper_name_configured_in_auxiliary_zookeepers:path', 'replica_name') ... +``` 你可以配置任何现有的 ZooKeeper 集群，系统会使用里面的目录来存取元数据（该目录在创建可复制表时指定）。 @@ -53,7 +95,9 @@ 对于非常大的集群，你可以把不同的 ZooKeeper 集群用于不同的分片。然而，即使 Yandex.Metrica 集群（大约300台服务器）也证明还不需要这么做。 -复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。 +复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。为复制表执行后台任务的线程数量，可以通过 [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) 进行设置。 + +`ReplicatedMergeTree` 引擎采用一个独立的线程池进行复制拉取。线程池的大小通过 [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) 进行限定，它可以在重启服务器时进行调整。 默认情况下，INSERT 语句仅等待一个副本写入成功后返回。如果数据只成功写入一个副本后该副本所在的服务器不再存在，则存储的数据会丢失。要启用数据写入多个副本才确认返回，使用 `insert_quorum` 选项。 @@ -75,6 +119,7 @@ - `zoo_path` — ZooKeeper 中该表的路径。 - `replica_name` — ZooKeeper 中的该表的副本名称。 +- `other_parameters` — 关于引擎的一系列参数，这个引擎即是用来创建复制的引擎，例如，`ReplacingMergeTree` 。 示例: @@ -90,7 +135,9 @@ ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) ``` -已弃用的建表语法示例： +

+ +已弃用的建表语法示例： ``` sql CREATE TABLE table_name @@ -101,17 +148,19 @@ CREATE TABLE table_name ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) ``` +
+ 如上例所示，这些参数可以包含宏替换的占位符，即大括号的部分。它们会被替换为配置文件里 ‘macros’ 那部分配置的值。示例： ``` xml 05 02 - example05-02-1.yandex.ru + example05-02-1 ``` -«ZooKeeper 中该表的路径»对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 +ZooKeeper 中该表的路径对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 这种情况下，路径包含下面这些部分： `/clickhouse/tables/` 是公共前缀，我们推荐使用这个。 @@ -119,7 +168,11 @@ CREATE TABLE table_name `{layer}-{shard}` 是分片标识部分。在此示例中，由于 Yandex.Metrica 集群使用了两级分片，所以它是由两部分组成的。但对于大多数情况来说，你只需保留 {shard} 占位符即可，它会替换展开为分片标识。 `table_name` 是该表在 ZooKeeper 中的名称。使其与 ClickHouse 中的表名相同比较好。 这里它被明确定义，跟 ClickHouse 表名不一样，它并不会被 RENAME 语句修改。 -*HINT*：你可以在前面添加一个数据库名称 `table_name` 也是 例如。 `db_name.table_name` +*HINT*：你也可以在 `table_name` 前面添加一个数据库名称。例如： `db_name.table_name` 。 + +两个内置的占位符 `{database}` 和 `{table}` 也可使用，它们可以展开成数据表名称和数据库名称（只有当这些宏指令在 `macros` 部分已经定义时才可以）。因此 ZooKeeper 路径可以指定为 `'/clickhouse/tables/{layer}-{shard}/{database}/{table}'` 。 + +当使用这些内置占位符时，请当心数据表重命名。 ZooKeeper 中的路径无法变更，而当数据表被重命名时，宏命令将展开为另一个路径，数据表将指向一个 ZooKeeper 上并不存在的路径，并因此转变为只读模式。 副本名称用于标识同一个表分片的不同副本。你可以使用服务器名称，如上例所示。同个分片中不同副本的副本名称要唯一。 @@ -127,6 +180,31 @@ CREATE TABLE table_name 使用大型集群时，我们建议使用宏替换，因为它可以降低出错的可能性。 +你可以在服务器的配置文件中指定 `Replicated` 数据表引擎的默认参数。例如： + +```xml +/clickhouse/tables/{shard}/{database}/{table} +{replica} +``` + +这样，你可以在建表时省略参数： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree +ORDER BY x; +``` + +它等价于： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') +ORDER BY x; +``` + 在每个副本服务器上运行 `CREATE TABLE` 查询。将创建新的复制表，或给现有表添加新副本。 如果其他副本上已包含了某些数据，在表上添加新副本，则在运行语句后，数据会从其他副本复制到新副本。换句话说，新副本会与其他副本同步。 @@ -164,7 +242,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 1. 在服务器上安装 ClickHouse。在包含分片标识符和副本的配置文件中正确定义宏配置，如果有用到的话， 2. 如果服务器上有非复制表则必须手动复制，可以从副本服务器上（在 `/var/lib/clickhouse/data/db_name/table_name/` 目录中）复制它们的数据。 3. 从副本服务器上中复制位于 `/var/lib/clickhouse/metadata/` 中的表定义信息。如果在表定义信息中显式指定了分片或副本标识符，请更正它以使其对应于该副本。（另外，启动服务器，然后会在 `/var/lib/clickhouse/metadata/` 中的.sql文件中生成所有的 `ATTACH TABLE` 语句。） - 4.要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` +4. 要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` 然后启动服务器（如果它已运行则重启）。数据会从副本中下载。 From ed8341025b49b31a674a27e58b1837a50bb0ba59 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Jun 2022 18:31:42 +0200 Subject: [PATCH 078/408] make code less bad --- src/Databases/DatabaseAtomic.cpp | 7 +- src/Databases/DatabaseOnDisk.cpp | 12 ++ src/Databases/DatabaseOrdinary.cpp | 32 ++-- src/Interpreters/DatabaseCatalog.cpp | 97 ++++++++---- src/Interpreters/DatabaseCatalog.h | 30 ++-- src/Interpreters/InterpreterCreateQuery.cpp | 156 ++++++++++--------- src/Storages/System/attachSystemTablesImpl.h | 1 + 7 files changed, 188 insertions(+), 147 deletions(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ea887c84111..a4fa1fa267b 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -292,7 +292,6 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); - bool locked_uuid = false; try { std::unique_lock lock{mutex}; @@ -302,9 +301,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) - DatabaseCatalog::instance().addUUIDMapping(query.uuid); - locked_uuid = true; + chassert(DatabaseCatalog::instance().hasUUIDMapping(query.uuid)); auto txn = query_context->getZooKeeperMetadataTransaction(); if (txn && !query_context->isInternalSubquery()) @@ -321,8 +318,6 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora catch (...) { fs::remove(table_metadata_tmp_path); - if (locked_uuid) - DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } if (table->storesDataOnDisk()) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 9484da8ec2d..64bc9a4a094 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -395,6 +395,18 @@ void DatabaseOnDisk::renameTable( if (auto * target_db = dynamic_cast(&to_database)) target_db->checkMetadataFilenameAvailability(to_table_name); + /// This place is actually quite dangerous. Since data directory is moved to store/ + /// DatabaseCatalog may try to clean it up as unused. We add UUID mapping to avoid this. + /// However, we may fail after data directory move, but before metadata file creation in the destination db. + /// In this case nothing will protect data directory (except 30-days timeout). + /// But this situation (when table in Ordinary database is partially renamed) require manual intervention anyway. + if (from_ordinary_to_atomic) + { + DatabaseCatalog::instance().addUUIDMapping(create.uuid); + if (table->storesDataOnDisk()) + LOG_INFO(log, "Moving table from {} to {}", table_data_relative_path, to_database.getTableDataPath(create)); + } + /// Notify the table that it is renamed. It will move data to new path (if it stores data on disk) and update StorageID table->rename(to_database.getTableDataPath(create), StorageID(create)); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index e44f5e5e628..0bdabf7ef79 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -176,24 +176,24 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables auto * create_query = ast->as(); create_query->setDatabase(database_name); + /// Even if we don't load the table we can still mark the uuid of it as taken. + if (create_query->uuid != UUIDHelpers::Nil) + { + /// A bit tricky way to distinguish ATTACH DATABASE and server startup (actually it's "force_attach" flag). + if (is_startup) + { + /// Server is starting up. Lock UUID used by permanently detached table. + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + } + else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } + } + if (fs::exists(full_path.string() + detached_suffix)) { - /// Even if we don't load the table we can still mark the uuid of it as taken. - if (create_query->uuid != UUIDHelpers::Nil) - { - /// A bit tricky way to distinguish ATTACH DATABASE and server startup. - if (is_startup) - { - /// Server is starting up. Lock UUID used by permanently detached table. - DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); - } - else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) - { - /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); - } - } - const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 116cc8aa359..159003138c2 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,7 @@ TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryT } auto table_id = StorageID(DatabaseCatalog::TEMPORARY_DATABASE, global_name, id); auto table = creator(table_id); + DatabaseCatalog::instance().addUUIDMapping(id); temporary_tables->createTable(getContext(), global_name, table, original_create); table->startup(); } @@ -207,19 +209,25 @@ void DatabaseCatalog::shutdownImpl() tables_marked_dropped.clear(); std::lock_guard lock(databases_mutex); + for (const auto & db : databases) + { + UUID db_uuid = db.second->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); + } assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { /// Ensure that all UUID mappings are empty (i.e. all mappings contain nullptr instead of a pointer to storage) const auto & not_empty_mapping = [] (const auto & mapping) { + auto & db = mapping.second.first; auto & table = mapping.second.second; - return table; + return db || table; }; auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); databases.clear(); - db_uuid_map.clear(); view_dependencies.clear(); } @@ -349,9 +357,10 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases.emplace(database_name, database); + NOEXCEPT_SCOPE; UUID db_uuid = database->getUUID(); if (db_uuid != UUIDHelpers::Nil) - db_uuid_map.emplace(db_uuid, database); + addUUIDMapping(db_uuid, database, nullptr); } @@ -365,7 +374,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); db = databases.find(database_name)->second; - db_uuid_map.erase(db->getUUID()); + UUID db_uuid = db->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); databases.erase(database_name); } @@ -450,21 +461,19 @@ DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const DatabasePtr DatabaseCatalog::getDatabase(const UUID & uuid) const { - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database UUID {} does not exist", toString(uuid)); - return it->second; + return db_and_table.first; } DatabasePtr DatabaseCatalog::tryGetDatabase(const UUID & uuid) const { assert(uuid != UUIDHelpers::Nil); - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) return {}; - return it->second; + return db_and_table.first; } bool DatabaseCatalog::isDatabaseExist(const String & database_name) const @@ -519,18 +528,22 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid) void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); - assert((database && table) || (!database && !table)); + assert(database || !table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); if (inserted) + { + /// Mapping must be locked before actually inserting something + chassert((!database && !table)); return; + } auto & prev_database = it->second.first; auto & prev_table = it->second.second; - assert((prev_database && prev_table) || (!prev_database && !prev_table)); + assert(prev_database || !prev_table); - if (!prev_table && table) + if (!prev_database && database) { /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. prev_database = database; @@ -538,8 +551,8 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & data return; } - /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error - if (table) + /// We are trying to replace existing mapping (prev_database != nullptr), it's logical error + if (database || table) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR @@ -732,6 +745,8 @@ DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table void DatabaseCatalog::loadMarkedAsDroppedTables() { + assert(!cleanup_task); + /// /clickhouse_root/metadata_dropped/ contains files with metadata of tables, /// which where marked as dropped by Atomic databases. /// Data directories of such tables still exists in store/ @@ -811,6 +826,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr time_t drop_time; if (table) { + chassert(hasUUIDMapping(table_id.uuid)); drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); table->is_dropped = true; } @@ -1142,7 +1158,7 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() } /// Order is important - if (!isProtectedUUIDDir(uuid) && !hasUUIDMapping(uuid)) + if (!hasUUIDMapping(uuid)) { /// We load uuids even for detached and permanently detached tables, /// so it looks safe enough to remove directory if we don't have uuid mapping for it. @@ -1219,32 +1235,45 @@ bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) } } -void DatabaseCatalog::addProtectedUUIDDir(const UUID & uuid) +static void maybeUnlockUUID(UUID uuid) { if (uuid == UUIDHelpers::Nil) return; - std::lock_guard lock{protected_uuid_dirs_mutex}; - bool inserted = protected_uuid_dirs.insert(uuid).second; - if (inserted) - return; - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " - "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); + chassert(DatabaseCatalog::instance().hasUUIDMapping(uuid)); + auto db_and_table = DatabaseCatalog::instance().tryGetByUUID(uuid); + if (!db_and_table.first && !db_and_table.second) + { + DatabaseCatalog::instance().removeUUIDMappingFinally(uuid); + return; + } + chassert(db_and_table.first || !db_and_table.second); } -void DatabaseCatalog::removeProtectedUUIDDir(const UUID & uuid) +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(UUID uuid_) + : uuid(uuid_) { - if (uuid == UUIDHelpers::Nil) - return; - std::lock_guard lock{protected_uuid_dirs_mutex}; - chassert(protected_uuid_dirs.contains(uuid)); - protected_uuid_dirs.erase(uuid); + if (uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(uuid); } -bool DatabaseCatalog::isProtectedUUIDDir(const UUID & uuid) +TemporaryLockForUUIDDirectory::~TemporaryLockForUUIDDirectory() { - std::lock_guard lock{protected_uuid_dirs_mutex}; - return protected_uuid_dirs.contains(uuid); + maybeUnlockUUID(uuid); +} + +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept + : uuid(rhs.uuid) +{ + rhs.uuid = UUIDHelpers::Nil; +} + +TemporaryLockForUUIDDirectory & TemporaryLockForUUIDDirectory::operator = (TemporaryLockForUUIDDirectory && rhs) noexcept +{ + maybeUnlockUUID(uuid); + uuid = rhs.uuid; + rhs.uuid = UUIDHelpers::Nil; + return *this; } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 7dde0643dd9..d82a0594c9a 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -207,10 +207,6 @@ public: bool hasUUIDMapping(const UUID & uuid); - void addProtectedUUIDDir(const UUID & uuid); - void removeProtectedUUIDDir(const UUID & uuid); - bool isProtectedUUIDDir(const UUID & uuid); - static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -275,14 +271,11 @@ private: static constexpr size_t reschedule_time_ms = 100; static constexpr time_t drop_error_cooldown_sec = 5; - using UUIDToDatabaseMap = std::unordered_map; - mutable std::mutex databases_mutex; ViewDependencies view_dependencies; Databases databases; - UUIDToDatabaseMap db_uuid_map; UUIDToStorageMap uuid_map; DependenciesInfos loading_dependencies; @@ -305,9 +298,6 @@ private: std::unordered_set tables_marked_dropped_ids; mutable std::mutex tables_marked_dropped_mutex; - std::unordered_set protected_uuid_dirs; - mutable std::mutex protected_uuid_dirs_mutex; - std::unique_ptr drop_task; static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; @@ -322,13 +312,23 @@ private: time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; - -class UUIDDirectoryProtector : private boost::noncopyable +/// This class is useful when creating a table or database. +/// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog. +/// But such object may start using a directory in store/ since its creation. +/// To avoid race with cleanupStoreDirectoryTask() we have to mark UUID as used first. +/// Then we can either add DatabasePtr/StoragePtr to the created UUID mapping +/// or remove the lock if creation failed. +/// See also addUUIDMapping(...) +class TemporaryLockForUUIDDirectory : private boost::noncopyable { - UUID uuid; + UUID uuid = UUIDHelpers::Nil; public: - UUIDDirectoryProtector(UUID uuid_) : uuid(uuid_) { DatabaseCatalog::instance().addProtectedUUIDDir(uuid); } - ~UUIDDirectoryProtector() { DatabaseCatalog::instance().removeProtectedUUIDDir(uuid); } + TemporaryLockForUUIDDirectory() = default; + TemporaryLockForUUIDDirectory(UUID uuid_); + ~TemporaryLockForUUIDDirectory(); + + TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept; + TemporaryLockForUUIDDirectory & operator = (TemporaryLockForUUIDDirectory && rhs) noexcept; }; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 20ebf96e617..00597e0e3cd 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -249,15 +249,22 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } - UUIDDirectoryProtector uuid_lock{create.uuid}; + bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); + bool need_lock_uuid = internal || need_write_metadata; + + /// Lock uuid, so we will known it's already in use. + /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); if (create.uuid != UUIDHelpers::Nil) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); - bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); - bool need_lock_uuid = internal || !create.attach; - if (need_write_metadata) { create.attach = true; @@ -281,13 +288,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// We attach database before loading it's tables, so do not allow concurrent DDL queries auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); - if (create.uuid != UUIDHelpers::Nil && need_lock_uuid) - { - /// Lock uuid, so we will known it's already in use. - /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); - DatabaseCatalog::instance().addUUIDMapping(create.uuid); - } - bool added = false; bool renamed = false; try @@ -1172,70 +1172,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties) { - std::unique_ptr guard; - - String data_path; - DatabasePtr database; - - bool need_add_to_database = !create.temporary; - if (need_add_to_database) - { - /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). - * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. - */ - guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - - database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); - assertOrSetUUID(create, database); - - String storage_name = create.is_dictionary ? "Dictionary" : "Table"; - auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; - - /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. - if (database->isTableExist(create.getTable(), getContext())) - { - /// TODO Check structure of table - if (create.if_not_exists) - return false; - else if (create.replace_view) - { - /// when executing CREATE OR REPLACE VIEW, drop current existing view - auto drop_ast = std::make_shared(); - drop_ast->setDatabase(create.getDatabase()); - drop_ast->setTable(create.getTable()); - drop_ast->no_ddl_lock = true; - - auto drop_context = Context::createCopy(context); - InterpreterDropQuery interpreter(drop_ast, drop_context); - interpreter.execute(); - } - else - throw Exception(storage_already_exists_error_code, - "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); - } - else if (!create.attach) - { - /// Checking that table may exists in detached/detached permanently state - try - { - database->checkMetadataFilenameAvailability(create.getTable()); - } - catch (const Exception &) - { - if (create.if_not_exists) - return false; - throw; - } - } - - - data_path = database->getTableDataPath(create); - - if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) - throw Exception(storage_already_exists_error_code, - "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); - } - else + if (create.temporary) { if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) return false; @@ -1246,6 +1183,65 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, return true; } + std::unique_ptr guard; + + String data_path; + DatabasePtr database; + + /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). + * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. + */ + guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); + + database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); + assertOrSetUUID(create, database); + + String storage_name = create.is_dictionary ? "Dictionary" : "Table"; + auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; + + /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. + if (database->isTableExist(create.getTable(), getContext())) + { + /// TODO Check structure of table + if (create.if_not_exists) + return false; + else if (create.replace_view) + { + /// when executing CREATE OR REPLACE VIEW, drop current existing view + auto drop_ast = std::make_shared(); + drop_ast->setDatabase(create.getDatabase()); + drop_ast->setTable(create.getTable()); + drop_ast->no_ddl_lock = true; + + auto drop_context = Context::createCopy(context); + InterpreterDropQuery interpreter(drop_ast, drop_context); + interpreter.execute(); + } + else + throw Exception(storage_already_exists_error_code, + "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); + } + else if (!create.attach) + { + /// Checking that table may exists in detached/detached permanently state + try + { + database->checkMetadataFilenameAvailability(create.getTable()); + } + catch (const Exception &) + { + if (create.if_not_exists) + return false; + throw; + } + } + + data_path = database->getTableDataPath(create); + + if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) + throw Exception(storage_already_exists_error_code, + "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); + bool from_path = create.attach_from_path.has_value(); String actual_data_path = data_path; if (from_path) @@ -1270,7 +1266,15 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database->checkDetachedTableNotInUse(create.uuid); } - UUIDDirectoryProtector uuid_lock{create.uuid}; + /// We should lock UUID on CREATE query (because for ATTACH it must be already locked previously). + /// But ATTACH without create.attach_short_syntax flag works like CREATE actually, that's why we check it. + bool need_lock_uuid = !create.attach_short_syntax; + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h index fcc1ab43a64..a1fae985d92 100644 --- a/src/Storages/System/attachSystemTablesImpl.h +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -22,6 +22,7 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages /// and path is actually not used auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); + DatabaseCatalog::instance().addUUIDMapping(table_id.uuid); String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid); system_database.attachTable(context, table_name, std::make_shared(table_id, std::forward(args)...), path); } From 52db1b35a10f744839311f18b2927221bc0118d6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 22 Jun 2022 16:25:57 +0000 Subject: [PATCH 079/408] improve performace of insertion to columns of type JSON --- src/AggregateFunctions/Helpers.h | 28 --- src/Columns/ColumnObject.cpp | 30 +-- src/Columns/ColumnObject.h | 2 + src/DataTypes/FieldToDataType.cpp | 79 ++++--- src/DataTypes/FieldToDataType.h | 10 +- src/DataTypes/IDataType.h | 27 +++ src/DataTypes/ObjectUtils.cpp | 2 +- .../Serializations/SerializationObject.cpp | 16 +- src/DataTypes/getLeastSupertype.cpp | 201 +++++++++--------- src/DataTypes/getLeastSupertype.h | 29 ++- src/Interpreters/RowRefs.cpp | 1 - src/Storages/MergeTree/KeyCondition.cpp | 2 +- tests/performance/json_type.xml | 17 ++ .../01825_type_json_parallel_insert.reference | 2 +- .../01825_type_json_parallel_insert.sql | 4 +- 15 files changed, 251 insertions(+), 199 deletions(-) create mode 100644 tests/performance/json_type.xml diff --git a/src/AggregateFunctions/Helpers.h b/src/AggregateFunctions/Helpers.h index 77660c54d32..6e140f4b9cf 100644 --- a/src/AggregateFunctions/Helpers.h +++ b/src/AggregateFunctions/Helpers.h @@ -3,34 +3,6 @@ #include #include -#define FOR_BASIC_NUMERIC_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(Float32) \ - M(Float64) - -#define FOR_NUMERIC_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(UInt128) \ - M(UInt256) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(Int128) \ - M(Int256) \ - M(Float32) \ - M(Float64) - namespace DB { struct Settings; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index d3e68273d03..b52a5aab256 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -138,7 +138,7 @@ public: type_indexes.insert(TypeToTypeIndex>); } - DataTypePtr getScalarType() const { return getLeastSupertype(type_indexes, true); } + DataTypePtr getScalarType() const { return getLeastSupertypeOrString(type_indexes); } bool haveNulls() const { return have_nulls; } bool needConvertField() const { return field_types.size() > 1; } @@ -167,6 +167,7 @@ FieldInfo getFieldInfo(const Field & field) ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr && data_, bool is_nullable_) : least_common_type(getDataTypeByColumn(*data_)) , is_nullable(is_nullable_) + , num_rows(data_->size()) { data.push_back(std::move(data_)); } @@ -176,15 +177,13 @@ ColumnObject::Subcolumn::Subcolumn( : least_common_type(std::make_shared()) , is_nullable(is_nullable_) , num_of_defaults_in_prefix(size_) + , num_rows(size_) { } size_t ColumnObject::Subcolumn::size() const { - size_t res = num_of_defaults_in_prefix; - for (const auto & part : data) - res += part->size(); - return res; + return num_rows; } size_t ColumnObject::Subcolumn::byteSize() const @@ -321,7 +320,7 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) { if (isConversionRequiredBetweenIntegers(*base_type, *least_common_base_type)) { - base_type = getLeastSupertype(DataTypes{std::move(base_type), least_common_base_type}, true); + base_type = getLeastSupertypeOrString(DataTypes{std::move(base_type), least_common_base_type}); type_changed = true; if (!least_common_base_type->equals(*base_type)) addNewColumnPart(createArrayOfType(std::move(base_type), value_dim)); @@ -332,12 +331,14 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) field = convertFieldToTypeOrThrow(field, *least_common_type.get()); data.back()->insert(field); + ++num_rows; } void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn & src, size_t start, size_t length) { assert(start + length <= src.size()); size_t end = start + length; + num_rows += length; if (data.empty()) { @@ -345,7 +346,7 @@ void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn & src, size_t star } else if (!least_common_type.get()->equals(*src.getLeastCommonType())) { - auto new_least_common_type = getLeastSupertype(DataTypes{least_common_type.get(), src.getLeastCommonType()}, true); + auto new_least_common_type = getLeastSupertypeOrString(DataTypes{least_common_type.get(), src.getLeastCommonType()}); if (!new_least_common_type->equals(*least_common_type.get())) addNewColumnPart(std::move(new_least_common_type)); } @@ -487,6 +488,8 @@ void ColumnObject::Subcolumn::insertDefault() ++num_of_defaults_in_prefix; else data.back()->insertDefault(); + + ++num_rows; } void ColumnObject::Subcolumn::insertManyDefaults(size_t length) @@ -495,12 +498,15 @@ void ColumnObject::Subcolumn::insertManyDefaults(size_t length) num_of_defaults_in_prefix += length; else data.back()->insertManyDefaults(length); + + num_rows += length; } void ColumnObject::Subcolumn::popBack(size_t n) { assert(n <= size()); + num_rows -= n; size_t num_removed = 0; for (auto it = data.rbegin(); it != data.rend(); ++it) { @@ -559,15 +565,11 @@ ColumnObject::Subcolumn ColumnObject::Subcolumn::recreateWithDefaultValues(const if (is_nullable) scalar_type = makeNullable(scalar_type); - Subcolumn new_subcolumn; + Subcolumn new_subcolumn(*this); new_subcolumn.least_common_type = LeastCommonType{createArrayOfType(scalar_type, field_info.num_dimensions)}; - new_subcolumn.is_nullable = is_nullable; - new_subcolumn.num_of_defaults_in_prefix = num_of_defaults_in_prefix; - new_subcolumn.data.reserve(data.size()); - for (const auto & part : data) - new_subcolumn.data.push_back(recreateColumnWithDefaultValues( - part, scalar_type, field_info.num_dimensions)); + for (auto & part : new_subcolumn.data) + part = recreateColumnWithDefaultValues(part, scalar_type, field_info.num_dimensions); return new_subcolumn; } diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 89e42183ea0..4dc5bb5ce24 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -146,6 +146,8 @@ public: /// least common type and we count number of defaults in prefix, /// which will be converted to the default type of final common type. size_t num_of_defaults_in_prefix = 0; + + size_t num_rows = 0; }; using Subcolumns = SubcolumnsTree; diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index 283d1b1e41a..00b4665af94 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -22,13 +22,14 @@ namespace ErrorCodes extern const int EMPTY_DATA_PASSED; } - -DataTypePtr FieldToDataType::operator() (const Null &) const +template +DataTypePtr FieldToDataType::operator() (const Null &) const { return std::make_shared(std::make_shared()); } -DataTypePtr FieldToDataType::operator() (const UInt64 & x) const +template +DataTypePtr FieldToDataType::operator() (const UInt64 & x) const { if (x <= std::numeric_limits::max()) return std::make_shared(); if (x <= std::numeric_limits::max()) return std::make_shared(); @@ -36,7 +37,8 @@ DataTypePtr FieldToDataType::operator() (const UInt64 & x) const return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int64 & x) const +template +DataTypePtr FieldToDataType::operator() (const Int64 & x) const { if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) return std::make_shared(); @@ -44,77 +46,90 @@ DataTypePtr FieldToDataType::operator() (const Int64 & x) const return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Float64 &) const +template +DataTypePtr FieldToDataType::operator() (const Float64 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UInt128 &) const +template +DataTypePtr FieldToDataType::operator() (const UInt128 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int128 &) const +template +DataTypePtr FieldToDataType::operator() (const Int128 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UInt256 &) const +template +DataTypePtr FieldToDataType::operator() (const UInt256 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const Int256 &) const +template +DataTypePtr FieldToDataType::operator() (const Int256 &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const UUID &) const +template +DataTypePtr FieldToDataType::operator() (const UUID &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const String &) const +template +DataTypePtr FieldToDataType::operator() (const String &) const { return std::make_shared(); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const DecimalField & x) const +template +DataTypePtr FieldToDataType::operator() (const DecimalField & x) const { using Type = DataTypeDecimal; return std::make_shared(Type::maxPrecision(), x.getScale()); } -DataTypePtr FieldToDataType::operator() (const Array & x) const +template +DataTypePtr FieldToDataType::operator() (const Array & x) const { DataTypes element_types; element_types.reserve(x.size()); for (const Field & elem : x) - element_types.emplace_back(applyVisitor(FieldToDataType(allow_convertion_to_string), elem)); + element_types.emplace_back(applyVisitor(*this, elem)); - return std::make_shared(getLeastSupertype(element_types, allow_convertion_to_string)); + return std::make_shared(getLeastSupertype(element_types)); } -DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const +template +DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const { if (tuple.empty()) throw Exception("Cannot infer type of an empty tuple", ErrorCodes::EMPTY_DATA_PASSED); @@ -123,12 +138,13 @@ DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const element_types.reserve(tuple.size()); for (const auto & element : tuple) - element_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), element)); + element_types.push_back(applyVisitor(*this, element)); return std::make_shared(element_types); } -DataTypePtr FieldToDataType::operator() (const Map & map) const +template +DataTypePtr FieldToDataType::operator() (const Map & map) const { DataTypes key_types; DataTypes value_types; @@ -139,30 +155,37 @@ DataTypePtr FieldToDataType::operator() (const Map & map) const { const auto & tuple = elem.safeGet(); assert(tuple.size() == 2); - key_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[0])); - value_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[1])); + key_types.push_back(applyVisitor(*this, tuple[0])); + value_types.push_back(applyVisitor(*this, tuple[1])); } return std::make_shared( - getLeastSupertype(key_types, allow_convertion_to_string), - getLeastSupertype(value_types, allow_convertion_to_string)); + getLeastSupertype(key_types), + getLeastSupertype(value_types)); } -DataTypePtr FieldToDataType::operator() (const Object &) const +template +DataTypePtr FieldToDataType::operator() (const Object &) const { /// TODO: Do we need different parameters for type Object? return std::make_shared("json", false); } -DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const +template +DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const { const auto & name = static_cast(x).name; return DataTypeFactory::instance().get(name); } -DataTypePtr FieldToDataType::operator()(const bool &) const +template +DataTypePtr FieldToDataType::operator()(const bool &) const { return DataTypeFactory::instance().get("Bool"); } +template class FieldToDataType; +template class FieldToDataType; +template class FieldToDataType; + } diff --git a/src/DataTypes/FieldToDataType.h b/src/DataTypes/FieldToDataType.h index 1922ac8b746..5e66fe420ad 100644 --- a/src/DataTypes/FieldToDataType.h +++ b/src/DataTypes/FieldToDataType.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -17,14 +18,10 @@ using DataTypePtr = std::shared_ptr; * Note that you still have to convert Field to corresponding data type before inserting to columns * (for example, this is necessary to convert elements of Array to common type). */ +template class FieldToDataType : public StaticVisitor { public: - FieldToDataType(bool allow_convertion_to_string_ = false) - : allow_convertion_to_string(allow_convertion_to_string_) - { - } - DataTypePtr operator() (const Null & x) const; DataTypePtr operator() (const UInt64 & x) const; DataTypePtr operator() (const UInt128 & x) const; @@ -45,9 +42,6 @@ public: DataTypePtr operator() (const UInt256 & x) const; DataTypePtr operator() (const Int256 & x) const; DataTypePtr operator() (const bool & x) const; - -private: - bool allow_convertion_to_string; }; } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 420ef61a13f..fce8906abe5 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -565,4 +565,31 @@ class DataTypeEnum; template inline constexpr bool IsDataTypeEnum> = true; +#define FOR_BASIC_NUMERIC_TYPES(M) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(Int8) \ + M(Int16) \ + M(Int32) \ + M(Int64) \ + M(Float32) \ + M(Float64) + +#define FOR_NUMERIC_TYPES(M) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(UInt128) \ + M(UInt256) \ + M(Int8) \ + M(Int16) \ + M(Int32) \ + M(Int64) \ + M(Int128) \ + M(Int256) \ + M(Float32) \ + M(Float64) } diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index df639ae7aab..b81c8c7a033 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -261,7 +261,7 @@ DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambi key.getPath(), subtypes[0]->getName(), subtypes[i]->getName()); tuple_paths.emplace_back(key); - tuple_types.emplace_back(getLeastSupertype(subtypes, /*allow_conversion_to_string=*/ true)); + tuple_types.emplace_back(getLeastSupertypeOrString(subtypes)); } if (tuple_paths.empty()) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 85831df271a..cf49fa8798d 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -61,29 +61,23 @@ void SerializationObject::deserializeTextImpl(IColumn & column, Reader & auto & [paths, values] = *result; assert(paths.size() == values.size()); - HashSet paths_set; - size_t column_size = column_object.size(); - + size_t old_column_size = column_object.size(); for (size_t i = 0; i < paths.size(); ++i) { auto field_info = getFieldInfo(values[i]); if (isNothing(field_info.scalar_type)) continue; - if (!paths_set.insert(paths[i].getPath()).second) - throw Exception(ErrorCodes::INCORRECT_DATA, - "Object has ambiguous path: {}", paths[i].getPath()); - if (!column_object.hasSubcolumn(paths[i])) { if (paths[i].hasNested()) - column_object.addNestedSubcolumn(paths[i], field_info, column_size); + column_object.addNestedSubcolumn(paths[i], field_info, old_column_size); else - column_object.addSubcolumn(paths[i], column_size); + column_object.addSubcolumn(paths[i], old_column_size); } auto & subcolumn = column_object.getSubcolumn(paths[i]); - assert(subcolumn.size() == column_size); + assert(subcolumn.size() == old_column_size); subcolumn.insert(std::move(values[i]), std::move(field_info)); } @@ -92,7 +86,7 @@ void SerializationObject::deserializeTextImpl(IColumn & column, Reader & const auto & subcolumns = column_object.getSubcolumns(); for (const auto & entry : subcolumns) { - if (!paths_set.has(entry->path.getPath())) + if (entry->data.size() == old_column_size) { bool inserted = column_object.tryInsertDefaultFromNested(entry); if (!inserted) diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index 14cfafb1ef3..8c6dba5a339 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -55,16 +55,24 @@ String getExceptionMessagePrefix(const DataTypes & types) return res.str(); } -DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_string) +template +DataTypePtr throwOrReturn(const DataTypes & types, std::string_view message_suffix, int error_code) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); + if constexpr (on_error == LeastSupertypeOnError::String) + return std::make_shared(); - throw Exception(String(message), error_code); - }; + if constexpr (on_error == LeastSupertypeOnError::Null) + return nullptr; + if (message_suffix.empty()) + throw Exception(error_code, getExceptionMessagePrefix(types)); + + throw Exception(error_code, "{} {}", getExceptionMessagePrefix(types), message_suffix); +} + +template +DataTypePtr getNumericType(const TypeIndexSet & types) +{ bool all_numbers = true; size_t max_bits_of_signed_integer = 0; @@ -107,14 +115,14 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ maximize(max_mantissa_bits_of_floating, 24); else if (type == TypeIndex::Float64) maximize(max_mantissa_bits_of_floating, 53); - else + else if (type != TypeIndex::Nothing) all_numbers = false; } if (max_bits_of_signed_integer || max_bits_of_unsigned_integer || max_mantissa_bits_of_floating) { if (!all_numbers) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); /// If there are signed and unsigned types of same bit-width, the result must be signed number with at least one more bit. /// Example, common of Int32, UInt32 = Int64. @@ -129,10 +137,9 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ if (min_bit_width_of_integer != 64) ++min_bit_width_of_integer; else - return throw_or_return( - getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," - " but there is no signed integer type, that can exactly represent all required unsigned integer values", + return throwOrReturn(types, + "because some of them are signed integers and some are unsigned integers," + " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); } @@ -145,8 +152,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_mantissa_bits <= 53) return std::make_shared(); else - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are integers and some are floating point," + return throwOrReturn(types, + " because some of them are integers and some are floating point," " but there is no floating point type, that can exactly represent all required integers", ErrorCodes::NO_COMMON_TYPE); } @@ -166,8 +173,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_bit_width_of_integer <= 256) return std::make_shared(); else - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," + throwOrReturn(types, + " because some of them are signed integers and some are unsigned integers," " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); } @@ -186,9 +193,8 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ else if (min_bit_width_of_integer <= 256) return std::make_shared(); else - return throw_or_return("Logical error: " + getExceptionMessagePrefix(types) - + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); - + throwOrReturn(types, + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); } } @@ -197,16 +203,9 @@ DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_ } -DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string) +template +DataTypePtr getLeastSupertype(const DataTypes & types) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); - - throw Exception(String(message), error_code); - }; - /// Trivial cases if (types.empty()) @@ -243,7 +242,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ non_nothing_types.emplace_back(type); if (non_nothing_types.size() < types.size()) - return getLeastSupertype(non_nothing_types, allow_conversion_to_string); + return getLeastSupertype(non_nothing_types); } /// For Arrays @@ -268,9 +267,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_array) { if (!all_arrays) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -294,7 +293,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ nested_types[elem_idx].reserve(types.size()); } else if (tuple_size != type_tuple->getElements().size()) - return throw_or_return(getExceptionMessagePrefix(types) + " because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); have_tuple = true; @@ -308,11 +307,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_tuple) { if (!all_tuples) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); DataTypes common_tuple_types(tuple_size); for (size_t elem_idx = 0; elem_idx < tuple_size; ++elem_idx) - common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx], allow_conversion_to_string); + common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx]); return std::make_shared(common_tuple_types); } @@ -342,11 +341,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_maps) { if (!all_maps) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); return std::make_shared( - getLeastSupertype(key_types, allow_conversion_to_string), - getLeastSupertype(value_types, allow_conversion_to_string)); + getLeastSupertype(key_types), + getLeastSupertype(value_types)); } } @@ -377,9 +376,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_low_cardinality) { if (have_not_low_cardinality) - return getLeastSupertype(nested_types, allow_conversion_to_string); + return getLeastSupertype(nested_types); else - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -405,7 +404,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ if (have_nullable) { - return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); + return std::make_shared(getLeastSupertype(nested_types)); } } @@ -425,7 +424,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ { bool all_strings = type_ids.size() == (have_string + have_fixed_string); if (!all_strings) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); return std::make_shared(); } @@ -442,8 +441,8 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ { bool all_date_or_datetime = type_ids.size() == (have_date + have_date32 + have_datetime + have_datetime64); if (!all_date_or_datetime) - return throw_or_return(getExceptionMessagePrefix(types) - + " because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", + return throwOrReturn(types, + "because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", ErrorCodes::NO_COMMON_TYPE); if (have_datetime64 == 0 && have_date32 == 0) @@ -520,8 +519,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ } if (num_supported != type_ids.size()) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them have no lossless conversion to Decimal", - ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "because some of them have no lossless conversion to Decimal", ErrorCodes::NO_COMMON_TYPE); UInt32 max_scale = 0; for (const auto & type : types) @@ -543,7 +541,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ } if (min_precision > DataTypeDecimal::maxPrecision()) - return throw_or_return(getExceptionMessagePrefix(types) + " because the least supertype is Decimal(" + return throwOrReturn(types, "because the least supertype is Decimal(" + toString(min_precision) + ',' + toString(max_scale) + ')', ErrorCodes::NO_COMMON_TYPE); @@ -557,68 +555,77 @@ DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_ /// For numeric types, the most complicated part. { - auto numeric_type = getNumericType(type_ids, allow_conversion_to_string); + auto numeric_type = getNumericType(type_ids); if (numeric_type) return numeric_type; } /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); + return throwOrReturn(types, "", ErrorCodes::NO_COMMON_TYPE); } -DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string) +DataTypePtr getLeastSupertypeOrString(const DataTypes & types) { - auto throw_or_return = [&](std::string_view message, int error_code) - { - if (allow_conversion_to_string) - return std::make_shared(); - - throw Exception(String(message), error_code); - }; - - TypeIndexSet types_set; - for (const auto & type : types) - { - if (WhichDataType(type).isNothing()) - continue; - - if (!WhichDataType(type).isSimple()) - throw Exception(ErrorCodes::NO_COMMON_TYPE, - "Cannot get common type by type ids with parametric type {}", typeToString(type)); - - types_set.insert(type); - } - - if (types_set.empty()) - return std::make_shared(); - - if (types.contains(TypeIndex::String)) - { - if (types.size() != 1) - return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String and some of them are not", ErrorCodes::NO_COMMON_TYPE); - - return std::make_shared(); - } - - /// For numeric types, the most complicated part. - auto numeric_type = getNumericType(types, allow_conversion_to_string); - if (numeric_type) - return numeric_type; - - /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); + return getLeastSupertype(types); } DataTypePtr tryGetLeastSupertype(const DataTypes & types) { - try - { - return getLeastSupertype(types); - } - catch (...) - { - return nullptr; - } + return getLeastSupertype(types); } +template +DataTypePtr getLeastSupertype(const TypeIndexSet & types) +{ + if (types.empty()) + return std::make_shared(); + + if (types.size() == 1) + { + WhichDataType which(*types.begin()); + if (which.isNothing()) + return std::make_shared(); + + #define DISPATCH(TYPE) \ + if (which.idx == TypeIndex::TYPE) \ + return std::make_shared>(); + + FOR_NUMERIC_TYPES(DISPATCH) + #undef DISPATCH + + if (which.isString()) + return std::make_shared(); + + throwOrReturn(types, "because cannot get common type by type indexes with non-simple types", ErrorCodes::NO_COMMON_TYPE); + } + + if (types.contains(TypeIndex::String)) + { + bool only_string = types.size() == 2 && types.contains(TypeIndex::Nothing); + if (!only_string) + return throwOrReturn(types, "because some of them are String and some of them are not", ErrorCodes::NO_COMMON_TYPE); + + return std::make_shared(); + } + + auto numeric_type = getNumericType(types); + if (numeric_type) + return numeric_type; + + return throwOrReturn(types, "", ErrorCodes::NO_COMMON_TYPE); +} + +DataTypePtr getLeastSupertypeOrString(const TypeIndexSet & types) +{ + return getLeastSupertype(types); +} + +DataTypePtr tryGetLeastSupertype(const TypeIndexSet & types) +{ + return getLeastSupertype(types); +} + +template DataTypePtr getLeastSupertype(const DataTypes & types); +template DataTypePtr getLeastSupertype(const TypeIndexSet & types); + } diff --git a/src/DataTypes/getLeastSupertype.h b/src/DataTypes/getLeastSupertype.h index 5444bb34d06..2ef4a0e6850 100644 --- a/src/DataTypes/getLeastSupertype.h +++ b/src/DataTypes/getLeastSupertype.h @@ -1,24 +1,39 @@ #pragma once - #include - namespace DB { +enum class LeastSupertypeOnError +{ + Throw, + String, + Null, +}; + /** Get data type that covers all possible values of passed data types. - * If there is no such data type, throws an exception - * or if 'allow_conversion_to_string' is true returns String as common type. + * If there is no such data type, throws an exception. * * Examples: least common supertype for UInt8, Int8 - Int16. * Examples: there is no least common supertype for Array(UInt8), Int8. */ -DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string = false); +template +DataTypePtr getLeastSupertype(const DataTypes & types); -using TypeIndexSet = std::unordered_set; -DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string = false); +/// Same as above but return String type instead of throwing exception. +/// All types can be casted to String, because they can be serialized to String. +DataTypePtr getLeastSupertypeOrString(const DataTypes & types); /// Same as above but return nullptr instead of throwing exception. DataTypePtr tryGetLeastSupertype(const DataTypes & types); +using TypeIndexSet = std::unordered_set; + +template +DataTypePtr getLeastSupertype(const TypeIndexSet & types); + +DataTypePtr getLeastSupertypeOrString(const TypeIndexSet & types); + +DataTypePtr tryGetLeastSupertype(const TypeIndexSet & types); + } diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index 97feed54c08..2a18c2c700a 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index f6baae723c9..40f23fe5294 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1300,7 +1300,7 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context, } else { - DataTypePtr common_type = tryGetLeastSupertype({key_expr_type_not_null, const_type}); + DataTypePtr common_type = tryGetLeastSupertype(DataTypes{key_expr_type_not_null, const_type}); if (!common_type) return false; diff --git a/tests/performance/json_type.xml b/tests/performance/json_type.xml new file mode 100644 index 00000000000..29e52f1e53f --- /dev/null +++ b/tests/performance/json_type.xml @@ -0,0 +1,17 @@ + + + 1 + + + CREATE TABLE t_json_1(data JSON) ENGINE = MergeTree ORDER BY tuple() + CREATE TABLE t_json_2(data JSON) ENGINE = MergeTree ORDER BY tuple() + CREATE TABLE t_json_3(data JSON) ENGINE = MergeTree ORDER BY tuple() + + INSERT INTO t_json_1 SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers(200000) + INSERT INTO t_json_2 SELECT '{"col' || toString(number % 100) || '":' || toString(number) || '}' FROM numbers(100000) + INSERT INTO t_json_3 SELECT materialize('{"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}') FROM numbers_mt(100000) + + DROP TABLE IF EXISTS t_json_1 + DROP TABLE IF EXISTS t_json_2 + DROP TABLE IF EXISTS t_json_3 + diff --git a/tests/queries/0_stateless/01825_type_json_parallel_insert.reference b/tests/queries/0_stateless/01825_type_json_parallel_insert.reference index ac512064a43..158d61d46f7 100644 --- a/tests/queries/0_stateless/01825_type_json_parallel_insert.reference +++ b/tests/queries/0_stateless/01825_type_json_parallel_insert.reference @@ -1 +1 @@ -Tuple(k1 Int8, k2 String) 3000000 +Tuple(k1 Int8, k2 String) 500000 diff --git a/tests/queries/0_stateless/01825_type_json_parallel_insert.sql b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql index 93d1eecfbd7..e443c9455d5 100644 --- a/tests/queries/0_stateless/01825_type_json_parallel_insert.sql +++ b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql @@ -1,10 +1,10 @@ -- Tags: long, no-backward-compatibility-check:22.3.2.1 DROP TABLE IF EXISTS t_json_parallel; -SET allow_experimental_object_type = 1, max_insert_threads = 20, max_threads = 20; +SET allow_experimental_object_type = 1, max_insert_threads = 20, max_threads = 20, min_insert_block_size_rows = 65536; CREATE TABLE t_json_parallel (data JSON) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO t_json_parallel SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers_mt(3000000); +INSERT INTO t_json_parallel SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers_mt(500000); SELECT any(toTypeName(data)), count() FROM t_json_parallel; DROP TABLE t_json_parallel; From ffab75baa7ec4242ef1d689095a3f64d6c0d4da3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:33:20 +0200 Subject: [PATCH 080/408] Checkout full repositories for performance tests --- .github/workflows/backport_branches.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index b93c1b61ffd..e88a121b5d3 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -143,6 +143,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive @@ -188,6 +190,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive From 3ab58fd14f20522de8cc3b1079e9388adaf6b477 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:16:01 +0200 Subject: [PATCH 081/408] Add docusaurus header to generated changelogs --- utils/changelog/changelog.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index c20c6cfd072..4e6bc7f4a6a 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -14,6 +14,7 @@ from typing import Dict, List, Optional, TextIO from fuzzywuzzy.fuzz import ratio # type: ignore from github import Github +from github.GithubException import UnknownObjectException from github.NamedUser import NamedUser from github.Issue import Issue from github.PullRequest import PullRequest @@ -66,7 +67,10 @@ class Description: r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", entry, ) - user_name = self.user.name if self.user.name else self.user.login + try: + user_name = self.user.name if self.user.name else self.user.login + except UnknownObjectException: + user_name = self.user.login return ( f"* {entry} [#{self.number}]({self.html_url}) " f"([{user_name}]({self.user.html_url}))." @@ -306,7 +310,11 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri def write_changelog(fd: TextIO, descriptions: Dict[str, List[Description]]): - fd.write(f"### ClickHouse release {TO_REF} FIXME as compared to {FROM_REF}\n\n") + year = date.today().year + fd.write( + f"---\nsidebar_position: 1\nsidebar_label: {year}\n---\n\n# {year} Changelog\n\n" + f"### ClickHouse release {TO_REF} FIXME as compared to {FROM_REF}\n\n" + ) seen_categories = [] # type: List[str] for category in categories_preferred_order: From 61d3e94d06668340e9fb06ae1aceb14a026172a3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:16:29 +0200 Subject: [PATCH 082/408] Regenerate v22.* changelogs --- docs/changelogs/v22.1.1.2542-prestable.md | 7 +++++++ docs/changelogs/v22.1.2.2-stable.md | 7 +++++++ docs/changelogs/v22.1.3.7-stable.md | 7 +++++++ docs/changelogs/v22.1.4.30-stable.md | 7 +++++++ docs/changelogs/v22.2.1.2139-prestable.md | 7 +++++++ docs/changelogs/v22.2.2.1-stable.md | 7 +++++++ docs/changelogs/v22.2.3.5-stable.md | 7 +++++++ docs/changelogs/v22.3.1.1262-prestable.md | 9 ++++++++- docs/changelogs/v22.3.2.2-lts.md | 7 +++++++ docs/changelogs/v22.3.3.44-lts.md | 7 +++++++ docs/changelogs/v22.3.4.20-lts.md | 7 +++++++ docs/changelogs/v22.3.5.5-lts.md | 7 +++++++ docs/changelogs/v22.3.6.5-lts.md | 7 +++++++ docs/changelogs/v22.4.1.2305-prestable.md | 7 +++++++ docs/changelogs/v22.4.2.1-stable.md | 7 +++++++ docs/changelogs/v22.4.3.3-stable.md | 7 +++++++ docs/changelogs/v22.4.4.7-stable.md | 7 +++++++ docs/changelogs/v22.4.5.9-stable.md | 7 +++++++ docs/changelogs/v22.5.1.2079-stable.md | 7 +++++++ docs/changelogs/v22.6.1.1985-stable.md | 7 +++++++ 20 files changed, 141 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index c9071422949..cbe8e781d3c 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.1.2542-prestable FIXME as compared to v21.12.1.9017-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.1.2.2-stable.md b/docs/changelogs/v22.1.2.2-stable.md index 450c640bc5e..de2dc6e3cee 100644 --- a/docs/changelogs/v22.1.2.2-stable.md +++ b/docs/changelogs/v22.1.2.2-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.2.2-stable FIXME as compared to v22.1.1.2542-prestable diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index f8bb8821031..72c7a9cc5da 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.3.7-stable FIXME as compared to v22.1.2.2-stable #### Improvement diff --git a/docs/changelogs/v22.1.4.30-stable.md b/docs/changelogs/v22.1.4.30-stable.md index 1ea56131481..7fa295cdcb0 100644 --- a/docs/changelogs/v22.1.4.30-stable.md +++ b/docs/changelogs/v22.1.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.4.30-stable FIXME as compared to v22.1.3.7-stable #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index 26ac24a7778..f93a3a690c4 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.1.2139-prestable FIXME as compared to v22.1.1.2542-prestable #### New Feature diff --git a/docs/changelogs/v22.2.2.1-stable.md b/docs/changelogs/v22.2.2.1-stable.md index c9158290753..0ea5be33c94 100644 --- a/docs/changelogs/v22.2.2.1-stable.md +++ b/docs/changelogs/v22.2.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.2.1-stable FIXME as compared to v22.2.1.2139-prestable diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index 90c0d22d570..24b535a7f91 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.3.5-stable FIXME as compared to v22.2.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index 03e81bd1808..d2f976f4517 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.1.1262-prestable FIXME as compared to v22.2.1.2139-prestable #### Backward Incompatible Change @@ -65,7 +72,7 @@ * Add setting to lower column case when reading parquet/ORC file. [#35145](https://github.com/ClickHouse/ClickHouse/pull/35145) ([shuchaome](https://github.com/shuchaome)). * Do not retry non-rertiable errors. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added disk_name to system.part_log. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)). -* Currently,ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Currently, ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). * Sometimes it is not enough for us to distinguish queries hierachy only by is_initial_query in system.query_log and system.processes. So distributed_depth is needed. [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)). * Support test mode for clickhouse-local. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)). * Return const for function getMacro if not in distributed query. Close [#34727](https://github.com/ClickHouse/ClickHouse/issues/34727). [#35289](https://github.com/ClickHouse/ClickHouse/pull/35289) ([李扬](https://github.com/taiyang-li)). diff --git a/docs/changelogs/v22.3.2.2-lts.md b/docs/changelogs/v22.3.2.2-lts.md index ef45265c7bd..b755db300c8 100644 --- a/docs/changelogs/v22.3.2.2-lts.md +++ b/docs/changelogs/v22.3.2.2-lts.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.2.2-lts FIXME as compared to v22.3.1.1262-prestable diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 4214512c533..4fa98940290 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.3.44-lts FIXME as compared to v22.3.2.2-lts #### Bug Fix diff --git a/docs/changelogs/v22.3.4.20-lts.md b/docs/changelogs/v22.3.4.20-lts.md index 4bb4f1bf0f4..d820adbdbec 100644 --- a/docs/changelogs/v22.3.4.20-lts.md +++ b/docs/changelogs/v22.3.4.20-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.4.20-lts FIXME as compared to v22.3.3.44-lts #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.3.5.5-lts.md b/docs/changelogs/v22.3.5.5-lts.md index d1c42807f41..7deff1be416 100644 --- a/docs/changelogs/v22.3.5.5-lts.md +++ b/docs/changelogs/v22.3.5.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.5.5-lts FIXME as compared to v22.3.4.20-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 16cf390c703..9d86d9e786c 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.6.5-lts FIXME as compared to v22.3.5.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index eb1ed6decd3..5c6948251b3 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.1.2305-prestable FIXME as compared to v22.3.1.1262-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.4.2.1-stable.md b/docs/changelogs/v22.4.2.1-stable.md index cd7ee75997c..fb77d3fee9b 100644 --- a/docs/changelogs/v22.4.2.1-stable.md +++ b/docs/changelogs/v22.4.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.2.1-stable FIXME as compared to v22.4.1.2305-prestable diff --git a/docs/changelogs/v22.4.3.3-stable.md b/docs/changelogs/v22.4.3.3-stable.md index 5ab7872f880..4baa63672ab 100644 --- a/docs/changelogs/v22.4.3.3-stable.md +++ b/docs/changelogs/v22.4.3.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.3.3-stable FIXME as compared to v22.4.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 9004ce2f6bc..525c0de1341 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.4.7-stable FIXME as compared to v22.4.3.3-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index ab43bf3eade..b5fffa56494 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.5.9-stable FIXME as compared to v22.4.4.7-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index aab8266c115..7614d678a2a 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.5.1.2079-stable FIXME as compared to v22.4.1.2305-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 583ffddf279..9c7ecc1dae3 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.6.1.1985-stable FIXME as compared to v22.5.1.2079-stable #### Backward Incompatible Change From f06c533eaf6d573aff18b1f4bfc053b5352b8a46 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:17:07 +0200 Subject: [PATCH 083/408] Add changelog for fresh v22.3.7.28-lts --- docs/changelogs/v22.3.7.28-lts.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/changelogs/v22.3.7.28-lts.md diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md new file mode 100644 index 00000000000..ecfd0ce2cf6 --- /dev/null +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -0,0 +1,24 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.3.7.28-lts FIXME as compared to v22.3.6.5-lts + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#37715](https://github.com/ClickHouse/ClickHouse/issues/37715): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37511](https://github.com/ClickHouse/ClickHouse/issues/37511): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#37843](https://github.com/ClickHouse/ClickHouse/issues/37843): Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#37940](https://github.com/ClickHouse/ClickHouse/issues/37940): Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#37926](https://github.com/ClickHouse/ClickHouse/issues/37926): Fix check asof join key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37172](https://github.com/ClickHouse/ClickHouse/issues/37172): Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#37457](https://github.com/ClickHouse/ClickHouse/issues/37457): Server might fail to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#36925](https://github.com/ClickHouse/ClickHouse/issues/36925): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +* Backported in [#37364](https://github.com/ClickHouse/ClickHouse/issues/37364): Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). + From 779b83262f770ba448852f9fcebfadbd3eda225e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 19:52:43 +0200 Subject: [PATCH 084/408] Add `not for changelog` category with titles as entries --- utils/changelog/changelog.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 4e6bc7f4a6a..e1797fb5dd3 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -284,7 +284,15 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri # Filter out the PR categories that are not for changelog. if re.match( - r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", + r"(?i)((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", + category, + ): + category = "NOT FOR CHANGELOG / INSIGNIFICANT" + return Description(item.number, item.user, item.html_url, item.title, category) + + # Filter out documentations changelog + if re.match( + r"(?i)doc", category, ): return None From 81becebfe3fbbfe9ae071739ac2f22147d63473d Mon Sep 17 00:00:00 2001 From: Laurie Li Date: Thu, 23 Jun 2022 09:29:46 +0800 Subject: [PATCH 085/408] Add "See Also" Signed-off-by: Laurie Li --- .../table-engines/mergetree-family/replication.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index 568484c6256..69d1cd9d942 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -277,4 +277,12 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 如果 ZooKeeper 中的数据丢失或损坏，如上所述，你可以通过将数据转移到非复制表来保存数据。 -[来源文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) +**参考** + +- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) +- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) + +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) From 4bf1fc47188c7c158f6661352c608370d779909e Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 23 Jun 2022 10:28:31 +0300 Subject: [PATCH 086/408] Add error code and message displaying in exceptions of KerberosInit; Correct code style in KerberosInit --- src/Access/KerberosInit.cpp | 54 ++++++++++++++++++++++--------------- src/Access/KerberosInit.h | 2 +- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index d3be559cba7..9ca45b12531 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #if USE_KRB5 #include #include @@ -36,23 +37,34 @@ struct K5Data class KerberosInit : boost::noncopyable { public: - int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + void init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct K5Data k5; + struct K5Data k5 {}; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; // Credentials structure including ticket, session key, and lifetime info. krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; + String fmtError(krb5_error_code code); }; } -int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) + +String KerberosInit::fmtError(krb5_error_code code) +{ + const char *msg; + msg = krb5_get_error_message(k5.ctx, code); + String fmt_error = fmt::format(" ({}, {})", code, msg); + krb5_free_error_message(k5.ctx, msg); + return fmt_error; +} + +void KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { auto * log = &Poco::Logger::get("KerberosInit"); - LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); + LOG_TRACE(log,"Trying to authenticate with Kerberos v5"); krb5_error_code ret; @@ -61,16 +73,15 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (!std::filesystem::exists(keytab_file)) throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); - memset(&k5, 0, sizeof(k5)); ret = krb5_init_context(&k5.ctx); if (ret) - throw Exception("Error while initializing Kerberos 5 library", ErrorCodes::KERBEROS_ERROR); + throw Exception(fmt::format("Error while initializing Kerberos 5 library ({})", ret), ErrorCodes::KERBEROS_ERROR); if (!cache_name.empty()) { ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); if (ret) - throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in resolving cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Resolved cache"); } else @@ -78,7 +89,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Resolve the default cache and get its type and default principal (if it is initialized). ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while getting default cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) @@ -88,7 +99,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Use the specified principal name. ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); if (ret) - throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when parsing principal name " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Cache related commands if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) @@ -96,8 +107,8 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Use an existing cache for the client principal if we can. ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); if (ret && ret != KRB5_CC_NOTFOUND) - throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); - if (!ret) + throw Exception("Error while searching for cache for " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + if (0 == ret) { LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; @@ -107,7 +118,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Create a new cache to avoid overwriting the initialized default cache. ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) - throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while generating new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } @@ -123,24 +134,24 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) - throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when unparsing name" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using principal: {}", k5.name); // Allocate a new initial credential options structure. ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) - throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in options allocation" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Resolve keytab ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) - throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in resolving keytab "+keytab_file + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using keytab: {}", keytab_file); // Set an output credential cache in initial credential options. ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) - throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in setting output credential cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Action: init or renew LOG_TRACE(log,"Trying to renew credentials"); @@ -153,7 +164,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) - throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in getting initial credentials" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); else LOG_TRACE(log,"Got initial credentials"); } @@ -163,7 +174,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) - throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when initializing cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Initialized cache"); // Store credentials in a credential cache. ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); @@ -177,11 +188,10 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Make a credential cache the primary cache for its collection. ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while switching to new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); } LOG_TRACE(log,"Authenticated to Kerberos v5"); - return 0; } KerberosInit::~KerberosInit() @@ -208,12 +218,12 @@ KerberosInit::~KerberosInit() } } -int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) { // Using mutex to prevent cache file corruptions static std::mutex kinit_mtx; std::unique_lock lck(kinit_mtx); KerberosInit k_init; - return k_init.init(keytab_file, principal, cache_name); + k_init.init(keytab_file, principal, cache_name); } #endif // USE_KRB5 diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 4731baf706f..5a11a275529 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -6,6 +6,6 @@ #if USE_KRB5 -int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); #endif // USE_KRB5 From 31bf1203d337e3127a319671d704fb324b6cfb01 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 23 Jun 2022 15:57:19 +0800 Subject: [PATCH 087/408] update codes --- src/DataTypes/NestedUtils.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 9cab49a509d..dbdba39fa84 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -121,7 +121,7 @@ Block flatten(const Block & block) const DataTypes & element_types = type_tuple->getElements(); const Strings & names = type_tuple->getElementNames(); const ColumnTuple * column_tuple; - if(isColumnConst(*elem.column)) + if (isColumnConst(*elem.column)) column_tuple = typeid_cast(&assert_cast(*elem.column).getDataColumn()); else column_tuple = typeid_cast(elem.column.get()); @@ -309,12 +309,12 @@ std::optional NestedColumnExtractHelper::extractColumn( auto new_column_name_prefix = Nested::concatenateName(column_name_prefix, nested_names.first); if (nested_names.second.empty()) { - if (nested_table->has(new_column_name_prefix, case_insentive)) + if (auto * column_ref = nested_table->findByName(new_column_name_prefix, case_insentive)) { - ColumnWithTypeAndName column = *nested_table->findByName(new_column_name_prefix, case_insentive); + ColumnWithTypeAndName column = *column_ref; if (case_insentive) column.name = original_column_name; - return {column}; + return {std::move(column)}; } else { From 96e6f9a2d02ba6cf560703f73d6acc971b3dd445 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 23 Jun 2022 16:10:01 +0800 Subject: [PATCH 088/408] fixed code style --- src/DataTypes/NestedUtils.h | 4 ++-- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index 9473d30497a..e7cda541f47 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -36,9 +36,9 @@ namespace Nested std::unordered_set getAllTableNames(const Block & block, bool to_lower_case = false); } -/// Use this class to extract element columns from columns of nested type in a block, e.g. named Tuple. +/// Use this class to extract element columns from columns of nested type in a block, e.g. named Tuple. /// It can extract a column from a multiple nested type column, e.g. named Tuple in named Tuple -/// Keeps some intermediate datas to avoid rebuild them multi-times. +/// Keeps some intermediate data to avoid rebuild them multi-times. class NestedColumnExtractHelper { public: diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 305009a070f..eefe4231b89 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,4 +1,3 @@ -#include "ArrowColumnToCHColumn.h" #include #include @@ -704,7 +703,7 @@ std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema { if (!allow_missing_columns) throw Exception{ErrorCodes::THERE_IS_NO_COLUMN, "Column '{}' is not presented in input data.", header_column.name}; - missing_columns.push_back(i); + missing_columns.push_back(i); } } } From 2c5a88fabaefb7b865e8cf6d08ad8609c4342b64 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 23 Jun 2022 09:59:13 +0200 Subject: [PATCH 089/408] deprecate Ordinary database --- docker/test/stateful/run.sh | 4 ++++ docker/test/stateless/run.sh | 4 ++++ src/Core/Settings.h | 3 ++- src/Databases/DatabaseAtomic.cpp | 6 +++--- src/Databases/DatabaseAtomic.h | 2 +- src/Databases/DatabaseFactory.cpp | 7 ++++++- src/Databases/DatabaseLazy.cpp | 4 ++-- src/Databases/DatabaseLazy.h | 2 +- src/Databases/DatabaseMemory.cpp | 2 +- src/Databases/DatabaseMemory.h | 2 +- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Databases/DatabaseOnDisk.h | 2 +- src/Databases/DatabaseReplicated.cpp | 4 ++-- src/Databases/DatabaseReplicated.h | 2 +- src/Databases/IDatabase.h | 2 +- .../MySQL/DatabaseMaterializedMySQL.cpp | 4 ++-- .../MySQL/DatabaseMaterializedMySQL.h | 2 +- src/Databases/MySQL/DatabaseMySQL.cpp | 2 +- src/Databases/MySQL/DatabaseMySQL.h | 2 +- .../DatabaseMaterializedPostgreSQL.cpp | 4 ++-- .../DatabaseMaterializedPostgreSQL.h | 2 +- .../PostgreSQL/DatabasePostgreSQL.cpp | 2 +- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 2 +- src/Interpreters/AsynchronousInsertQueue.h | 1 - src/Interpreters/HashJoin.h | 6 +++--- src/Interpreters/InterpreterCreateQuery.cpp | 6 ++---- src/Interpreters/InterpreterDropQuery.cpp | 18 +++++++++--------- src/Interpreters/InterpreterDropQuery.h | 2 +- src/Interpreters/loadMetadata.cpp | 7 ------- src/Parsers/ASTDropQuery.cpp | 4 ++-- src/Parsers/ASTDropQuery.h | 2 +- src/Parsers/ParserDropQuery.cpp | 6 +++--- src/Storages/IStorage.h | 2 +- .../StorageMaterializedPostgreSQL.cpp | 4 ++-- .../PostgreSQL/StorageMaterializedPostgreSQL.h | 2 +- src/Storages/StorageMaterializedView.cpp | 4 ++-- src/Storages/StorageMaterializedView.h | 2 +- src/Storages/WindowView/StorageWindowView.cpp | 6 +++--- src/Storages/WindowView/StorageWindowView.h | 2 +- tests/clickhouse-test | 4 ++-- tests/config/users.d/database_ordinary.xml | 2 +- .../configs/users.xml | 1 - .../configs/users_disable_bytes_settings.xml | 1 - .../configs/users_disable_rows_settings.xml | 1 - .../01047_window_view_parser_inner_table.sql | 1 + .../0_stateless/01048_window_view_parser.sql | 1 + .../01053_drop_database_mat_view.sql | 1 + .../0_stateless/01085_window_view_attach.sql | 1 + .../0_stateless/01086_window_view_cleanup.sh | 2 +- .../0_stateless/01109_exchange_tables.sql | 4 +--- .../0_stateless/01114_database_atomic.sh | 4 ++-- .../01148_zookeeper_path_macros_unfolding.sql | 1 + .../01155_rename_move_materialized_view.sql | 2 ++ .../0_stateless/01162_strange_mutations.sh | 10 +++++----- .../0_stateless/01190_full_attach_syntax.sql | 1 + .../01192_rename_database_zookeeper.sh | 8 ++++---- .../01224_no_superfluous_dict_reload.sql | 1 + ...01225_show_create_table_from_dictionary.sql | 1 + .../01249_bad_arguments_for_bloom_filter.sql | 1 + ...320_create_sync_race_condition_zookeeper.sh | 2 +- .../01516_create_table_primary_key.sql | 1 + .../01517_drop_mv_with_inner_table.sql | 1 + .../0_stateless/01600_detach_permanently.sh | 2 +- .../0_stateless/01601_detach_permanently.sql | 1 + .../0_stateless/01603_rename_overwrite_bug.sql | 1 + .../01810_max_part_removal_threads_long.sh | 2 +- .../0_stateless/02096_rename_atomic_hang.sql | 1 + 67 files changed, 108 insertions(+), 93 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 5f55bb9fa21..d77978c904b 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -120,6 +120,10 @@ function run_tests() ADDITIONAL_OPTIONS+=('--replicated-database') fi + if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') + fi + set +e clickhouse-test -j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --hung-check --print-time \ --skip 00168_parallel_processing_on_replicas "${ADDITIONAL_OPTIONS[@]}" \ diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 52bf8a60669..075f588cae3 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -115,6 +115,10 @@ function run_tests() ADDITIONAL_OPTIONS+=("$RUN_BY_HASH_TOTAL") fi + if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') + fi + set +e clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \ --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f1fd9d20f00..883281c94f3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -445,7 +445,6 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Seconds, wait_for_window_view_fire_signal_timeout, 10, "Timeout for waiting for window view fire signal in event time processing", 0) \ M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ - M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \ M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \ M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \ @@ -591,6 +590,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, use_local_cache_for_remote_storage, true, "Use local cache for remote storage like HDFS or S3, it's used for remote table engine only", 0) \ \ M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (without condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ + M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ @@ -637,6 +637,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) MAKE_OBSOLETE(M, UInt64, background_schedule_pool_size, 128) \ MAKE_OBSOLETE(M, UInt64, background_message_broker_schedule_pool_size, 16) \ MAKE_OBSOLETE(M, UInt64, background_distributed_schedule_pool_size, 16) \ + MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \ /** The section above is for obsolete settings. Do not add anything there. */ diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ea887c84111..9e7d29d89b1 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -109,13 +109,13 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String & return table; } -void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool sync) { auto table = tryGetTable(table_name, local_context); /// Remove the inner table (if any) to avoid deadlock /// (due to attempt to execute DROP from the worker thread) if (table) - table->dropInnerTableIfAny(no_delay, local_context); + table->dropInnerTableIfAny(sync, local_context); else throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuote(database_name), backQuote(table_name)); @@ -145,7 +145,7 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. /// Cleanup is performed outside of database to allow easily DROP DATABASE without waiting for cleanup to complete. - DatabaseCatalog::instance().enqueueDroppedTableCleanup(table->getStorageID(), table, table_metadata_path_drop, no_delay); + DatabaseCatalog::instance().enqueueDroppedTableCleanup(table->getStorageID(), table, table_metadata_path_drop, sync); } void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_name, IDatabase & to_database, diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index b748e53244d..cba9593a601 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -35,7 +35,7 @@ public: bool exchange, bool dictionary) override; - void dropTable(ContextPtr context, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr context, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & name) override; diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 5cc334eaad4..182d9fe686b 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -138,8 +138,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String throw Exception(ErrorCodes::BAD_ARGUMENTS, "Database engine `{}` cannot have table overrides", engine_name); if (engine_name == "Ordinary") + { + if (!create.attach && !context->getSettingsRef().allow_deprecated_database_ordinary) + throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Ordinary database engine is deprecated"); return std::make_shared(database_name, metadata_path, context); - else if (engine_name == "Atomic") + } + + if (engine_name == "Atomic") return std::make_shared(database_name, metadata_path, uuid, context); else if (engine_name == "Memory") return std::make_shared(database_name, context); diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index b024c73d578..84e1a132669 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -77,10 +77,10 @@ void DatabaseLazy::createTable( void DatabaseLazy::dropTable( ContextPtr local_context, const String & table_name, - bool no_delay) + bool sync) { SCOPE_EXIT_MEMORY_SAFE({ clearExpiredTables(); }); - DatabaseOnDisk::dropTable(local_context, table_name, no_delay); + DatabaseOnDisk::dropTable(local_context, table_name, sync); } void DatabaseLazy::renameTable( diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 3a7d7b14be1..8d4799945fb 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -37,7 +37,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; void renameTable( ContextPtr context, diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 6df5b70c827..664bb015925 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -52,7 +52,7 @@ void DatabaseMemory::createTable( void DatabaseMemory::dropTable( ContextPtr /*context*/, const String & table_name, - bool /*no_delay*/) + bool /*sync*/) { std::unique_lock lock{mutex}; auto table = detachTableUnlocked(table_name, lock); diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index b854d9be1f3..6b22b5a4ec2 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -32,7 +32,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override; ASTPtr getCreateDatabaseQuery() const override; diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 9484da8ec2d..67aa712d557 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -281,7 +281,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri } } -void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_name, bool /*no_delay*/) +void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_name, bool /*sync*/) { String table_metadata_path = getObjectMetadataPath(table_name); String table_metadata_path_drop = table_metadata_path + drop_suffix; diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index a118c8da678..462b602a8b8 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -43,7 +43,7 @@ public: void dropTable( ContextPtr context, const String & table_name, - bool no_delay) override; + bool sync) override; void renameTable( ContextPtr context, diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 5c701c8d90c..0d20dfb2770 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -811,7 +811,7 @@ void DatabaseReplicated::shutdown() } -void DatabaseReplicated::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseReplicated::dropTable(ContextPtr local_context, const String & table_name, bool sync) { auto txn = local_context->getZooKeeperMetadataTransaction(); assert(!ddl_worker->isCurrentlyActive() || txn || startsWith(table_name, ".inner_id.")); @@ -820,7 +820,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); txn->addOp(zkutil::makeRemoveRequest(metadata_zk_path, -1)); } - DatabaseAtomic::dropTable(local_context, table_name, no_delay); + DatabaseAtomic::dropTable(local_context, table_name, sync); } void DatabaseReplicated::renameTable(ContextPtr local_context, const String & table_name, IDatabase & to_database, diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 45a9d12981c..3aa2aa378b7 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -30,7 +30,7 @@ public: String getEngineName() const override { return "Replicated"; } /// If current query is initial, then the following methods add metadata updating ZooKeeper operations to current ZooKeeperMetadataTransaction. - void dropTable(ContextPtr, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr, const String & table_name, bool sync) override; void renameTable(ContextPtr context, const String & table_name, IDatabase & to_database, const String & to_table_name, bool exchange, bool dictionary) override; void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 38c85cf3d05..4de64a2b24f 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -198,7 +198,7 @@ public: virtual void dropTable( /// NOLINT ContextPtr /*context*/, const String & /*name*/, - [[maybe_unused]] bool no_delay = false) + [[maybe_unused]] bool sync = false) { throw Exception("There is no DROP TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 13f55eab9e2..a81e07f0173 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -80,10 +80,10 @@ void DatabaseMaterializedMySQL::createTable(ContextPtr context_, const String & DatabaseAtomic::createTable(context_, name, table, query); } -void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & name, bool no_delay) +void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & name, bool sync) { checkIsInternalQuery(context_, "DROP TABLE"); - DatabaseAtomic::dropTable(context_, name, no_delay); + DatabaseAtomic::dropTable(context_, name, sync); } void DatabaseMaterializedMySQL::attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.h b/src/Databases/MySQL/DatabaseMaterializedMySQL.h index 32686784f2a..a6810f29d87 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.h @@ -52,7 +52,7 @@ public: void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override; - void dropTable(ContextPtr context_, const String & name, bool no_delay) override; + void dropTable(ContextPtr context_, const String & name, bool sync) override; void attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) override; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 58be682bd73..95098ba9cbd 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -447,7 +447,7 @@ void DatabaseMySQL::detachTablePermanently(ContextPtr, const String & table_name table_iter->second.second->is_dropped = true; } -void DatabaseMySQL::dropTable(ContextPtr local_context, const String & table_name, bool /*no_delay*/) +void DatabaseMySQL::dropTable(ContextPtr local_context, const String & table_name, bool /*sync*/) { detachTablePermanently(local_context, table_name); } diff --git a/src/Databases/MySQL/DatabaseMySQL.h b/src/Databases/MySQL/DatabaseMySQL.h index 1ee090ecd52..6a5105e2dba 100644 --- a/src/Databases/MySQL/DatabaseMySQL.h +++ b/src/Databases/MySQL/DatabaseMySQL.h @@ -82,7 +82,7 @@ public: void detachTablePermanently(ContextPtr context, const String & table_name) override; - void dropTable(ContextPtr context, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr context, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index 60d2fa0d2c8..8c4be49ef5c 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -389,10 +389,10 @@ void DatabaseMaterializedPostgreSQL::stopReplication() } -void DatabaseMaterializedPostgreSQL::dropTable(ContextPtr local_context, const String & table_name, bool no_delay) +void DatabaseMaterializedPostgreSQL::dropTable(ContextPtr local_context, const String & table_name, bool sync) { /// Modify context into nested_context and pass query to Atomic database. - DatabaseAtomic::dropTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, no_delay); + DatabaseAtomic::dropTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, sync); } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h index 08420f4ba5e..ac2bcedca60 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h @@ -55,7 +55,7 @@ public: StoragePtr detachTable(ContextPtr context, const String & table_name) override; - void dropTable(ContextPtr local_context, const String & name, bool no_delay) override; + void dropTable(ContextPtr local_context, const String & name, bool sync) override; void drop(ContextPtr local_context) override; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 1bcc203beb9..fed4d1ee64d 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -264,7 +264,7 @@ void DatabasePostgreSQL::createTable(ContextPtr local_context, const String & ta } -void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /* no_delay */) +void DatabasePostgreSQL::dropTable(ContextPtr, const String & table_name, bool /* sync */) { std::lock_guard lock{mutex}; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 3397dcc8076..3d5e7bd444e 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -53,7 +53,7 @@ public: StoragePtr tryGetTable(const String & name, ContextPtr context) const override; void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; - void dropTable(ContextPtr, const String & table_name, bool no_delay) override; + void dropTable(ContextPtr, const String & table_name, bool sync) override; void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & table_name) override; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index db3cb3049fd..8a4e8dad8dd 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index b2a7aedaa5a..17d09b25ea1 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include @@ -339,7 +339,7 @@ public: /// We keep correspondence between used_flags and hash table internal buffer. /// Hash table cannot be modified during HashJoin lifetime and must be protected with lock. - void setLock(RWLockImpl::LockHolder rwlock_holder) + void setLock(TableLockHolder rwlock_holder) { storage_join_lock = rwlock_holder; } @@ -394,7 +394,7 @@ private: /// Should be set via setLock to protect hash table from modification from StorageJoin /// If set HashJoin instance is not available for modification (addJoinedBlock) - RWLockImpl::LockHolder storage_join_lock = nullptr; + TableLockHolder storage_join_lock = nullptr; void dataMapInit(MapsVariant &); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index b29f7372d38..7c5c6e15638 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -150,10 +150,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// When attaching old-style database during server startup, we must always use Ordinary engine if (create.attach) throw Exception("Database engine must be specified for ATTACH DATABASE query", ErrorCodes::UNKNOWN_DATABASE_ENGINE); - bool old_style_database = getContext()->getSettingsRef().default_database_engine.value == DefaultDatabaseEngine::Ordinary; auto engine = std::make_shared(); auto storage = std::make_shared(); - engine->name = old_style_database ? "Ordinary" : "Atomic"; + engine->name = "Atomic"; engine->no_empty_args = true; storage->set(storage->engine, engine); create.set(create.storage, storage); @@ -196,8 +195,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (create_from_user) { - const auto & default_engine = getContext()->getSettingsRef().default_database_engine.value; - if (create.uuid == UUIDHelpers::Nil && default_engine == DefaultDatabaseEngine::Atomic) + if (create.uuid == UUIDHelpers::Nil) create.uuid = UUIDHelpers::generateV4(); /// Will enable Atomic engine for nested database } else if (attach_from_user && create.uuid == UUIDHelpers::Nil) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 41b65e73efa..ac731ec6f4b 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -62,7 +62,7 @@ BlockIO InterpreterDropQuery::execute() } if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) - drop.no_delay = true; + drop.sync = true; if (drop.table) return executeToTable(drop); @@ -89,7 +89,7 @@ BlockIO InterpreterDropQuery::executeToTable(ASTDropQuery & query) DatabasePtr database; UUID table_to_wait_on = UUIDHelpers::Nil; auto res = executeToTableImpl(getContext(), query, database, table_to_wait_on); - if (query.no_delay) + if (query.sync) waitForTableToBeActuallyDroppedOrDetached(query, database, table_to_wait_on); return res; } @@ -244,7 +244,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue DatabaseCatalog::instance().tryRemoveLoadingDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, is_drop_or_detach_database); - database->dropTable(context_, table_id.table_name, query.no_delay); + database->dropTable(context_, table_id.table_name, query.sync); } db = database; @@ -300,7 +300,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) } catch (...) { - if (query.no_delay) + if (query.sync) { for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); @@ -308,7 +308,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) throw; } - if (query.no_delay) + if (query.sync) { for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); @@ -345,7 +345,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, query_for_table.kind = query.kind; query_for_table.if_exists = true; query_for_table.setDatabase(database_name); - query_for_table.no_delay = query.no_delay; + query_for_table.sync = query.sync; /// Flush should not be done if shouldBeEmptyOnDetach() == false, /// since in this case getTablesIterator() may do some additional work, @@ -368,7 +368,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, } } - if (!drop && query.no_delay) + if (!drop && query.sync) { /// Avoid "some tables are still in use" when sync mode is enabled for (const auto & table_uuid : uuids_to_wait) @@ -428,7 +428,7 @@ void InterpreterDropQuery::extendQueryLogElemImpl(QueryLogElement & elem, const elem.query_kind = "Drop"; } -void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool no_delay) +void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync) { if (DatabaseCatalog::instance().tryGetTable(target_table_id, current_context)) { @@ -437,7 +437,7 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr drop_query->setDatabase(target_table_id.database_name); drop_query->setTable(target_table_id.table_name); drop_query->kind = kind; - drop_query->no_delay = no_delay; + drop_query->sync = sync; drop_query->if_exists = true; ASTPtr ast_drop_query = drop_query; /// FIXME We have to use global context to execute DROP query for inner table diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 1a38abcdff9..2b65039954b 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -26,7 +26,7 @@ public: void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const override; - static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool no_delay); + static void executeDropQuery(ASTDropQuery::Kind kind, ContextPtr global_context, ContextPtr current_context, const StorageID & target_table_id, bool sync); private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index de920eaddbf..f6cfbe452c6 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include @@ -71,12 +70,6 @@ static void loadDatabase( ReadBufferFromFile in(database_metadata_file, 1024); readStringUntilEOF(database_attach_query, in); } - else if (fs::exists(fs::path(database_path))) - { - /// TODO Remove this code (it's required for compatibility with versions older than 20.7) - /// Database exists, but .sql file is absent. It's old-style Ordinary database (e.g. system or default) - database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database) + " ENGINE = Ordinary"; - } else { /// It's first server run and we need create default and system databases. diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index 9e815ee75de..11c1dd4c47a 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -72,8 +72,8 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState if (permanently) settings.ostr << " PERMANENTLY"; - if (no_delay) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " NO DELAY" << (settings.hilite ? hilite_none : ""); + if (sync) + settings.ostr << (settings.hilite ? hilite_keyword : "") << " SYNC" << (settings.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index ef2b609fbac..b4c96353a09 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -31,7 +31,7 @@ public: /// Same as above bool is_view{false}; - bool no_delay{false}; + bool sync{false}; // We detach the object permanently, so it will not be reattached back during server restart. bool permanently{false}; diff --git a/src/Parsers/ParserDropQuery.cpp b/src/Parsers/ParserDropQuery.cpp index e7263aa47f5..f40a39e6b2f 100644 --- a/src/Parsers/ParserDropQuery.cpp +++ b/src/Parsers/ParserDropQuery.cpp @@ -31,7 +31,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons bool temporary = false; bool is_dictionary = false; bool is_view = false; - bool no_delay = false; + bool sync = false; bool permanently = false; if (s_database.ignore(pos, expected)) @@ -83,7 +83,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons /// actually for TRUNCATE NO DELAY / SYNC means nothing if (s_no_delay.ignore(pos, expected) || s_sync.ignore(pos, expected)) - no_delay = true; + sync = true; auto query = std::make_shared(); node = query; @@ -93,7 +93,7 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons query->temporary = temporary; query->is_dictionary = is_dictionary; query->is_view = is_view; - query->no_delay = no_delay; + query->sync = sync; query->permanently = permanently; query->database = database; query->table = table; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a655da4473b..6dd329db02b 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -396,7 +396,7 @@ public: */ virtual void drop() {} - virtual void dropInnerTableIfAny(bool /* no_delay */, ContextPtr /* context */) {} + virtual void dropInnerTableIfAny(bool /* sync */, ContextPtr /* context */) {} /** Clear the table data and leave it empty. * Must be called under exclusive lock (lockExclusively). diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index 190ffabe2c1..cc80d567d1d 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -240,7 +240,7 @@ void StorageMaterializedPostgreSQL::shutdown() } -void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool sync, ContextPtr local_context) { /// If it is a table with database engine MaterializedPostgreSQL - return, because delition of /// internal tables is managed there. @@ -252,7 +252,7 @@ void StorageMaterializedPostgreSQL::dropInnerTableIfAny(bool no_delay, ContextPt auto nested_table = tryGetNested() != nullptr; if (nested_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, getNestedStorageID(), sync); } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index bb3836a8853..f1eea33d4b0 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -84,7 +84,7 @@ public: void shutdown() override; /// Used only for single MaterializedPostgreSQL storage. - void dropInnerTableIfAny(bool no_delay, ContextPtr local_context) override; + void dropInnerTableIfAny(bool sync, ContextPtr local_context) override; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index d0685c263f8..2ece0af3359 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -215,10 +215,10 @@ void StorageMaterializedView::drop() dropInnerTableIfAny(true, getContext()); } -void StorageMaterializedView::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageMaterializedView::dropInnerTableIfAny(bool sync, ContextPtr local_context) { if (has_inner_table && tryGetTargetTable()) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync); } void StorageMaterializedView::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 8aec0313ecb..0adf394876c 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -42,7 +42,7 @@ public: SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; void drop() override; - void dropInnerTableIfAny(bool no_delay, ContextPtr local_context) override; + void dropInnerTableIfAny(bool sync, ContextPtr local_context) override; void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index cfb19869074..d9780eb8446 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1606,7 +1606,7 @@ void StorageWindowView::drop() dropInnerTableIfAny(true, getContext()); } -void StorageWindowView::dropInnerTableIfAny(bool no_delay, ContextPtr local_context) +void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) { if (!std::exchange(has_inner_table, false)) return; @@ -1614,10 +1614,10 @@ void StorageWindowView::dropInnerTableIfAny(bool no_delay, ContextPtr local_cont try { InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, no_delay); + ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync); if (has_inner_target_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, no_delay); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync); } catch (...) { diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 86cc80ee8ea..96c034b9590 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -120,7 +120,7 @@ public: void checkTableCanBeDropped() const override; - void dropInnerTableIfAny(bool no_delay, ContextPtr context) override; + void dropInnerTableIfAny(bool sync, ContextPtr context) override; void drop() override; diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3e0d4e822b4..189bb1e687f 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1489,9 +1489,9 @@ def collect_build_flags(args): result.append(BuildFlags.RELEASE) value = clickhouse_execute( - args, "SELECT value FROM system.settings WHERE name = 'default_database_engine'" + args, "SELECT value FROM system.settings WHERE name = 'allow_deprecated_database_ordinary'" ) - if value == b"Ordinary": + if value == b"1": result.append(BuildFlags.ORDINARY_DATABASE) value = int( diff --git a/tests/config/users.d/database_ordinary.xml b/tests/config/users.d/database_ordinary.xml index b3b81ee25ff..8ffd2f27a62 100644 --- a/tests/config/users.d/database_ordinary.xml +++ b/tests/config/users.d/database_ordinary.xml @@ -1,7 +1,7 @@ - Ordinary + 1 diff --git a/tests/integration/test_materialized_mysql_database/configs/users.xml b/tests/integration/test_materialized_mysql_database/configs/users.xml index 4b7f5a1b109..0e116f115fe 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users.xml @@ -3,7 +3,6 @@ 1 - Atomic 1 0 diff --git a/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml b/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml index f590ebff6b4..a00b6ca6b9a 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users_disable_bytes_settings.xml @@ -3,7 +3,6 @@ 1 - Atomic 1 0 diff --git a/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml b/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml index e0fa0a9097b..3a7cc2537e5 100644 --- a/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml +++ b/tests/integration/test_materialized_mysql_database/configs/users_disable_rows_settings.xml @@ -3,7 +3,6 @@ 1 - Atomic 0 1 diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql index 8b978e6094c..2d9911287a3 100644 --- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql +++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql @@ -2,6 +2,7 @@ SET allow_experimental_window_view = 1; DROP DATABASE IF EXISTS test_01047; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01047 ENGINE=Ordinary; DROP TABLE IF EXISTS test_01047.mt; diff --git a/tests/queries/0_stateless/01048_window_view_parser.sql b/tests/queries/0_stateless/01048_window_view_parser.sql index 95190ddafa1..4c329f99f6e 100644 --- a/tests/queries/0_stateless/01048_window_view_parser.sql +++ b/tests/queries/0_stateless/01048_window_view_parser.sql @@ -2,6 +2,7 @@ SET allow_experimental_window_view = 1; DROP DATABASE IF EXISTS test_01048; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01048 ENGINE=Ordinary; DROP TABLE IF EXISTS test_01048.mt; diff --git a/tests/queries/0_stateless/01053_drop_database_mat_view.sql b/tests/queries/0_stateless/01053_drop_database_mat_view.sql index 67a488f7245..e9936d7d3b2 100644 --- a/tests/queries/0_stateless/01053_drop_database_mat_view.sql +++ b/tests/queries/0_stateless/01053_drop_database_mat_view.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP DATABASE IF EXISTS some_tests; +set allow_deprecated_database_ordinary=1; CREATE DATABASE some_tests ENGINE=Ordinary; -- Different inner table name with Atomic create table some_tests.my_table ENGINE = MergeTree(day, (day), 8192) as select today() as day, 'mystring' as str; diff --git a/tests/queries/0_stateless/01085_window_view_attach.sql b/tests/queries/0_stateless/01085_window_view_attach.sql index 604bf5dd198..bb47e0dc6b9 100644 --- a/tests/queries/0_stateless/01085_window_view_attach.sql +++ b/tests/queries/0_stateless/01085_window_view_attach.sql @@ -3,6 +3,7 @@ SET allow_experimental_window_view = 1; DROP DATABASE IF EXISTS test_01085; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01085 ENGINE=Ordinary; DROP TABLE IF EXISTS test_01085.mt; diff --git a/tests/queries/0_stateless/01086_window_view_cleanup.sh b/tests/queries/0_stateless/01086_window_view_cleanup.sh index a7b976bf4e0..c85455616e1 100755 --- a/tests/queries/0_stateless/01086_window_view_cleanup.sh +++ b/tests/queries/0_stateless/01086_window_view_cleanup.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --multiquery <&1| grep -Ev "Removing leftovers from table|removed by another replica" $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" if [[ $engine == *"ReplicatedMergeTree"* ]]; then - $CLICKHOUSE_CLIENT --enable_positional_arguments 0 -q "ALTER TABLE test - UPDATE test = (SELECT groupArray(id) FROM t1)[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "statement with subquery may be nondeterministic" - $CLICKHOUSE_CLIENT --enable_positional_arguments 0 --allow_nondeterministic_mutations=1 --mutations_sync=1 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT 0 -q "ALTER TABLE test + UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 'dummy')[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "statement with subquery may be nondeterministic" + $CLICKHOUSE_CLIENT --allow_nondeterministic_mutations=1 --mutations_sync=1 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1)[n - 99] WHERE 1" elif [[ $engine == *"Join"* ]]; then - $CLICKHOUSE_CLIENT --enable_positional_arguments 0 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1)[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "Table engine Join supports only DELETE mutations" else - $CLICKHOUSE_CLIENT --enable_positional_arguments 0 --mutations_sync=1 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT --mutations_sync=1 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1)[n - 99] WHERE 1" fi $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql index ed05950ff98..e0ffe7ede66 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.sql +++ b/tests/queries/0_stateless/01190_full_attach_syntax.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP DATABASE IF EXISTS test_01190; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01190 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01190; diff --git a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh index c6686bde672..641e26870d4 100755 --- a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh +++ b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh @@ -11,8 +11,8 @@ $CLICKHOUSE_CLIENT --check_table_dependencies=0 -q "DROP DATABASE IF EXISTS test $CLICKHOUSE_CLIENT --check_table_dependencies=0 -q "DROP DATABASE IF EXISTS test_01192_renamed" $CLICKHOUSE_CLIENT --check_table_dependencies=0 -q "DROP DATABASE IF EXISTS test_01192_atomic" -$CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" 2>&1| grep -F "does not support" > /dev/null && echo "ok" -$CLICKHOUSE_CLIENT --default_database_engine=Atomic -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" +$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001' ENGINE=Ordinary" 2>&1| grep -F "does not support" > /dev/null && echo "ok" +$CLICKHOUSE_CLIENT -q "CREATE DATABASE test_01192 UUID '00001192-0000-4000-8000-000000000001'" # 2. check metadata $CLICKHOUSE_CLIENT --show_table_uuid_in_table_create_query_if_not_nil=1 -q "SHOW CREATE DATABASE test_01192" @@ -35,14 +35,14 @@ $CLICKHOUSE_CLIENT -q "SHOW CREATE DATABASE test_01192_renamed" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_renamed.mt" # 5. check moving tables from Ordinary to Atomic (can be used to "alter" database engine) -$CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01192" +$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "CREATE DATABASE test_01192 ENGINE=Ordinary" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.mt AS test_01192_renamed.mt ENGINE=MergeTree ORDER BY n" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.rmt AS test_01192_renamed.mt ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/', '1') ORDER BY n" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW test_01192.mv TO test_01192.rmt AS SELECT * FROM test_01192.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01192.mt SELECT number FROM numbers(10)" && echo "inserted" -$CLICKHOUSE_CLIENT --default_database_engine=Atomic -q "CREATE DATABASE test_01192_atomic" +$CLICKHOUSE_CLIENT -q "CREATE DATABASE test_01192_atomic" $CLICKHOUSE_CLIENT -q "DROP DATABASE test_01192_renamed" # it's blocking $CLICKHOUSE_CLIENT -q "RENAME TABLE test_01192.mt TO test_01192_atomic.mt, test_01192.rmt TO test_01192_atomic.rmt, test_01192.mv TO test_01192_atomic.mv" && echo "renamed" diff --git a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql index 6f5deb91ee4..5db92e70650 100644 --- a/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql +++ b/tests/queries/0_stateless/01224_no_superfluous_dict_reload.sql @@ -2,6 +2,7 @@ DROP DATABASE IF EXISTS dict_db_01224; DROP DATABASE IF EXISTS dict_db_01224_dictionary; +set allow_deprecated_database_ordinary=1; CREATE DATABASE dict_db_01224 ENGINE=Ordinary; -- Different internal dictionary name with Atomic CREATE DATABASE dict_db_01224_dictionary Engine=Dictionary; diff --git a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql index 006ff952ee9..09cde642ed2 100644 --- a/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql +++ b/tests/queries/0_stateless/01225_show_create_table_from_dictionary.sql @@ -2,6 +2,7 @@ DROP DATABASE IF EXISTS dict_db_01225; DROP DATABASE IF EXISTS dict_db_01225_dictionary; +set allow_deprecated_database_ordinary=1; CREATE DATABASE dict_db_01225 ENGINE=Ordinary; -- Different internal dictionary name with Atomic CREATE DATABASE dict_db_01225_dictionary Engine=Dictionary; diff --git a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql index c2be4d04e5f..d187a2e4d4e 100644 --- a/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql +++ b/tests/queries/0_stateless/01249_bad_arguments_for_bloom_filter.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP DATABASE IF EXISTS test_01249; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01249 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01249; diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index 758ec4825e0..ef45e8e63bc 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01320" -$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Different bahaviour of DROP with Atomic +$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Different bahaviour of DROP with Atomic function thread1() { diff --git a/tests/queries/0_stateless/01516_create_table_primary_key.sql b/tests/queries/0_stateless/01516_create_table_primary_key.sql index b3c4acd50ff..b2b9f288eab 100644 --- a/tests/queries/0_stateless/01516_create_table_primary_key.sql +++ b/tests/queries/0_stateless/01516_create_table_primary_key.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP DATABASE IF EXISTS test_01516; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01516 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01516; diff --git a/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql b/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql index 5e31faa70da..67a2009b913 100644 --- a/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql +++ b/tests/queries/0_stateless/01517_drop_mv_with_inner_table.sql @@ -32,6 +32,7 @@ show tables from db_01517_atomic_sync; -- Ordinary --- drop database if exists db_01517_ordinary; +set allow_deprecated_database_ordinary=1; create database db_01517_ordinary Engine=Ordinary; create table db_01517_ordinary.source (key Int) engine=Null; diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh index 949fe0e6ea2..c32a255448e 100755 --- a/tests/queries/0_stateless/01600_detach_permanently.sh +++ b/tests/queries/0_stateless/01600_detach_permanently.sh @@ -15,7 +15,7 @@ mkdir -p "${WORKING_FOLDER_01600}" clickhouse_local() { local query="$1" shift - ${CLICKHOUSE_LOCAL} --query "$query" "$@" --path="${WORKING_FOLDER_01600}" + ${CLICKHOUSE_LOCAL} --allow_deprecated_database_ordinary=1 --query "$query" "$@" --path="${WORKING_FOLDER_01600}" } test_detach_attach_sequence() { diff --git a/tests/queries/0_stateless/01601_detach_permanently.sql b/tests/queries/0_stateless/01601_detach_permanently.sql index 97797a59af5..95c80e77213 100644 --- a/tests/queries/0_stateless/01601_detach_permanently.sql +++ b/tests/queries/0_stateless/01601_detach_permanently.sql @@ -72,6 +72,7 @@ SELECT '-----------------------'; SELECT 'database ordinary tests'; DROP DATABASE IF EXISTS test1601_detach_permanently_ordinary; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test1601_detach_permanently_ordinary Engine=Ordinary; create table test1601_detach_permanently_ordinary.test_name_reuse (number UInt64) engine=MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/01603_rename_overwrite_bug.sql b/tests/queries/0_stateless/01603_rename_overwrite_bug.sql index 82f9996991f..acf9f520709 100644 --- a/tests/queries/0_stateless/01603_rename_overwrite_bug.sql +++ b/tests/queries/0_stateless/01603_rename_overwrite_bug.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP database IF EXISTS test_1603_rename_bug_ordinary; +set allow_deprecated_database_ordinary=1; create database test_1603_rename_bug_ordinary engine=Ordinary; create table test_1603_rename_bug_ordinary.foo engine=Memory as select * from numbers(100); create table test_1603_rename_bug_ordinary.bar engine=Log as select * from numbers(200); diff --git a/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh b/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh index f5ab71d8d34..b1f30a41924 100755 --- a/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh +++ b/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh @@ -10,7 +10,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q "create database ordinary_$CLICKHOUSE_DATABASE engine=Ordinary" +$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -nm -q "create database ordinary_$CLICKHOUSE_DATABASE engine=Ordinary" # MergeTree $CLICKHOUSE_CLIENT -nm -q """ diff --git a/tests/queries/0_stateless/02096_rename_atomic_hang.sql b/tests/queries/0_stateless/02096_rename_atomic_hang.sql index 96261bfe127..dec5f3f9506 100644 --- a/tests/queries/0_stateless/02096_rename_atomic_hang.sql +++ b/tests/queries/0_stateless/02096_rename_atomic_hang.sql @@ -2,6 +2,7 @@ drop database if exists db_hang; drop database if exists db_hang_temp; +set allow_deprecated_database_ordinary=1; create database db_hang engine=Ordinary; use db_hang; create table db_hang.test(A Int64) Engine=MergeTree order by A; From f00e6b5a7a50bc0dc616b5df81034e6cd27e03d2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 23 Jun 2022 10:37:52 +0200 Subject: [PATCH 090/408] deprecate old MergeTree syntax --- src/Core/Settings.h | 1 + .../MergeTree/registerStorageMergeTree.cpp | 5 ++++ .../test_cluster_copier/configs/users.xml | 1 + .../configs_three_nodes/users.xml | 1 + .../configs_two_nodes/users.xml | 1 + .../configs/config_allow_databases.xml | 5 ++++ .../test_cross_replication/test.py | 2 +- .../test_delayed_replica_failover/test.py | 2 +- .../configs/users.d/query_log.xml | 1 + .../configs_secure/users.d/query_log.xml | 1 + .../test_extreme_deduplication/test.py | 4 ++-- .../test_https_replication/test.py | 2 +- .../test_insert_into_distributed/test.py | 24 +++++++++---------- .../test.py | 2 +- .../enable_distributed_inserts_batching.xml | 1 + .../test_merge_tree_empty_parts/test.py | 4 ++-- .../test_mutations_with_merge_tree/test.py | 2 +- tests/integration/test_random_inserts/test.py | 2 +- .../test_replace_partition/test.py | 6 ++--- .../test_replicated_database/test.py | 4 ++-- .../test_replication_credentials/test.py | 2 +- .../test.py | 4 ++-- tests/integration/test_union_header/test.py | 2 +- .../integration/test_zookeeper_config/test.py | 4 ++-- .../test_zookeeper_config/test_password.py | 4 ++-- tests/performance/parallel_final.xml | 1 + tests/performance/scalar2.xml | 4 ++++ .../queries/0_stateless/00030_alter_table.sql | 1 + .../0_stateless/00043_summing_empty_part.sql | 1 + .../00046_stored_aggregates_simple.sql | 1 + .../00047_stored_aggregates_complex.sql | 1 + .../00048_a_stored_aggregates_merge.sql | 1 + .../00048_b_stored_aggregates_merge.sql | 1 + .../0_stateless/00061_merge_tree_alter.sql | 1 + ...icated_merge_tree_alter_zookeeper_long.sql | 1 + .../0_stateless/00079_defaulted_columns.sql | 1 + .../0_stateless/00084_summing_merge_tree.sql | 1 + .../0_stateless/00098_shard_i_union_all.sql | 1 + ...s_and_insert_without_explicit_database.sql | 1 + .../00121_drop_column_zookeeper.sql | 1 + ...4_shard_distributed_with_many_replicas.sql | 1 + .../00140_prewhere_column_order.sql | 1 + .../00141_parse_timestamp_as_datetime.sql | 1 + .../00146_summing_merge_tree_nested_map.sql | 1 + .../00147_alter_nested_default.sql | 1 + ..._summing_merge_tree_aggregate_function.sql | 1 + ..._merge_tree_nested_map_multiple_values.sql | 1 + .../queries/0_stateless/00155_long_merges.sh | 4 ++-- .../0_stateless/00168_buffer_defaults.sql | 1 + ...00191_aggregating_merge_tree_and_final.sql | 1 + .../0_stateless/00193_parallel_replicas.sql | 1 + ...ated_drop_on_non_leader_zookeeper_long.sql | 1 + .../00253_insert_recursive_defaults.sql | 1 + .../00278_insert_already_sorted.sql | 1 + tests/queries/0_stateless/00282_merging.sql | 1 + .../00345_index_accurate_comparison.sql | 1 + ..._column_aggregate_function_insert_from.sql | 1 + .../00394_new_nested_column_keeps_offsets.sql | 1 + .../00394_replaceall_vector_fixed.sql | 1 + tests/queries/0_stateless/00395_nullable.sql | 1 + .../0_stateless/00427_alter_primary_key.sh | 1 + .../0_stateless/00440_nulls_merge_tree.sql | 1 + ...olumn_in_partition_concurrent_zookeeper.sh | 2 +- tests/queries/0_stateless/00453_cast_enum.sql | 1 + .../00495_reading_const_zero_column.sql | 1 + .../0_stateless/00504_mergetree_arrays_rw.sql | 1 + .../0_stateless/00506_union_distributed.sql | 1 + ...rd_desc_table_functions_and_subqueries.sql | 1 + ...ication_after_drop_partition_zookeeper.sql | 1 + .../00531_aggregate_over_nullable.sql | 1 + .../0_stateless/00543_null_and_prewhere.sql | 2 ++ .../0_stateless/00561_storage_join.sql | 1 + .../00563_complex_in_expression.sql | 1 + ..._column_values_with_default_expression.sql | 1 + ...t_database_when_create_materializ_view.sql | 1 + ...column_exception_when_drop_depen_column.sh | 2 +- ...75_merge_and_index_with_function_in_in.sql | 1 + .../0_stateless/00584_view_union_all.sql | 1 + .../00594_alias_in_distributed.sql | 1 + .../00597_push_down_predicate_long.sql | 1 + .../0_stateless/00609_mv_index_in_in.sql | 1 + ...iew_forward_alter_partition_statements.sql | 1 + ...l_and_remote_node_in_distributed_query.sql | 1 + .../00615_nullable_alter_optimize.sql | 1 + .../00621_regression_for_in_operator.sql | 1 + .../0_stateless/00623_in_partition_key.sql | 1 + ...plicated_truncate_table_zookeeper_long.sql | 1 + ...0648_replacing_empty_set_from_prewhere.sql | 1 + .../0_stateless/00652_mergetree_mutations.sh | 2 +- .../00652_replicated_mutations_zookeeper.sh | 4 ++-- .../0_stateless/00678_shard_funnel_window.sql | 1 + .../00712_prewhere_with_alias_bug_2.sql | 1 + .../0_stateless/00712_prewhere_with_final.sql | 1 + ...00712_prewhere_with_sampling_and_alias.sql | 1 + .../00717_merge_and_distributed.sql | 1 + .../0_stateless/00729_prewhere_array_join.sql | 1 + ...731_long_merge_tree_select_opened_files.sh | 2 +- .../00732_decimal_summing_merge_tree.sql | 1 + .../00748_insert_array_with_null.sql | 1 + .../00752_low_cardinality_mv_2.sql | 1 + ..._system_columns_and_system_tables_long.sql | 1 + .../00754_alter_modify_order_by.sql | 1 + ...ify_order_by_replicated_zookeeper_long.sql | 1 + ...materialized_view_with_column_defaults.sql | 1 + ...800_low_cardinality_distributed_insert.sql | 1 + .../0_stateless/00806_alter_update.sql | 1 + .../0_stateless/00829_bitmap_function.sql | 1 + .../00834_hints_for_type_function_typos.sh | 4 ++-- ...crash_when_distributed_modify_order_by.sql | 1 + ...plicated_merge_tree_optimize_final_long.sh | 4 ++-- ...o_distributed_with_materialized_column.sql | 1 + .../00981_in_subquery_with_tuple.sh | 2 +- ...1008_materialized_view_henyihanwobushi.sql | 1 + .../01053_drop_database_mat_view.sql | 1 + ...1062_alter_on_mutataion_zookeeper_long.sql | 1 + ...ndary_index_with_old_format_merge_tree.sql | 1 + .../01076_predicate_optimizer_with_view.sql | 1 + .../01089_alter_settings_old_format.sql | 1 + .../01101_prewhere_after_alter.sql | 1 + .../01125_generate_random_qoega.sql | 1 + ...126_month_partitioning_consistent_code.sql | 1 + ..._month_partitioning_consistency_select.sql | 1 + ...ery_in_aggregate_function_JustStranger.sql | 1 + .../01355_alter_column_with_order.sql | 1 + .../01387_clear_column_default_depends.sql | 1 + .../01417_freeze_partition_verbose.sh | 2 +- .../01430_modify_sample_by_zookeeper_long.sql | 1 + ...1455_shard_leaf_max_rows_bytes_to_read.sql | 1 + .../01648_mutations_and_escaping.sql | 1 + .../01713_table_ttl_old_syntax_zookeeper.sql | 1 + ...778_test_LowCardinality_FixedString_pk.sql | 1 + .../0_stateless/01798_uniq_theta_sketch.sql | 1 + .../01925_broken_partition_id_zookeeper.sql | 1 + .../0_stateless/02250_hints_for_columns.sh | 2 +- .../00040_aggregating_materialized_view.sql | 1 + .../00041_aggregating_materialized_view.sql | 1 + .../00054_merge_tree_partitions.sql | 1 + .../00071_merge_tree_optimize_aio.sql | 1 + 138 files changed, 173 insertions(+), 50 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 883281c94f3..1ee46ed4caa 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -591,6 +591,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (without condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ + M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 43e1af21eac..5ad39aa826a 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -303,6 +303,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) arg_idx, e.message(), getMergeTreeVerboseHelp(is_extended_storage_def)); } } + else if (!args.attach && !args.local_context.lock()->getSettingsRef().allow_deprecated_syntax_for_merge_tree) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "This syntax for *MergeTree engine is deprecated. " + "Use extended storage definition syntax with ORDER BY/PRIMARY KEY clause."); + } /// For Replicated. String zookeeper_path; diff --git a/tests/integration/test_cluster_copier/configs/users.xml b/tests/integration/test_cluster_copier/configs/users.xml index 492cf4d7ee6..2542642f6df 100644 --- a/tests/integration/test_cluster_copier/configs/users.xml +++ b/tests/integration/test_cluster_copier/configs/users.xml @@ -5,6 +5,7 @@ 1 5 + 1 diff --git a/tests/integration/test_cluster_copier/configs_three_nodes/users.xml b/tests/integration/test_cluster_copier/configs_three_nodes/users.xml index ce3538a31b8..f017daff974 100644 --- a/tests/integration/test_cluster_copier/configs_three_nodes/users.xml +++ b/tests/integration/test_cluster_copier/configs_three_nodes/users.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_cluster_copier/configs_two_nodes/users.xml b/tests/integration/test_cluster_copier/configs_two_nodes/users.xml index ce3538a31b8..f017daff974 100644 --- a/tests/integration/test_cluster_copier/configs_two_nodes/users.xml +++ b/tests/integration/test_cluster_copier/configs_two_nodes/users.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_config_substitutions/configs/config_allow_databases.xml b/tests/integration/test_config_substitutions/configs/config_allow_databases.xml index 98008306787..be727360dcf 100644 --- a/tests/integration/test_config_substitutions/configs/config_allow_databases.xml +++ b/tests/integration/test_config_substitutions/configs/config_allow_databases.xml @@ -1,4 +1,9 @@ + + + 1 + + diff --git a/tests/integration/test_cross_replication/test.py b/tests/integration/test_cross_replication/test.py index 143b8823bf2..2a73acadafd 100644 --- a/tests/integration/test_cross_replication/test.py +++ b/tests/integration/test_cross_replication/test.py @@ -37,7 +37,7 @@ def started_cluster(): CREATE DATABASE shard_{shard}; CREATE TABLE shard_{shard}.replicated(date Date, id UInt32, shard_id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_delayed_replica_failover/test.py b/tests/integration/test_delayed_replica_failover/test.py index 387d6a12f48..a480ee3f278 100644 --- a/tests/integration/test_delayed_replica_failover/test.py +++ b/tests/integration/test_delayed_replica_failover/test.py @@ -32,7 +32,7 @@ def started_cluster(): node.query( """ CREATE TABLE replicated (d Date, x UInt32) ENGINE = - ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)""".format( + ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}') PARTITION BY toYYYYMM(d) ORDER BY d""".format( shard=shard, instance=node.name ) ) diff --git a/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml b/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml index 26db7f54514..ef8abbd9174 100644 --- a/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml +++ b/tests/integration/test_distributed_ddl/configs/users.d/query_log.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml b/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml index 26db7f54514..ef8abbd9174 100644 --- a/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml +++ b/tests/integration/test_distributed_ddl/configs_secure/users.d/query_log.xml @@ -3,6 +3,7 @@ 1 + 1 diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py index 2c8772aad4e..71f783d37c9 100644 --- a/tests/integration/test_extreme_deduplication/test.py +++ b/tests/integration/test_extreme_deduplication/test.py @@ -40,7 +40,7 @@ def test_deduplication_window_in_seconds(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id""" ) node.query("INSERT INTO simple VALUES (0, 0)") @@ -77,7 +77,7 @@ def test_deduplication_works_in_case_of_intensive_inserts(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id""" ) node1.query("INSERT INTO simple VALUES (0, 0)") diff --git a/tests/integration/test_https_replication/test.py b/tests/integration/test_https_replication/test.py index 4cf9f19b870..301487aa6cf 100644 --- a/tests/integration/test_https_replication/test.py +++ b/tests/integration/test_https_replication/test.py @@ -19,7 +19,7 @@ def _fill_nodes(nodes, shard): CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index b8d94d2a043..aff31ec365c 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -52,7 +52,7 @@ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'defaul ) remote.query( - "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x" ) instance_test_inserts_batching.query( """ @@ -61,7 +61,7 @@ CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', ) instance_test_inserts_local_cluster.query( - "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)" + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x" ) instance_test_inserts_local_cluster.query( """ @@ -71,12 +71,12 @@ CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_ node1.query( """ -CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node1', date, id, 8192) +CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node1') PARTITION BY toYYYYMM(date) ORDER BY id """ ) node2.query( """ -CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node2', date, id, 8192) +CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node2') PARTITION BY toYYYYMM(date) ORDER BY id """ ) @@ -94,12 +94,12 @@ CREATE TABLE distributed (date Date, id UInt32) ENGINE = Distributed('shard_with shard1.query( """ -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x""" ) shard2.query( """ -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY x""" ) shard1.query( @@ -143,7 +143,7 @@ CREATE TABLE distributed_one_replica_no_internal_replication (date Date, id UInt node2.query( """ -CREATE TABLE single_replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/single_replicated', 'node2', date, id, 8192) +CREATE TABLE single_replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/single_replicated', 'node2') PARTITION BY toYYYYMM(date) ORDER BY id """ ) @@ -228,11 +228,11 @@ def test_inserts_batching(started_cluster): # 4. Full batch of inserts after ALTER (that have different block structure). # 5. What was left to insert with the column structure before ALTER. expected = """\ -20000101_20000101_1_1_0\t[1] -20000101_20000101_2_2_0\t[2,3,4] -20000101_20000101_3_3_0\t[5,6,7] -20000101_20000101_4_4_0\t[10,11,12] -20000101_20000101_5_5_0\t[8,9] +20000101__1_1_0\t[1] +20000101__2_2_0\t[2,3,4] +20000101__3_3_0\t[5,6,7] +20000101__4_4_0\t[10,11,12] +20000101__5_5_0\t[8,9] """ assert TSV(result) == TSV(expected) diff --git a/tests/integration/test_insert_into_distributed_sync_async/test.py b/tests/integration/test_insert_into_distributed_sync_async/test.py index e0c454feee6..12423cc4747 100755 --- a/tests/integration/test_insert_into_distributed_sync_async/test.py +++ b/tests/integration/test_insert_into_distributed_sync_async/test.py @@ -23,7 +23,7 @@ def started_cluster(): for node in (node1, node2): node.query( """ -CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree(date, (date, val), 8192); +CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY (date, val); """ ) diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml index de0c930b8ab..295d1c8d3cc 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/configs/enable_distributed_inserts_batching.xml @@ -3,6 +3,7 @@ 1 3 + 1 diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py index 7ca275e96de..57bf49e6803 100644 --- a/tests/integration/test_merge_tree_empty_parts/test.py +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -25,7 +25,7 @@ def started_cluster(): def test_empty_parts_alter_delete(started_cluster): node1.query( "CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" ) node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')") @@ -44,7 +44,7 @@ def test_empty_parts_alter_delete(started_cluster): def test_empty_parts_summing(started_cluster): node1.query( "CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)" + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1') PARTITION BY toYYYYMM(d) ORDER BY key" ) node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)") diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index d1843017b9f..7831cde7dea 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -17,7 +17,7 @@ def started_cluster(): try: cluster.start() instance_test_mutations.query( - """CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)""" + """CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY (a, date)""" ) instance_test_mutations.query( """INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1) SETTINGS force_index_by_date = 0, force_primary_key = 0""" diff --git a/tests/integration/test_random_inserts/test.py b/tests/integration/test_random_inserts/test.py index 4d6aaa9276d..9ac0c5b024c 100644 --- a/tests/integration/test_random_inserts/test.py +++ b/tests/integration/test_random_inserts/test.py @@ -44,7 +44,7 @@ def test_random_inserts(started_cluster): node1.query( """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, i UInt32, s String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, i, 8192)""" + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY i""" ) with PartitionManager() as pm_random_drops: diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index 7ce79d9aca8..579b22286b9 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -20,13 +20,13 @@ def _fill_nodes(nodes, shard): CREATE DATABASE test; CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY id; CREATE TABLE other_table(date Date, id UInt32, dummy UInt32) - ENGINE = MergeTree(date, id, 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(date) ORDER BY id; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 92c109974e1..99fb5bc1e89 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -393,7 +393,7 @@ def test_alters_from_different_replicas(started_cluster): main_node.query( "CREATE TABLE testdb.concurrent_test " "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);" + "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" ) main_node.query( @@ -443,7 +443,7 @@ def test_alters_from_different_replicas(started_cluster): " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" " `AddedNested2.B` Array(UInt64)\\n)\\n" - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" + "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" ) assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) diff --git a/tests/integration/test_replication_credentials/test.py b/tests/integration/test_replication_credentials/test.py index e5313cb6bd4..79588fcd38b 100644 --- a/tests/integration/test_replication_credentials/test.py +++ b/tests/integration/test_replication_credentials/test.py @@ -10,7 +10,7 @@ def _fill_nodes(nodes, shard): """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( shard=shard, replica=node.name ) diff --git a/tests/integration/test_send_request_to_leader_replica/test.py b/tests/integration/test_send_request_to_leader_replica/test.py index 60df18bf7d3..b56e1315672 100644 --- a/tests/integration/test_send_request_to_leader_replica/test.py +++ b/tests/integration/test_send_request_to_leader_replica/test.py @@ -40,7 +40,7 @@ def started_cluster(): node.query( """ CREATE TABLE sometable(date Date, id UInt32, value Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ), @@ -51,7 +51,7 @@ def started_cluster(): node.query( """ CREATE TABLE someothertable(date Date, id UInt32, value Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ), diff --git a/tests/integration/test_union_header/test.py b/tests/integration/test_union_header/test.py index f883057c1d8..2e7f6cb399a 100644 --- a/tests/integration/test_union_header/test.py +++ b/tests/integration/test_union_header/test.py @@ -27,7 +27,7 @@ def started_cluster(): log_type UInt32, account_id String ) - ENGINE = MergeTree(event_date, (event_time, account_id), 8192); + ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_time, account_id); """ ) diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index d3d90ca0d4f..65f82c2286b 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -48,7 +48,7 @@ def test_chroot_with_same_root(started_cluster): node.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ) @@ -68,7 +68,7 @@ def test_chroot_with_different_root(started_cluster): node.query( """ CREATE TABLE simple_different (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node.name ) diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index 580b426db6f..71f059b3277 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -35,7 +35,7 @@ def test_identity(started_cluster): node1.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; """.format( replica=node1.name ) @@ -45,6 +45,6 @@ def test_identity(started_cluster): node2.query( """ CREATE TABLE simple (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1', date, id, 8192); + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1') PARTITION BY toYYYYMM(date) ORDER BY id; """ ) diff --git a/tests/performance/parallel_final.xml b/tests/performance/parallel_final.xml index ca84ed52a04..d7ea0240105 100644 --- a/tests/performance/parallel_final.xml +++ b/tests/performance/parallel_final.xml @@ -4,6 +4,7 @@ 1024 1 20G + 1 diff --git a/tests/performance/scalar2.xml b/tests/performance/scalar2.xml index eb427536646..7a122dbec95 100644 --- a/tests/performance/scalar2.xml +++ b/tests/performance/scalar2.xml @@ -1,4 +1,8 @@ + + 1 + + CREATE TABLE tbl0 (`ds` Date, `x1` String, `x2` UInt32, `x3` UInt32, `x4` UInt32, `bm` AggregateFunction(groupBitmap, UInt32)) ENGINE = MergeTree PARTITION BY (ds, x1) ORDER BY (x2, x3, x4) SETTINGS index_granularity = 1 CREATE TABLE tbl (`ds` Date, `y1` UInt32, `x4` UInt32, `y2` UInt32, `y3` UInt32, `bm` AggregateFunction(groupBitmap, UInt32), `y4` UInt32 DEFAULT 0) ENGINE = MergeTree PARTITION BY (ds) ORDER BY (x4, y2, y3) SETTINGS index_granularity = 8192, max_parts_in_total = 10000000 diff --git a/tests/queries/0_stateless/00030_alter_table.sql b/tests/queries/0_stateless/00030_alter_table.sql index 5fc45575a4a..fb9b3de4067 100644 --- a/tests/queries/0_stateless/00030_alter_table.sql +++ b/tests/queries/0_stateless/00030_alter_table.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS alter_test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192); INSERT INTO alter_test VALUES (1, '2014-01-01', 2, 3, [1,2,3], ['a','b','c'], 4); diff --git a/tests/queries/0_stateless/00043_summing_empty_part.sql b/tests/queries/0_stateless/00043_summing_empty_part.sql index 68fc4b5b1c4..40cecabf378 100644 --- a/tests/queries/0_stateless/00043_summing_empty_part.sql +++ b/tests/queries/0_stateless/00043_summing_empty_part.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS empty_summing; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE empty_summing (d Date, k UInt64, v Int8) ENGINE=SummingMergeTree(d, k, 8192); INSERT INTO empty_summing VALUES ('2015-01-01', 1, 10); diff --git a/tests/queries/0_stateless/00046_stored_aggregates_simple.sql b/tests/queries/0_stateless/00046_stored_aggregates_simple.sql index 8b1ef5ba48d..2a4ee9fa5d3 100644 --- a/tests/queries/0_stateless/00046_stored_aggregates_simple.sql +++ b/tests/queries/0_stateless/00046_stored_aggregates_simple.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS stored_aggregates; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE stored_aggregates ( d Date, diff --git a/tests/queries/0_stateless/00047_stored_aggregates_complex.sql b/tests/queries/0_stateless/00047_stored_aggregates_complex.sql index 63728f131b0..2e416f91d5d 100644 --- a/tests/queries/0_stateless/00047_stored_aggregates_complex.sql +++ b/tests/queries/0_stateless/00047_stored_aggregates_complex.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS stored_aggregates; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE stored_aggregates ( d Date, diff --git a/tests/queries/0_stateless/00048_a_stored_aggregates_merge.sql b/tests/queries/0_stateless/00048_a_stored_aggregates_merge.sql index 0138a75c19a..0213ebf46d6 100644 --- a/tests/queries/0_stateless/00048_a_stored_aggregates_merge.sql +++ b/tests/queries/0_stateless/00048_a_stored_aggregates_merge.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS stored_aggregates; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE stored_aggregates ( d Date, diff --git a/tests/queries/0_stateless/00048_b_stored_aggregates_merge.sql b/tests/queries/0_stateless/00048_b_stored_aggregates_merge.sql index 79617692ebd..708794eabff 100644 --- a/tests/queries/0_stateless/00048_b_stored_aggregates_merge.sql +++ b/tests/queries/0_stateless/00048_b_stored_aggregates_merge.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS stored_aggregates; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE stored_aggregates ( d Date, diff --git a/tests/queries/0_stateless/00061_merge_tree_alter.sql b/tests/queries/0_stateless/00061_merge_tree_alter.sql index 822386baa47..ee5694518d9 100644 --- a/tests/queries/0_stateless/00061_merge_tree_alter.sql +++ b/tests/queries/0_stateless/00061_merge_tree_alter.sql @@ -1,6 +1,7 @@ -- Tags: no-backward-compatibility-check DROP TABLE IF EXISTS alter_00061; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_00061 (d Date, k UInt64, i32 Int32) ENGINE=MergeTree(d, k, 8192); INSERT INTO alter_00061 VALUES ('2015-01-01', 10, 42); diff --git a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper_long.sql b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper_long.sql index e8d1a713c21..4475421d36f 100644 --- a/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper_long.sql +++ b/tests/queries/0_stateless/00062_replicated_merge_tree_alter_zookeeper_long.sql @@ -6,6 +6,7 @@ DROP TABLE IF EXISTS replicated_alter2; SET replication_alter_partitions_sync = 2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE replicated_alter1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_00062/alter', 'r1', d, k, 8192); CREATE TABLE replicated_alter2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_00062/alter', 'r2', d, k, 8192); diff --git a/tests/queries/0_stateless/00079_defaulted_columns.sql b/tests/queries/0_stateless/00079_defaulted_columns.sql index 617dc2435c4..04dfb7057d2 100644 --- a/tests/queries/0_stateless/00079_defaulted_columns.sql +++ b/tests/queries/0_stateless/00079_defaulted_columns.sql @@ -13,6 +13,7 @@ drop table defaulted; create table defaulted (col1 Int8, col2 UInt64 default (SELECT dummy+99 from system.one)) engine=Memory; --{serverError 116} +set allow_deprecated_syntax_for_merge_tree=1; create table defaulted (payload String, date materialized today(), key materialized 0 * rand()) engine=MergeTree(date, key, 8192); desc table defaulted; insert into defaulted (payload) values ('hello clickhouse'); diff --git a/tests/queries/0_stateless/00084_summing_merge_tree.sql b/tests/queries/0_stateless/00084_summing_merge_tree.sql index 5be1371fbcc..429fde5c2b5 100644 --- a/tests/queries/0_stateless/00084_summing_merge_tree.sql +++ b/tests/queries/0_stateless/00084_summing_merge_tree.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS summing_merge_tree; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE summing_merge_tree (d Date, a String, x UInt32, y UInt64, z Float64) ENGINE = SummingMergeTree(d, a, 8192); INSERT INTO summing_merge_tree VALUES ('2000-01-01', 'Hello', 1, 2, 3); diff --git a/tests/queries/0_stateless/00098_shard_i_union_all.sql b/tests/queries/0_stateless/00098_shard_i_union_all.sql index 5151f9ad0e5..58db30a8f9a 100644 --- a/tests/queries/0_stateless/00098_shard_i_union_all.sql +++ b/tests/queries/0_stateless/00098_shard_i_union_all.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS report1; DROP TABLE IF EXISTS report2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE report1(id UInt32, event_date Date, priority UInt32, description String) ENGINE = MergeTree(event_date, intHash32(id), (id, event_date, intHash32(id)), 8192); CREATE TABLE report2(id UInt32, event_date Date, priority UInt32, description String) ENGINE = MergeTree(event_date, intHash32(id), (id, event_date, intHash32(id)), 8192); diff --git a/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql b/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql index b05b49ba33a..95d46032601 100644 --- a/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql +++ b/tests/queries/0_stateless/00101_materialized_views_and_insert_without_explicit_database.sql @@ -8,6 +8,7 @@ DROP TABLE IF EXISTS test_table; DROP TABLE IF EXISTS test_view; DROP TABLE IF EXISTS test_view_filtered; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('America/Los_Angeles'), UTCEventTime DateTime('UTC')) ENGINE = MergeTree(EventDate, CounterID, 8192); CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('America/Los_Angeles')) ENGINE = Memory AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; CREATE MATERIALIZED VIEW test_view_filtered (EventDate Date, CounterID UInt32) ENGINE = Memory POPULATE AS SELECT CounterID, EventDate FROM test_table WHERE EventDate < '2013-01-01'; diff --git a/tests/queries/0_stateless/00121_drop_column_zookeeper.sql b/tests/queries/0_stateless/00121_drop_column_zookeeper.sql index fd32f3d32d2..f62f11c60fd 100644 --- a/tests/queries/0_stateless/00121_drop_column_zookeeper.sql +++ b/tests/queries/0_stateless/00121_drop_column_zookeeper.sql @@ -2,6 +2,7 @@ -- Tag no-replicated-database: Old syntax is not allowed DROP TABLE IF EXISTS alter_00121; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_00121 (d Date, x UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test/alter_00121/t1', 'r1', d, (d), 8192); INSERT INTO alter_00121 VALUES ('2014-01-01', 1); diff --git a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql index e29a166c1ee..901b818cbc0 100644 --- a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql +++ b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql @@ -4,6 +4,7 @@ SET allow_experimental_parallel_reading_from_replicas = 0; SET max_parallel_replicas = 2; DROP TABLE IF EXISTS report; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE report(id UInt32, event_date Date, priority UInt32, description String) ENGINE = MergeTree(event_date, intHash32(id), (id, event_date, intHash32(id)), 8192); INSERT INTO report(id,event_date,priority,description) VALUES (1, '2015-01-01', 1, 'foo')(2, '2015-02-01', 2, 'bar')(3, '2015-03-01', 3, 'foo')(4, '2015-04-01', 4, 'bar')(5, '2015-05-01', 5, 'foo'); diff --git a/tests/queries/0_stateless/00140_prewhere_column_order.sql b/tests/queries/0_stateless/00140_prewhere_column_order.sql index d949b6f780b..61c2fbcf39c 100644 --- a/tests/queries/0_stateless/00140_prewhere_column_order.sql +++ b/tests/queries/0_stateless/00140_prewhere_column_order.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS prewhere; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE prewhere (d Date, a String, b String) ENGINE = MergeTree(d, d, 8192); INSERT INTO prewhere VALUES ('2015-01-01', 'hello', 'world'); diff --git a/tests/queries/0_stateless/00141_parse_timestamp_as_datetime.sql b/tests/queries/0_stateless/00141_parse_timestamp_as_datetime.sql index 4780884ca02..dbd251f878c 100644 --- a/tests/queries/0_stateless/00141_parse_timestamp_as_datetime.sql +++ b/tests/queries/0_stateless/00141_parse_timestamp_as_datetime.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS default; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE default (d Date DEFAULT toDate(t), t DateTime) ENGINE = MergeTree(d, t, 8192); INSERT INTO default (t) VALUES ('1234567890'); SELECT toStartOfMonth(d), toUInt32(t) FROM default; diff --git a/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql b/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql index ab9e8c9b177..e759fbd85d4 100644 --- a/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql +++ b/tests/queries/0_stateless/00146_summing_merge_tree_nested_map.sql @@ -1,5 +1,6 @@ drop table if exists nested_map; +set allow_deprecated_syntax_for_merge_tree=1; create table nested_map (d default today(), k UInt64, payload default rand(), SomeMap Nested(ID UInt32, Num Int64)) engine=SummingMergeTree(d, k, 8192); insert into nested_map (k, `SomeMap.ID`, `SomeMap.Num`) values (0,[1],[100]),(1,[1],[100]),(2,[1],[100]),(3,[1,2],[100,150]); diff --git a/tests/queries/0_stateless/00147_alter_nested_default.sql b/tests/queries/0_stateless/00147_alter_nested_default.sql index 54c99545364..070204aef65 100644 --- a/tests/queries/0_stateless/00147_alter_nested_default.sql +++ b/tests/queries/0_stateless/00147_alter_nested_default.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS alter_00147; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_00147 (d Date DEFAULT toDate('2015-01-01'), n Nested(x String)) ENGINE = MergeTree(d, d, 8192); INSERT INTO alter_00147 (`n.x`) VALUES (['Hello', 'World']); diff --git a/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql b/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql index eab6b9da465..04edf709bde 100644 --- a/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql +++ b/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql @@ -29,6 +29,7 @@ select count() from summing_merge_tree_aggregate_function; drop table summing_merge_tree_aggregate_function; ---- sum + uniq + uniqExact +set allow_deprecated_syntax_for_merge_tree=1; create table summing_merge_tree_aggregate_function ( d materialized today(), k UInt64, diff --git a/tests/queries/0_stateless/00148_summing_merge_tree_nested_map_multiple_values.sql b/tests/queries/0_stateless/00148_summing_merge_tree_nested_map_multiple_values.sql index 68c27cd726f..7c5757cd526 100644 --- a/tests/queries/0_stateless/00148_summing_merge_tree_nested_map_multiple_values.sql +++ b/tests/queries/0_stateless/00148_summing_merge_tree_nested_map_multiple_values.sql @@ -1,5 +1,6 @@ drop table if exists nested_map_multiple_values; +set allow_deprecated_syntax_for_merge_tree=1; create table nested_map_multiple_values (d materialized today(), k UInt64, payload materialized rand(), SomeMap Nested(ID UInt32, Num1 Int64, Num2 Float64)) engine=SummingMergeTree(d, k, 8192); insert into nested_map_multiple_values values (0,[1],[100],[1.0]),(1,[1],[100],[1.0]),(2,[1],[100],[1.0]),(3,[1,2],[100,150],[1.0,1.5]); diff --git a/tests/queries/0_stateless/00155_long_merges.sh b/tests/queries/0_stateless/00155_long_merges.sh index 15ad0892a42..88f9daf6798 100755 --- a/tests/queries/0_stateless/00155_long_merges.sh +++ b/tests/queries/0_stateless/00155_long_merges.sh @@ -11,9 +11,9 @@ function create { $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS aggregating_00155" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS replacing_00155" - $CLICKHOUSE_CLIENT --query="CREATE TABLE summing_00155 (d Date DEFAULT today(), x UInt64, s UInt64 DEFAULT 1) ENGINE = SummingMergeTree(d, x, 8192)" + $CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE summing_00155 (d Date DEFAULT today(), x UInt64, s UInt64 DEFAULT 1) ENGINE = SummingMergeTree(d, x, 8192)" $CLICKHOUSE_CLIENT --query="CREATE TABLE collapsing_00155 (d Date DEFAULT today(), x UInt64, s Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(d, x, 8192, s)" - $CLICKHOUSE_CLIENT --query="CREATE TABLE aggregating_00155 (d Date DEFAULT today(), x UInt64, s AggregateFunction(sum, UInt64)) ENGINE = AggregatingMergeTree(d, x, 8192)" + $CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE aggregating_00155 (d Date DEFAULT today(), x UInt64, s AggregateFunction(sum, UInt64)) ENGINE = AggregatingMergeTree(d, x, 8192)" $CLICKHOUSE_CLIENT --query="CREATE TABLE replacing_00155 (d Date DEFAULT today(), x UInt64, s Int8 DEFAULT 1, v UInt64) ENGINE = ReplacingMergeTree(d, (x), 8192, v)" } diff --git a/tests/queries/0_stateless/00168_buffer_defaults.sql b/tests/queries/0_stateless/00168_buffer_defaults.sql index 8e0008adf4d..ce1dea8aaa8 100644 --- a/tests/queries/0_stateless/00168_buffer_defaults.sql +++ b/tests/queries/0_stateless/00168_buffer_defaults.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS mt_00168; DROP TABLE IF EXISTS mt_00168_buffer; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mt_00168 (EventDate Date, UTCEventTime DateTime, MoscowEventDate Date DEFAULT toDate(UTCEventTime)) ENGINE = MergeTree(EventDate, UTCEventTime, 8192); CREATE TABLE mt_00168_buffer AS mt_00168 ENGINE = Buffer(currentDatabase(), mt_00168, 16, 10, 100, 10000, 1000000, 10000000, 100000000); DESC TABLE mt_00168; diff --git a/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql b/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql index 776edeeb43c..8160d4dee9e 100644 --- a/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql +++ b/tests/queries/0_stateless/00191_aggregating_merge_tree_and_final.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS aggregating_00191; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE aggregating_00191 (d Date DEFAULT '2000-01-01', k UInt64, u AggregateFunction(uniq, UInt64)) ENGINE = AggregatingMergeTree(d, k, 8192); INSERT INTO aggregating_00191 (k, u) SELECT intDiv(number, 100) AS k, uniqState(toUInt64(number % 100)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000) GROUP BY k; diff --git a/tests/queries/0_stateless/00193_parallel_replicas.sql b/tests/queries/0_stateless/00193_parallel_replicas.sql index 6c5b50972cc..2549ada2a78 100644 --- a/tests/queries/0_stateless/00193_parallel_replicas.sql +++ b/tests/queries/0_stateless/00193_parallel_replicas.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS parallel_replicas; DROP TABLE IF EXISTS parallel_replicas_backup; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE parallel_replicas (d Date DEFAULT today(), x UInt32, u UInt64, s String) ENGINE = MergeTree(d, cityHash64(u, s), (x, d, cityHash64(u, s)), 8192); INSERT INTO parallel_replicas (x, u, s) VALUES (1, 2, 'A'),(3, 4, 'B'),(5, 6, 'C'),(7, 8, 'D'),(9,10,'E'); INSERT INTO parallel_replicas (x, u, s) VALUES (11, 12, 'F'),(13, 14, 'G'),(15, 16, 'H'),(17, 18, 'I'),(19,20,'J'); diff --git a/tests/queries/0_stateless/00236_replicated_drop_on_non_leader_zookeeper_long.sql b/tests/queries/0_stateless/00236_replicated_drop_on_non_leader_zookeeper_long.sql index f6eb4b2f8c1..78319c3edd4 100644 --- a/tests/queries/0_stateless/00236_replicated_drop_on_non_leader_zookeeper_long.sql +++ b/tests/queries/0_stateless/00236_replicated_drop_on_non_leader_zookeeper_long.sql @@ -6,6 +6,7 @@ SET replication_alter_partitions_sync = 2; DROP TABLE IF EXISTS attach_r1; DROP TABLE IF EXISTS attach_r2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE attach_r1 (d Date) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00236/01/attach', 'r1', d, d, 8192); CREATE TABLE attach_r2 (d Date) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00236/01/attach', 'r2', d, d, 8192); diff --git a/tests/queries/0_stateless/00253_insert_recursive_defaults.sql b/tests/queries/0_stateless/00253_insert_recursive_defaults.sql index 37bfe0aa188..c0edc447125 100644 --- a/tests/queries/0_stateless/00253_insert_recursive_defaults.sql +++ b/tests/queries/0_stateless/00253_insert_recursive_defaults.sql @@ -5,6 +5,7 @@ SELECT * FROM defaults; DROP TABLE defaults; DROP TABLE IF EXISTS elog_cut; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE elog_cut ( date Date DEFAULT toDate(uts), diff --git a/tests/queries/0_stateless/00278_insert_already_sorted.sql b/tests/queries/0_stateless/00278_insert_already_sorted.sql index 735b46d22dd..b3de48dc155 100644 --- a/tests/queries/0_stateless/00278_insert_already_sorted.sql +++ b/tests/queries/0_stateless/00278_insert_already_sorted.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS sorted; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE sorted (d Date DEFAULT '2000-01-01', x UInt64) ENGINE = MergeTree(d, x, 8192); INSERT INTO sorted (x) SELECT intDiv(number, 100000) AS x FROM system.numbers LIMIT 1000000; diff --git a/tests/queries/0_stateless/00282_merging.sql b/tests/queries/0_stateless/00282_merging.sql index 30aec1bc439..a49cde87134 100644 --- a/tests/queries/0_stateless/00282_merging.sql +++ b/tests/queries/0_stateless/00282_merging.sql @@ -73,6 +73,7 @@ SELECT * FROM merge ORDER BY _part_index, x; DROP TABLE merge; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE IF NOT EXISTS merge (d Date DEFAULT '2000-01-01', x UInt64) ENGINE = MergeTree(d, x, 8192); SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; diff --git a/tests/queries/0_stateless/00345_index_accurate_comparison.sql b/tests/queries/0_stateless/00345_index_accurate_comparison.sql index 138a93990cf..aafe2a0ae69 100644 --- a/tests/queries/0_stateless/00345_index_accurate_comparison.sql +++ b/tests/queries/0_stateless/00345_index_accurate_comparison.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS index; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE index ( key Int32, diff --git a/tests/queries/0_stateless/00384_column_aggregate_function_insert_from.sql b/tests/queries/0_stateless/00384_column_aggregate_function_insert_from.sql index 0c614602866..a723bc9b59c 100644 --- a/tests/queries/0_stateless/00384_column_aggregate_function_insert_from.sql +++ b/tests/queries/0_stateless/00384_column_aggregate_function_insert_from.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS aggregates; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE aggregates (d Date, s AggregateFunction(uniq, UInt64)) ENGINE = MergeTree(d, d, 8192); INSERT INTO aggregates diff --git a/tests/queries/0_stateless/00394_new_nested_column_keeps_offsets.sql b/tests/queries/0_stateless/00394_new_nested_column_keeps_offsets.sql index 7c377529416..f058a852e91 100644 --- a/tests/queries/0_stateless/00394_new_nested_column_keeps_offsets.sql +++ b/tests/queries/0_stateless/00394_new_nested_column_keeps_offsets.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS alter_00394; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_00394 (d Date, k UInt64, i32 Int32, n Nested(ui8 UInt8, s String)) ENGINE=MergeTree(d, k, 8192); INSERT INTO alter_00394 VALUES ('2015-01-01', 3, 30, [1,2,3], ['1','12','123']); diff --git a/tests/queries/0_stateless/00394_replaceall_vector_fixed.sql b/tests/queries/0_stateless/00394_replaceall_vector_fixed.sql index 47ab3fe4a1c..ada48add403 100644 --- a/tests/queries/0_stateless/00394_replaceall_vector_fixed.sql +++ b/tests/queries/0_stateless/00394_replaceall_vector_fixed.sql @@ -14,6 +14,7 @@ ORDER BY str ASC; DROP TABLE replaceall; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE replaceall (date Date DEFAULT today(), fs FixedString(16)) ENGINE = MergeTree(date, (date, fs), 8192); INSERT INTO replaceall (fs) VALUES ('54db0d43009d\0\0\0\0'), ('fe2b58224766cf10'), ('54db0d43009d\0\0\0\0'), ('fe2b58224766cf10'); diff --git a/tests/queries/0_stateless/00395_nullable.sql b/tests/queries/0_stateless/00395_nullable.sql index 71dc045ad09..83d27830f4e 100644 --- a/tests/queries/0_stateless/00395_nullable.sql +++ b/tests/queries/0_stateless/00395_nullable.sql @@ -8,6 +8,7 @@ SELECT NULL + NULL; SELECT '----- MergeTree engine -----'; DROP TABLE IF EXISTS test1_00395; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test1_00395( col1 UInt64, col2 Nullable(UInt64), col3 String, col4 Nullable(String), diff --git a/tests/queries/0_stateless/00427_alter_primary_key.sh b/tests/queries/0_stateless/00427_alter_primary_key.sh index 4ad1166bfa4..1269e2ad6e3 100755 --- a/tests/queries/0_stateless/00427_alter_primary_key.sh +++ b/tests/queries/0_stateless/00427_alter_primary_key.sh @@ -8,6 +8,7 @@ function perform() { local query=$1 TZ=UTC $CLICKHOUSE_CLIENT \ + --allow_deprecated_syntax_for_merge_tree=1 \ --use_client_time_zone=1 \ --input_format_values_interpret_expressions=0 \ --query "$query" 2>/dev/null diff --git a/tests/queries/0_stateless/00440_nulls_merge_tree.sql b/tests/queries/0_stateless/00440_nulls_merge_tree.sql index 7281f960509..dd9473027b4 100644 --- a/tests/queries/0_stateless/00440_nulls_merge_tree.sql +++ b/tests/queries/0_stateless/00440_nulls_merge_tree.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS nulls; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE nulls (d Date, x Nullable(UInt64)) ENGINE = MergeTree(d, d, 8192); INSERT INTO nulls SELECT toDate('2000-01-01'), number % 10 != 0 ? number : NULL FROM system.numbers LIMIT 10000; SELECT count() FROM nulls WHERE x IS NULL; diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh index a441c9a6761..2f79365f756 100755 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -ch="$CLICKHOUSE_CLIENT --stacktrace -q" +ch="$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 --stacktrace -q" $ch "DROP TABLE IF EXISTS clear_column1" $ch "DROP TABLE IF EXISTS clear_column2" diff --git a/tests/queries/0_stateless/00453_cast_enum.sql b/tests/queries/0_stateless/00453_cast_enum.sql index 70b7e2b1d6d..384db50c7c4 100644 --- a/tests/queries/0_stateless/00453_cast_enum.sql +++ b/tests/queries/0_stateless/00453_cast_enum.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS cast_enums; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE cast_enums ( type Enum8('session' = 1, 'pageview' = 2, 'click' = 3), diff --git a/tests/queries/0_stateless/00495_reading_const_zero_column.sql b/tests/queries/0_stateless/00495_reading_const_zero_column.sql index da529c5b9b7..0af201b66e9 100644 --- a/tests/queries/0_stateless/00495_reading_const_zero_column.sql +++ b/tests/queries/0_stateless/00495_reading_const_zero_column.sql @@ -1,4 +1,5 @@ drop table if exists one_table; +set allow_deprecated_syntax_for_merge_tree=1; create table one_table (date Date, one UInt64) engine = MergeTree(date, (date, one), 8192); insert into one_table select today(), toUInt64(1) from system.numbers limit 100000; SET preferred_block_size_bytes = 8192; diff --git a/tests/queries/0_stateless/00504_mergetree_arrays_rw.sql b/tests/queries/0_stateless/00504_mergetree_arrays_rw.sql index 766f9dfb368..7c939d060ea 100644 --- a/tests/queries/0_stateless/00504_mergetree_arrays_rw.sql +++ b/tests/queries/0_stateless/00504_mergetree_arrays_rw.sql @@ -1,4 +1,5 @@ +set allow_deprecated_syntax_for_merge_tree=1; drop table if exists test_ins_arr; create table test_ins_arr (date Date, val Array(UInt64)) engine = MergeTree(date, (date), 8192); insert into test_ins_arr select toDate('2017-10-02'), [number, 42] from system.numbers limit 10000; diff --git a/tests/queries/0_stateless/00506_union_distributed.sql b/tests/queries/0_stateless/00506_union_distributed.sql index 08282d94d8b..8a5dba00c27 100644 --- a/tests/queries/0_stateless/00506_union_distributed.sql +++ b/tests/queries/0_stateless/00506_union_distributed.sql @@ -8,6 +8,7 @@ DROP TABLE IF EXISTS union1; DROP TABLE IF EXISTS union2; DROP TABLE IF EXISTS union3; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE union1 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = MergeTree(date, (a, date), 8192); CREATE TABLE union2 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = Distributed(test_shard_localhost, currentDatabase(), 'union1'); CREATE TABLE union3 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = Distributed(test_shard_localhost, currentDatabase(), 'union2'); diff --git a/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.sql b/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.sql index 64ceb394b17..e358072248a 100644 --- a/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.sql +++ b/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.sql @@ -1,6 +1,7 @@ -- Tags: shard drop table if exists tab; +set allow_deprecated_syntax_for_merge_tree=1; create table tab (date Date, val UInt64, val2 UInt8 default 42, val3 UInt8 default val2 + 1, val4 UInt64 alias val) engine = MergeTree(date, (date, val), 8192); desc tab; select '-'; diff --git a/tests/queries/0_stateless/00516_deduplication_after_drop_partition_zookeeper.sql b/tests/queries/0_stateless/00516_deduplication_after_drop_partition_zookeeper.sql index 6c76adb86d0..08dccac9d25 100644 --- a/tests/queries/0_stateless/00516_deduplication_after_drop_partition_zookeeper.sql +++ b/tests/queries/0_stateless/00516_deduplication_after_drop_partition_zookeeper.sql @@ -2,6 +2,7 @@ -- Tag no-replicated-database: Old syntax is not allowed DROP TABLE IF EXISTS deduplication_by_partition; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE deduplication_by_partition(d Date, x UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_00516/deduplication_by_partition', 'r1', d, x, 8192); diff --git a/tests/queries/0_stateless/00531_aggregate_over_nullable.sql b/tests/queries/0_stateless/00531_aggregate_over_nullable.sql index ff485b4251a..1680bb90bb1 100644 --- a/tests/queries/0_stateless/00531_aggregate_over_nullable.sql +++ b/tests/queries/0_stateless/00531_aggregate_over_nullable.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS agg_over_nullable; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE agg_over_nullable ( partition Date, timestamp DateTime, diff --git a/tests/queries/0_stateless/00543_null_and_prewhere.sql b/tests/queries/0_stateless/00543_null_and_prewhere.sql index 793d65925a1..5f50397862a 100644 --- a/tests/queries/0_stateless/00543_null_and_prewhere.sql +++ b/tests/queries/0_stateless/00543_null_and_prewhere.sql @@ -1,3 +1,5 @@ + +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test ( dt Date, diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index 913ecec6f4a..9927592465a 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -1,5 +1,6 @@ drop table IF EXISTS joinbug; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE joinbug ( event_date Date MATERIALIZED toDate(created, 'Asia/Istanbul'), id UInt64, diff --git a/tests/queries/0_stateless/00563_complex_in_expression.sql b/tests/queries/0_stateless/00563_complex_in_expression.sql index cd80b9c3a7a..bd053e0d020 100644 --- a/tests/queries/0_stateless/00563_complex_in_expression.sql +++ b/tests/queries/0_stateless/00563_complex_in_expression.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test_00563; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_00563 ( dt Date, site_id Int32, site_key String ) ENGINE = MergeTree(dt, (site_id, site_key, dt), 8192); INSERT INTO test_00563 (dt,site_id, site_key) VALUES ('2018-1-29', 100, 'key'); SELECT * FROM test_00563 WHERE toInt32(site_id) IN (100); diff --git a/tests/queries/0_stateless/00564_initial_column_values_with_default_expression.sql b/tests/queries/0_stateless/00564_initial_column_values_with_default_expression.sql index d26621dce7a..3fff20a1fe1 100644 --- a/tests/queries/0_stateless/00564_initial_column_values_with_default_expression.sql +++ b/tests/queries/0_stateless/00564_initial_column_values_with_default_expression.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE IF NOT EXISTS test( id UInt32, track UInt8, codec String, content String, rdate Date DEFAULT '2018-02-03', track_id String DEFAULT concat(concat(concat(toString(track), '-'), codec), content) ) ENGINE=MergeTree(rdate, (id, track_id), 8192); INSERT INTO test(id, track, codec) VALUES(1, 0, 'h264'); diff --git a/tests/queries/0_stateless/00571_non_exist_database_when_create_materializ_view.sql b/tests/queries/0_stateless/00571_non_exist_database_when_create_materializ_view.sql index c384ef37fc7..46fc0dd586d 100644 --- a/tests/queries/0_stateless/00571_non_exist_database_when_create_materializ_view.sql +++ b/tests/queries/0_stateless/00571_non_exist_database_when_create_materializ_view.sql @@ -8,6 +8,7 @@ DROP DATABASE IF EXISTS none; DROP TABLE IF EXISTS test_00571; DROP TABLE IF EXISTS test_materialized_00571; +set allow_deprecated_syntax_for_merge_tree=1; CREATE DATABASE none; CREATE TABLE test_00571 ( date Date, platform Enum8('a' = 0, 'b' = 1, 'c' = 2), app Enum8('a' = 0, 'b' = 1) ) ENGINE = MergeTree(date, (platform, app), 8192); CREATE MATERIALIZED VIEW test_materialized_00571 ENGINE = MergeTree(date, (platform, app), 8192) POPULATE AS SELECT date, platform, app FROM (SELECT * FROM test_00571); diff --git a/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh b/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh index 42bf37cbda5..706c0819125 100755 --- a/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh +++ b/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) exception_pattern="Code: 44.*Cannot drop column \`id\`, because column \`id2\` depends on it" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_00575;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_00575 (dt Date DEFAULT now(), id UInt32, id2 UInt32 DEFAULT id + 1) ENGINE = MergeTree(dt, dt, 8192);" +${CLICKHOUSE_CLIENT} --allow_deprecated_syntax_for_merge_tree=1 --query "CREATE TABLE test_00575 (dt Date DEFAULT now(), id UInt32, id2 UInt32 DEFAULT id + 1) ENGINE = MergeTree(dt, dt, 8192);" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_00575(dt,id) VALUES ('2018-02-22',3), ('2018-02-22',4), ('2018-02-22',5);" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_00575 ORDER BY id;" echo "$(${CLICKHOUSE_CLIENT} --query "ALTER TABLE test_00575 DROP COLUMN id;" --server_logs_file=/dev/null 2>&1 | grep -c "$exception_pattern")" diff --git a/tests/queries/0_stateless/00575_merge_and_index_with_function_in_in.sql b/tests/queries/0_stateless/00575_merge_and_index_with_function_in_in.sql index e491ef37ada..6f0ddd9fae3 100644 --- a/tests/queries/0_stateless/00575_merge_and_index_with_function_in_in.sql +++ b/tests/queries/0_stateless/00575_merge_and_index_with_function_in_in.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS t_00575; +set allow_deprecated_syntax_for_merge_tree=1; create table t_00575(d Date) engine MergeTree(d, d, 8192); insert into t_00575 values ('2018-02-20'); diff --git a/tests/queries/0_stateless/00584_view_union_all.sql b/tests/queries/0_stateless/00584_view_union_all.sql index 2e4d7ea66db..a86dfaec6c8 100644 --- a/tests/queries/0_stateless/00584_view_union_all.sql +++ b/tests/queries/0_stateless/00584_view_union_all.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS Test_00584; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE Test_00584 ( createdDate Date, str String, diff --git a/tests/queries/0_stateless/00594_alias_in_distributed.sql b/tests/queries/0_stateless/00594_alias_in_distributed.sql index b75dfdf7967..250ede2bb10 100644 --- a/tests/queries/0_stateless/00594_alias_in_distributed.sql +++ b/tests/queries/0_stateless/00594_alias_in_distributed.sql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS alias_local10; DROP TABLE IF EXISTS alias10; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alias_local10 ( Id Int8, EventDate Date DEFAULT '2000-01-01', diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql index 40ea8b48caa..a77f3730ac2 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql +++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql @@ -7,6 +7,7 @@ SET joined_subquery_requires_alias = 0; DROP TABLE IF EXISTS test_00597; DROP TABLE IF EXISTS test_view_00597; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_00597(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); CREATE VIEW test_view_00597 AS SELECT * FROM test_00597; diff --git a/tests/queries/0_stateless/00609_mv_index_in_in.sql b/tests/queries/0_stateless/00609_mv_index_in_in.sql index 002c94dd576..bd9f35350c1 100644 --- a/tests/queries/0_stateless/00609_mv_index_in_in.sql +++ b/tests/queries/0_stateless/00609_mv_index_in_in.sql @@ -6,6 +6,7 @@ DROP TABLE IF EXISTS test_mv_00609; create table test_00609 (a Int8) engine=Memory; insert into test_00609 values (1); +set allow_deprecated_syntax_for_merge_tree=1; create materialized view test_mv_00609 uuid '00000609-1000-4000-8000-000000000001' Engine=MergeTree(date, (a), 8192) populate as select a, toDate('2000-01-01') date from test_00609; select * from test_mv_00609; -- OK diff --git a/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql b/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql index 6f5ba07e5db..8830204ecb5 100644 --- a/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql +++ b/tests/queries/0_stateless/00610_materialized_view_forward_alter_partition_statements.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS tab_00610; DROP TABLE IF EXISTS mv_00610; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE tab_00610(d Date, x UInt32) ENGINE MergeTree(d, x, 8192); CREATE MATERIALIZED VIEW mv_00610(d Date, y UInt64) ENGINE MergeTree(d, y, 8192) AS SELECT d, x + 1 AS y FROM tab_00610; diff --git a/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql b/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql index 5d373927714..17cc76670b3 100644 --- a/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql +++ b/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql @@ -1,6 +1,7 @@ -- Tags: distributed drop table if exists tab; +set allow_deprecated_syntax_for_merge_tree=1; create table tab (date Date, time DateTime, data String) ENGINE = MergeTree(date, (time, data), 8192); insert into tab values ('2018-01-21','2018-01-21 15:12:13','test'); select time FROM remote('127.0.0.{1,2}', currentDatabase(), tab) WHERE date = '2018-01-21' limit 2; diff --git a/tests/queries/0_stateless/00615_nullable_alter_optimize.sql b/tests/queries/0_stateless/00615_nullable_alter_optimize.sql index 5c0a192e826..26ff3b78d5e 100644 --- a/tests/queries/0_stateless/00615_nullable_alter_optimize.sql +++ b/tests/queries/0_stateless/00615_nullable_alter_optimize.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test_00615; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_00615 ( dt Date, diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.sql b/tests/queries/0_stateless/00621_regression_for_in_operator.sql index 84355a04df4..273f930a90f 100644 --- a/tests/queries/0_stateless/00621_regression_for_in_operator.sql +++ b/tests/queries/0_stateless/00621_regression_for_in_operator.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS regression_for_in_operator_view; DROP TABLE IF EXISTS regression_for_in_operator; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE regression_for_in_operator (d Date, v UInt32, g String) ENGINE=MergeTree(d, d, 8192); CREATE MATERIALIZED VIEW regression_for_in_operator_view ENGINE=AggregatingMergeTree(d, (d,g), 8192) AS SELECT d, g, maxState(v) FROM regression_for_in_operator GROUP BY d, g; diff --git a/tests/queries/0_stateless/00623_in_partition_key.sql b/tests/queries/0_stateless/00623_in_partition_key.sql index 0141151e369..6cb33f96f13 100644 --- a/tests/queries/0_stateless/00623_in_partition_key.sql +++ b/tests/queries/0_stateless/00623_in_partition_key.sql @@ -1,4 +1,5 @@ drop table if exists test54378; +set allow_deprecated_syntax_for_merge_tree=1; create table test54378 (part_date Date, pk_date Date, date Date) Engine=MergeTree(part_date, pk_date, 8192); insert into test54378 values ('2018-04-19', '2018-04-19', '2018-04-19'); diff --git a/tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper_long.sql b/tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper_long.sql index 4a3640e2dd7..44a7d8ca60f 100644 --- a/tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper_long.sql +++ b/tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper_long.sql @@ -4,6 +4,7 @@ DROP TABLE IF EXISTS replicated_truncate1; DROP TABLE IF EXISTS replicated_truncate2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE replicated_truncate1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_00623/truncate', 'r1', d, k, 8192); CREATE TABLE replicated_truncate2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/test_00623/truncate', 'r2', d, k, 8192); diff --git a/tests/queries/0_stateless/00648_replacing_empty_set_from_prewhere.sql b/tests/queries/0_stateless/00648_replacing_empty_set_from_prewhere.sql index 1f873c94b18..bbeb4dd3148 100644 --- a/tests/queries/0_stateless/00648_replacing_empty_set_from_prewhere.sql +++ b/tests/queries/0_stateless/00648_replacing_empty_set_from_prewhere.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS final_test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE final_test (id String, version Date) ENGINE = ReplacingMergeTree(version, id, 8192); INSERT INTO final_test (id, version) VALUES ('2018-01-01', '2018-01-01'); SELECT * FROM final_test FINAL PREWHERE id == '2018-01-02'; diff --git a/tests/queries/0_stateless/00652_mergetree_mutations.sh b/tests/queries/0_stateless/00652_mergetree_mutations.sh index 0cba8b94040..535cbe99dfe 100755 --- a/tests/queries/0_stateless/00652_mergetree_mutations.sh +++ b/tests/queries/0_stateless/00652_mergetree_mutations.sh @@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations(d Date, x UInt32, s String, a UInt32 ALIAS x + 1, m MATERIALIZED x + 2) ENGINE MergeTree(d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE mutations(d Date, x UInt32, s String, a UInt32 ALIAS x + 1, m MATERIALIZED x + 2) ENGINE MergeTree(d, intDiv(x, 10), 8192)" # Test a mutation on empty table ${CLICKHOUSE_CLIENT} --query="ALTER TABLE mutations DELETE WHERE x = 1" diff --git a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh index f070e34f2a1..7a6c7609660 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh @@ -12,8 +12,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1', d, intDiv(x, 10), 8192)" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r2', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r2', d, intDiv(x, 10), 8192)" # Test a mutation on empty table ${CLICKHOUSE_CLIENT} --query="ALTER TABLE mutations_r1 DELETE WHERE x = 1 SETTINGS mutations_sync = 2" diff --git a/tests/queries/0_stateless/00678_shard_funnel_window.sql b/tests/queries/0_stateless/00678_shard_funnel_window.sql index 8ea07a1c4ba..73e48923283 100644 --- a/tests/queries/0_stateless/00678_shard_funnel_window.sql +++ b/tests/queries/0_stateless/00678_shard_funnel_window.sql @@ -1,6 +1,7 @@ -- Tags: shard DROP TABLE IF EXISTS remote_test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE remote_test(uid String, its UInt32, action_code String, day Date) ENGINE = MergeTree(day, (uid, its), 8192); INSERT INTO remote_test SELECT toString(number) AS uid, number % 3 AS its, toString(number % 3) AS action_code, '2000-01-01' FROM system.numbers LIMIT 10000; SELECT level, COUNT() FROM (SELECT uid, windowFunnel(3600)(toUInt32(its), action_code != '', action_code = '2') AS level FROM remote('127.0.0.{2,3}', currentDatabase(), remote_test) GROUP BY uid) GROUP BY level; diff --git a/tests/queries/0_stateless/00712_prewhere_with_alias_bug_2.sql b/tests/queries/0_stateless/00712_prewhere_with_alias_bug_2.sql index 97d5e33633a..beb986adebc 100644 --- a/tests/queries/0_stateless/00712_prewhere_with_alias_bug_2.sql +++ b/tests/queries/0_stateless/00712_prewhere_with_alias_bug_2.sql @@ -1,5 +1,6 @@ drop table if exists table; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE table (a UInt32, date Date, b UInt64, c UInt64, str String, d Int8, arr Array(UInt64), arr_alias Array(UInt64) ALIAS arr) ENGINE = MergeTree(date, intHash32(c), (a, date, intHash32(c), b), 8192); SELECT alias2 AS alias3 diff --git a/tests/queries/0_stateless/00712_prewhere_with_final.sql b/tests/queries/0_stateless/00712_prewhere_with_final.sql index 4528d4f61f0..6b49e523f70 100644 --- a/tests/queries/0_stateless/00712_prewhere_with_final.sql +++ b/tests/queries/0_stateless/00712_prewhere_with_final.sql @@ -1,4 +1,5 @@ drop table if exists trepl; +set allow_deprecated_syntax_for_merge_tree=1; create table trepl(d Date,a Int32, b Int32) engine = ReplacingMergeTree(d, (a,b), 8192); insert into trepl values ('2018-09-19', 1, 1); select b from trepl FINAL prewhere a < 1000; diff --git a/tests/queries/0_stateless/00712_prewhere_with_sampling_and_alias.sql b/tests/queries/0_stateless/00712_prewhere_with_sampling_and_alias.sql index d011e1b368d..7c8ae4eed7c 100644 --- a/tests/queries/0_stateless/00712_prewhere_with_sampling_and_alias.sql +++ b/tests/queries/0_stateless/00712_prewhere_with_sampling_and_alias.sql @@ -1,4 +1,5 @@ drop table if exists t_00712_2; +set allow_deprecated_syntax_for_merge_tree=1; create table t_00712_2 (date Date, counter UInt64, sampler UInt64, alias_col alias date + 1) engine = MergeTree(date, intHash32(sampler), (counter, date, intHash32(sampler)), 8192); insert into t_00712_2 values ('2018-01-01', 1, 1); select alias_col from t_00712_2 sample 1 / 2 where date = '2018-01-01' and counter = 1 and sampler = 1; diff --git a/tests/queries/0_stateless/00717_merge_and_distributed.sql b/tests/queries/0_stateless/00717_merge_and_distributed.sql index f7b4a2b24d3..f27f6a75688 100644 --- a/tests/queries/0_stateless/00717_merge_and_distributed.sql +++ b/tests/queries/0_stateless/00717_merge_and_distributed.sql @@ -7,6 +7,7 @@ DROP TABLE IF EXISTS test_local_2; DROP TABLE IF EXISTS test_distributed_1; DROP TABLE IF EXISTS test_distributed_2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_local_1 (date Date, value UInt32) ENGINE = MergeTree(date, date, 8192); CREATE TABLE test_local_2 (date Date, value UInt32) ENGINE = MergeTree(date, date, 8192); CREATE TABLE test_distributed_1 AS test_local_1 ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_local_1, rand()); diff --git a/tests/queries/0_stateless/00729_prewhere_array_join.sql b/tests/queries/0_stateless/00729_prewhere_array_join.sql index ba10dd38bd2..5ac79c150c6 100644 --- a/tests/queries/0_stateless/00729_prewhere_array_join.sql +++ b/tests/queries/0_stateless/00729_prewhere_array_join.sql @@ -1,6 +1,7 @@ SET send_logs_level = 'fatal'; drop table if exists t1_00729; +set allow_deprecated_syntax_for_merge_tree=1; create table t1_00729 (id UInt64, val Array(String),nid UInt64, eDate Date)ENGINE = MergeTree(eDate, (id, eDate), 8192); insert into t1_00729 (id,val,nid,eDate) values (1,['background','foreground','heading','image'],1,'2018-09-27'); diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index d9a3631a7dd..2510517a740 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -settings="--log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" +settings="--log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1 --allow_deprecated_syntax_for_merge_tree=1" # Test insert logging on each block and checkPacket() method diff --git a/tests/queries/0_stateless/00732_decimal_summing_merge_tree.sql b/tests/queries/0_stateless/00732_decimal_summing_merge_tree.sql index 1524776c16e..96e16a1300e 100644 --- a/tests/queries/0_stateless/00732_decimal_summing_merge_tree.sql +++ b/tests/queries/0_stateless/00732_decimal_summing_merge_tree.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS decimal_sum; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE decimal_sum ( date Date, diff --git a/tests/queries/0_stateless/00748_insert_array_with_null.sql b/tests/queries/0_stateless/00748_insert_array_with_null.sql index 78c564abad3..ca36352c2cf 100644 --- a/tests/queries/0_stateless/00748_insert_array_with_null.sql +++ b/tests/queries/0_stateless/00748_insert_array_with_null.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS arraytest; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE arraytest ( created_date Date DEFAULT toDate(created_at), created_at DateTime DEFAULT now(), strings Array(String) DEFAULT emptyArrayString()) ENGINE = MergeTree(created_date, cityHash64(created_at), (created_date, cityHash64(created_at)), 8192); INSERT INTO arraytest (created_at, strings) VALUES (now(), ['aaaaa', 'bbbbb', 'ccccc']); diff --git a/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql b/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql index 71950469a47..83c6b1c1a6b 100644 --- a/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql +++ b/tests/queries/0_stateless/00752_low_cardinality_mv_2.sql @@ -1,6 +1,7 @@ drop table if exists radacct; drop table if exists mv_traffic_by_tadig15min; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE radacct ( radacctid UInt64, f3gppchargingid Nullable(String), f3gppggsnaddress Nullable(String), f3gppggsnmccmnc Nullable(String), f3gppgprsqos Nullable(String), f3gppimeisv Nullable(String), f3gppimsi Nullable(UInt64), f3gppimsimccmnc Nullable(String), f3gpploci Nullable(String), f3gppnsapi Nullable(String), f3gpprattype Nullable(String), f3gppsgsnaddress Nullable(String), f3gppsgsnmccmnc Nullable(String), acctdelaytime Nullable(UInt32), acctinputoctets Nullable(UInt64), acctinputpackets Nullable(UInt64), acctoutputoctets Nullable(UInt64), acctoutputpackets Nullable(UInt64), acctsessionid String, acctstatustype Nullable(String), acctuniqueid String, calledstationid Nullable(String), callingstationid Nullable(String), framedipaddress Nullable(String), nasidentifier Nullable(String), nasipaddress Nullable(String), acctstarttime Nullable(DateTime), acctstoptime Nullable(DateTime), acctsessiontime Nullable(UInt32), acctterminatecause Nullable(String), acctstartdelay Nullable(UInt32), acctstopdelay Nullable(UInt32), connectinfo_start Nullable(String), connectinfo_stop Nullable(String), timestamp DateTime, username Nullable(String), realm Nullable(String), f3gppimsi_int UInt64, f3gppsgsnaddress_int Nullable(UInt32), timestamp_date Date, tac Nullable(String), mnc Nullable(String), tadig LowCardinality(String), country LowCardinality(String), tadig_op_ip Nullable(String) DEFAULT CAST('TADIG NOT FOUND', 'Nullable(String)'), mcc Nullable(UInt16) MATERIALIZED toUInt16OrNull(substring(f3gppsgsnmccmnc, 1, 6))) ENGINE = MergeTree(timestamp_date, (timestamp, radacctid, acctuniqueid), 8192); insert into radacct values (1, 'a', 'b', 'c', 'd', 'e', 2, 'a', 'b', 'c', 'd', 'e', 'f', 3, 4, 5, 6, 7, 'a', 'Stop', 'c', 'd', 'e', 'f', 'g', 'h', '2018-10-10 15:54:21', '2018-10-10 15:54:21', 8, 'a', 9, 10, 'a', 'b', '2018-10-10 15:54:21', 'a', 'b', 11, 12, '2018-10-10', 'a', 'b', 'c', 'd', 'e'); diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index a2d19c22500..e1392d299dc 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -49,6 +49,7 @@ FORMAT PrettyCompactNoEscapes; DROP TABLE IF EXISTS check_system_tables; -- Check MergeTree declaration in old format +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE check_system_tables ( Event Date, diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.sql b/tests/queries/0_stateless/00754_alter_modify_order_by.sql index cb81f868e7b..234bd61902b 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.sql @@ -2,6 +2,7 @@ SET send_logs_level = 'fatal'; SET optimize_on_insert = 0; DROP TABLE IF EXISTS old_style; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE old_style(d Date, x UInt32) ENGINE MergeTree(d, x, 8192); ALTER TABLE old_style ADD COLUMN y UInt32, MODIFY ORDER BY (x, y); -- { serverError 36} DROP TABLE old_style; diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql index c859c7b9921..29d0ef79b91 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql @@ -6,6 +6,7 @@ SET optimize_on_insert = 0; SET send_logs_level = 'fatal'; DROP TABLE IF EXISTS old_style; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE old_style(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_00754/old_style', 'r1', d, x, 8192); ALTER TABLE old_style ADD COLUMN y UInt32, MODIFY ORDER BY (x, y); -- { serverError 36 } DROP TABLE old_style; diff --git a/tests/queries/0_stateless/00794_materialized_view_with_column_defaults.sql b/tests/queries/0_stateless/00794_materialized_view_with_column_defaults.sql index a756f5c3537..43dcb322f1c 100644 --- a/tests/queries/0_stateless/00794_materialized_view_with_column_defaults.sql +++ b/tests/queries/0_stateless/00794_materialized_view_with_column_defaults.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS table_view; DROP TABLE IF EXISTS source_table; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE source_table ( date Date, datetime DateTime, diff --git a/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql b/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql index 600793853ca..7ddf1caa85f 100644 --- a/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql +++ b/tests/queries/0_stateless/00800_low_cardinality_distributed_insert.sql @@ -5,6 +5,7 @@ SET insert_distributed_sync = 1; DROP TABLE IF EXISTS low_cardinality; DROP TABLE IF EXISTS low_cardinality_all; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192); CREATE TABLE low_cardinality_all (d Date, x UInt32, s LowCardinality(String)) ENGINE = Distributed(test_shard_localhost, currentDatabase(), low_cardinality, sipHash64(s)); diff --git a/tests/queries/0_stateless/00806_alter_update.sql b/tests/queries/0_stateless/00806_alter_update.sql index c4fe3969df3..c9b1bfb2d9d 100644 --- a/tests/queries/0_stateless/00806_alter_update.sql +++ b/tests/queries/0_stateless/00806_alter_update.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS alter_update_00806; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_update_00806 (d Date, e Enum8('foo'=1, 'bar'=2)) Engine = MergeTree(d, (d), 8192); INSERT INTO alter_update_00806 (d, e) VALUES ('2018-01-01', 'foo'); INSERT INTO alter_update_00806 (d, e) VALUES ('2018-01-02', 'bar'); diff --git a/tests/queries/0_stateless/00829_bitmap_function.sql b/tests/queries/0_stateless/00829_bitmap_function.sql index fde0176de5b..6a21f5caf0f 100644 --- a/tests/queries/0_stateless/00829_bitmap_function.sql +++ b/tests/queries/0_stateless/00829_bitmap_function.sql @@ -71,6 +71,7 @@ SELECT bitmapToArray(bitmapAnd(groupBitmapState(uid), bitmapBuild(CAST([1, 2, 3] -- bitmap state test DROP TABLE IF EXISTS bitmap_state_test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE bitmap_state_test ( pickup_date Date, diff --git a/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh b/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh index 6640e0003cd..41a35d908d1 100755 --- a/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh +++ b/tests/queries/0_stateless/00834_hints_for_type_function_typos.sh @@ -26,5 +26,5 @@ $CLICKHOUSE_CLIENT -q "select * FROM numberss(10);" 2>&1 | grep "Maybe you meant $CLICKHOUSE_CLIENT -q "select * FROM anothernumbers(10);" 2>&1 | grep -v "Maybe you meant: \['numbers'\,'numbers_mt'\]." &>/dev/null $CLICKHOUSE_CLIENT -q "select * FROM mynumbers(10);" 2>&1 | grep "Maybe you meant: \['numbers'\]." &>/dev/null -$CLICKHOUSE_CLIENT -q "CREATE TABLE stored_aggregates (d Date, Uniq AggregateFunction(uniq, UInt64)) ENGINE = MergeTre(d, d, 8192);" 2>&1 | grep "Maybe you meant: \['MergeTree'\]." &>/dev/null -$CLICKHOUSE_CLIENT -q "CREATE TABLE stored_aggregates (d Date, Uniq AgregateFunction(uniq, UInt64)) ENGINE = MergeTree(d, d, 8192);" 2>&1 | grep "Maybe you meant: \['AggregateFunction'\]." &>/dev/null +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE stored_aggregates (d Date, Uniq AggregateFunction(uniq, UInt64)) ENGINE = MergeTre(d, d, 8192);" 2>&1 | grep "Maybe you meant: \['MergeTree'\]." &>/dev/null +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE stored_aggregates (d Date, Uniq AgregateFunction(uniq, UInt64)) ENGINE = MergeTree(d, d, 8192);" 2>&1 | grep "Maybe you meant: \['AggregateFunction'\]." &>/dev/null diff --git a/tests/queries/0_stateless/00910_crash_when_distributed_modify_order_by.sql b/tests/queries/0_stateless/00910_crash_when_distributed_modify_order_by.sql index 6dea97d1f27..00811d8ab89 100644 --- a/tests/queries/0_stateless/00910_crash_when_distributed_modify_order_by.sql +++ b/tests/queries/0_stateless/00910_crash_when_distributed_modify_order_by.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS union1; DROP TABLE IF EXISTS union2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE union1 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = MergeTree(date, (a, date), 8192); CREATE TABLE union2 ( date Date, a Int32, b Int32, c Int32, d Int32) ENGINE = Distributed(test_shard_localhost, currentDatabase(), 'union1'); ALTER TABLE union2 MODIFY ORDER BY a; -- { serverError 48 } diff --git a/tests/queries/0_stateless/00925_zookeeper_empty_replicated_merge_tree_optimize_final_long.sh b/tests/queries/0_stateless/00925_zookeeper_empty_replicated_merge_tree_optimize_final_long.sh index b0757829424..9e82cb4d30b 100755 --- a/tests/queries/0_stateless/00925_zookeeper_empty_replicated_merge_tree_optimize_final_long.sh +++ b/tests/queries/0_stateless/00925_zookeeper_empty_replicated_merge_tree_optimize_final_long.sh @@ -8,8 +8,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replicated_optimize1;" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replicated_optimize2;" -$CLICKHOUSE_CLIENT -q "CREATE TABLE replicated_optimize1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00925/optimize', 'r1', d, k, 8192);" -$CLICKHOUSE_CLIENT -q "CREATE TABLE replicated_optimize2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00925/optimize', 'r2', d, k, 8192);" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE replicated_optimize1 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00925/optimize', 'r1', d, k, 8192);" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE replicated_optimize2 (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00925/optimize', 'r2', d, k, 8192);" num_tries=0 while [[ $($CLICKHOUSE_CLIENT -q "SELECT is_leader FROM system.replicas WHERE database=currentDatabase() AND table='replicated_optimize1'") -ne 1 ]]; do diff --git a/tests/queries/0_stateless/00952_insert_into_distributed_with_materialized_column.sql b/tests/queries/0_stateless/00952_insert_into_distributed_with_materialized_column.sql index 3a9a9154abe..05382b855b1 100644 --- a/tests/queries/0_stateless/00952_insert_into_distributed_with_materialized_column.sql +++ b/tests/queries/0_stateless/00952_insert_into_distributed_with_materialized_column.sql @@ -15,6 +15,7 @@ SET insert_allow_materialized_columns=0; SELECT 'insert_distributed_sync=0'; SET insert_distributed_sync=0; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE local_00952 (date Date, value Date MATERIALIZED toDate('2017-08-01')) ENGINE = MergeTree(date, date, 8192); CREATE TABLE distributed_00952 AS local_00952 ENGINE = Distributed('test_cluster_two_shards', currentDatabase(), local_00952, rand()); diff --git a/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh b/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh index 99173062595..a899886e463 100755 --- a/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh +++ b/tests/queries/0_stateless/00981_in_subquery_with_tuple.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bug"; -$CLICKHOUSE_CLIENT --query="CREATE TABLE bug (d Date, s String) ENGINE = MergeTree(d, s, 8192)"; +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE bug (d Date, s String) ENGINE = MergeTree(d, s, 8192)"; $CLICKHOUSE_CLIENT --query="INSERT INTO bug VALUES ('2019-08-09', 'hello'), ('2019-08-10', 'world'), ('2019-08-11', 'world'), ('2019-08-12', 'hello')"; #SET force_primary_key = 1; diff --git a/tests/queries/0_stateless/01008_materialized_view_henyihanwobushi.sql b/tests/queries/0_stateless/01008_materialized_view_henyihanwobushi.sql index 8def546cd2e..1e91f6daf65 100644 --- a/tests/queries/0_stateless/01008_materialized_view_henyihanwobushi.sql +++ b/tests/queries/0_stateless/01008_materialized_view_henyihanwobushi.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS foo; DROP TABLE IF EXISTS bar; DROP TABLE IF EXISTS view_foo_bar; +set allow_deprecated_syntax_for_merge_tree=1; create table foo (ddate Date, id Int64, n String) ENGINE = ReplacingMergeTree(ddate, (id), 8192); create table bar (ddate Date, id Int64, n String, foo_id Int64) ENGINE = ReplacingMergeTree(ddate, (id), 8192); insert into bar (id, n, foo_id) values (1, 'bar_n_1', 1); diff --git a/tests/queries/0_stateless/01053_drop_database_mat_view.sql b/tests/queries/0_stateless/01053_drop_database_mat_view.sql index e9936d7d3b2..7651ac4885c 100644 --- a/tests/queries/0_stateless/01053_drop_database_mat_view.sql +++ b/tests/queries/0_stateless/01053_drop_database_mat_view.sql @@ -4,6 +4,7 @@ DROP DATABASE IF EXISTS some_tests; set allow_deprecated_database_ordinary=1; CREATE DATABASE some_tests ENGINE=Ordinary; -- Different inner table name with Atomic +set allow_deprecated_syntax_for_merge_tree=1; create table some_tests.my_table ENGINE = MergeTree(day, (day), 8192) as select today() as day, 'mystring' as str; show tables from some_tests; create materialized view some_tests.my_materialized_view ENGINE = MergeTree(day, (day), 8192) as select * from some_tests.my_table; diff --git a/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper_long.sql b/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper_long.sql index aba84f43033..3777ebb1af3 100644 --- a/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper_long.sql +++ b/tests/queries/0_stateless/01062_alter_on_mutataion_zookeeper_long.sql @@ -61,6 +61,7 @@ DROP TABLE IF EXISTS test_alter_on_mutation; DROP TABLE IF EXISTS nested_alter; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE nested_alter (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT '0') ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_01062/nested_alter', 'r2', d, k, 8192); INSERT INTO nested_alter VALUES ('2015-01-01', 6,38,'2014-07-15 13:26:50',[10,20,30],['asd','qwe','qwe'],['2000-01-01','2000-01-01','2000-01-03'],'100500'); diff --git a/tests/queries/0_stateless/01071_prohibition_secondary_index_with_old_format_merge_tree.sql b/tests/queries/0_stateless/01071_prohibition_secondary_index_with_old_format_merge_tree.sql index bbb607fe89f..992973c97e8 100644 --- a/tests/queries/0_stateless/01071_prohibition_secondary_index_with_old_format_merge_tree.sql +++ b/tests/queries/0_stateless/01071_prohibition_secondary_index_with_old_format_merge_tree.sql @@ -1,5 +1,6 @@ -- Tags: no-parallel +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE old_syntax_01071_test (date Date, id UInt8) ENGINE = MergeTree(date, id, 8192); ALTER TABLE old_syntax_01071_test ADD INDEX id_minmax id TYPE minmax GRANULARITY 1; -- { serverError 36 } CREATE TABLE new_syntax_01071_test (date Date, id UInt8) ENGINE = MergeTree() ORDER BY id; diff --git a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql index d652cbeea5d..cfa25179d05 100644 --- a/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql +++ b/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS test; DROP TABLE IF EXISTS test_view; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); CREATE VIEW test_view AS SELECT * FROM test; diff --git a/tests/queries/0_stateless/01089_alter_settings_old_format.sql b/tests/queries/0_stateless/01089_alter_settings_old_format.sql index 15d3ac508cb..7e7674f4d43 100644 --- a/tests/queries/0_stateless/01089_alter_settings_old_format.sql +++ b/tests/queries/0_stateless/01089_alter_settings_old_format.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS old_format_mt; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE old_format_mt ( event_date Date, key UInt64, diff --git a/tests/queries/0_stateless/01101_prewhere_after_alter.sql b/tests/queries/0_stateless/01101_prewhere_after_alter.sql index 11ebac7448d..976eb586a9a 100644 --- a/tests/queries/0_stateless/01101_prewhere_after_alter.sql +++ b/tests/queries/0_stateless/01101_prewhere_after_alter.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS test_a; DROP TABLE IF EXISTS test_b; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_a ( OldColumn String DEFAULT '', diff --git a/tests/queries/0_stateless/01125_generate_random_qoega.sql b/tests/queries/0_stateless/01125_generate_random_qoega.sql index 7fb586ad2b5..9088e411a80 100644 --- a/tests/queries/0_stateless/01125_generate_random_qoega.sql +++ b/tests/queries/0_stateless/01125_generate_random_qoega.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS mass_table_117; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mass_table_117 (`dt` Date, `site_id` Int32, `site_key` String) ENGINE = MergeTree(dt, (site_id, site_key, dt), 8192); INSERT INTO mass_table_117 SELECT * FROM generateRandom('`dt` Date,`site_id` Int32,`site_key` String', 1, 10, 2) LIMIT 100; SELECT count(), sum(cityHash64(*)) FROM mass_table_117; diff --git a/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql b/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql index c9bfbbe5111..f5f04178d5e 100644 --- a/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql +++ b/tests/queries/0_stateless/01126_month_partitioning_consistent_code.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS mt; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mt (d Date, x UInt8) ENGINE = MergeTree(d, x, 8192); INSERT INTO mt VALUES (52392, 1), (62677, 2); DROP TABLE mt; diff --git a/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql index eb5ea091ca3..2a1d04e6074 100644 --- a/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql +++ b/tests/queries/0_stateless/01127_month_partitioning_consistency_select.sql @@ -1,6 +1,7 @@ -- Tags: no-parallel DROP TABLE IF EXISTS mt; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mt (d Date, x String) ENGINE = MergeTree(d, x, 8192); INSERT INTO mt VALUES ('2106-02-07', 'Hello'), ('1970-01-01', 'World'); diff --git a/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql index 9659db7973d..97b0182f06f 100644 --- a/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql +++ b/tests/queries/0_stateless/01253_subquery_in_aggregate_function_JustStranger.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS test_table; DROP TABLE IF EXISTS test_table_sharded; +set allow_deprecated_syntax_for_merge_tree=1; create table test_table_sharded( date Date, diff --git a/tests/queries/0_stateless/01355_alter_column_with_order.sql b/tests/queries/0_stateless/01355_alter_column_with_order.sql index c0c85e389ea..0b1b4c42cce 100644 --- a/tests/queries/0_stateless/01355_alter_column_with_order.sql +++ b/tests/queries/0_stateless/01355_alter_column_with_order.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS alter_test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192); ALTER TABLE alter_test ADD COLUMN Added1 UInt32 FIRST; diff --git a/tests/queries/0_stateless/01387_clear_column_default_depends.sql b/tests/queries/0_stateless/01387_clear_column_default_depends.sql index 21a41f09df6..733daafa91c 100644 --- a/tests/queries/0_stateless/01387_clear_column_default_depends.sql +++ b/tests/queries/0_stateless/01387_clear_column_default_depends.sql @@ -31,6 +31,7 @@ DROP TABLE test; -- The original report from Mikhail Petrov DROP TABLE IF EXISTS Test; +set allow_deprecated_syntax_for_merge_tree=1; create table Test (impression_id String,impression_id_compressed FixedString(16) DEFAULT UUIDStringToNum(substring(impression_id, 1, 36)), impression_id_hashed UInt16 DEFAULT reinterpretAsUInt16(impression_id_compressed), event_date Date ) ENGINE = MergeTree(event_date, impression_id_hashed, (event_date, impression_id_hashed), 8192); alter table Test clear column impression_id in partition '202001'; DROP TABLE Test; diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh index 1f67100a4b6..12f104b5337 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose.sh @@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze SELECT number, toStri # also for old syntax ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_old_syntax;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_old_syntax (dt Date, value String) ENGINE = MergeTree(dt, (value), 8192);" +${CLICKHOUSE_CLIENT} --allow_deprecated_syntax_for_merge_tree=1 --query "CREATE TABLE table_for_freeze_old_syntax (dt Date, value String) ENGINE = MergeTree(dt, (value), 8192);" ${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_old_syntax SELECT toDate('2021-03-01'), toString(number) from numbers(10);" ${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ diff --git a/tests/queries/0_stateless/01430_modify_sample_by_zookeeper_long.sql b/tests/queries/0_stateless/01430_modify_sample_by_zookeeper_long.sql index 223c27804df..752bc6b377f 100644 --- a/tests/queries/0_stateless/01430_modify_sample_by_zookeeper_long.sql +++ b/tests/queries/0_stateless/01430_modify_sample_by_zookeeper_long.sql @@ -37,6 +37,7 @@ ATTACH TABLE modify_sample_replicated; SELECT count(), min(y), max(y), sum(y), uniqExact(y) FROM modify_sample_replicated SAMPLE 0.1; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE modify_sample_old (d Date DEFAULT '2000-01-01', x UInt8, y UInt64) ENGINE = MergeTree(d, (x, y), 8192); ALTER TABLE modify_sample_old MODIFY SAMPLE BY x; -- { serverError 36 } diff --git a/tests/queries/0_stateless/01455_shard_leaf_max_rows_bytes_to_read.sql b/tests/queries/0_stateless/01455_shard_leaf_max_rows_bytes_to_read.sql index a271295a0cd..620daeb9f35 100644 --- a/tests/queries/0_stateless/01455_shard_leaf_max_rows_bytes_to_read.sql +++ b/tests/queries/0_stateless/01455_shard_leaf_max_rows_bytes_to_read.sql @@ -20,6 +20,7 @@ SELECT count() FROM (SELECT * FROM remote('127.0.0.2', system.numbers) LIMIT 100 DROP TABLE IF EXISTS test_local; DROP TABLE IF EXISTS test_distributed; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_local (date Date, value UInt32) ENGINE = MergeTree(date, date, 8192); CREATE TABLE test_distributed AS test_local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_local, rand()); diff --git a/tests/queries/0_stateless/01648_mutations_and_escaping.sql b/tests/queries/0_stateless/01648_mutations_and_escaping.sql index 689da842f16..18bd0b8ee3b 100644 --- a/tests/queries/0_stateless/01648_mutations_and_escaping.sql +++ b/tests/queries/0_stateless/01648_mutations_and_escaping.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS mutations_and_escaping_1648; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mutations_and_escaping_1648 (d Date, e Enum8('foo'=1, 'bar'=2)) Engine = MergeTree(d, (d), 8192); INSERT INTO mutations_and_escaping_1648 (d, e) VALUES ('2018-01-01', 'foo'); INSERT INTO mutations_and_escaping_1648 (d, e) VALUES ('2018-01-02', 'bar'); diff --git a/tests/queries/0_stateless/01713_table_ttl_old_syntax_zookeeper.sql b/tests/queries/0_stateless/01713_table_ttl_old_syntax_zookeeper.sql index 1709a2d412a..11346a812f2 100644 --- a/tests/queries/0_stateless/01713_table_ttl_old_syntax_zookeeper.sql +++ b/tests/queries/0_stateless/01713_table_ttl_old_syntax_zookeeper.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS ttl_table; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE ttl_table ( date Date, diff --git a/tests/queries/0_stateless/01778_test_LowCardinality_FixedString_pk.sql b/tests/queries/0_stateless/01778_test_LowCardinality_FixedString_pk.sql index 1a0a1d35f76..78a9b35a40e 100644 --- a/tests/queries/0_stateless/01778_test_LowCardinality_FixedString_pk.sql +++ b/tests/queries/0_stateless/01778_test_LowCardinality_FixedString_pk.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test_01778; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_01778 ( `key` LowCardinality(FixedString(3)), diff --git a/tests/queries/0_stateless/01798_uniq_theta_sketch.sql b/tests/queries/0_stateless/01798_uniq_theta_sketch.sql index eace83d5cfa..59d5888c0a1 100644 --- a/tests/queries/0_stateless/01798_uniq_theta_sketch.sql +++ b/tests/queries/0_stateless/01798_uniq_theta_sketch.sql @@ -107,6 +107,7 @@ EXPLAIN SYNTAX select uniqTheta(-bitNot(-x)) from (select number % 2 as x from n DROP TABLE IF EXISTS stored_aggregates; -- simple +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE stored_aggregates ( d Date, diff --git a/tests/queries/0_stateless/01925_broken_partition_id_zookeeper.sql b/tests/queries/0_stateless/01925_broken_partition_id_zookeeper.sql index 7d151f063e1..9c6aa3146ee 100644 --- a/tests/queries/0_stateless/01925_broken_partition_id_zookeeper.sql +++ b/tests/queries/0_stateless/01925_broken_partition_id_zookeeper.sql @@ -19,6 +19,7 @@ DROP TABLE IF EXISTS broken_partition; DROP TABLE IF EXISTS old_partition_key; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE old_partition_key (sd Date, dh UInt64, ak UInt32, ed Date) ENGINE=MergeTree(sd, dh, (ak, ed, dh), 8192); ALTER TABLE old_partition_key DROP PARTITION ID '20210325_0_13241_6_12747'; --{serverError 248} diff --git a/tests/queries/0_stateless/02250_hints_for_columns.sh b/tests/queries/0_stateless/02250_hints_for_columns.sh index 45fd2f238b1..ea08d8c5535 100755 --- a/tests/queries/0_stateless/02250_hints_for_columns.sh +++ b/tests/queries/0_stateless/02250_hints_for_columns.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS t" -$CLICKHOUSE_CLIENT --query="CREATE TABLE t (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 --query="CREATE TABLE t (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" $CLICKHOUSE_CLIENT --query="ALTER TABLE t DROP COLUMN ToDro" 2>&1 | grep -q "Maybe you meant: \['ToDrop'\]" && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/1_stateful/00040_aggregating_materialized_view.sql b/tests/queries/1_stateful/00040_aggregating_materialized_view.sql index 555a8b64d75..99fed62683f 100644 --- a/tests/queries/1_stateful/00040_aggregating_materialized_view.sql +++ b/tests/queries/1_stateful/00040_aggregating_materialized_view.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test.basic_00040; +set allow_deprecated_syntax_for_merge_tree=1; CREATE MATERIALIZED VIEW test.basic_00040 ENGINE = AggregatingMergeTree(StartDate, (CounterID, StartDate), 8192) POPULATE AS diff --git a/tests/queries/1_stateful/00041_aggregating_materialized_view.sql b/tests/queries/1_stateful/00041_aggregating_materialized_view.sql index 0e59d2a88a9..ea99b78902c 100644 --- a/tests/queries/1_stateful/00041_aggregating_materialized_view.sql +++ b/tests/queries/1_stateful/00041_aggregating_materialized_view.sql @@ -9,6 +9,7 @@ CREATE TABLE test.visits_null UserID UInt64 ) ENGINE = Null; +set allow_deprecated_syntax_for_merge_tree=1; CREATE MATERIALIZED VIEW test.basic ENGINE = AggregatingMergeTree(StartDate, (CounterID, StartDate), 8192) AS SELECT diff --git a/tests/queries/1_stateful/00054_merge_tree_partitions.sql b/tests/queries/1_stateful/00054_merge_tree_partitions.sql index 73020952977..33f7e4e8666 100644 --- a/tests/queries/1_stateful/00054_merge_tree_partitions.sql +++ b/tests/queries/1_stateful/00054_merge_tree_partitions.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS test.partitions; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test.partitions (EventDate Date, CounterID UInt32) ENGINE = MergeTree(EventDate, CounterID, 8192); INSERT INTO test.partitions SELECT EventDate + UserID % 365 AS EventDate, CounterID FROM test.hits WHERE CounterID = 1704509; diff --git a/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql b/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql index 16c0097bf21..e0575122644 100644 --- a/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql +++ b/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test.hits_snippet; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test.hits_snippet(EventTime DateTime('Asia/Dubai'), EventDate Date, CounterID UInt32, UserID UInt64, URL String, Referer String) ENGINE = MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; From cd8e5c7c49d4246014ce4cf423d087278745cfd3 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Thu, 23 Jun 2022 17:43:54 +0800 Subject: [PATCH 091/408] update headers --- src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp | 3 +-- src/Storages/Hive/StorageHive.cpp | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index eefe4231b89..f922ff048b2 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,5 +1,4 @@ -#include -#include +#include "ArrowColumnToCHColumn.h" #if USE_ARROW || USE_ORC || USE_PARQUET diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 6d298c0033c..b717d373598 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -1,4 +1,3 @@ -#include #include #if USE_HIVE From 944c247345da5014e22c72fe6e388eb7a3668669 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 22 Jun 2022 19:23:22 +0000 Subject: [PATCH 092/408] DISTINCT in order optimization + try use the optimization for final distinct in case of sorted stream (sorting inside and among chunks) + sorting description contains only columns from sorting key which are in header as well --- src/Processors/QueryPlan/DistinctStep.cpp | 45 +++++++++++++------ src/Processors/QueryPlan/DistinctStep.h | 1 - .../QueryPlan/ReadFromMergeTree.cpp | 12 +++-- ...m.cpp => DistinctSortedChunkTransform.cpp} | 20 ++++----- ...sform.h => DistinctSortedChunkTransform.h} | 10 ++--- src/Storages/ReadInOrderOptimizer.cpp | 8 ++-- src/Storages/SelectQueryInfo.h | 6 +-- ...7_distinct_in_order_optimization.reference | 6 +-- .../02317_distinct_in_order_optimization.sql | 4 +- 9 files changed, 64 insertions(+), 48 deletions(-) rename src/Processors/Transforms/{DistinctPrimaryKeyTransform.cpp => DistinctSortedChunkTransform.cpp} (88%) rename src/Processors/Transforms/{DistinctPrimaryKeyTransform.h => DistinctSortedChunkTransform.h} (82%) diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 9e8eabc53c8..c5e478eca8c 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -1,5 +1,6 @@ #include -#include +#include +#include #include #include #include @@ -87,21 +88,39 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil if (!pre_distinct) pipeline.resize(1); - if (pre_distinct && optimize_distinct_in_order) + if (optimize_distinct_in_order) { - if (SortDescription distinct_sort_desc = getSortDescription(input_streams.front().sort_description, columns); - !distinct_sort_desc.empty()) + const auto & input_stream = input_streams.back(); + SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns); + if (!distinct_sort_desc.empty()) { - pipeline.addSimpleTransform( - [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr - { - if (stream_type != QueryPipelineBuilder::StreamType::Main) - return nullptr; + /// pre-distinct for sorted chunks + if (pre_distinct) + { + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; - return std::make_shared( - header, set_size_limits, limit_hint, distinct_sort_desc, columns); - }); - return; + return std::make_shared( + header, set_size_limits, limit_hint, distinct_sort_desc, columns); + }); + return; + } + /// final distinct for sorted stream (sorting inside and among chunks) + if (input_stream.has_single_port) + { + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; + + return std::make_shared(header, distinct_sort_desc, set_size_limits, limit_hint, columns); + }); + return; + } } } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index 7009739e3ad..7c34aa01dd9 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include namespace DB { diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index d6425929d08..e20585ef64d 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -118,12 +118,16 @@ ReadFromMergeTree::ReadFromMergeTree( /// Add explicit description. setStepDescription(data.getStorageID().getFullNameNotQuoted()); - Names sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); - + /// build sort description for output stream SortDescription sort_description; - for (const auto & column : sorting_key_columns) + const Names sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); + Block const & header = output_stream->header; + for (const auto & column_name : sorting_key_columns) { - sort_description.emplace_back(column, 1); + if (std::find_if(header.begin(), header.end(), [&](ColumnWithTypeAndName const & col) { return col.name == column_name; }) + == header.end()) + break; + sort_description.emplace_back(column_name, 1); } output_stream->sort_description = std::move(sort_description); } diff --git a/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp similarity index 88% rename from src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp rename to src/Processors/Transforms/DistinctSortedChunkTransform.cpp index b3e3e15d963..213fcd6264a 100644 --- a/src/Processors/Transforms/DistinctPrimaryKeyTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp @@ -1,4 +1,4 @@ -#include +#include namespace DB { @@ -8,7 +8,7 @@ namespace ErrorCodes extern const int SET_SIZE_LIMIT_EXCEEDED; } -DistinctPrimaryKeyTransform::DistinctPrimaryKeyTransform( +DistinctSortedChunkTransform::DistinctSortedChunkTransform( const Block & header_, const SizeLimits & output_size_limits_, UInt64 limit_hint_, @@ -46,7 +46,7 @@ DistinctPrimaryKeyTransform::DistinctPrimaryKeyTransform( current_key.reserve(sorted_columns.size()); } -void DistinctPrimaryKeyTransform::initChunkProcessing(const Columns & input_columns) +void DistinctSortedChunkTransform::initChunkProcessing(const Columns & input_columns) { sorted_columns.clear(); for (size_t pos : sorted_columns_pos) @@ -60,7 +60,7 @@ void DistinctPrimaryKeyTransform::initChunkProcessing(const Columns & input_colu data.init(ClearableSetVariants::chooseMethod(other_columns, other_columns_sizes)); } -size_t DistinctPrimaryKeyTransform::ordinaryDistinctOnRange(IColumn::Filter & filter, size_t range_begin, size_t range_end) +size_t DistinctSortedChunkTransform::ordinaryDistinctOnRange(IColumn::Filter & filter, size_t range_begin, size_t range_end) { size_t count = 0; switch (data.type) @@ -81,7 +81,7 @@ size_t DistinctPrimaryKeyTransform::ordinaryDistinctOnRange(IColumn::Filter & fi } template -size_t DistinctPrimaryKeyTransform::buildFilterForRange( +size_t DistinctSortedChunkTransform::buildFilterForRange( Method & method, IColumn::Filter & filter, size_t range_begin, size_t range_end, ClearableSetVariants & variants) { typename Method::State state(other_columns, other_columns_sizes, nullptr); @@ -100,7 +100,7 @@ size_t DistinctPrimaryKeyTransform::buildFilterForRange( return count; } -void DistinctPrimaryKeyTransform::setCurrentKey(const size_t row_pos) +void DistinctSortedChunkTransform::setCurrentKey(const size_t row_pos) { current_key.clear(); for (auto const & col : sorted_columns) @@ -110,7 +110,7 @@ void DistinctPrimaryKeyTransform::setCurrentKey(const size_t row_pos) } } -bool DistinctPrimaryKeyTransform::isCurrentKey(const size_t row_pos) +bool DistinctSortedChunkTransform::isCurrentKey(const size_t row_pos) { for (size_t i = 0; i < sorted_columns.size(); ++i) { @@ -121,7 +121,7 @@ bool DistinctPrimaryKeyTransform::isCurrentKey(const size_t row_pos) return true; } -size_t DistinctPrimaryKeyTransform::getRangeEnd(size_t range_begin, size_t range_end) +size_t DistinctSortedChunkTransform::getRangeEnd(size_t range_begin, size_t range_end) { size_t low = range_begin; size_t high = range_end - 1; @@ -139,7 +139,7 @@ size_t DistinctPrimaryKeyTransform::getRangeEnd(size_t range_begin, size_t range return range_end; } -size_t DistinctPrimaryKeyTransform::getStartPosition(const size_t chunk_rows) +size_t DistinctSortedChunkTransform::getStartPosition(const size_t chunk_rows) { if (!current_key.empty()) // current_key is empty on very first transform() call { @@ -149,7 +149,7 @@ size_t DistinctPrimaryKeyTransform::getStartPosition(const size_t chunk_rows) return 0; } -void DistinctPrimaryKeyTransform::transform(Chunk & chunk) +void DistinctSortedChunkTransform::transform(Chunk & chunk) { const size_t chunk_rows = chunk.getNumRows(); size_t output_rows = 0; diff --git a/src/Processors/Transforms/DistinctPrimaryKeyTransform.h b/src/Processors/Transforms/DistinctSortedChunkTransform.h similarity index 82% rename from src/Processors/Transforms/DistinctPrimaryKeyTransform.h rename to src/Processors/Transforms/DistinctSortedChunkTransform.h index d862e4a28ce..3a6f46d699e 100644 --- a/src/Processors/Transforms/DistinctPrimaryKeyTransform.h +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.h @@ -10,21 +10,19 @@ namespace DB { /// -/// DISTINCT optimization for MergeTree family engines -/// Applied in case of DISTINCT is done over primary key(prefix) columns -/// It leverages their sorting property +/// DISTINCT optimization for sorted chunks /// -class DistinctPrimaryKeyTransform : public ISimpleTransform +class DistinctSortedChunkTransform : public ISimpleTransform { public: - DistinctPrimaryKeyTransform( + DistinctSortedChunkTransform( const Block & header_, const SizeLimits & output_size_limits_, UInt64 limit_hint_, const SortDescription & sorted_columns_descr_, const Names & source_columns_); - String getName() const override { return "DistinctPrimaryKeyTransform"; } + String getName() const override { return "DistinctSortedChunkTransform"; } protected: void transform(Chunk & chunk) override; diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index e3f1c79719f..3ff4baa0b11 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -1,15 +1,13 @@ #include -#include -#include #include #include -#include #include #include -#include +#include +#include #include -#include +#include namespace DB { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 339e7b609d8..bdb4c392c48 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -95,12 +95,10 @@ struct InputOrderInfo InputOrderInfo( const SortDescription & order_key_fixed_prefix_descr_, const SortDescription & order_key_prefix_descr_, - int direction_, - UInt64 limit_) + int direction_, UInt64 limit_) : order_key_fixed_prefix_descr(order_key_fixed_prefix_descr_) , order_key_prefix_descr(order_key_prefix_descr_) - , direction(direction_) - , limit(limit_) + , direction(direction_), limit(limit_) { } diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index 24c2edca708..8ab6966665a 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -30,7 +30,7 @@ ExpressionTransform (Distinct) DistinctTransform (Distinct) - DistinctPrimaryKeyTransform + DistinctSortedChunkTransform (Expression) ExpressionTransform (ReadFromMergeTree) @@ -42,7 +42,7 @@ ExpressionTransform (Distinct) DistinctTransform (Distinct) - DistinctPrimaryKeyTransform + DistinctSortedChunkTransform (Expression) ExpressionTransform (ReadFromMergeTree) @@ -65,7 +65,7 @@ single-threaded distinct 0 multi-threaded distinct 0 -skip part of chunk since it contians values from previous one +skip part of chunk since it contains values from previous one single-threaded distinct 0 1 diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index f718986ae70..519b511dddb 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -33,7 +33,7 @@ select distinct * from distinct_in_order settings max_block_size=10, max_threads select 'multi-threaded distinct'; select distinct * from distinct_in_order settings max_block_size=10; -select 'skip part of chunk since it contians values from previous one'; +select 'skip part of chunk since it contains values from previous one'; drop table if exists distinct_in_order sync; create table distinct_in_order (a int) engine=MergeTree() order by a settings index_granularity=10; insert into distinct_in_order (a) select * from zeros(10); @@ -54,4 +54,4 @@ select distinct a,b from distinct_in_order order by b; select 'distinct with key prefix and non-sorted column'; select distinct a,c from distinct_in_order order by c; -drop table if exists distinct_in_order sync; +-- drop table if exists distinct_in_order sync; From d2f3c5b83643674c0bd6f9ae51b8fb48cfca94af Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:45:54 +0200 Subject: [PATCH 093/408] Add sleeping after received `second rate limit exceeded` --- utils/changelog/changelog.py | 41 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index e1797fb5dd3..9d1dabbabe1 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -10,11 +10,12 @@ from datetime import date, datetime, timedelta from queue import Empty, Queue from subprocess import CalledProcessError, DEVNULL from threading import Thread +from time import sleep from typing import Dict, List, Optional, TextIO from fuzzywuzzy.fuzz import ratio # type: ignore from github import Github -from github.GithubException import UnknownObjectException +from github.GithubException import RateLimitExceededException, UnknownObjectException from github.NamedUser import NamedUser from github.Issue import Issue from github.PullRequest import PullRequest @@ -67,10 +68,17 @@ class Description: r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", entry, ) - try: - user_name = self.user.name if self.user.name else self.user.login - except UnknownObjectException: - user_name = self.user.login + # It's possible that we face a secondary rate limit. + # In this case we should sleep until we get it + while True: + try: + user_name = self.user.name if self.user.name else self.user.login + break + except UnknownObjectException: + user_name = self.user.login + break + except RateLimitExceededException: + sleep_on_rate_limit() return ( f"* {entry} [#{self.number}]({self.html_url}) " f"([{user_name}]({self.user.html_url}))." @@ -118,6 +126,11 @@ class Worker(Thread): self.queue.task_done() +def sleep_on_rate_limit(time: int = 20): + logging.warning("Faced rate limit, sleeping %s", time) + sleep(time) + + def get_pull_cached( repo: Repository, number: int, updated_at: Optional[datetime] = None ) -> PullRequest: @@ -130,7 +143,12 @@ def get_pull_cached( if cache_updated > updated_at: with open(pr_cache_file, "rb") as prfd: return GitHub.load(prfd) # type: ignore - pr = repo.get_pull(number) + while True: + try: + pr = repo.get_pull(number) + break + except RateLimitExceededException: + sleep_on_rate_limit() with open(pr_cache_file, "wb") as prfd: GitHub.dump(pr, prfd) # type: ignore return pr @@ -414,9 +432,16 @@ def main(): api_prs = GitHub.search_issues(query=query, sort="created") logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query) - pr_numbers = list(api_prs) + issues = [] # type: List[Issue] + while True: + try: + for issue in api_prs: + issues.append(issue) + break + except RateLimitExceededException: + sleep_on_rate_limit() - descriptions = get_descriptions(repo, pr_numbers, args.jobs) + descriptions = get_descriptions(repo, issues, args.jobs) write_changelog(args.output, descriptions) From 0b871a20500f2c6ccab132a5196583479fd51bd6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:47:26 +0200 Subject: [PATCH 094/408] Regenerate v22.* with `not for changelog` category --- docs/changelogs/v22.1.1.2542-prestable.md | 248 ++++++++++++++++++++++ docs/changelogs/v22.2.1.2139-prestable.md | 161 ++++++++++++++ docs/changelogs/v22.2.3.5-stable.md | 4 + docs/changelogs/v22.3.1.1262-prestable.md | 121 +++++++++++ docs/changelogs/v22.3.3.44-lts.md | 8 + docs/changelogs/v22.3.6.5-lts.md | 4 + docs/changelogs/v22.3.7.28-lts.md | 7 + docs/changelogs/v22.4.1.2305-prestable.md | 205 ++++++++++++++++++ docs/changelogs/v22.4.4.7-stable.md | 4 + docs/changelogs/v22.4.5.9-stable.md | 4 + docs/changelogs/v22.5.1.2079-stable.md | 201 ++++++++++++++++++ docs/changelogs/v22.6.1.1985-stable.md | 161 ++++++++++++++ 12 files changed, 1128 insertions(+) diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index cbe8e781d3c..a0c6f61d5a6 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -218,6 +218,254 @@ sidebar_label: 2022 * NO CL ENTRY: 'Added Superwall to adopters list'. [#33573](https://github.com/ClickHouse/ClickHouse/pull/33573) ([Justin Hilliard](https://github.com/jahilliard)). * NO CL ENTRY: 'Revert "Ignore parse failure of opentelemetry header"'. [#33594](https://github.com/ClickHouse/ClickHouse/pull/33594) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merging [#26074](https://github.com/ClickHouse/ClickHouse/issues/26074) [#26559](https://github.com/ClickHouse/ClickHouse/pull/26559) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Add performance tests to new CI [#31467](https://github.com/ClickHouse/ClickHouse/pull/31467) ([alesapin](https://github.com/alesapin)). +* refactor CI tests [#31882](https://github.com/ClickHouse/ClickHouse/pull/31882) ([Constantine Peresypkin](https://github.com/pkit)). +* Add ability to drain connections synchronously [#31965](https://github.com/ClickHouse/ClickHouse/pull/31965) ([Azat Khuzhin](https://github.com/azat)). +* Add docker container for AWS lambda [#32205](https://github.com/ClickHouse/ClickHouse/pull/32205) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* test for [#28280](https://github.com/ClickHouse/ClickHouse/issues/28280) [#32235](https://github.com/ClickHouse/ClickHouse/pull/32235) ([Denny Crane](https://github.com/den-crane)). +* Rename window functions in WindowView [#32324](https://github.com/ClickHouse/ClickHouse/pull/32324) ([vxider](https://github.com/Vxider)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-local: fix CREATE DATABASE with Atomic engine [#32417](https://github.com/ClickHouse/ClickHouse/pull/32417) ([Azat Khuzhin](https://github.com/azat)). +* Add a test [#16171](https://github.com/ClickHouse/ClickHouse/issues/16171) [#32421](https://github.com/ClickHouse/ClickHouse/pull/32421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix images names in integration tests [#32438](https://github.com/ClickHouse/ClickHouse/pull/32438) ([alesapin](https://github.com/alesapin)). +* Fix build check empty report [#32440](https://github.com/ClickHouse/ClickHouse/pull/32440) ([alesapin](https://github.com/alesapin)). +* Fix ASTFuzzer [#32447](https://github.com/ClickHouse/ClickHouse/pull/32447) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix attaching gdb in tests [#32448](https://github.com/ClickHouse/ClickHouse/pull/32448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix unit tests (ubsan) on master [#32459](https://github.com/ClickHouse/ClickHouse/pull/32459) ([alesapin](https://github.com/alesapin)). +* Remove dependency between integration and functional tests [#32461](https://github.com/ClickHouse/ClickHouse/pull/32461) ([alesapin](https://github.com/alesapin)). +* Add automatic workflow rerun [#32462](https://github.com/ClickHouse/ClickHouse/pull/32462) ([alesapin](https://github.com/alesapin)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update odbc-bridge.md [#32475](https://github.com/ClickHouse/ClickHouse/pull/32475) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test_prometheus_endpoint [#32476](https://github.com/ClickHouse/ClickHouse/pull/32476) ([Vladimir C](https://github.com/vdimir)). +* 02122_parallel_formatting: Address grep binary warnings [#32477](https://github.com/ClickHouse/ClickHouse/pull/32477) ([Raúl Marín](https://github.com/Algunenano)). +* 01950_kill_large_group_by_query: Increase timeout [#32479](https://github.com/ClickHouse/ClickHouse/pull/32479) ([Raúl Marín](https://github.com/Algunenano)). +* Fix backport workflow [#32485](https://github.com/ClickHouse/ClickHouse/pull/32485) ([alesapin](https://github.com/alesapin)). +* Make 01675_distributed_bytes_to_delay_insert less flaky (use http over client) [#32492](https://github.com/ClickHouse/ClickHouse/pull/32492) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02050_client_profile_events flakiness [#32493](https://github.com/ClickHouse/ClickHouse/pull/32493) ([Azat Khuzhin](https://github.com/azat)). +* Fix integration tests docker images path [#32494](https://github.com/ClickHouse/ClickHouse/pull/32494) ([alesapin](https://github.com/alesapin)). +* Split long tests into multiple checks [#32496](https://github.com/ClickHouse/ClickHouse/pull/32496) ([alesapin](https://github.com/alesapin)). +* Add test for clickhouse-client history navigation [#32497](https://github.com/ClickHouse/ClickHouse/pull/32497) ([Amos Bird](https://github.com/amosbird)). +* Rename window-view function to time window function [#32498](https://github.com/ClickHouse/ClickHouse/pull/32498) ([vxider](https://github.com/Vxider)). +* Disable flaky tests with Window View [#32500](https://github.com/ClickHouse/ClickHouse/pull/32500) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky window view tests [#32503](https://github.com/ClickHouse/ClickHouse/pull/32503) ([vxider](https://github.com/Vxider)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to fix attaching gdb 2 [#32511](https://github.com/ClickHouse/ClickHouse/pull/32511) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: use basename of the test for *.sh tests [#32533](https://github.com/ClickHouse/ClickHouse/pull/32533) ([Azat Khuzhin](https://github.com/azat)). +* Fix processing initial table (--table/stdin) in clickhouse-local [#32534](https://github.com/ClickHouse/ClickHouse/pull/32534) ([Azat Khuzhin](https://github.com/azat)). +* Suppress UBSan errors for avg() function [#32535](https://github.com/ClickHouse/ClickHouse/pull/32535) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR for MATERIALIZED VIEW over table functions (i.e. numbers()) [#32571](https://github.com/ClickHouse/ClickHouse/pull/32571) ([Azat Khuzhin](https://github.com/azat)). +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* remove unused headers in test [#32578](https://github.com/ClickHouse/ClickHouse/pull/32578) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove arcadia build support [#32580](https://github.com/ClickHouse/ClickHouse/pull/32580) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup perf test runner [#32582](https://github.com/ClickHouse/ClickHouse/pull/32582) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] perf: do not fail in case of slow queries (to avoid hiding possible issues) [#32583](https://github.com/ClickHouse/ClickHouse/pull/32583) ([Azat Khuzhin](https://github.com/azat)). +* Add test for issue 28316 [#32585](https://github.com/ClickHouse/ClickHouse/pull/32585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bad test [#32571](https://github.com/ClickHouse/ClickHouse/issues/32571) [#32588](https://github.com/ClickHouse/ClickHouse/pull/32588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix perf test `reinterpret_as` [#32589](https://github.com/ClickHouse/ClickHouse/pull/32589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecent error message [#32591](https://github.com/ClickHouse/ClickHouse/pull/32591) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange code in TCPHandler [#32592](https://github.com/ClickHouse/ClickHouse/pull/32592) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CI: upload logs if task failed [#32618](https://github.com/ClickHouse/ClickHouse/pull/32618) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: avoid failing with UNKNOWN status in some cases [#32625](https://github.com/ClickHouse/ClickHouse/pull/32625) ([Azat Khuzhin](https://github.com/azat)). +* avoid inconsistent state in Block [#32641](https://github.com/ClickHouse/ClickHouse/pull/32641) ([gulige](https://github.com/gulige)). +* Rerun workflows more times [#32642](https://github.com/ClickHouse/ClickHouse/pull/32642) ([alesapin](https://github.com/alesapin)). +* Remove flaky check from master and split asan test [#32645](https://github.com/ClickHouse/ClickHouse/pull/32645) ([alesapin](https://github.com/alesapin)). +* Split database replicated [#32650](https://github.com/ClickHouse/ClickHouse/pull/32650) ([alesapin](https://github.com/alesapin)). +* Improve exceptions usage in access control [#32662](https://github.com/ClickHouse/ClickHouse/pull/32662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32667](https://github.com/ClickHouse/ClickHouse/pull/32667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test_s3_zero_copy_concurrent_merge [#32694](https://github.com/ClickHouse/ClickHouse/pull/32694) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Check trusted contributors in lowercase [#32696](https://github.com/ClickHouse/ClickHouse/pull/32696) ([Vladimir C](https://github.com/vdimir)). +* Mention ClickHouse CLA in CONTRIBUTING.md [#32697](https://github.com/ClickHouse/ClickHouse/pull/32697) ([Ivan Blinkov](https://github.com/blinkov)). +* Rerun release workflows [#32701](https://github.com/ClickHouse/ClickHouse/pull/32701) ([alesapin](https://github.com/alesapin)). +* plausible analytic blog post [#32719](https://github.com/ClickHouse/ClickHouse/pull/32719) ([Tom Risse](https://github.com/flickerbox-tom)). +* Update backport.py [#32720](https://github.com/ClickHouse/ClickHouse/pull/32720) ([Kruglov Pavel](https://github.com/Avogar)). +* Pass -no-pie to the linker call to disable it [#32731](https://github.com/ClickHouse/ClickHouse/pull/32731) ([Raúl Marín](https://github.com/Algunenano)). +* Add severity table to support case form [#32736](https://github.com/ClickHouse/ClickHouse/pull/32736) ([Cody Baker](https://github.com/codyrobert)). +* tests/integration: fix wait_start()/start_clickhouse() [#32739](https://github.com/ClickHouse/ClickHouse/pull/32739) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration: fix waiting of mysql client container [#32740](https://github.com/ClickHouse/ClickHouse/pull/32740) ([Azat Khuzhin](https://github.com/azat)). +* Better control build artifacts [#32745](https://github.com/ClickHouse/ClickHouse/pull/32745) ([alesapin](https://github.com/alesapin)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build performance [#32766](https://github.com/ClickHouse/ClickHouse/pull/32766) ([alesapin](https://github.com/alesapin)). +* Add test for issue 26912 [#32771](https://github.com/ClickHouse/ClickHouse/pull/32771) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for CAST to named tuple just in case. [#32776](https://github.com/ClickHouse/ClickHouse/pull/32776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance build [#32796](https://github.com/ClickHouse/ClickHouse/pull/32796) ([alesapin](https://github.com/alesapin)). +* Trying another stress servers [#32798](https://github.com/ClickHouse/ClickHouse/pull/32798) ([alesapin](https://github.com/alesapin)). +* More diagnostics in gdb script [#32801](https://github.com/ClickHouse/ClickHouse/pull/32801) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CacheDictionary dictionary source access race fix [#32805](https://github.com/ClickHouse/ClickHouse/pull/32805) ([Maksim Kita](https://github.com/kitaisreal)). +* Update NuRaft [#32808](https://github.com/ClickHouse/ClickHouse/pull/32808) ([alesapin](https://github.com/alesapin)). +* Add retries to curl in performance tests. [#32809](https://github.com/ClickHouse/ClickHouse/pull/32809) ([alesapin](https://github.com/alesapin)). +* Made Azure blob storage to obey ENABLE_LIBRARIES flag [#32815](https://github.com/ClickHouse/ClickHouse/pull/32815) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove recursive submodules [#32820](https://github.com/ClickHouse/ClickHouse/pull/32820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* test for summap nullable(0) [#32822](https://github.com/ClickHouse/ClickHouse/pull/32822) ([Denny Crane](https://github.com/den-crane)). +* Update CHANGELOG.md [#32823](https://github.com/ClickHouse/ClickHouse/pull/32823) ([Denny Crane](https://github.com/den-crane)). +* Make `test_storage_s3` less flaky [#32825](https://github.com/ClickHouse/ClickHouse/pull/32825) ([Azat Khuzhin](https://github.com/azat)). +* Fix error message for reading from url [#32826](https://github.com/ClickHouse/ClickHouse/pull/32826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert docs release [#32827](https://github.com/ClickHouse/ClickHouse/pull/32827) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32828](https://github.com/ClickHouse/ClickHouse/pull/32828) ([alesapin](https://github.com/alesapin)). +* Trigger also for workflows changes [#32830](https://github.com/ClickHouse/ClickHouse/pull/32830) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32831](https://github.com/ClickHouse/ClickHouse/pull/32831) ([alesapin](https://github.com/alesapin)). +* Add dispatch event to docs release [#32832](https://github.com/ClickHouse/ClickHouse/pull/32832) ([alesapin](https://github.com/alesapin)). +* Remove repository before checkout [#32834](https://github.com/ClickHouse/ClickHouse/pull/32834) ([alesapin](https://github.com/alesapin)). +* Fix release branches filter [#32837](https://github.com/ClickHouse/ClickHouse/pull/32837) ([alesapin](https://github.com/alesapin)). +* StorageLiveView fix function style [#32838](https://github.com/ClickHouse/ClickHouse/pull/32838) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flaky test: order for test_s3_zero_copy_replication [#32846](https://github.com/ClickHouse/ClickHouse/pull/32846) ([Vladimir C](https://github.com/vdimir)). +* Update TRUSTED_CONTIBUTORS [#32850](https://github.com/ClickHouse/ClickHouse/pull/32850) ([Vladimir C](https://github.com/vdimir)). +* Followup docs check workflow [#32857](https://github.com/ClickHouse/ClickHouse/pull/32857) ([alesapin](https://github.com/alesapin)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use a proper syntax for multiline env in GITHUB_ENV [#32864](https://github.com/ClickHouse/ClickHouse/pull/32864) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix multiline SSH_KEY in GITHUB_ENV file [#32865](https://github.com/ClickHouse/ClickHouse/pull/32865) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* warm up for test_kafka_json_as_string_no_kdc [#32885](https://github.com/ClickHouse/ClickHouse/pull/32885) ([Ilya Golshtein](https://github.com/ilejn)). +* Rename files and code from BlobStorage to AzureBlobStorage [#32886](https://github.com/ClickHouse/ClickHouse/pull/32886) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add dockerhub proxy to CI [#32905](https://github.com/ClickHouse/ClickHouse/pull/32905) ([alesapin](https://github.com/alesapin)). +* Update 01069_window_view_proc_tumble_watch.py [#32908](https://github.com/ClickHouse/ClickHouse/pull/32908) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update test azure_blob_storage::test_table_manipulations [#32909](https://github.com/ClickHouse/ClickHouse/pull/32909) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More retries in perf tests. [#32910](https://github.com/ClickHouse/ClickHouse/pull/32910) ([alesapin](https://github.com/alesapin)). +* fix wrong testcase in gtest_Parser [#32912](https://github.com/ClickHouse/ClickHouse/pull/32912) ([SuperDJY](https://github.com/cmsxbc)). +* Disable percpu arena in jemalloc in case of non deterministic CPU count [#32916](https://github.com/ClickHouse/ClickHouse/pull/32916) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Add replxx/jemalloc into fasttest build of clickhouse [#32923](https://github.com/ClickHouse/ClickHouse/pull/32923) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration/cleanup_environment: fix obtaining PYTEST_CLEANUP_CONTAINERS [#32931](https://github.com/ClickHouse/ClickHouse/pull/32931) ([Azat Khuzhin](https://github.com/azat)). +* fix typo [#32936](https://github.com/ClickHouse/ClickHouse/pull/32936) ([Denny Crane](https://github.com/den-crane)). +* Fix build issue related to azure blob storage [another try] [#32948](https://github.com/ClickHouse/ClickHouse/pull/32948) ([Amos Bird](https://github.com/amosbird)). +* test for base64encode_trailing_bytes [#31797](https://github.com/ClickHouse/ClickHouse/issues/31797) [#32954](https://github.com/ClickHouse/ClickHouse/pull/32954) ([Denny Crane](https://github.com/den-crane)). +* simplify looping in S2 functions [#32960](https://github.com/ClickHouse/ClickHouse/pull/32960) ([Bharat Nallan](https://github.com/bharatnc)). +* Sync release CI branches in master [#32967](https://github.com/ClickHouse/ClickHouse/pull/32967) ([alesapin](https://github.com/alesapin)). +* Loops remove postfix increment [#32970](https://github.com/ClickHouse/ClickHouse/pull/32970) ([Maksim Kita](https://github.com/kitaisreal)). +* Containers iteration fix erase [#32972](https://github.com/ClickHouse/ClickHouse/pull/32972) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#32400](https://github.com/ClickHouse/ClickHouse/issues/32400) [#32974](https://github.com/ClickHouse/ClickHouse/pull/32974) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Decrease log level for some s3 messages. [#32979](https://github.com/ClickHouse/ClickHouse/pull/32979) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to add woboq to new CI [#32983](https://github.com/ClickHouse/ClickHouse/pull/32983) ([alesapin](https://github.com/alesapin)). +* RabbitMQ tests fix [#32985](https://github.com/ClickHouse/ClickHouse/pull/32985) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#32987](https://github.com/ClickHouse/ClickHouse/pull/32987) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo [#32992](https://github.com/ClickHouse/ClickHouse/pull/32992) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix weird permission on log directories [#32994](https://github.com/ClickHouse/ClickHouse/pull/32994) ([Håvard Kvålen](https://github.com/havardk)). +* Add Keeper Jepsen check to new CI [#32998](https://github.com/ClickHouse/ClickHouse/pull/32998) ([alesapin](https://github.com/alesapin)). +* Add backport workflow rerun [#33005](https://github.com/ClickHouse/ClickHouse/pull/33005) ([alesapin](https://github.com/alesapin)). +* Remove the possibility of adding columns with table overrides [#33017](https://github.com/ClickHouse/ClickHouse/pull/33017) ([Stig Bakken](https://github.com/stigsb)). +* Merge [#33023](https://github.com/ClickHouse/ClickHouse/issues/33023) [#33027](https://github.com/ClickHouse/ClickHouse/pull/33027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cron expression [#33035](https://github.com/ClickHouse/ClickHouse/pull/33035) ([alesapin](https://github.com/alesapin)). +* Make `can be tested` great again [#33036](https://github.com/ClickHouse/ClickHouse/pull/33036) ([alesapin](https://github.com/alesapin)). +* Fix exception in azure write buffer desctructor [#33039](https://github.com/ClickHouse/ClickHouse/pull/33039) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add manual run for woboq [#33042](https://github.com/ClickHouse/ClickHouse/pull/33042) ([alesapin](https://github.com/alesapin)). +* Update Cassandra driver [#33043](https://github.com/ClickHouse/ClickHouse/pull/33043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Decrease default timeout for integration test [#33044](https://github.com/ClickHouse/ClickHouse/pull/33044) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix merge [#33021](https://github.com/ClickHouse/ClickHouse/issues/33021) [#33049](https://github.com/ClickHouse/ClickHouse/pull/33049) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update banner for january 20 release webinar [#33052](https://github.com/ClickHouse/ClickHouse/pull/33052) ([Cody Baker](https://github.com/codyrobert)). +* Update harmful library [#33058](https://github.com/ClickHouse/ClickHouse/pull/33058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33025](https://github.com/ClickHouse/ClickHouse/issues/33025) [#33063](https://github.com/ClickHouse/ClickHouse/pull/33063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add alternative LRUCache version [#33098](https://github.com/ClickHouse/ClickHouse/pull/33098) ([lgbo](https://github.com/lgbo-ustc)). +* Fix test that was dependent on time zone [#33122](https://github.com/ClickHouse/ClickHouse/pull/33122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang tidy [#33131](https://github.com/ClickHouse/ClickHouse/pull/33131) ([alesapin](https://github.com/alesapin)). +* Fix PVS check [#33135](https://github.com/ClickHouse/ClickHouse/pull/33135) ([alesapin](https://github.com/alesapin)). +* Upload build artifact in case of build failures [#33136](https://github.com/ClickHouse/ClickHouse/pull/33136) ([alesapin](https://github.com/alesapin)). +* Fix woboq codebrowser [#33137](https://github.com/ClickHouse/ClickHouse/pull/33137) ([alesapin](https://github.com/alesapin)). +* Longer timeout for server start in fuzzer [#33138](https://github.com/ClickHouse/ClickHouse/pull/33138) ([alesapin](https://github.com/alesapin)). +* Fix azure blob storage tests failures [#33140](https://github.com/ClickHouse/ClickHouse/pull/33140) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix functions with sparse arguments in case when result type doesn't support sparse serialization [#33146](https://github.com/ClickHouse/ClickHouse/pull/33146) ([Anton Popov](https://github.com/CurtizJ)). +* Add pretty print unknow event [#33148](https://github.com/ClickHouse/ClickHouse/pull/33148) ([alesapin](https://github.com/alesapin)). +* Better pr info for unknown events [#33149](https://github.com/ClickHouse/ClickHouse/pull/33149) ([alesapin](https://github.com/alesapin)). +* Better jepsen [#33150](https://github.com/ClickHouse/ClickHouse/pull/33150) ([alesapin](https://github.com/alesapin)). +* Debugging PR info [#33151](https://github.com/ClickHouse/ClickHouse/pull/33151) ([alesapin](https://github.com/alesapin)). +* Fix shared hermetic builds on Arch linux [#33153](https://github.com/ClickHouse/ClickHouse/pull/33153) ([Azat Khuzhin](https://github.com/azat)). +* Fix ccache with ENABLE_CHECK_HEAVY_BUILDS (ccache 4.0 and 4.1 only affected) [#33154](https://github.com/ClickHouse/ClickHouse/pull/33154) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup trash from Kafka, HDFS and Azure [#33160](https://github.com/ClickHouse/ClickHouse/pull/33160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix Date32 test [#33161](https://github.com/ClickHouse/ClickHouse/pull/33161) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make test_input_format_parallel_parsing_memory_tracking less flaky [#33163](https://github.com/ClickHouse/ClickHouse/pull/33163) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_async_drain_connection flakiness [#33164](https://github.com/ClickHouse/ClickHouse/pull/33164) ([Azat Khuzhin](https://github.com/azat)). +* Fix clang-tidy [#33168](https://github.com/ClickHouse/ClickHouse/pull/33168) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01622_defaults_for_url_engine [#33169](https://github.com/ClickHouse/ClickHouse/pull/33169) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* materialized postgresql better startup [#33177](https://github.com/ClickHouse/ClickHouse/pull/33177) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible uncaught exception in clickhouse-local [#33189](https://github.com/ClickHouse/ClickHouse/pull/33189) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix tests with event_time/event_date = today(), and add a style check [#33198](https://github.com/ClickHouse/ClickHouse/pull/33198) ([Azat Khuzhin](https://github.com/azat)). +* Fix 01370_client_autocomplete_word_break_characters test logic [#33202](https://github.com/ClickHouse/ClickHouse/pull/33202) ([Azat Khuzhin](https://github.com/azat)). +* More debug in jepsen test [#33207](https://github.com/ClickHouse/ClickHouse/pull/33207) ([alesapin](https://github.com/alesapin)). +* Fix some flaky integration tests [#33215](https://github.com/ClickHouse/ClickHouse/pull/33215) ([alesapin](https://github.com/alesapin)). +* Remove old trash [#33217](https://github.com/ClickHouse/ClickHouse/pull/33217) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cancel lambda [#33222](https://github.com/ClickHouse/ClickHouse/pull/33222) ([alesapin](https://github.com/alesapin)). +* Fix keeper log messages [#33224](https://github.com/ClickHouse/ClickHouse/pull/33224) ([alesapin](https://github.com/alesapin)). +* Fix clang tidy [#33235](https://github.com/ClickHouse/ClickHouse/pull/33235) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01650 [#33250](https://github.com/ClickHouse/ClickHouse/pull/33250) ([alesapin](https://github.com/alesapin)). +* Add retries for github api [#33252](https://github.com/ClickHouse/ClickHouse/pull/33252) ([alesapin](https://github.com/alesapin)). +* Add retries to AST fuzzer download and fix flaky test. [#33256](https://github.com/ClickHouse/ClickHouse/pull/33256) ([alesapin](https://github.com/alesapin)). +* Do not ignore eof in expect tests [#33263](https://github.com/ClickHouse/ClickHouse/pull/33263) ([Azat Khuzhin](https://github.com/azat)). +* Fix parsing symbols from resources (for shared builds) [#33264](https://github.com/ClickHouse/ClickHouse/pull/33264) ([Azat Khuzhin](https://github.com/azat)). +* test for [#24410](https://github.com/ClickHouse/ClickHouse/issues/24410) [#33265](https://github.com/ClickHouse/ClickHouse/pull/33265) ([Denny Crane](https://github.com/den-crane)). +* Wait for RabbitMQ container to actually start when it was restarted in test on purpose [#33266](https://github.com/ClickHouse/ClickHouse/pull/33266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Mark max_alter_threads as obsolete [#33268](https://github.com/ClickHouse/ClickHouse/pull/33268) ([Denny Crane](https://github.com/den-crane)). +* Fix azure tests flackyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Test for [#26920](https://github.com/ClickHouse/ClickHouse/issues/26920) [#33272](https://github.com/ClickHouse/ClickHouse/pull/33272) ([Denny Crane](https://github.com/den-crane)). +* Fix test_storage_kafka failures by adjusting retention.ms [#33278](https://github.com/ClickHouse/ClickHouse/pull/33278) ([Azat Khuzhin](https://github.com/azat)). +* Disable FunctionConvertFromString::canBeExecutedOnDefaultArguments [#33286](https://github.com/ClickHouse/ClickHouse/pull/33286) ([Vladimir C](https://github.com/vdimir)). +* Add IAST::QueryKind enum and use it in query limiter [#33294](https://github.com/ClickHouse/ClickHouse/pull/33294) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix clang tidy 3 [#33296](https://github.com/ClickHouse/ClickHouse/pull/33296) ([alesapin](https://github.com/alesapin)). +* Fix jepsen check [#33304](https://github.com/ClickHouse/ClickHouse/pull/33304) ([alesapin](https://github.com/alesapin)). +* Add test for broken connection [#33305](https://github.com/ClickHouse/ClickHouse/pull/33305) ([nvartolomei](https://github.com/nvartolomei)). +* Fix launcher not being defined for ccache < 4.0 and slightly cleanup [#33306](https://github.com/ClickHouse/ClickHouse/pull/33306) ([nvartolomei](https://github.com/nvartolomei)). +* Return early if azure blob storage is not used, otherwise it might with fail with irrelevant errors [#33307](https://github.com/ClickHouse/ClickHouse/pull/33307) ([nvartolomei](https://github.com/nvartolomei)). +* Minor refactor of read buffers with compression [#33318](https://github.com/ClickHouse/ClickHouse/pull/33318) ([Nikolay Degterinsky](https://github.com/evillique)). +* Rename Committed data part state to Active [#33327](https://github.com/ClickHouse/ClickHouse/pull/33327) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny follow-up to pr 33098 [#33334](https://github.com/ClickHouse/ClickHouse/pull/33334) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix inner table parser in window view [#33340](https://github.com/ClickHouse/ClickHouse/pull/33340) ([vxider](https://github.com/Vxider)). +* Cover CustomSeparated/Template formats for Kafka [#33343](https://github.com/ClickHouse/ClickHouse/pull/33343) ([Azat Khuzhin](https://github.com/azat)). +* update copyright in docs [#33353](https://github.com/ClickHouse/ClickHouse/pull/33353) ([Bharat Nallan](https://github.com/bharatnc)). +* UserDefinedExecutableFunction fix exception [#33362](https://github.com/ClickHouse/ClickHouse/pull/33362) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to merge [#23230](https://github.com/ClickHouse/ClickHouse/issues/23230) [#33368](https://github.com/ClickHouse/ClickHouse/pull/33368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Print alter command names in EXPLAIN AST [#33383](https://github.com/ClickHouse/ClickHouse/pull/33383) ([Stig Bakken](https://github.com/stigsb)). +* DictionaryStructure fixes [#33388](https://github.com/ClickHouse/ClickHouse/pull/33388) ([Maksim Kita](https://github.com/kitaisreal)). +* tests/stress: add core dumps into artifacts [#33389](https://github.com/ClickHouse/ClickHouse/pull/33389) ([Azat Khuzhin](https://github.com/azat)). +* Update success.html [#33397](https://github.com/ClickHouse/ClickHouse/pull/33397) ([Adri Fernandez](https://github.com/iladriano)). +* Remove obsolete and unsupported Docker Builder [#33345](https://github.com/ClickHouse/ClickHouse/issues/33345) [#33399](https://github.com/ClickHouse/ClickHouse/pull/33399) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe improve robustness of the "query profiler" test [#33422](https://github.com/ClickHouse/ClickHouse/pull/33422) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix misleading log message in StorageWindowView [#33432](https://github.com/ClickHouse/ClickHouse/pull/33432) ([flynn](https://github.com/ucasfl)). +* Preparation for [#33167](https://github.com/ClickHouse/ClickHouse/issues/33167). [#33444](https://github.com/ClickHouse/ClickHouse/pull/33444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rerun docs release check as well [#33447](https://github.com/ClickHouse/ClickHouse/pull/33447) ([alesapin](https://github.com/alesapin)). +* Update AsynchronousReadIndirectBufferFromRemoteFS.h [#33452](https://github.com/ClickHouse/ClickHouse/pull/33452) ([xiedeyantu](https://github.com/xiedeyantu)). +* Tests visualizer better [#33453](https://github.com/ClickHouse/ClickHouse/pull/33453) ([Dmitry Khorkin](https://github.com/DimaAmega)). +* Function monthName small fix [#33464](https://github.com/ClickHouse/ClickHouse/pull/33464) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for operator priorities in SQL [#33487](https://github.com/ClickHouse/ClickHouse/pull/33487) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new team members to company page [#33506](https://github.com/ClickHouse/ClickHouse/pull/33506) ([Cody Baker](https://github.com/codyrobert)). +* fix variable name in h3 misc funcs [#33510](https://github.com/ClickHouse/ClickHouse/pull/33510) ([Bharat Nallan](https://github.com/bharatnc)). +* H3 functions bad arguments possible crash fix [#33512](https://github.com/ClickHouse/ClickHouse/pull/33512) ([Maksim Kita](https://github.com/kitaisreal)). +* Build with ENABLE_TESTS fix [#33513](https://github.com/ClickHouse/ClickHouse/pull/33513) ([Maksim Kita](https://github.com/kitaisreal)). +* IAST QueryKind style fix [#33514](https://github.com/ClickHouse/ClickHouse/pull/33514) ([Maksim Kita](https://github.com/kitaisreal)). +* Don't print exception twice in client in case of exception in parallel parsing [#33524](https://github.com/ClickHouse/ClickHouse/pull/33524) ([Kruglov Pavel](https://github.com/Avogar)). +* HashTable constant fix [#33525](https://github.com/ClickHouse/ClickHouse/pull/33525) ([Maksim Kita](https://github.com/kitaisreal)). +* Add admixer blog post [#33528](https://github.com/ClickHouse/ClickHouse/pull/33528) ([Cody Baker](https://github.com/codyrobert)). +* Fix clickhouse local interactive exception case [#33535](https://github.com/ClickHouse/ClickHouse/pull/33535) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parsing queries with FROM INFILE statement again [#33556](https://github.com/ClickHouse/ClickHouse/pull/33556) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logic for upload upload_master_static_binaries [#33560](https://github.com/ClickHouse/ClickHouse/pull/33560) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Don't upload splitted binaries to builds.clickhouse.com [#33562](https://github.com/ClickHouse/ClickHouse/pull/33562) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix minor bug in TTL and flaky test [#33564](https://github.com/ClickHouse/ClickHouse/pull/33564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* skip MySQLCreateRewritten unit testsuite when enable_mysql is off [#33577](https://github.com/ClickHouse/ClickHouse/pull/33577) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Fix test `02156_storage_merge_prewhere` [#33580](https://github.com/ClickHouse/ClickHouse/pull/33580) ([Anton Popov](https://github.com/CurtizJ)). +* Update mongodb.md [#33585](https://github.com/ClickHouse/ClickHouse/pull/33585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Restore existing static builds links [#33597](https://github.com/ClickHouse/ClickHouse/pull/33597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix pylint for run_check.py [#33600](https://github.com/ClickHouse/ClickHouse/pull/33600) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flacky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make ZooKeeper client better interpret keeper server connection reject [#33602](https://github.com/ClickHouse/ClickHouse/pull/33602) ([alesapin](https://github.com/alesapin)). +* Fix broken workflow dependencies [#33608](https://github.com/ClickHouse/ClickHouse/pull/33608) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Force rebuild images in CI [#33609](https://github.com/ClickHouse/ClickHouse/pull/33609) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* YetAnother workflow fix [#33610](https://github.com/ClickHouse/ClickHouse/pull/33610) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Metrics lambda: fix deletion [#33616](https://github.com/ClickHouse/ClickHouse/pull/33616) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add an exception message example [#33623](https://github.com/ClickHouse/ClickHouse/pull/33623) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix two flaky integration tests [#33625](https://github.com/ClickHouse/ClickHouse/pull/33625) ([alesapin](https://github.com/alesapin)). +* Update version to 22.1 not 21.13 [#33626](https://github.com/ClickHouse/ClickHouse/pull/33626) ([alesapin](https://github.com/alesapin)). +* do not construct std::string if there is no error [#33628](https://github.com/ClickHouse/ClickHouse/pull/33628) ([Sergei Trifonov](https://github.com/serxa)). +* Fix test 00900_long_parquet_load [#33629](https://github.com/ClickHouse/ClickHouse/pull/33629) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix error codes order [#33633](https://github.com/ClickHouse/ClickHouse/pull/33633) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix s3 integration tests [#33641](https://github.com/ClickHouse/ClickHouse/pull/33641) ([Kruglov Pavel](https://github.com/Avogar)). +* test for [#33592](https://github.com/ClickHouse/ClickHouse/issues/33592) [#33658](https://github.com/ClickHouse/ClickHouse/pull/33658) ([Denny Crane](https://github.com/den-crane)). +* Update "gosu" version [#33660](https://github.com/ClickHouse/ClickHouse/pull/33660) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01814_distributed_push_down_limit flakiness [#33662](https://github.com/ClickHouse/ClickHouse/pull/33662) ([Azat Khuzhin](https://github.com/azat)). +* Add amosbird to trusted user. [#33663](https://github.com/ClickHouse/ClickHouse/pull/33663) ([Amos Bird](https://github.com/amosbird)). +* Fix pr ci restart [#33667](https://github.com/ClickHouse/ClickHouse/pull/33667) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + #### New Feature / New Tool * Tool for collecting diagnostics data. [#33175](https://github.com/ClickHouse/ClickHouse/pull/33175) ([Alexander Burmak](https://github.com/Alex-Burmak)). diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index f93a3a690c4..379fb8988c8 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -215,3 +215,164 @@ sidebar_label: 2022 * NO CL ENTRY: 'Add support agreement page and snippets.'. [#34512](https://github.com/ClickHouse/ClickHouse/pull/34512) ([Tom Risse](https://github.com/flickerbox-tom)). * NO CL ENTRY: 'Add Gigasheet to adopters'. [#34589](https://github.com/ClickHouse/ClickHouse/pull/34589) ([Brian Hunter](https://github.com/bjhunter)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix status of SKIPPED integration tests (was incorrectly marked as ERROR) [#33162](https://github.com/ClickHouse/ClickHouse/pull/33162) ([Azat Khuzhin](https://github.com/azat)). +* Client interactive suggest (extract info from CREATE queries) [#33201](https://github.com/ClickHouse/ClickHouse/pull/33201) ([Azat Khuzhin](https://github.com/azat)). +* Fix flushing of in-memory parts [#33234](https://github.com/ClickHouse/ClickHouse/pull/33234) ([Anton Popov](https://github.com/CurtizJ)). +* improvements to tests for h3kRing and h3ToChildren funcs [#33311](https://github.com/ClickHouse/ClickHouse/pull/33311) ([Bharat Nallan](https://github.com/bharatnc)). +* Simplify different block structure (i.e. after ALTER) support for Buffer [#33324](https://github.com/ClickHouse/ClickHouse/pull/33324) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary rename fix [#33526](https://github.com/ClickHouse/ClickHouse/pull/33526) ([Maksim Kita](https://github.com/kitaisreal)). +* RFC: Split headers, move SystemLog into module, more forward declarations [#33534](https://github.com/ClickHouse/ClickHouse/pull/33534) ([Azat Khuzhin](https://github.com/azat)). +* add check for h3 empty column arguments [#33552](https://github.com/ClickHouse/ClickHouse/pull/33552) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix testflows tests [#33575](https://github.com/ClickHouse/ClickHouse/pull/33575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Ignore parse failure of opentelemetry header. Another try. [#33595](https://github.com/ClickHouse/ClickHouse/pull/33595) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix mongo no_auth compose [#33634](https://github.com/ClickHouse/ClickHouse/pull/33634) ([Ilya Yatsishin](https://github.com/qoega)). +* AsynchronousMetrics: Ignore inaccessible sensors [#33639](https://github.com/ClickHouse/ClickHouse/pull/33639) ([Raúl Marín](https://github.com/Algunenano)). +* Slightly reduce memory usage for parsing collections (by using move ctor) [#33665](https://github.com/ClickHouse/ClickHouse/pull/33665) ([Azat Khuzhin](https://github.com/azat)). +* Merge [#33563](https://github.com/ClickHouse/ClickHouse/issues/33563) [#33669](https://github.com/ClickHouse/ClickHouse/pull/33669) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release 22.1 [#33673](https://github.com/ClickHouse/ClickHouse/pull/33673) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix sumIf rewrite [#33677](https://github.com/ClickHouse/ClickHouse/pull/33677) ([flynn](https://github.com/ucasfl)). +* DictionarySourceCoordinator update interface [#33682](https://github.com/ClickHouse/ClickHouse/pull/33682) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix LDAP and Kerberos config handling [#33689](https://github.com/ClickHouse/ClickHouse/pull/33689) ([Denis Glazachev](https://github.com/traceon)). +* Add some helping comments for API endpoints [#33700](https://github.com/ClickHouse/ClickHouse/pull/33700) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix hive tests [#33703](https://github.com/ClickHouse/ClickHouse/pull/33703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* H3 remove functions degsToRads, radsToDegs [#33707](https://github.com/ClickHouse/ClickHouse/pull/33707) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove unused variable [#33731](https://github.com/ClickHouse/ClickHouse/pull/33731) ([vxider](https://github.com/Vxider)). +* Add test issue_31009 [#33739](https://github.com/ClickHouse/ClickHouse/pull/33739) ([Vladimir C](https://github.com/vdimir)). +* Cleanup build: .gitignore more debian directories, libpqxx-cmake without configure_file [#33742](https://github.com/ClickHouse/ClickHouse/pull/33742) ([Ilya Yatsishin](https://github.com/qoega)). +* Better exception text on suspicious broken parts [#33743](https://github.com/ClickHouse/ClickHouse/pull/33743) ([alesapin](https://github.com/alesapin)). +* Fix release_branches workflow for some cases [#33744](https://github.com/ClickHouse/ClickHouse/pull/33744) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new team members to company page [#33755](https://github.com/ClickHouse/ClickHouse/pull/33755) ([Cody Baker](https://github.com/codyrobert)). +* Remove duplicated header [#33760](https://github.com/ClickHouse/ClickHouse/pull/33760) ([vxider](https://github.com/Vxider)). +* Hotfix of missing header [#33765](https://github.com/ClickHouse/ClickHouse/pull/33765) ([Amos Bird](https://github.com/amosbird)). +* rewrite bitHammingDistance with FunctionBinaryArithmetic [#33772](https://github.com/ClickHouse/ClickHouse/pull/33772) ([flynn](https://github.com/ucasfl)). +* Use tb64senc for base64Decode on aarch64 [#33779](https://github.com/ClickHouse/ClickHouse/pull/33779) ([Vladimir C](https://github.com/vdimir)). +* Support USE_* build flags in tests' tags [#33780](https://github.com/ClickHouse/ClickHouse/pull/33780) ([Vladimir C](https://github.com/vdimir)). +* Use workflow names in approve lambda [#33789](https://github.com/ClickHouse/ClickHouse/pull/33789) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Dictionaries remove unnecessary copy of keys during read [#33791](https://github.com/ClickHouse/ClickHouse/pull/33791) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove debug logging from TableFunctionFile [#33796](https://github.com/ClickHouse/ClickHouse/pull/33796) ([Kruglov Pavel](https://github.com/Avogar)). +* RangeHashedDictionary handle invalid intervals [#33827](https://github.com/ClickHouse/ClickHouse/pull/33827) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries added Date32 type support [#33831](https://github.com/ClickHouse/ClickHouse/pull/33831) ([Maksim Kita](https://github.com/kitaisreal)). +* TypeId better naming [#33832](https://github.com/ClickHouse/ClickHouse/pull/33832) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsConversion fix typo [#33841](https://github.com/ClickHouse/ClickHouse/pull/33841) ([Maksim Kita](https://github.com/kitaisreal)). +* More cmake external modules cleanups [#33842](https://github.com/ClickHouse/ClickHouse/pull/33842) ([Azat Khuzhin](https://github.com/azat)). +* Remove harmful code and check what will happen [#33844](https://github.com/ClickHouse/ClickHouse/pull/33844) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix DESCRIBE TABLE query formatting [#33846](https://github.com/ClickHouse/ClickHouse/pull/33846) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix build with ENABLE_EXAMPLES [#33848](https://github.com/ClickHouse/ClickHouse/pull/33848) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Trademark Policy Page [#33851](https://github.com/ClickHouse/ClickHouse/pull/33851) ([Cody Baker](https://github.com/codyrobert)). +* updates to min2 and max2 funcs [#33852](https://github.com/ClickHouse/ClickHouse/pull/33852) ([Bharat Nallan](https://github.com/bharatnc)). +* Use correct logging level [#33857](https://github.com/ClickHouse/ClickHouse/pull/33857) ([李扬](https://github.com/taiyang-li)). +* Fix zookeeper library dependency from interpreters (by marking library STATIC) [#33860](https://github.com/ClickHouse/ClickHouse/pull/33860) ([Azat Khuzhin](https://github.com/azat)). +* FunctionBase64Conversion warning fix [#33863](https://github.com/ClickHouse/ClickHouse/pull/33863) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary ATTACH, DETACH added test [#33870](https://github.com/ClickHouse/ClickHouse/pull/33870) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix perf comparison rerun [#33872](https://github.com/ClickHouse/ClickHouse/pull/33872) ([alesapin](https://github.com/alesapin)). +* Improve postgresql integration test [#33880](https://github.com/ClickHouse/ClickHouse/pull/33880) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix cmake for mac. [#33882](https://github.com/ClickHouse/ClickHouse/pull/33882) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* SQLUserDefinedFunctions invalid lambda additional fixes [#33889](https://github.com/ClickHouse/ClickHouse/pull/33889) ([Maksim Kita](https://github.com/kitaisreal)). +* Update upcoming and past webinar links on homepage [#33890](https://github.com/ClickHouse/ClickHouse/pull/33890) ([Cody Baker](https://github.com/codyrobert)). +* disable animation on docs menu [#33903](https://github.com/ClickHouse/ClickHouse/pull/33903) ([SuperDJY](https://github.com/cmsxbc)). +* Add more retries in CI for AWS "spot" instances [#33907](https://github.com/ClickHouse/ClickHouse/pull/33907) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionMathUnary remove macro usage [#33916](https://github.com/ClickHouse/ClickHouse/pull/33916) ([Maksim Kita](https://github.com/kitaisreal)). +* explicitly checkAndGetColumn for remaining H3 funcs [#33921](https://github.com/ClickHouse/ClickHouse/pull/33921) ([Bharat Nallan](https://github.com/bharatnc)). +* SQLUserDefinedFunctions invalid lambda additional cases [#33924](https://github.com/ClickHouse/ClickHouse/pull/33924) ([Maksim Kita](https://github.com/kitaisreal)). +* when not DEBUG set USE_DEBUG_HELPERS OFF [#33925](https://github.com/ClickHouse/ClickHouse/pull/33925) ([Ben](https://github.com/benbiti)). +* Fix test_replica_is_active flaky test [#33926](https://github.com/ClickHouse/ClickHouse/pull/33926) ([alesapin](https://github.com/alesapin)). +* Fix bug in keeper which can lead to inconsistent snapshots [#33941](https://github.com/ClickHouse/ClickHouse/pull/33941) ([alesapin](https://github.com/alesapin)). +* Fix keeper data dumper build [#33942](https://github.com/ClickHouse/ClickHouse/pull/33942) ([alesapin](https://github.com/alesapin)). +* Remove MAKE_STATIC_LIBRARIES (in favor of USE_STATIC_LIBRARIES) [#33946](https://github.com/ClickHouse/ClickHouse/pull/33946) ([Azat Khuzhin](https://github.com/azat)). +* Revert glibc compatibility (via .symver) in favor of hermetic build (bundled libc) [#33950](https://github.com/ClickHouse/ClickHouse/pull/33950) ([Azat Khuzhin](https://github.com/azat)). +* add c++expr script for C++ one-liners [#33962](https://github.com/ClickHouse/ClickHouse/pull/33962) ([Sergei Trifonov](https://github.com/serxa)). +* docker: fix root squashed data dirs [#33963](https://github.com/ClickHouse/ClickHouse/pull/33963) ([Constantine Peresypkin](https://github.com/pkit)). +* Disable parallel run for 00985_merge_stack_overflow [#33976](https://github.com/ClickHouse/ClickHouse/pull/33976) ([Azat Khuzhin](https://github.com/azat)). +* Cancel lambda [#33990](https://github.com/ClickHouse/ClickHouse/pull/33990) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* reduce signal_pipe_buf_size [#33996](https://github.com/ClickHouse/ClickHouse/pull/33996) ([save-my-heart](https://github.com/save-my-heart)). +* make systemd to use EnvironmentFile [#34024](https://github.com/ClickHouse/ClickHouse/pull/34024) ([Denny Crane](https://github.com/den-crane)). +* CurrentlyExecuting: Require mutex usage explicitly [#34034](https://github.com/ClickHouse/ClickHouse/pull/34034) ([Raúl Marín](https://github.com/Algunenano)). +* Add symlinks to keeper [#34042](https://github.com/ClickHouse/ClickHouse/pull/34042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DecimalPaddedPODArray [#34052](https://github.com/ClickHouse/ClickHouse/pull/34052) ([Maksim Kita](https://github.com/kitaisreal)). +* Less flaky test_inconsistent_parts_if_drop_while_replica_not_active [#34057](https://github.com/ClickHouse/ClickHouse/pull/34057) ([Raúl Marín](https://github.com/Algunenano)). +* Small test for `FINAL` [#34058](https://github.com/ClickHouse/ClickHouse/pull/34058) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Slightly optimize ColumnArray::get()/operator[] (by using reserve over resize) [#34074](https://github.com/ClickHouse/ClickHouse/pull/34074) ([Azat Khuzhin](https://github.com/azat)). +* Tiny cleanup of AggregateFunctionSimpleState/AggregateFunctionState [#34075](https://github.com/ClickHouse/ClickHouse/pull/34075) ([Azat Khuzhin](https://github.com/azat)). +* Fix builds [#34090](https://github.com/ClickHouse/ClickHouse/pull/34090) ([Alexander Tokmakov](https://github.com/tavplubix)). +* add c++expr script examples [#34112](https://github.com/ClickHouse/ClickHouse/pull/34112) ([Sergei Trifonov](https://github.com/serxa)). +* Make clickhouse-diagnostics also work for altinity release [#34116](https://github.com/ClickHouse/ClickHouse/pull/34116) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Small improvement in schema inference from stdin in local [#34117](https://github.com/ClickHouse/ClickHouse/pull/34117) ([Kruglov Pavel](https://github.com/Avogar)). +* Slightly optimize Array/Tuple/Map [#34126](https://github.com/ClickHouse/ClickHouse/pull/34126) ([Azat Khuzhin](https://github.com/azat)). +* Sort block refactoring [#34143](https://github.com/ClickHouse/ClickHouse/pull/34143) ([Maksim Kita](https://github.com/kitaisreal)). +* pdqsort performance check [#34145](https://github.com/ClickHouse/ClickHouse/pull/34145) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#33957](https://github.com/ClickHouse/ClickHouse/issues/33957) [#34146](https://github.com/ClickHouse/ClickHouse/pull/34146) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add func tests run with s3 [#34153](https://github.com/ClickHouse/ClickHouse/pull/34153) ([alesapin](https://github.com/alesapin)). +* Close [#10197](https://github.com/ClickHouse/ClickHouse/issues/10197) [#34159](https://github.com/ClickHouse/ClickHouse/pull/34159) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cmake leftovers cleanup [#34167](https://github.com/ClickHouse/ClickHouse/pull/34167) ([Azat Khuzhin](https://github.com/azat)). +* bitsetsort peformance check [#34175](https://github.com/ClickHouse/ClickHouse/pull/34175) ([Maksim Kita](https://github.com/kitaisreal)). +* Add authorisation for dockerhub proxy container [#34183](https://github.com/ClickHouse/ClickHouse/pull/34183) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue [#34189](https://github.com/ClickHouse/ClickHouse/pull/34189) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some perf tests [#34191](https://github.com/ClickHouse/ClickHouse/pull/34191) ([Kruglov Pavel](https://github.com/Avogar)). +* Clean up: insert_deduplication_token setting for INSERT statement [#34192](https://github.com/ClickHouse/ClickHouse/pull/34192) ([Igor Nikonov](https://github.com/devcrafter)). +* Add func tests run with s3 and fix several bugs [#34215](https://github.com/ClickHouse/ClickHouse/pull/34215) ([alesapin](https://github.com/alesapin)). +* Revert "Revert "Add pool to WriteBufferFromS3"" [#34219](https://github.com/ClickHouse/ClickHouse/pull/34219) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Revert "Revert "Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue"" [#34221](https://github.com/ClickHouse/ClickHouse/pull/34221) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add const to make clang-tidy happy [#34222](https://github.com/ClickHouse/ClickHouse/pull/34222) ([Vitaly Baranov](https://github.com/vitlibar)). +* Bump fmtlib from 7.0.0 to 8.1.1 [#34223](https://github.com/ClickHouse/ClickHouse/pull/34223) ([Azat Khuzhin](https://github.com/azat)). +* Add submodule minizip [#34226](https://github.com/ClickHouse/ClickHouse/pull/34226) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update list-versions.sh, update version_date.tsv [#34240](https://github.com/ClickHouse/ClickHouse/pull/34240) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add missing fmt::runtime() in MergeTreeBackgroundExecutor (fixes the build) [#34245](https://github.com/ClickHouse/ClickHouse/pull/34245) ([Azat Khuzhin](https://github.com/azat)). +* Update clickhouse-keeper.md [#34264](https://github.com/ClickHouse/ClickHouse/pull/34264) ([Andrew](https://github.com/andycol)). +* print query id when using `--interactive` with `--queries-file` in client/local [#34265](https://github.com/ClickHouse/ClickHouse/pull/34265) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update and add new team members [#34268](https://github.com/ClickHouse/ClickHouse/pull/34268) ([Cody Baker](https://github.com/codyrobert)). +* Add a test for shebang [#34274](https://github.com/ClickHouse/ClickHouse/pull/34274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fractional_progress_bar test [#34282](https://github.com/ClickHouse/ClickHouse/pull/34282) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes for [#34267](https://github.com/ClickHouse/ClickHouse/issues/34267) [#34284](https://github.com/ClickHouse/ClickHouse/pull/34284) ([Anton Popov](https://github.com/CurtizJ)). +* Add support policy page content. [#34309](https://github.com/ClickHouse/ClickHouse/pull/34309) ([Tom Risse](https://github.com/flickerbox-tom)). +* Probably fix data race in WriteBufferFromS3 destructor. [#34316](https://github.com/ClickHouse/ClickHouse/pull/34316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Decrease severity for "Reading ... ranges ..." log message to Trace [#34319](https://github.com/ClickHouse/ClickHouse/pull/34319) ([Azat Khuzhin](https://github.com/azat)). +* use LowCardinality type for _file and _path in some Storages, continue of [#34317](https://github.com/ClickHouse/ClickHouse/issues/34317) [#34333](https://github.com/ClickHouse/ClickHouse/pull/34333) ([flynn](https://github.com/ucasfl)). +* Function mapPopulateSeries added additional performance test [#34339](https://github.com/ClickHouse/ClickHouse/pull/34339) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for propagating OpenTelemetry context via gRPC protocol [#34341](https://github.com/ClickHouse/ClickHouse/pull/34341) ([Vitaly Baranov](https://github.com/vitlibar)). +* explicitly check and get columns for s2 funcs [#34344](https://github.com/ClickHouse/ClickHouse/pull/34344) ([Bharat Nallan](https://github.com/bharatnc)). +* Small improvements [#34351](https://github.com/ClickHouse/ClickHouse/pull/34351) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix USE database for clickhouse-local [#34357](https://github.com/ClickHouse/ClickHouse/pull/34357) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy issue [#34365](https://github.com/ClickHouse/ClickHouse/pull/34365) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Delete empty file DateOrDateTimeFunctionsConvertion.cpp [#34371](https://github.com/ClickHouse/ClickHouse/pull/34371) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix consecutive backward seeks in seekable read buffers [#34376](https://github.com/ClickHouse/ClickHouse/pull/34376) ([Anton Popov](https://github.com/CurtizJ)). +* Enable one more check for clang-tidy [#34388](https://github.com/ClickHouse/ClickHouse/pull/34388) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Method called on already moved [#34398](https://github.com/ClickHouse/ClickHouse/pull/34398) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Fix wrong destruction order in CreatingSetsTransform. [#34406](https://github.com/ClickHouse/ClickHouse/pull/34406) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Split and rename compression fields in gRPC [#34408](https://github.com/ClickHouse/ClickHouse/pull/34408) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixing test_storage_postgresql [#34410](https://github.com/ClickHouse/ClickHouse/pull/34410) ([Kseniia Sumarokova](https://github.com/kssenii)). +* asynchronous_inserts engine AsynchronousInserts -> SystemAsynchronousInserts [#34429](https://github.com/ClickHouse/ClickHouse/pull/34429) ([filimonov](https://github.com/filimonov)). +* clang-tidy move fix build [#34431](https://github.com/ClickHouse/ClickHouse/pull/34431) ([Maksim Kita](https://github.com/kitaisreal)). +* `static-files-disk-uploader`: add a mode to create symlinks [#34432](https://github.com/ClickHouse/ClickHouse/pull/34432) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bug in URL engine [#34448](https://github.com/ClickHouse/ClickHouse/pull/34448) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix style [#34450](https://github.com/ClickHouse/ClickHouse/pull/34450) ([alesapin](https://github.com/alesapin)). +* Added test 33734 [#34454](https://github.com/ClickHouse/ClickHouse/pull/34454) ([Maksim Kita](https://github.com/kitaisreal)). +* Update http_max_tries setting default [#34457](https://github.com/ClickHouse/ClickHouse/pull/34457) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use `cpp_bin_float_double` in `set_multiplier` for `wide_integer_from_builtin` for aarch64 [#34463](https://github.com/ClickHouse/ClickHouse/pull/34463) ([Vladimir C](https://github.com/vdimir)). +* test for [#13907](https://github.com/ClickHouse/ClickHouse/issues/13907) toColumnTypeName_toLowCardinality_const [#34471](https://github.com/ClickHouse/ClickHouse/pull/34471) ([Denny Crane](https://github.com/den-crane)). +* Remove invalid IOS setting for RocksDB CMAKE to fix Apple M1 build [#34472](https://github.com/ClickHouse/ClickHouse/pull/34472) ([Geoff Genz](https://github.com/genzgd)). +* clang-tidy reported potential chance for divide by zero exception [#34473](https://github.com/ClickHouse/ClickHouse/pull/34473) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* accessing nested_column after already moved to data [#34475](https://github.com/ClickHouse/ClickHouse/pull/34475) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Avoid unnecessary copying of `Settings` [#34476](https://github.com/ClickHouse/ClickHouse/pull/34476) ([Anton Popov](https://github.com/CurtizJ)). +* Update buildPushingToViewsChain.h [#34515](https://github.com/ClickHouse/ClickHouse/pull/34515) ([William.Walliams.Wong](https://github.com/wangzhen11aaa)). +* More gdb introspection on CI [#34517](https://github.com/ClickHouse/ClickHouse/pull/34517) ([Azat Khuzhin](https://github.com/azat)). +* add BeforeLambdaBody to .clang-format [#34533](https://github.com/ClickHouse/ClickHouse/pull/34533) ([Vladimir C](https://github.com/vdimir)). +* Remove very old cruft [#34547](https://github.com/ClickHouse/ClickHouse/pull/34547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tiny fixes for client/local suggestions [#34550](https://github.com/ClickHouse/ClickHouse/pull/34550) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix gtest_archive_reader_and_writer in case of !USE_MINIZIP [#34554](https://github.com/ClickHouse/ClickHouse/pull/34554) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionFile added performance test [#34555](https://github.com/ClickHouse/ClickHouse/pull/34555) ([Maksim Kita](https://github.com/kitaisreal)). +* check and get columns in geoToH3 func [#34557](https://github.com/ClickHouse/ClickHouse/pull/34557) ([Bharat Nallan](https://github.com/bharatnc)). +* try fix data race in StorageLog [#34558](https://github.com/ClickHouse/ClickHouse/pull/34558) ([flynn](https://github.com/ucasfl)). +* Postpone a bit nightly builds to the least loaded time [#34569](https://github.com/ClickHouse/ClickHouse/pull/34569) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Always remove unused actions from addMissingDefaults [#34577](https://github.com/ClickHouse/ClickHouse/pull/34577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix indentation in ARRAY JOIN formatting [#34578](https://github.com/ClickHouse/ClickHouse/pull/34578) ([Azat Khuzhin](https://github.com/azat)). +* Fix deadlock in OvercommitTracker [#34591](https://github.com/ClickHouse/ClickHouse/pull/34591) ([Dmitry Novik](https://github.com/novikd)). +* ASTCreateQuery: Remove usused `tables` member [#34610](https://github.com/ClickHouse/ClickHouse/pull/34610) ([Raúl Marín](https://github.com/Algunenano)). +* Add test for [#19222](https://github.com/ClickHouse/ClickHouse/issues/19222) [#34615](https://github.com/ClickHouse/ClickHouse/pull/34615) ([Raúl Marín](https://github.com/Algunenano)). +* Attempt to fix freeBSD build [#34617](https://github.com/ClickHouse/ClickHouse/pull/34617) ([Raúl Marín](https://github.com/Algunenano)). +* Fix quadratic complexity while adding subcolumns [#34623](https://github.com/ClickHouse/ClickHouse/pull/34623) ([Anton Popov](https://github.com/CurtizJ)). +* Fix typo in welchttest.md [#34634](https://github.com/ClickHouse/ClickHouse/pull/34634) ([achimbab](https://github.com/achimbab)). +* Improve the release.py script [#34659](https://github.com/ClickHouse/ClickHouse/pull/34659) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable memory overcommit if timeout is not set [#34663](https://github.com/ClickHouse/ClickHouse/pull/34663) ([Dmitry Novik](https://github.com/novikd)). + diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index 24b535a7f91..2eb8a6b63cc 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -11,3 +11,7 @@ sidebar_label: 2022 * Backported in [#34848](https://github.com/ClickHouse/ClickHouse/issues/34848): Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index d2f976f4517..0058396d634 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -151,3 +151,124 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "Change timezone in Docker"'. [#35243](https://github.com/ClickHouse/ClickHouse/pull/35243) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix 00900_long_parquet_load"'. [#35301](https://github.com/ClickHouse/ClickHouse/pull/35301) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* [RFC] Update jemalloc to 5.3RC [#33057](https://github.com/ClickHouse/ClickHouse/pull/33057) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless setting experimental_query_deduplication_send_all_part_uuids [#34387](https://github.com/ClickHouse/ClickHouse/pull/34387) ([nvartolomei](https://github.com/nvartolomei)). +* Apply join_use_nulls on types before join [#34529](https://github.com/ClickHouse/ClickHouse/pull/34529) ([Vladimir C](https://github.com/vdimir)). +* Adjust 01681_arg_min_max_if_fix [#34612](https://github.com/ClickHouse/ClickHouse/pull/34612) ([Vladimir C](https://github.com/vdimir)). +* Check overflow in addSeconds/Minues/etc functions [#34619](https://github.com/ClickHouse/ClickHouse/pull/34619) ([Vladimir C](https://github.com/vdimir)). +* Update version after release [#34677](https://github.com/ClickHouse/ClickHouse/pull/34677) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `positionUTF8` on aarch64 [#34683](https://github.com/ClickHouse/ClickHouse/pull/34683) ([Vladimir C](https://github.com/vdimir)). +* Update version_date.tsv after v22.2.2.1-stable [#34685](https://github.com/ClickHouse/ClickHouse/pull/34685) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix ZooKepper paths in zero_copy_schema_converter.py [#34686](https://github.com/ClickHouse/ClickHouse/pull/34686) ([ianton-ru](https://github.com/ianton-ru)). +* Changelog for version 22.2 [#34687](https://github.com/ClickHouse/ClickHouse/pull/34687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow up to PR [#33698](https://github.com/ClickHouse/ClickHouse/issues/33698) [#34689](https://github.com/ClickHouse/ClickHouse/pull/34689) ([Vladimir C](https://github.com/vdimir)). +* Always update ProfileEvents (even on exceptions) [#34690](https://github.com/ClickHouse/ClickHouse/pull/34690) ([nvartolomei](https://github.com/nvartolomei)). +* Function encodeURLComponent minor fixes [#34695](https://github.com/ClickHouse/ClickHouse/pull/34695) ([Maksim Kita](https://github.com/kitaisreal)). +* CMake llvm add status message if not used [#34698](https://github.com/ClickHouse/ClickHouse/pull/34698) ([Maksim Kita](https://github.com/kitaisreal)). +* Adding noexcept for move constructor SharedContextHolder [#34699](https://github.com/ClickHouse/ClickHouse/pull/34699) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add new team members to company page [#34701](https://github.com/ClickHouse/ClickHouse/pull/34701) ([Cody Baker](https://github.com/codyrobert)). +* Fix flaky 00502_custom_partitioning_local [#34714](https://github.com/ClickHouse/ClickHouse/pull/34714) ([Azat Khuzhin](https://github.com/azat)). +* Workaround for a bug in NuRaft library [#34715](https://github.com/ClickHouse/ClickHouse/pull/34715) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible memory_tracker use-after-free (for async s3 writes) for merges/mutations [#34717](https://github.com/ClickHouse/ClickHouse/pull/34717) ([Azat Khuzhin](https://github.com/azat)). +* Add taiyang-li to trusted contributors [#34729](https://github.com/ClickHouse/ClickHouse/pull/34729) ([Vladimir C](https://github.com/vdimir)). +* rename name in ci check [#34730](https://github.com/ClickHouse/ClickHouse/pull/34730) ([flynn](https://github.com/ucasfl)). +* Performance tests fix H3 [#34731](https://github.com/ClickHouse/ClickHouse/pull/34731) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable async writes to S3. [#34734](https://github.com/ClickHouse/ClickHouse/pull/34734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add a patch release for stable/lts packages to release.py [#34740](https://github.com/ClickHouse/ClickHouse/pull/34740) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Print only total profile events for --profile-events-delay-ms=-1 [#34749](https://github.com/ClickHouse/ClickHouse/pull/34749) ([Azat Khuzhin](https://github.com/azat)). +* Fix `parallel_reading_from_replicas` with `clickhouse-bechmark` [#34751](https://github.com/ClickHouse/ClickHouse/pull/34751) ([Azat Khuzhin](https://github.com/azat)). +* Submodules are mandatory [#34753](https://github.com/ClickHouse/ClickHouse/pull/34753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for version 22.2 [#34758](https://github.com/ClickHouse/ClickHouse/pull/34758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Sort additional_html_urls in test report in CI [#34770](https://github.com/ClickHouse/ClickHouse/pull/34770) ([Vladimir C](https://github.com/vdimir)). +* Fix undefined __pthread_mutex_lock/unlock for glibc 2.34+/DISABLE_HERMETIC_BUILD [#34771](https://github.com/ClickHouse/ClickHouse/pull/34771) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc: fix includes order to avoid overlaps [#34783](https://github.com/ClickHouse/ClickHouse/pull/34783) ([Azat Khuzhin](https://github.com/azat)). +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). +* Update links for 22.2 and 22.3 release webinars [#34814](https://github.com/ClickHouse/ClickHouse/pull/34814) ([Cody Baker](https://github.com/codyrobert)). +* Add opensee blog post [#34817](https://github.com/ClickHouse/ClickHouse/pull/34817) ([Cody Baker](https://github.com/codyrobert)). +* Try remove trash from StorageFileLog [#34819](https://github.com/ClickHouse/ClickHouse/pull/34819) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix generating system.build_options [#34823](https://github.com/ClickHouse/ClickHouse/pull/34823) ([Azat Khuzhin](https://github.com/azat)). +* Comment output header for team keys [#34828](https://github.com/ClickHouse/ClickHouse/pull/34828) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix untuple condition in IN function [#34836](https://github.com/ClickHouse/ClickHouse/pull/34836) ([Vladimir C](https://github.com/vdimir)). +* Improve certificate reloader [#34887](https://github.com/ClickHouse/ClickHouse/pull/34887) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update DiskS3.cpp [#34889](https://github.com/ClickHouse/ClickHouse/pull/34889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Clang tidy improvements [#34893](https://github.com/ClickHouse/ClickHouse/pull/34893) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version_date.tsv after v22.2.3.5-stable [#34898](https://github.com/ClickHouse/ClickHouse/pull/34898) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Add check_prerequisites to release.py [#34901](https://github.com/ClickHouse/ClickHouse/pull/34901) ([Vladimir C](https://github.com/vdimir)). +* Fix filelog storage data paths [#34912](https://github.com/ClickHouse/ClickHouse/pull/34912) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.1.4.30-stable [#34913](https://github.com/ClickHouse/ClickHouse/pull/34913) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix alignment for aligned_alloc() to fix build for glibc2.35+ [#34919](https://github.com/ClickHouse/ClickHouse/pull/34919) ([Azat Khuzhin](https://github.com/azat)). +* Stop processing multiqueries in clickhouse-client/local on SIGINT [#34923](https://github.com/ClickHouse/ClickHouse/pull/34923) ([Azat Khuzhin](https://github.com/azat)). +* Fix some code comments style [#34935](https://github.com/ClickHouse/ClickHouse/pull/34935) ([hongbin](https://github.com/xlwh)). +* Adjust couple of tests for aarch64 [#34951](https://github.com/ClickHouse/ClickHouse/pull/34951) ([Vladimir C](https://github.com/vdimir)). +* Add benchmarks [#34960](https://github.com/ClickHouse/ClickHouse/pull/34960) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Catch exception in ~WriteBufferFromS3 [#34963](https://github.com/ClickHouse/ClickHouse/pull/34963) ([Vladimir C](https://github.com/vdimir)). +* Disable memory checking test with thread sanitizer [#34964](https://github.com/ClickHouse/ClickHouse/pull/34964) ([alesapin](https://github.com/alesapin)). +* Print number of available logical cores in configuration [#34972](https://github.com/ClickHouse/ClickHouse/pull/34972) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix possible "nmalloc >= ndalloc" jemalloc assertion [#34973](https://github.com/ClickHouse/ClickHouse/pull/34973) ([Azat Khuzhin](https://github.com/azat)). +* Add some benchmarks [#34974](https://github.com/ClickHouse/ClickHouse/pull/34974) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix typo [#34975](https://github.com/ClickHouse/ClickHouse/pull/34975) ([Nikita Evsyukov](https://github.com/NikitaEvs)). +* Fix signal-unsafe TSan report in client [#34988](https://github.com/ClickHouse/ClickHouse/pull/34988) ([Azat Khuzhin](https://github.com/azat)). +* Fix non-MergeTree engines for system.*_log [#34989](https://github.com/ClickHouse/ClickHouse/pull/34989) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_storage_postgresql::test_concurrent_queries [#34991](https://github.com/ClickHouse/ClickHouse/pull/34991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix race between INSERT async_insert=1 and system.asynchronous_inserts [#34992](https://github.com/ClickHouse/ClickHouse/pull/34992) ([Azat Khuzhin](https://github.com/azat)). +* Add option to clickhouse-test to skip aarch64 build [#34995](https://github.com/ClickHouse/ClickHouse/pull/34995) ([Vladimir C](https://github.com/vdimir)). +* clickhouse obfuscator aarch64 fix [#35005](https://github.com/ClickHouse/ClickHouse/pull/35005) ([Vladimir C](https://github.com/vdimir)). +* Fix tutorial link [#35014](https://github.com/ClickHouse/ClickHouse/pull/35014) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* ExecutableUDF function deterministic in scope of query fix [#35018](https://github.com/ClickHouse/ClickHouse/pull/35018) ([Maksim Kita](https://github.com/kitaisreal)). +* fasttest: add timings for "Test time" column in the results table [#35019](https://github.com/ClickHouse/ClickHouse/pull/35019) ([Azat Khuzhin](https://github.com/azat)). +* Always write preprocessed config in XML format [#35020](https://github.com/ClickHouse/ClickHouse/pull/35020) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix read old records from logs [#35021](https://github.com/ClickHouse/ClickHouse/pull/35021) ([alesapin](https://github.com/alesapin)). +* Fix trim function [#35046](https://github.com/ClickHouse/ClickHouse/pull/35046) ([Vladimir C](https://github.com/vdimir)). +* Skip 01086_odbc_roundtrip for aarch, disable force_tests [#35058](https://github.com/ClickHouse/ClickHouse/pull/35058) ([Vladimir C](https://github.com/vdimir)). +* Remove useless define [#35066](https://github.com/ClickHouse/ClickHouse/pull/35066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Take flush_time into account for scheduling background flush of the Buffer [#35071](https://github.com/ClickHouse/ClickHouse/pull/35071) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix failed tests [#35073](https://github.com/ClickHouse/ClickHouse/pull/35073) ([Kruglov Pavel](https://github.com/Avogar)). +* Do not hide exceptions during mutations [#35080](https://github.com/ClickHouse/ClickHouse/pull/35080) ([Azat Khuzhin](https://github.com/azat)). +* Put downloaded artifacts to a temorary path [#35088](https://github.com/ClickHouse/ClickHouse/pull/35088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of duplicate query planing. [#35094](https://github.com/ClickHouse/ClickHouse/pull/35094) ([Amos Bird](https://github.com/amosbird)). +* Temporary supress move partition long for storage S3 [#35097](https://github.com/ClickHouse/ClickHouse/pull/35097) ([alesapin](https://github.com/alesapin)). +* Fix inconsistency in DiskLocal [#35099](https://github.com/ClickHouse/ClickHouse/pull/35099) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to build llvm for Aarch64 [#35103](https://github.com/ClickHouse/ClickHouse/pull/35103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix build fail with gcc [#35123](https://github.com/ClickHouse/ClickHouse/pull/35123) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix hardcoded page size [#35129](https://github.com/ClickHouse/ClickHouse/pull/35129) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Delete nodes with attributes `remove` or `replace` if they didn't merge [#35141](https://github.com/ClickHouse/ClickHouse/pull/35141) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Upgrade icu to icu-release-70-1 [#35142](https://github.com/ClickHouse/ClickHouse/pull/35142) ([Yong Wang](https://github.com/kashwy)). +* Merging [#33398](https://github.com/ClickHouse/ClickHouse/issues/33398) [#35157](https://github.com/ClickHouse/ClickHouse/pull/35157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typos [#35174](https://github.com/ClickHouse/ClickHouse/pull/35174) ([Zhang Yifan](https://github.com/zhangyifan27)). +* Adjust max_memory_usage in external_aggregation.sql [#35194](https://github.com/ClickHouse/ClickHouse/pull/35194) ([Vladimir C](https://github.com/vdimir)). +* MaterializedMySQL protocol example fix [#35198](https://github.com/ClickHouse/ClickHouse/pull/35198) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#35199](https://github.com/ClickHouse/ClickHouse/pull/35199) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#35200](https://github.com/ClickHouse/ClickHouse/pull/35200) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Segment script to of docs pages [#35203](https://github.com/ClickHouse/ClickHouse/pull/35203) ([Rich Raposa](https://github.com/rfraposa)). +* Cache fix [#35209](https://github.com/ClickHouse/ClickHouse/pull/35209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use C++14 aliases for some type traits [#35212](https://github.com/ClickHouse/ClickHouse/pull/35212) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove redundant configs for TestFlows [#35223](https://github.com/ClickHouse/ClickHouse/pull/35223) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correct some integration tests [#35224](https://github.com/ClickHouse/ClickHouse/pull/35224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in Docker [#35225](https://github.com/ClickHouse/ClickHouse/pull/35225) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone example in server config [#35226](https://github.com/ClickHouse/ClickHouse/pull/35226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust timezone in performance tests [#35227](https://github.com/ClickHouse/ClickHouse/pull/35227) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35228](https://github.com/ClickHouse/ClickHouse/pull/35228) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in stateful tests. [#35229](https://github.com/ClickHouse/ClickHouse/pull/35229) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moved submodules from ClickHouse-Extras to ClickHouse [#35232](https://github.com/ClickHouse/ClickHouse/pull/35232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35242](https://github.com/ClickHouse/ClickHouse/pull/35242) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CachedReadBufferFromRemoteFS.cpp [#35245](https://github.com/ClickHouse/ClickHouse/pull/35245) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#34682](https://github.com/ClickHouse/ClickHouse/issues/34682) [#35247](https://github.com/ClickHouse/ClickHouse/pull/35247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clang tidy for header files [#35248](https://github.com/ClickHouse/ClickHouse/pull/35248) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible 01506_buffer_table_alter_block_structure_2 flakiness [#35249](https://github.com/ClickHouse/ClickHouse/pull/35249) ([Azat Khuzhin](https://github.com/azat)). +* Mark build action as failed if it was retried and there was no succeeded builds [#35260](https://github.com/ClickHouse/ClickHouse/pull/35260) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test [#35261](https://github.com/ClickHouse/ClickHouse/pull/35261) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logical error in remote fs cache [#35275](https://github.com/ClickHouse/ClickHouse/pull/35275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix code style and other minor corrections after implementing allow_no_password. [#35276](https://github.com/ClickHouse/ClickHouse/pull/35276) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add remote fs cache optimization [#35278](https://github.com/ClickHouse/ClickHouse/pull/35278) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove utils/github-hook/hook.py [#35280](https://github.com/ClickHouse/ClickHouse/pull/35280) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test [#35282](https://github.com/ClickHouse/ClickHouse/pull/35282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix 00284_external_aggregation.sql [#35293](https://github.com/ClickHouse/ClickHouse/pull/35293) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 00900_long_parquet_load [#35299](https://github.com/ClickHouse/ClickHouse/pull/35299) ([Vladimir C](https://github.com/vdimir)). +* Maybe fix use-after-free inside S3 upload thread [#35305](https://github.com/ClickHouse/ClickHouse/pull/35305) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Modified python packages [#35306](https://github.com/ClickHouse/ClickHouse/pull/35306) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix disappeared host and port from client help message [#35309](https://github.com/ClickHouse/ClickHouse/pull/35309) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog 22.3 [#35344](https://github.com/ClickHouse/ClickHouse/pull/35344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace a few uses of enable_if for SFINAE by concepts [#35347](https://github.com/ClickHouse/ClickHouse/pull/35347) ([Robert Schulze](https://github.com/rschu1ze)). + diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 4fa98940290..4cd48eefa5a 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -19,3 +19,11 @@ sidebar_label: 2022 * Backported in [#35856](https://github.com/ClickHouse/ClickHouse/issues/35856): Respect only quota & period from groups, ignore shares (which are not really limit the number of the cores which can be used). [#35815](https://github.com/ClickHouse/ClickHouse/pull/35815) ([filimonov](https://github.com/filimonov)). * Backported in [#35938](https://github.com/ClickHouse/ClickHouse/issues/35938): Avoid processing per-column TTL multiple times. [#35820](https://github.com/ClickHouse/ClickHouse/pull/35820) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* Backport release to 22.3 [#35909](https://github.com/ClickHouse/ClickHouse/pull/35909) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 9d86d9e786c..4b4772c611a 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -11,3 +11,7 @@ sidebar_label: 2022 * Backported in [#36795](https://github.com/ClickHouse/ClickHouse/issues/36795): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md index ecfd0ce2cf6..14cb8628f09 100644 --- a/docs/changelogs/v22.3.7.28-lts.md +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -22,3 +22,10 @@ sidebar_label: 2022 * Backported in [#36925](https://github.com/ClickHouse/ClickHouse/issues/36925): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). * Backported in [#37364](https://github.com/ClickHouse/ClickHouse/issues/37364): Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index 5c6948251b3..c202b0b9331 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -243,3 +243,208 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "clang-tidy report issues with Medium priority"'. [#35941](https://github.com/ClickHouse/ClickHouse/pull/35941) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix crash in ParallelReadBuffer"'. [#36210](https://github.com/ClickHouse/ClickHouse/pull/36210) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve backup and restore (experimental) [#33985](https://github.com/ClickHouse/ClickHouse/pull/33985) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not leave any queries after test finish (and add a check into clickhouse-test) [#34924](https://github.com/ClickHouse/ClickHouse/pull/34924) ([Azat Khuzhin](https://github.com/azat)). +* libxml2 package under contrib directory is upgraded to 2.9.13 [#35034](https://github.com/ClickHouse/ClickHouse/pull/35034) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add bugfix validate check [#35124](https://github.com/ClickHouse/ClickHouse/pull/35124) ([Vladimir C](https://github.com/vdimir)). +* curl package upgraded to 7.81.0 [#35130](https://github.com/ClickHouse/ClickHouse/pull/35130) ([Deleted user](https://github.com/ghost)). +* Add test for [#26965](https://github.com/ClickHouse/ClickHouse/issues/26965) [#35159](https://github.com/ClickHouse/ClickHouse/pull/35159) ([palegre-tiny](https://github.com/palegre-tiny)). +* clang-tidy report issues with Medium priority [#35184](https://github.com/ClickHouse/ClickHouse/pull/35184) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add build with GCC [#35204](https://github.com/ClickHouse/ClickHouse/pull/35204) ([Azat Khuzhin](https://github.com/azat)). +* Fix-Clang-Tidy-Errors [#35297](https://github.com/ClickHouse/ClickHouse/pull/35297) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Other fix for 00900_long_parquet_load [#35302](https://github.com/ClickHouse/ClickHouse/pull/35302) ([Vladimir C](https://github.com/vdimir)). +* Add more checks with remoteHostsFilter [#35355](https://github.com/ClickHouse/ClickHouse/pull/35355) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#35356](https://github.com/ClickHouse/ClickHouse/pull/35356) ([Anton Popov](https://github.com/CurtizJ)). +* Function h3GetRes0Indexes crash fix [#35358](https://github.com/ClickHouse/ClickHouse/pull/35358) ([Maksim Kita](https://github.com/kitaisreal)). +* Function proporationsZTest formatting fix [#35369](https://github.com/ClickHouse/ClickHouse/pull/35369) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version after release [#35374](https://github.com/ClickHouse/ClickHouse/pull/35374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update SECURITY.md [#35375](https://github.com/ClickHouse/ClickHouse/pull/35375) ([Ivan Blinkov](https://github.com/blinkov)). +* Update version_date.tsv after v22.3.2.2-lts [#35377](https://github.com/ClickHouse/ClickHouse/pull/35377) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Function proportionsZTest fix style check [#35380](https://github.com/ClickHouse/ClickHouse/pull/35380) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace more uses of std::enable_if for SFINAE by concepts [#35383](https://github.com/ClickHouse/ClickHouse/pull/35383) ([Robert Schulze](https://github.com/rschu1ze)). +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Validate some thoughts over making sets [#35395](https://github.com/ClickHouse/ClickHouse/pull/35395) ([Amos Bird](https://github.com/amosbird)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Rename some variables in keeper [#35431](https://github.com/ClickHouse/ClickHouse/pull/35431) ([alesapin](https://github.com/alesapin)). +* fix 02177_issue_31009_pt2.sql [#35445](https://github.com/ClickHouse/ClickHouse/pull/35445) ([Vladimir C](https://github.com/vdimir)). +* mysqlxx PoolWithFailover style fix [#35462](https://github.com/ClickHouse/ClickHouse/pull/35462) ([Maksim Kita](https://github.com/kitaisreal)). +* Resubmit [#21474](https://github.com/ClickHouse/ClickHouse/issues/21474) [#35467](https://github.com/ClickHouse/ClickHouse/pull/35467) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Explicitly set allow_introspection_functions to 0 [#35470](https://github.com/ClickHouse/ClickHouse/pull/35470) ([Nikita Taranov](https://github.com/nickitat)). +* Merging [#30325](https://github.com/ClickHouse/ClickHouse/issues/30325) [#35478](https://github.com/ClickHouse/ClickHouse/pull/35478) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix show create for information_schema [#35480](https://github.com/ClickHouse/ClickHouse/pull/35480) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add retries in backward compatibility check [#35482](https://github.com/ClickHouse/ClickHouse/pull/35482) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve backward compatibility check and stress tests [#35499](https://github.com/ClickHouse/ClickHouse/pull/35499) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix too strict assertion in DDLWorker [#35503](https://github.com/ClickHouse/ClickHouse/pull/35503) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add more validations in mask operations [#35507](https://github.com/ClickHouse/ClickHouse/pull/35507) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Don't put red cross if jepsen check couldn't wait for build [#35522](https://github.com/ClickHouse/ClickHouse/pull/35522) ([alesapin](https://github.com/alesapin)). +* Fix ClickHouse name typo in caches.md [#35526](https://github.com/ClickHouse/ClickHouse/pull/35526) ([erikbaan](https://github.com/erikbaan)). +* Simplify strip for new packages [#35533](https://github.com/ClickHouse/ClickHouse/pull/35533) ([alesapin](https://github.com/alesapin)). +* Add workflow dispatch [#35535](https://github.com/ClickHouse/ClickHouse/pull/35535) ([alesapin](https://github.com/alesapin)). +* fix clang tidy warning, add nullptr check [#35540](https://github.com/ClickHouse/ClickHouse/pull/35540) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Slightly better integration tests: test_backup_with_other_granularity test_azure_blob_storage_zero_copy_replication [#35543](https://github.com/ClickHouse/ClickHouse/pull/35543) ([Ilya Yatsishin](https://github.com/qoega)). +* ExternalModelsLoader refactoring [#35546](https://github.com/ClickHouse/ClickHouse/pull/35546) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed style check [#35550](https://github.com/ClickHouse/ClickHouse/pull/35550) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup test data [#35555](https://github.com/ClickHouse/ClickHouse/pull/35555) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Revert "Decrease data_type_max_parse_depth a little to avoid stack overflow in coroutines" [#35556](https://github.com/ClickHouse/ClickHouse/pull/35556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test `test_s3_zero_copy_replication` [#35574](https://github.com/ClickHouse/ClickHouse/pull/35574) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logging in `test_distributed_respect_user_timeouts` [#35575](https://github.com/ClickHouse/ClickHouse/pull/35575) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Increase fiber stack size a bit in attempt to fix stack overflow in tests with address sanitizer [#35578](https://github.com/ClickHouse/ClickHouse/pull/35578) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't run 01318_long_unsuccessful_mutation_zookeeper test in backward compatibility check [#35580](https://github.com/ClickHouse/ClickHouse/pull/35580) ([Kruglov Pavel](https://github.com/Avogar)). +* Make some tests more stable [#35599](https://github.com/ClickHouse/ClickHouse/pull/35599) ([Kruglov Pavel](https://github.com/Avogar)). +* Check all logs for crashes, logical errors, etc in backward compatibility check [#35613](https://github.com/ClickHouse/ClickHouse/pull/35613) ([Kruglov Pavel](https://github.com/Avogar)). +* Update comment about mismatching checksums [#35621](https://github.com/ClickHouse/ClickHouse/pull/35621) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push only to the new CI DB [#35622](https://github.com/ClickHouse/ClickHouse/pull/35622) ([alesapin](https://github.com/alesapin)). +* Fix optin.cplusplus.UninitializedObject issue [#35626](https://github.com/ClickHouse/ClickHouse/pull/35626) ([larryluogit](https://github.com/larryluogit)). +* test for partition_by using ignore() [#35636](https://github.com/ClickHouse/ClickHouse/pull/35636) ([Denny Crane](https://github.com/den-crane)). +* test for crash _join_with_nullable_lowcardinality [#35551](https://github.com/ClickHouse/ClickHouse/issues/35551) [#35638](https://github.com/ClickHouse/ClickHouse/pull/35638) ([Denny Crane](https://github.com/den-crane)). +* Use compile-time check for `Exception` messages [#35655](https://github.com/ClickHouse/ClickHouse/pull/35655) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky test `01532_primary_key_without_order_by_zookeeper` [#35657](https://github.com/ClickHouse/ClickHouse/pull/35657) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix multiple flaky tests [#35659](https://github.com/ClickHouse/ClickHouse/pull/35659) ([alesapin](https://github.com/alesapin)). +* Fix flaky test `01091_num_threads` [#35663](https://github.com/ClickHouse/ClickHouse/pull/35663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve black check: show diff in the output [#35665](https://github.com/ClickHouse/ClickHouse/pull/35665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use float devision for avg after optimize_fuse_sum_count_avg [#35666](https://github.com/ClickHouse/ClickHouse/pull/35666) ([Vladimir C](https://github.com/vdimir)). +* Disable random settings in Fast Test [#35669](https://github.com/ClickHouse/ClickHouse/pull/35669) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix randomization of max_read_buffer_size [#35675](https://github.com/ClickHouse/ClickHouse/pull/35675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove outdated links from CI [#35676](https://github.com/ClickHouse/ClickHouse/pull/35676) ([alesapin](https://github.com/alesapin)). +* Fix flaky tests 02149_read_in_order_fixed_prefix and 02177_issue_31009 [#35679](https://github.com/ClickHouse/ClickHouse/pull/35679) ([Kruglov Pavel](https://github.com/Avogar)). +* Rerun failed jobs only for failed workflowa [#35685](https://github.com/ClickHouse/ClickHouse/pull/35685) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flaky 01037_polygon_dicts_correctness_fast [#35688](https://github.com/ClickHouse/ClickHouse/pull/35688) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fail CI checks in case of errors in checks (not failed tests) [#35718](https://github.com/ClickHouse/ClickHouse/pull/35718) ([alesapin](https://github.com/alesapin)). +* Try to fix test_global_overcommit_tracker flakyness [#35719](https://github.com/ClickHouse/ClickHouse/pull/35719) ([Dmitry Novik](https://github.com/novikd)). +* Try to run stateful tests in parallel [#35720](https://github.com/ClickHouse/ClickHouse/pull/35720) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Format `test_allowed_url_from_config/test.py` with black [#35721](https://github.com/ClickHouse/ClickHouse/pull/35721) ([Antonio Andelic](https://github.com/antonio2368)). +* Resurrect automatic labelling [#35722](https://github.com/ClickHouse/ClickHouse/pull/35722) ([alesapin](https://github.com/alesapin)). +* Revert "Fix enable LLVM for JIT compilation in CMake" [#35725](https://github.com/ClickHouse/ClickHouse/pull/35725) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not require writable source directory for generating krb5 error tables [#35734](https://github.com/ClickHouse/ClickHouse/pull/35734) ([Azat Khuzhin](https://github.com/azat)). +* Update ci checks server. [#35737](https://github.com/ClickHouse/ClickHouse/pull/35737) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Tests fixes [#35745](https://github.com/ClickHouse/ClickHouse/pull/35745) ([Azat Khuzhin](https://github.com/azat)). +* Use common IOThreadPool for S3 and URL [#35746](https://github.com/ClickHouse/ClickHouse/pull/35746) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix stylecheck [#35754](https://github.com/ClickHouse/ClickHouse/pull/35754) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* extended a description of clickhouse-client compression parameter [#35758](https://github.com/ClickHouse/ClickHouse/pull/35758) ([Denny Crane](https://github.com/den-crane)). +* Resurrect build hash [#35766](https://github.com/ClickHouse/ClickHouse/pull/35766) ([alesapin](https://github.com/alesapin)). +* Fix 00484_preferred_max_column_in_block_size_bytes [#35768](https://github.com/ClickHouse/ClickHouse/pull/35768) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse-keeper: correctly handle closed client connection [#35772](https://github.com/ClickHouse/ClickHouse/pull/35772) ([Azat Khuzhin](https://github.com/azat)). +* ci: replace directory system log tables artifacts with tsv [#35773](https://github.com/ClickHouse/ClickHouse/pull/35773) ([Azat Khuzhin](https://github.com/azat)). +* One more try to resurrect build hash [#35774](https://github.com/ClickHouse/ClickHouse/pull/35774) ([alesapin](https://github.com/alesapin)). +* Refactoring QueryPipeline [#35789](https://github.com/ClickHouse/ClickHouse/pull/35789) ([Amos Bird](https://github.com/amosbird)). +* Delete duplicate code [#35798](https://github.com/ClickHouse/ClickHouse/pull/35798) ([xiedeyantu](https://github.com/xiedeyantu)). +* remove unused variable [#35800](https://github.com/ClickHouse/ClickHouse/pull/35800) ([flynn](https://github.com/ucasfl)). +* Make `SortDescription::column_name` always non-empty [#35805](https://github.com/ClickHouse/ClickHouse/pull/35805) ([Nikita Taranov](https://github.com/nickitat)). +* Fix latest_error referenced before assignment [#35807](https://github.com/ClickHouse/ClickHouse/pull/35807) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to fix some integration tests [#35808](https://github.com/ClickHouse/ClickHouse/pull/35808) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better error message for incorrect *MergeTree arguments [#35814](https://github.com/ClickHouse/ClickHouse/pull/35814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix missing `noexcept(false)` flag on destructor which causes builds to fail [#35817](https://github.com/ClickHouse/ClickHouse/pull/35817) ([tcoyvwac](https://github.com/tcoyvwac)). +* Try remove unneed variable [#35833](https://github.com/ClickHouse/ClickHouse/pull/35833) ([flynn](https://github.com/ucasfl)). +* Refactoring of hints for column descriptor [#35852](https://github.com/ClickHouse/ClickHouse/pull/35852) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix automatic bucket prefix for master [#35854](https://github.com/ClickHouse/ClickHouse/pull/35854) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Format changes for new docs [#35858](https://github.com/ClickHouse/ClickHouse/pull/35858) ([Rich Raposa](https://github.com/rfraposa)). +* Properly cancel the query after client format error [#35867](https://github.com/ClickHouse/ClickHouse/pull/35867) ([Azat Khuzhin](https://github.com/azat)). +* Drop modernize-replace-auto-ptr from .clang-tidy [#35868](https://github.com/ClickHouse/ClickHouse/pull/35868) ([Robert Schulze](https://github.com/rschu1ze)). +* fix service start with systemd [#35869](https://github.com/ClickHouse/ClickHouse/pull/35869) ([Denny Crane](https://github.com/den-crane)). +* fix postgres test [#35885](https://github.com/ClickHouse/ClickHouse/pull/35885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update poco: Fix polling of socket with negative timeout (when poll() interrupted by EINTR) [#35899](https://github.com/ClickHouse/ClickHouse/pull/35899) ([Azat Khuzhin](https://github.com/azat)). +* More logs on unsuccessful part removal [#35904](https://github.com/ClickHouse/ClickHouse/pull/35904) ([alesapin](https://github.com/alesapin)). +* Executable user defined functions prevent executing during analysis [#35917](https://github.com/ClickHouse/ClickHouse/pull/35917) ([Maksim Kita](https://github.com/kitaisreal)). +* JIT ProfileEvents added test [#35918](https://github.com/ClickHouse/ClickHouse/pull/35918) ([Maksim Kita](https://github.com/kitaisreal)). +* Clang tidy issues [#35919](https://github.com/ClickHouse/ClickHouse/pull/35919) ([Deleted user](https://github.com/ghost)). +* Fix race in cached buffer [#35922](https://github.com/ClickHouse/ClickHouse/pull/35922) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix building ubuntu image from deb-repo [#35931](https://github.com/ClickHouse/ClickHouse/pull/35931) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move credentials for integration tests into a separate file [#35936](https://github.com/ClickHouse/ClickHouse/pull/35936) ([Lalit Srikant](https://github.com/LAL2211)). +* [Snyk] Security upgrade node from 8 to 16.14.2 [#35942](https://github.com/ClickHouse/ClickHouse/pull/35942) ([Snyk bot](https://github.com/snyk-bot)). +* Fix keeper image for `arm64` [#35945](https://github.com/ClickHouse/ClickHouse/pull/35945) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make some replicated DDL queries faster [#35946](https://github.com/ClickHouse/ClickHouse/pull/35946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update Contentsquare company case [#35964](https://github.com/ClickHouse/ClickHouse/pull/35964) ([François Violette](https://github.com/fviolette)). +* Small code changes in ZooKeeper client/Keeper code [#35967](https://github.com/ClickHouse/ClickHouse/pull/35967) ([Antonio Andelic](https://github.com/antonio2368)). +* parallel reading files for FileLog Engine [#35980](https://github.com/ClickHouse/ClickHouse/pull/35980) ([flynn](https://github.com/ucasfl)). +* Fix data race in StorageURL [#35984](https://github.com/ClickHouse/ClickHouse/pull/35984) ([Antonio Andelic](https://github.com/antonio2368)). +* Improve logs analysis in stress test checks [#35985](https://github.com/ClickHouse/ClickHouse/pull/35985) ([Azat Khuzhin](https://github.com/azat)). +* Print labels to log in run_check.py [#35991](https://github.com/ClickHouse/ClickHouse/pull/35991) ([Vladimir C](https://github.com/vdimir)). +* Set `ENABLE_BUILD_PATH_MAPPING` to `OFF` by default, if `CMAKE_BUILD_TYPE` is set to `Debug` [#35998](https://github.com/ClickHouse/ClickHouse/pull/35998) ([Denis Glazachev](https://github.com/traceon)). +* Make test 00159_parallel_formatting_tsv_and_friends.sh more stable [#36001](https://github.com/ClickHouse/ClickHouse/pull/36001) ([Kruglov Pavel](https://github.com/Avogar)). +* Update version_date.tsv after v22.3.3.44-lts [#36002](https://github.com/ClickHouse/ClickHouse/pull/36002) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test `test_system_merges/test.py::test_merge_simple` [#36004](https://github.com/ClickHouse/ClickHouse/pull/36004) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Another fix for settings randomization [#36005](https://github.com/ClickHouse/ClickHouse/pull/36005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove some TestFlows that are irrelevant. [#36006](https://github.com/ClickHouse/ClickHouse/pull/36006) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix 02248_nullable_custom_types_to_string [#36008](https://github.com/ClickHouse/ClickHouse/pull/36008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 00155_long_merges [#36009](https://github.com/ClickHouse/ClickHouse/pull/36009) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unused code from TestFlows [#36010](https://github.com/ClickHouse/ClickHouse/pull/36010) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix data race in StorgeFileLog [#36015](https://github.com/ClickHouse/ClickHouse/pull/36015) ([flynn](https://github.com/ucasfl)). +* Fix Backward comapatibility check [#36026](https://github.com/ClickHouse/ClickHouse/pull/36026) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use storage_snapshot for projection analysis [#36034](https://github.com/ClickHouse/ClickHouse/pull/36034) ([Amos Bird](https://github.com/amosbird)). +* Nightly coverity [#36044](https://github.com/ClickHouse/ClickHouse/pull/36044) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix typo [#36047](https://github.com/ClickHouse/ClickHouse/pull/36047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test [#36054](https://github.com/ClickHouse/ClickHouse/pull/36054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trailing whitespace in default client_name [#36056](https://github.com/ClickHouse/ClickHouse/pull/36056) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-client: fix query cancellation if any result was not received yet [#36057](https://github.com/ClickHouse/ClickHouse/pull/36057) ([Azat Khuzhin](https://github.com/azat)). +* Add debug and fix cancel_rerun lambda [#36064](https://github.com/ClickHouse/ClickHouse/pull/36064) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix style in run_check.py [#36065](https://github.com/ClickHouse/ClickHouse/pull/36065) ([Vladimir C](https://github.com/vdimir)). +* Try to turn off always green backward compatibility checks [#36067](https://github.com/ClickHouse/ClickHouse/pull/36067) ([Kruglov Pavel](https://github.com/Avogar)). +* Less dependencies from disks in buffers [#36070](https://github.com/ClickHouse/ClickHouse/pull/36070) ([alesapin](https://github.com/alesapin)). +* Backups: improve arguments handling and file removing [#36072](https://github.com/ClickHouse/ClickHouse/pull/36072) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add tonickkozlov to trusted users for auto ci run [#36080](https://github.com/ClickHouse/ClickHouse/pull/36080) ([nvartolomei](https://github.com/nvartolomei)). +* Try to avoid timeoutes in parallel parsing tests [#36083](https://github.com/ClickHouse/ClickHouse/pull/36083) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix BUILD_NAME issue in build jobs [#36085](https://github.com/ClickHouse/ClickHouse/pull/36085) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove incorrect assertion [#36086](https://github.com/ClickHouse/ClickHouse/pull/36086) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of fs paths keeper [#36106](https://github.com/ClickHouse/ClickHouse/pull/36106) ([alesapin](https://github.com/alesapin)). +* fix typo [#36110](https://github.com/ClickHouse/ClickHouse/pull/36110) ([Denny Crane](https://github.com/den-crane)). +* Fix scan report filename [#36112](https://github.com/ClickHouse/ClickHouse/pull/36112) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new/delete overloads with align [#36125](https://github.com/ClickHouse/ClickHouse/pull/36125) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix code style in registerBackupEnginesFileAndDisk.cpp [#36127](https://github.com/ClickHouse/ClickHouse/pull/36127) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix flaky tests 00971 and 01003 [#36128](https://github.com/ClickHouse/ClickHouse/pull/36128) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 01171_mv_select_insert_isolation_long.sh [#36131](https://github.com/ClickHouse/ClickHouse/pull/36131) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build report [#36132](https://github.com/ClickHouse/ClickHouse/pull/36132) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix after [#35475](https://github.com/ClickHouse/ClickHouse/issues/35475) [#36135](https://github.com/ClickHouse/ClickHouse/pull/36135) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor fix for transactions [#36136](https://github.com/ClickHouse/ClickHouse/pull/36136) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProfileEvents fixes [#36137](https://github.com/ClickHouse/ClickHouse/pull/36137) ([Azat Khuzhin](https://github.com/azat)). +* Minor improvement for hung check [#36138](https://github.com/ClickHouse/ClickHouse/pull/36138) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Upload ccache for the first run in PR [#36139](https://github.com/ClickHouse/ClickHouse/pull/36139) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix test 02241_remote_filesystem_cache_on_insert for database Ordinary [#36150](https://github.com/ClickHouse/ClickHouse/pull/36150) ([Kseniia Sumarokova](https://github.com/kssenii)). +* remove unused array in h3Res0Indexes func [#36154](https://github.com/ClickHouse/ClickHouse/pull/36154) ([Bharat Nallan](https://github.com/bharatnc)). +* fix typo in cmake message [#36155](https://github.com/ClickHouse/ClickHouse/pull/36155) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix UT error sometimes [#36157](https://github.com/ClickHouse/ClickHouse/pull/36157) ([zhanglistar](https://github.com/zhanglistar)). +* Fix a logical error left after debugging [#36159](https://github.com/ClickHouse/ClickHouse/pull/36159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Some fixes for ReplicatedMergeTree [#36163](https://github.com/ClickHouse/ClickHouse/pull/36163) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-client: properly cancel query in case of error during formatting data [#36164](https://github.com/ClickHouse/ClickHouse/pull/36164) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert "Fix possible mutation stuck due to race with DROP_RANGE" [#36190](https://github.com/ClickHouse/ClickHouse/pull/36190) ([Azat Khuzhin](https://github.com/azat)). +* Use atomic instead of mutex + condvar in ParallelReadBuffer [#36192](https://github.com/ClickHouse/ClickHouse/pull/36192) ([Kruglov Pavel](https://github.com/Avogar)). +* Follow-up to [#36138](https://github.com/ClickHouse/ClickHouse/issues/36138) [#36194](https://github.com/ClickHouse/ClickHouse/pull/36194) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Revert reverting "Fix crash in ParallelReadBuffer" [#36212](https://github.com/ClickHouse/ClickHouse/pull/36212) ([Kruglov Pavel](https://github.com/Avogar)). +* Make stateless tests with s3 always green [#36214](https://github.com/ClickHouse/ClickHouse/pull/36214) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add Tyler Hannan to contributors [#36216](https://github.com/ClickHouse/ClickHouse/pull/36216) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix the repeated call of func to get the table when drop table [#36248](https://github.com/ClickHouse/ClickHouse/pull/36248) ([xiedeyantu](https://github.com/xiedeyantu)). +* Split test 01675_data_type_coroutine into 2 tests to prevent possible timeouts [#36250](https://github.com/ClickHouse/ClickHouse/pull/36250) ([Kruglov Pavel](https://github.com/Avogar)). +* Merge TRUSTED_CONTRIBUTORS in lambda and import in check [#36252](https://github.com/ClickHouse/ClickHouse/pull/36252) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix exception "File segment can be completed only by downloader" in tests [#36253](https://github.com/ClickHouse/ClickHouse/pull/36253) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests report parser [#36257](https://github.com/ClickHouse/ClickHouse/pull/36257) ([alesapin](https://github.com/alesapin)). +* fix crash when you use clickhouse-git-import with invalid parameter [#36262](https://github.com/ClickHouse/ClickHouse/pull/36262) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix cancel-lambda for closed PRs [#36269](https://github.com/ClickHouse/ClickHouse/pull/36269) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Place new logic of schema inference in insert select from table function under setting [#36275](https://github.com/ClickHouse/ClickHouse/pull/36275) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove trash from CPUID [#36310](https://github.com/ClickHouse/ClickHouse/pull/36310) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove remaining parts of Arcadia [#36312](https://github.com/ClickHouse/ClickHouse/pull/36312) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy-14 (part 1) [#36320](https://github.com/ClickHouse/ClickHouse/pull/36320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add dispatch trigger for debug CI [#36329](https://github.com/ClickHouse/ClickHouse/pull/36329) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Revert "support UNSIGNED modifier with unused parameters of INT" [#36337](https://github.com/ClickHouse/ClickHouse/pull/36337) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Simplify perf scripts by using schema inference [#36346](https://github.com/ClickHouse/ClickHouse/pull/36346) ([Azat Khuzhin](https://github.com/azat)). +* Do not randomize "priority" setting [#36358](https://github.com/ClickHouse/ClickHouse/pull/36358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration tests [#36361](https://github.com/ClickHouse/ClickHouse/pull/36361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove abbreviation [#36362](https://github.com/ClickHouse/ClickHouse/pull/36362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow test [#36363](https://github.com/ClickHouse/ClickHouse/pull/36363) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error in sanity checks [#36365](https://github.com/ClickHouse/ClickHouse/pull/36365) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add nodiscard attribute to immutable methods of `IColumn` to avoid errors [#36368](https://github.com/ClickHouse/ClickHouse/pull/36368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange trash in Keeper [#36369](https://github.com/ClickHouse/ClickHouse/pull/36369) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#36374](https://github.com/ClickHouse/ClickHouse/pull/36374) ([flynn](https://github.com/ucasfl)). +* Fix test 01161_all_system_tables under s3 storage [#36388](https://github.com/ClickHouse/ClickHouse/pull/36388) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor refactor to prefer C++ Standard Algorithms [#36393](https://github.com/ClickHouse/ClickHouse/pull/36393) ([tcoyvwac](https://github.com/tcoyvwac)). +* jemalloc: enable logging in debug build [#36398](https://github.com/ClickHouse/ClickHouse/pull/36398) ([Azat Khuzhin](https://github.com/azat)). +* Respect library type for contrib libraries [#36399](https://github.com/ClickHouse/ClickHouse/pull/36399) ([Azat Khuzhin](https://github.com/azat)). +* Add more harmful variables for OSX [#36400](https://github.com/ClickHouse/ClickHouse/pull/36400) ([Azat Khuzhin](https://github.com/azat)). +* Activate clang-tidy warning "readability-container-contains" [#36402](https://github.com/ClickHouse/ClickHouse/pull/36402) ([Robert Schulze](https://github.com/rschu1ze)). +* Move down data_range construction in filterMarksUsingIndex and filterMarksUsingMergedIndex [#36410](https://github.com/ClickHouse/ClickHouse/pull/36410) ([Tian Xinhui](https://github.com/xinhuitian)). +* Fix stress test [#36450](https://github.com/ClickHouse/ClickHouse/pull/36450) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 525c0de1341..71e077ac071 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -12,3 +12,7 @@ sidebar_label: 2022 * Backported in [#36524](https://github.com/ClickHouse/ClickHouse/issues/36524): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36673](https://github.com/ClickHouse/ClickHouse/issues/36673): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index b5fffa56494..636ad2ed3ac 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -13,3 +13,7 @@ sidebar_label: 2022 * Backported in [#36794](https://github.com/ClickHouse/ClickHouse/issues/36794): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#36926](https://github.com/ClickHouse/ClickHouse/issues/36926): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index 7614d678a2a..dfdcad64561 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -187,3 +187,204 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "BLAKE3 hash function documentation"'. [#37092](https://github.com/ClickHouse/ClickHouse/pull/37092) ([Rich Raposa](https://github.com/rfraposa)). * NO CL ENTRY: 'Revert "Remove height restrictions from the query div in play web tool."'. [#37261](https://github.com/ClickHouse/ClickHouse/pull/37261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Cleanup GROUPING SETS formating [#32999](https://github.com/ClickHouse/ClickHouse/pull/32999) ([Dmitry Novik](https://github.com/novikd)). +* Minor renames [#35272](https://github.com/ClickHouse/ClickHouse/pull/35272) ([Anton Popov](https://github.com/CurtizJ)). +* clickhouse-test: fix left-queries-check, to fix test log parser [#35865](https://github.com/ClickHouse/ClickHouse/pull/35865) ([Azat Khuzhin](https://github.com/azat)). +* Regression test for CHECKSUM_DOESNT_MATCH error because of per-column TTL bug [#35971](https://github.com/ClickHouse/ClickHouse/pull/35971) ([Azat Khuzhin](https://github.com/azat)). +* Fix performance tests [#35976](https://github.com/ClickHouse/ClickHouse/pull/35976) ([Maksim Kita](https://github.com/kitaisreal)). +* Backup for replicated tables and other improvements [#36198](https://github.com/ClickHouse/ClickHouse/pull/36198) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to fix some trash in zero copy replication [#36299](https://github.com/ClickHouse/ClickHouse/pull/36299) ([alesapin](https://github.com/alesapin)). +* Speed up build a little [#36319](https://github.com/ClickHouse/ClickHouse/pull/36319) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace remove-erase idiom by C++20 erase()/erase_if() [#36348](https://github.com/ClickHouse/ClickHouse/pull/36348) ([Robert Schulze](https://github.com/rschu1ze)). +* Strict taskstats parser [#36351](https://github.com/ClickHouse/ClickHouse/pull/36351) ([Azat Khuzhin](https://github.com/azat)). +* Draft changelog for version 22.4 [#36397](https://github.com/ClickHouse/ClickHouse/pull/36397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update int-uint.md [#36404](https://github.com/ClickHouse/ClickHouse/pull/36404) ([hardstep33](https://github.com/hardstep33)). +* Return back [#36126](https://github.com/ClickHouse/ClickHouse/issues/36126) [#36423](https://github.com/ClickHouse/ClickHouse/pull/36423) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed warnings of clang-tidy check "bugprone-branch-clone" [#36431](https://github.com/ClickHouse/ClickHouse/pull/36431) ([Robert Schulze](https://github.com/rschu1ze)). +* Clang tidy fixes [#36444](https://github.com/ClickHouse/ClickHouse/pull/36444) ([Robert Schulze](https://github.com/rschu1ze)). +* fixed /common/example cow_compositions.cpp clone the inner column when there is no need [#36453](https://github.com/ClickHouse/ClickHouse/pull/36453) ([zombee0](https://github.com/zombee0)). +* Refactoring dependency for ParserAttachAccessEntity [#36468](https://github.com/ClickHouse/ClickHouse/pull/36468) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* more specific warning text about low disk space [#36472](https://github.com/ClickHouse/ClickHouse/pull/36472) ([Sergei Trifonov](https://github.com/serxa)). +* Fixed missing enum values for ClientInfo::Interface [#36482](https://github.com/ClickHouse/ClickHouse/pull/36482) ([Vasily Nemkov](https://github.com/Enmk)). +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update version after release [#36502](https://github.com/ClickHouse/ClickHouse/pull/36502) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Followup on [#36172](https://github.com/ClickHouse/ClickHouse/issues/36172) password hash salt feature [#36510](https://github.com/ClickHouse/ClickHouse/pull/36510) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update version_date.tsv after v22.4.2.1-stable [#36533](https://github.com/ClickHouse/ClickHouse/pull/36533) ([github-actions[bot]](https://github.com/apps/github-actions)). +* fix log should print 'from' path [#36535](https://github.com/ClickHouse/ClickHouse/pull/36535) ([xiedeyantu](https://github.com/xiedeyantu)). +* Add function bin tests for Int/UInt128/UInt256 [#36537](https://github.com/ClickHouse/ClickHouse/pull/36537) ([Memo](https://github.com/Joeywzr)). +* Fix 01161_all_system_tables [#36539](https://github.com/ClickHouse/ClickHouse/pull/36539) ([Antonio Andelic](https://github.com/antonio2368)). +* Update PULL_REQUEST_TEMPLATE.md [#36543](https://github.com/ClickHouse/ClickHouse/pull/36543) ([Ivan Blinkov](https://github.com/blinkov)). +* Fixed integer overflow in toStartOfInterval [#36546](https://github.com/ClickHouse/ClickHouse/pull/36546) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix issue with broken git ownership [#36548](https://github.com/ClickHouse/ClickHouse/pull/36548) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix version autodetection for docker_server.py [#36552](https://github.com/ClickHouse/ClickHouse/pull/36552) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Snyk fixes [#36554](https://github.com/ClickHouse/ClickHouse/pull/36554) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use just index to split performance tests by group [#36559](https://github.com/ClickHouse/ClickHouse/pull/36559) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update quantiledeterministic.md [#36560](https://github.com/ClickHouse/ClickHouse/pull/36560) ([ifinik](https://github.com/ifinik)). +* CodeQL Experiment - Exclude contrib Dependencies [#36561](https://github.com/ClickHouse/ClickHouse/pull/36561) ([Julio Jimenez](https://github.com/juliojimenez)). +* Small refactoring of Processors and QueryPipeline [#36579](https://github.com/ClickHouse/ClickHouse/pull/36579) ([Amos Bird](https://github.com/amosbird)). +* Simplify 01834_alias_columns_laziness_filimonov test [#36585](https://github.com/ClickHouse/ClickHouse/pull/36585) ([Azat Khuzhin](https://github.com/azat)). +* bash-completion: add completion for send_logs_level [#36586](https://github.com/ClickHouse/ClickHouse/pull/36586) ([Azat Khuzhin](https://github.com/azat)). +* client: add a message on reconnect (under warning log level) [#36587](https://github.com/ClickHouse/ClickHouse/pull/36587) ([Azat Khuzhin](https://github.com/azat)). +* Fix strange whitespace (or I do not know YAML) [#36590](https://github.com/ClickHouse/ClickHouse/pull/36590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Strict taskstats parser (resubmit) [#36602](https://github.com/ClickHouse/ClickHouse/pull/36602) ([Azat Khuzhin](https://github.com/azat)). +* Exclude test `02271_fix_column_matcher_and_column_transformer` from bc check [#36607](https://github.com/ClickHouse/ClickHouse/pull/36607) ([alesapin](https://github.com/alesapin)). +* Ancient cmake version cleanup [#36612](https://github.com/ClickHouse/ClickHouse/pull/36612) ([Robert Schulze](https://github.com/rschu1ze)). +* Cleanup clang-tidy integration. [#36613](https://github.com/ClickHouse/ClickHouse/pull/36613) ([Robert Schulze](https://github.com/rschu1ze)). +* ParallelReadBuffer small improvements [#36619](https://github.com/ClickHouse/ClickHouse/pull/36619) ([Antonio Andelic](https://github.com/antonio2368)). +* Lambda cancel sync [#36622](https://github.com/ClickHouse/ClickHouse/pull/36622) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Version history benchmarks [#36628](https://github.com/ClickHouse/ClickHouse/pull/36628) ([Ilya Yatsishin](https://github.com/qoega)). +* fix typo [#36629](https://github.com/ClickHouse/ClickHouse/pull/36629) ([Sergei Trifonov](https://github.com/serxa)). +* Tiny Mutator code cleanup [#36630](https://github.com/ClickHouse/ClickHouse/pull/36630) ([Azat Khuzhin](https://github.com/azat)). +* Disble test postgresql replica with asan [#36631](https://github.com/ClickHouse/ClickHouse/pull/36631) ([alesapin](https://github.com/alesapin)). +* Minor Coverity defects fixes [#36632](https://github.com/ClickHouse/ClickHouse/pull/36632) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Properly wait for queries in 01502_long_log_tinylog_deadlock_race test [#36634](https://github.com/ClickHouse/ClickHouse/pull/36634) ([Azat Khuzhin](https://github.com/azat)). +* remove unneeded if statement [#36636](https://github.com/ClickHouse/ClickHouse/pull/36636) ([flynn](https://github.com/ucasfl)). +* Fix SortingStep::updateOutputStream() [#36638](https://github.com/ClickHouse/ClickHouse/pull/36638) ([Nikita Taranov](https://github.com/nickitat)). +* CodeQL - Run Daily Analysis [#36640](https://github.com/ClickHouse/ClickHouse/pull/36640) ([Julio Jimenez](https://github.com/juliojimenez)). +* Simplify check_branch, prefetch target branches [#36641](https://github.com/ClickHouse/ClickHouse/pull/36641) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Make backward compatible integration tests runner [#36643](https://github.com/ClickHouse/ClickHouse/pull/36643) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix some flaky tests [#36644](https://github.com/ClickHouse/ClickHouse/pull/36644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update version_date.tsv after v22.4.3.3-stable [#36651](https://github.com/ClickHouse/ClickHouse/pull/36651) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test [#36652](https://github.com/ClickHouse/ClickHouse/pull/36652) ([Amos Bird](https://github.com/amosbird)). +* add missing pandas package [#36653](https://github.com/ClickHouse/ClickHouse/pull/36653) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Fix style issue reported by black formatter [#36655](https://github.com/ClickHouse/ClickHouse/pull/36655) ([Alexander Gololobov](https://github.com/davenger)). +* Fix formatting in drop cache system query [#36658](https://github.com/ClickHouse/ClickHouse/pull/36658) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix stress test after 36639 [#36660](https://github.com/ClickHouse/ClickHouse/pull/36660) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.3.4.20-lts [#36668](https://github.com/ClickHouse/ClickHouse/pull/36668) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update 00170_s3_cache.sql [#36669](https://github.com/ClickHouse/ClickHouse/pull/36669) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better version of SeekableReadBufferWithSize [#36676](https://github.com/ClickHouse/ClickHouse/pull/36676) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better logs for virtual parts [#36680](https://github.com/ClickHouse/ClickHouse/pull/36680) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Check socket is connected in HTTPSession [#36683](https://github.com/ClickHouse/ClickHouse/pull/36683) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typo [#36684](https://github.com/ClickHouse/ClickHouse/pull/36684) ([flynn](https://github.com/ucasfl)). +* Remove excessive logging from S3 [#36689](https://github.com/ClickHouse/ClickHouse/pull/36689) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tests report [#36701](https://github.com/ClickHouse/ClickHouse/pull/36701) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use correct nodes in test_keeper_znode_time [#36711](https://github.com/ClickHouse/ClickHouse/pull/36711) ([Antonio Andelic](https://github.com/antonio2368)). +* Reorganize source files so that base won't depend on Common [#36715](https://github.com/ClickHouse/ClickHouse/pull/36715) ([Amos Bird](https://github.com/amosbird)). +* tests: fix 02015_async_inserts_stress_long flakiness [#36731](https://github.com/ClickHouse/ClickHouse/pull/36731) ([Azat Khuzhin](https://github.com/azat)). +* Add an extra check for RAFT config change [#36736](https://github.com/ClickHouse/ClickHouse/pull/36736) ([Antonio Andelic](https://github.com/antonio2368)). +* Update 00170_s3_cache.sql [#36743](https://github.com/ClickHouse/ClickHouse/pull/36743) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ClickHouseDictionarySource context copy [#36744](https://github.com/ClickHouse/ClickHouse/pull/36744) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix build SeekableReadBufferWithSize -> SeekableReadBuffer [#36745](https://github.com/ClickHouse/ClickHouse/pull/36745) ([Vladimir C](https://github.com/vdimir)). +* Better error message from NuRaft in case of CRC mismatch [#36746](https://github.com/ClickHouse/ClickHouse/pull/36746) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix certs (finishing pr [#36457](https://github.com/ClickHouse/ClickHouse/issues/36457)) [#36747](https://github.com/ClickHouse/ClickHouse/pull/36747) ([Nikita Taranov](https://github.com/nickitat)). +* Another fix for Hung Check [#36752](https://github.com/ClickHouse/ClickHouse/pull/36752) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Deduce `UInt8` type for bools from json instead of `UInt64` [#36756](https://github.com/ClickHouse/ClickHouse/pull/36756) ([Anton Popov](https://github.com/CurtizJ)). +* Add small script for keeper check [#36758](https://github.com/ClickHouse/ClickHouse/pull/36758) ([alesapin](https://github.com/alesapin)). +* Fix ungrammatical error message [#36760](https://github.com/ClickHouse/ClickHouse/pull/36760) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test `test_ddl_worker_non_leader` [#36765](https://github.com/ClickHouse/ClickHouse/pull/36765) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix evaluateConstantExpression for subqueries [#36766](https://github.com/ClickHouse/ClickHouse/pull/36766) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clickhouse-test for server without `is_all_data_sent` in system.processes [#36767](https://github.com/ClickHouse/ClickHouse/pull/36767) ([Azat Khuzhin](https://github.com/azat)). +* Init thread pools for clickhouse-local [#36778](https://github.com/ClickHouse/ClickHouse/pull/36778) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix performance test [#36779](https://github.com/ClickHouse/ClickHouse/pull/36779) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception messages while socket timeouts [#36781](https://github.com/ClickHouse/ClickHouse/pull/36781) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve docs_check to be used in clickhouse-docs [#36796](https://github.com/ClickHouse/ClickHouse/pull/36796) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* pull poco #58 [#36798](https://github.com/ClickHouse/ClickHouse/pull/36798) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Tiny cleanup [#36799](https://github.com/ClickHouse/ClickHouse/pull/36799) ([Azat Khuzhin](https://github.com/azat)). +* Use mutex per worker in ParallelReadBuffer [#36801](https://github.com/ClickHouse/ClickHouse/pull/36801) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv after v22.3.5.5-lts [#36805](https://github.com/ClickHouse/ClickHouse/pull/36805) ([github-actions[bot]](https://github.com/apps/github-actions)). +* play.html: add button to transpose table [#36811](https://github.com/ClickHouse/ClickHouse/pull/36811) ([Vladimir C](https://github.com/vdimir)). +* Fix style error in DocsCheck [#36813](https://github.com/ClickHouse/ClickHouse/pull/36813) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set is_all_data_sent on exceptions too [#36816](https://github.com/ClickHouse/ClickHouse/pull/36816) ([Azat Khuzhin](https://github.com/azat)). +* Clang -Tidy Fixes [#36817](https://github.com/ClickHouse/ClickHouse/pull/36817) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Protection from incorrect build [#36819](https://github.com/ClickHouse/ClickHouse/pull/36819) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid recreation of system.asynchronous_metric_log (due to difference in codec) [#36820](https://github.com/ClickHouse/ClickHouse/pull/36820) ([Azat Khuzhin](https://github.com/azat)). +* Removed forceful drop cache command, fix detached status state [#36825](https://github.com/ClickHouse/ClickHouse/pull/36825) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log hash table's cache messages with TRACE level [#36830](https://github.com/ClickHouse/ClickHouse/pull/36830) ([Nikita Taranov](https://github.com/nickitat)). +* Cleanup: Remove switches for obsolete GCC version [#36831](https://github.com/ClickHouse/ClickHouse/pull/36831) ([Robert Schulze](https://github.com/rschu1ze)). +* Add functions from CREATE FUNCTION to completion [#36834](https://github.com/ClickHouse/ClickHouse/pull/36834) ([Azat Khuzhin](https://github.com/azat)). +* Validate that function had been passed in CREATE FUNCTION [#36835](https://github.com/ClickHouse/ClickHouse/pull/36835) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unimplemented RAID1 support [#36836](https://github.com/ClickHouse/ClickHouse/pull/36836) ([Azat Khuzhin](https://github.com/azat)). +* Disable `merge_tree_metadata_cache` by default [#36838](https://github.com/ClickHouse/ClickHouse/pull/36838) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace uses of SFINAE by C++20 concepts [#36839](https://github.com/ClickHouse/ClickHouse/pull/36839) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix performance test (4) [#36840](https://github.com/ClickHouse/ClickHouse/pull/36840) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "preconditions" from performance tests (overengineering, unneeded feature) [#36841](https://github.com/ClickHouse/ClickHouse/pull/36841) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove inherited create() method + disallow copying of storages [#36844](https://github.com/ClickHouse/ClickHouse/pull/36844) ([Robert Schulze](https://github.com/rschu1ze)). +* tests: disable 02260_alter_compact_part_drop_nested_column for <=22.4 [#36845](https://github.com/ClickHouse/ClickHouse/pull/36845) ([Azat Khuzhin](https://github.com/azat)). +* Fix check black [#36850](https://github.com/ClickHouse/ClickHouse/pull/36850) ([Kruglov Pavel](https://github.com/Avogar)). +* Aggregator JIT compilation lock fix [#36852](https://github.com/ClickHouse/ClickHouse/pull/36852) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Other Query Time Microseconds Profile Event [#36853](https://github.com/ClickHouse/ClickHouse/pull/36853) ([Ilya Yatsishin](https://github.com/qoega)). +* Reproduce and a little bit better fix for LC dict right offset. [#36856](https://github.com/ClickHouse/ClickHouse/pull/36856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Terminate if S3 buffer is not finalized [#36857](https://github.com/ClickHouse/ClickHouse/pull/36857) ([alesapin](https://github.com/alesapin)). +* Activated a bunch of LLVM 12/13/14 clang-tidy warnings [#36862](https://github.com/ClickHouse/ClickHouse/pull/36862) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace `Timeout` with `Tests not finished` [#36863](https://github.com/ClickHouse/ClickHouse/pull/36863) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backup improvements [#36864](https://github.com/ClickHouse/ClickHouse/pull/36864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Relax cmake check for CFLAGS/CXXFLAGS/LDFLAGS [#36869](https://github.com/ClickHouse/ClickHouse/pull/36869) ([Azat Khuzhin](https://github.com/azat)). +* Fix insertion of complex JSONs with nested arrays [2] [#36873](https://github.com/ClickHouse/ClickHouse/pull/36873) ([Anton Popov](https://github.com/CurtizJ)). +* Add column to system.filesystem_cache_log [#36874](https://github.com/ClickHouse/ClickHouse/pull/36874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log into filesystem_cache_log when cache not even attempted [#36876](https://github.com/ClickHouse/ClickHouse/pull/36876) ([Kseniia Sumarokova](https://github.com/kssenii)). +* fix typo in comment [#36880](https://github.com/ClickHouse/ClickHouse/pull/36880) ([Sergei Trifonov](https://github.com/serxa)). +* Add some CurrentMetrics for fs cache [#36882](https://github.com/ClickHouse/ClickHouse/pull/36882) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add some asynchronous metrics for fs cache [#36883](https://github.com/ClickHouse/ClickHouse/pull/36883) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Save system logs in functionsl tests if server crashed [#36885](https://github.com/ClickHouse/ClickHouse/pull/36885) ([Alexander Tokmakov](https://github.com/tavplubix)). +* quick tmp fix for stress test [#36900](https://github.com/ClickHouse/ClickHouse/pull/36900) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Invert .clang-tidy [#36911](https://github.com/ClickHouse/ClickHouse/pull/36911) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace make_pair()/make_tuple() by pair()/tuple() [#36913](https://github.com/ClickHouse/ClickHouse/pull/36913) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove log message on client reconnects (reverts [#36587](https://github.com/ClickHouse/ClickHouse/issues/36587)) [#36915](https://github.com/ClickHouse/ClickHouse/pull/36915) ([Azat Khuzhin](https://github.com/azat)). +* Fix profile events in fs cached buffer [#36916](https://github.com/ClickHouse/ClickHouse/pull/36916) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Benchmark was not loaded properly [#36918](https://github.com/ClickHouse/ClickHouse/pull/36918) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Force keeper snapshot equality on different replicas [#36947](https://github.com/ClickHouse/ClickHouse/pull/36947) ([alesapin](https://github.com/alesapin)). +* Remove strange code [#36951](https://github.com/ClickHouse/ClickHouse/pull/36951) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nested exception [#36952](https://github.com/ClickHouse/ClickHouse/pull/36952) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clangtidies [#36960](https://github.com/ClickHouse/ClickHouse/pull/36960) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve stress tests report a little bit [#36961](https://github.com/ClickHouse/ClickHouse/pull/36961) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Follow up for ConcurrentHashJoin [#36970](https://github.com/ClickHouse/ClickHouse/pull/36970) ([Vladimir C](https://github.com/vdimir)). +* Fix formats docs [#36972](https://github.com/ClickHouse/ClickHouse/pull/36972) ([Kruglov Pavel](https://github.com/Avogar)). +* Support secure connection in clickhouse-test [#36977](https://github.com/ClickHouse/ClickHouse/pull/36977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix installation_id filter, minimize docker images diff [#36978](https://github.com/ClickHouse/ClickHouse/pull/36978) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version_date.tsv after v22.3.6.5-lts [#36985](https://github.com/ClickHouse/ClickHouse/pull/36985) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update version_date.tsv after v22.4.5.9-stable [#36986](https://github.com/ClickHouse/ClickHouse/pull/36986) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Removing ReplacingWindowColumnTransform in WindowView [#36998](https://github.com/ClickHouse/ClickHouse/pull/36998) ([vxider](https://github.com/Vxider)). +* Merging [#34765](https://github.com/ClickHouse/ClickHouse/issues/34765) [#37002](https://github.com/ClickHouse/ClickHouse/pull/37002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove CodeQL [#37006](https://github.com/ClickHouse/ClickHouse/pull/37006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete code [#37009](https://github.com/ClickHouse/ClickHouse/pull/37009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix flaky test [#37010](https://github.com/ClickHouse/ClickHouse/pull/37010) ([Nikita Taranov](https://github.com/nickitat)). +* fix output error in LabelsCheck [#37016](https://github.com/ClickHouse/ClickHouse/pull/37016) ([wuxiaobai24](https://github.com/wuxiaobai24)). +* remove useless code [#37020](https://github.com/ClickHouse/ClickHouse/pull/37020) ([flynn](https://github.com/ucasfl)). +* Merging [#34932](https://github.com/ClickHouse/ClickHouse/issues/34932). [#37023](https://github.com/ClickHouse/ClickHouse/pull/37023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix benchmark build code [#37025](https://github.com/ClickHouse/ClickHouse/pull/37025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add benchmark script for testing clouds [#37027](https://github.com/ClickHouse/ClickHouse/pull/37027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration mongodb test [#37035](https://github.com/ClickHouse/ClickHouse/pull/37035) ([Kruglov Pavel](https://github.com/Avogar)). +* Print stacks if we cannot terminate server in stress tests [#37052](https://github.com/ClickHouse/ClickHouse/pull/37052) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove last mentions of data streams [#37053](https://github.com/ClickHouse/ClickHouse/pull/37053) ([Anton Popov](https://github.com/CurtizJ)). +* Changelog script [#37057](https://github.com/ClickHouse/ClickHouse/pull/37057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* This evening I started using Grammarly. [#37058](https://github.com/ClickHouse/ClickHouse/pull/37058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update sanity checks warning message text [#37063](https://github.com/ClickHouse/ClickHouse/pull/37063) ([Sergei Trifonov](https://github.com/serxa)). +* Continue fixing [#36199](https://github.com/ClickHouse/ClickHouse/issues/36199). [#37071](https://github.com/ClickHouse/ClickHouse/pull/37071) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* just fix a number [#37072](https://github.com/ClickHouse/ClickHouse/pull/37072) ([jiahui-97](https://github.com/jiahui-97)). +* Disable thread fuzzer after server restart [#37079](https://github.com/ClickHouse/ClickHouse/pull/37079) ([alesapin](https://github.com/alesapin)). +* Option to force cross_to_inner_join_rewrite [#37085](https://github.com/ClickHouse/ClickHouse/pull/37085) ([Vladimir C](https://github.com/vdimir)). +* fix wrong argument in proxy resolver of DiskS3 [#37100](https://github.com/ClickHouse/ClickHouse/pull/37100) ([flynn](https://github.com/ucasfl)). +* tests: fix 01119_optimize_trivial_insert_select (due to max_threads randomization) [#37101](https://github.com/ClickHouse/ClickHouse/pull/37101) ([Azat Khuzhin](https://github.com/azat)). +* Support SELECT query in WindowView [#37105](https://github.com/ClickHouse/ClickHouse/pull/37105) ([vxider](https://github.com/Vxider)). +* Fix workflow style check [#37113](https://github.com/ClickHouse/ClickHouse/pull/37113) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for columns size in Block::cloneWithColumns [#37124](https://github.com/ClickHouse/ClickHouse/pull/37124) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update documentation and defaults for memory overcommit [#37129](https://github.com/ClickHouse/ClickHouse/pull/37129) ([Dmitry Novik](https://github.com/novikd)). +* Update default remote fs read method in ReadSettings [#37130](https://github.com/ClickHouse/ClickHouse/pull/37130) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update Exception Message for allowed auth types [#37132](https://github.com/ClickHouse/ClickHouse/pull/37132) ([Marcelo Rodriguez](https://github.com/e-mars)). +* tests/integration: fix possible race for iptables user rules inside containers [#37138](https://github.com/ClickHouse/ClickHouse/pull/37138) ([Azat Khuzhin](https://github.com/azat)). +* Fix fasttest ccache permissions [#37143](https://github.com/ClickHouse/ClickHouse/pull/37143) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add const qualifier for few methods from Context [#37154](https://github.com/ClickHouse/ClickHouse/pull/37154) ([Azat Khuzhin](https://github.com/azat)). +* Reload listen_reuse_port/listen_backlog from config [#37156](https://github.com/ClickHouse/ClickHouse/pull/37156) ([Azat Khuzhin](https://github.com/azat)). +* Enable DNS cache for HTTPSClientSession. [#37157](https://github.com/ClickHouse/ClickHouse/pull/37157) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix async reads from remote fs internal setting not being always turned on [#37164](https://github.com/ClickHouse/ClickHouse/pull/37164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backups Improvements 5 [#37168](https://github.com/ClickHouse/ClickHouse/pull/37168) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add S3Requests metric [#37200](https://github.com/ClickHouse/ClickHouse/pull/37200) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix coverity build problem with LD_PRELOAD [#37203](https://github.com/ClickHouse/ClickHouse/pull/37203) ([Boris Kuschel](https://github.com/bkuschel)). +* Relax log level for some checks in check thread [#37208](https://github.com/ClickHouse/ClickHouse/pull/37208) ([alesapin](https://github.com/alesapin)). +* update poco [#37209](https://github.com/ClickHouse/ClickHouse/pull/37209) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Ignore harmful env variables in clickhouse binaries (reexec w/o them) [#37211](https://github.com/ClickHouse/ClickHouse/pull/37211) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong comment in IVolume.h [#37218](https://github.com/ClickHouse/ClickHouse/pull/37218) ([Sergei Trifonov](https://github.com/serxa)). +* Cmake cleanup pt2 [#37222](https://github.com/ClickHouse/ClickHouse/pull/37222) ([Robert Schulze](https://github.com/rschu1ze)). +* Temporarily fix flaky test `01825_type_json_insert_select.sql` [#37245](https://github.com/ClickHouse/ClickHouse/pull/37245) ([Anton Popov](https://github.com/CurtizJ)). +* Pass need_filter, has_null_map to joinRightColumns [#37256](https://github.com/ClickHouse/ClickHouse/pull/37256) ([Vladimir C](https://github.com/vdimir)). +* Activate more clangtidies [#37259](https://github.com/ClickHouse/ClickHouse/pull/37259) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix docker cleaner in workflows [#37271](https://github.com/ClickHouse/ClickHouse/pull/37271) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tidy build fix [#37291](https://github.com/ClickHouse/ClickHouse/pull/37291) ([Alexander Gololobov](https://github.com/davenger)). +* Update run-check.py to match PR template, add comments [#37301](https://github.com/ClickHouse/ClickHouse/pull/37301) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Release without prestable [#37306](https://github.com/ClickHouse/ClickHouse/pull/37306) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixed typos [#37322](https://github.com/ClickHouse/ClickHouse/pull/37322) ([Alexander Gololobov](https://github.com/davenger)). + diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 9c7ecc1dae3..eeb4078eb04 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -197,3 +197,164 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "More parallel execution for queries with `FINAL`"'. [#38094](https://github.com/ClickHouse/ClickHouse/pull/38094) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'Revert "Revert "add d3js based trace visualizer as gantt chart""'. [#38129](https://github.com/ClickHouse/ClickHouse/pull/38129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix collision of S3 operation log revision [#34915](https://github.com/ClickHouse/ClickHouse/pull/34915) ([ianton-ru](https://github.com/ianton-ru)). +* Change timezone in stateless tests [#35231](https://github.com/ClickHouse/ClickHouse/pull/35231) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code in ReplicatedMergeTreeRestartingThread [#36113](https://github.com/ClickHouse/ClickHouse/pull/36113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge release and performance builds [#36538](https://github.com/ClickHouse/ClickHouse/pull/36538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add support for preprocessing ZooKeeper operations in `clickhouse-keeper` [#37036](https://github.com/ClickHouse/ClickHouse/pull/37036) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix jemalloc compatibility with LLVM libunwind [#37078](https://github.com/ClickHouse/ClickHouse/pull/37078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Separate object storage operations from disks [#37139](https://github.com/ClickHouse/ClickHouse/pull/37139) ([alesapin](https://github.com/alesapin)). +* Add test for WATCH LIMIT query in WindowView [#37219](https://github.com/ClickHouse/ClickHouse/pull/37219) ([vxider](https://github.com/Vxider)). +* Improve changelog script [#37249](https://github.com/ClickHouse/ClickHouse/pull/37249) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rework AccessControl's notifications. [#37269](https://github.com/ClickHouse/ClickHouse/pull/37269) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add ability to pass QueryKind via clickhouse-client/local (useful for debugging) [#37290](https://github.com/ClickHouse/ClickHouse/pull/37290) ([Azat Khuzhin](https://github.com/azat)). +* Various cmake cleanups [#37300](https://github.com/ClickHouse/ClickHouse/pull/37300) ([Robert Schulze](https://github.com/rschu1ze)). +* Try to fix some trash [#37303](https://github.com/ClickHouse/ClickHouse/pull/37303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [bug-fix] root_dir is not set in copyThroughBuffers [#37319](https://github.com/ClickHouse/ClickHouse/pull/37319) ([lingo-xp](https://github.com/lingo-xp)). +* Speed up test 00157_cache_dictionary [#37320](https://github.com/ClickHouse/ClickHouse/pull/37320) ([Kruglov Pavel](https://github.com/Avogar)). +* Add changelog for 22.5 [#37339](https://github.com/ClickHouse/ClickHouse/pull/37339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release [#37346](https://github.com/ClickHouse/ClickHouse/pull/37346) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable clang-tidy readability-identifier-length [#37347](https://github.com/ClickHouse/ClickHouse/pull/37347) ([Robert Schulze](https://github.com/rschu1ze)). +* Tags and release [#37348](https://github.com/ClickHouse/ClickHouse/pull/37348) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* New stable tag [#37349](https://github.com/ClickHouse/ClickHouse/pull/37349) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixes for eigen library build [#37369](https://github.com/ClickHouse/ClickHouse/pull/37369) ([Alexander Gololobov](https://github.com/davenger)). +* Do not fail CI if events clickhouse is down [#37371](https://github.com/ClickHouse/ClickHouse/pull/37371) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Ugly hacks for performance.tgz artifacts [#37373](https://github.com/ClickHouse/ClickHouse/pull/37373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix docker server images building [#37374](https://github.com/ClickHouse/ClickHouse/pull/37374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix debug symbols in packages [#37379](https://github.com/ClickHouse/ClickHouse/pull/37379) ([Azat Khuzhin](https://github.com/azat)). +* Enable -Wc++98-compat-extra-semi [#37382](https://github.com/ClickHouse/ClickHouse/pull/37382) ([Robert Schulze](https://github.com/rschu1ze)). +* Mark all `operator bool()` as explicit [#37388](https://github.com/ClickHouse/ClickHouse/pull/37388) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes for transactions [#37398](https://github.com/ClickHouse/ClickHouse/pull/37398) ([Alexander Tokmakov](https://github.com/tavplubix)). +* WindowTransform::moveRowNumber fix [#37400](https://github.com/ClickHouse/ClickHouse/pull/37400) ([Nikolay](https://github.com/ndchikin)). +* Fix possible memory leaks in system.certificates implementation [#37407](https://github.com/ClickHouse/ClickHouse/pull/37407) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove unused code in WindowView [#37426](https://github.com/ClickHouse/ClickHouse/pull/37426) ([vxider](https://github.com/Vxider)). +* Add join_algorithm='parallel_hash' to stress test [#37436](https://github.com/ClickHouse/ClickHouse/pull/37436) ([Vladimir C](https://github.com/vdimir)). +* Perf test for Norm and Distance functions for arrays and tuples [#37437](https://github.com/ClickHouse/ClickHouse/pull/37437) ([Alexander Gololobov](https://github.com/davenger)). +* FunctionBinaryRepresentation style fixes [#37438](https://github.com/ClickHouse/ClickHouse/pull/37438) ([Maksim Kita](https://github.com/kitaisreal)). +* CompressedWriteBuffer added comment [#37442](https://github.com/ClickHouse/ClickHouse/pull/37442) ([Maksim Kita](https://github.com/kitaisreal)). +* BinaryFunctionVectorized remove macro [#37447](https://github.com/ClickHouse/ClickHouse/pull/37447) ([Maksim Kita](https://github.com/kitaisreal)). +* RangeHashedDictionary added test [#37449](https://github.com/ClickHouse/ClickHouse/pull/37449) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix failed thread stateless tests [#37455](https://github.com/ClickHouse/ClickHouse/pull/37455) ([Kruglov Pavel](https://github.com/Avogar)). +* tests: fix table in 01710_projection_aggregation_in_order [#37468](https://github.com/ClickHouse/ClickHouse/pull/37468) ([Azat Khuzhin](https://github.com/azat)). +* Sync workflows paths for PR and DocsCheck [#37477](https://github.com/ClickHouse/ClickHouse/pull/37477) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set compiler flags for ASM same as for C/CXX [#37479](https://github.com/ClickHouse/ClickHouse/pull/37479) ([Azat Khuzhin](https://github.com/azat)). +* Dynamic dispatch infrastructure style fixes [#37480](https://github.com/ClickHouse/ClickHouse/pull/37480) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactorings of LIKE/MATCH code [#37491](https://github.com/ClickHouse/ClickHouse/pull/37491) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve changelog.py script, get changelogs for 2021 [#37496](https://github.com/ClickHouse/ClickHouse/pull/37496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to run aarch64 performance tests [#37497](https://github.com/ClickHouse/ClickHouse/pull/37497) ([Ilya Yatsishin](https://github.com/qoega)). +* Renamed arrayXXNorm/arrayXXDistance functions to XXNorm/XXDistance and fixed some overflow cases [#37502](https://github.com/ClickHouse/ClickHouse/pull/37502) ([Alexander Gololobov](https://github.com/davenger)). +* Bump cctz to 2022-05-15 [#37518](https://github.com/ClickHouse/ClickHouse/pull/37518) ([Robert Schulze](https://github.com/rschu1ze)). +* Add failed builds to the build report [#37527](https://github.com/ClickHouse/ClickHouse/pull/37527) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Functions normalizeUTF8 unstable performance tests fix [#37528](https://github.com/ClickHouse/ClickHouse/pull/37528) ([Maksim Kita](https://github.com/kitaisreal)). +* Use a separate mutex for query_factories_info in Context. [#37532](https://github.com/ClickHouse/ClickHouse/pull/37532) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update SECURITY.md [#37541](https://github.com/ClickHouse/ClickHouse/pull/37541) ([Ivan Blinkov](https://github.com/blinkov)). +* Refactor read metrics and callbacks [#37543](https://github.com/ClickHouse/ClickHouse/pull/37543) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#37398](https://github.com/ClickHouse/ClickHouse/issues/37398) [#37547](https://github.com/ClickHouse/ClickHouse/pull/37547) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix root CMakeLists.txt search [#37552](https://github.com/ClickHouse/ClickHouse/pull/37552) ([Sergei Trifonov](https://github.com/serxa)). +* Cleanup StorageHDFS (unused variables prevent build with clang 12) [#37554](https://github.com/ClickHouse/ClickHouse/pull/37554) ([Michail Safronov](https://github.com/msaf1980)). +* Fix failed assertion in cache [#37566](https://github.com/ClickHouse/ClickHouse/pull/37566) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Slightly better jepsen tests [#37568](https://github.com/ClickHouse/ClickHouse/pull/37568) ([alesapin](https://github.com/alesapin)). +* Remove unused MergeTreeDataMergerMutator::chooseMergeAlgorithm() [#37574](https://github.com/ClickHouse/ClickHouse/pull/37574) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless files [#37579](https://github.com/ClickHouse/ClickHouse/pull/37579) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Flush and shutdown temporary table before drop [#37585](https://github.com/ClickHouse/ClickHouse/pull/37585) ([vxider](https://github.com/Vxider)). +* tests: fix 01317_no_password_in_command_line flakiness (and make it race free) [#37597](https://github.com/ClickHouse/ClickHouse/pull/37597) ([Azat Khuzhin](https://github.com/azat)). +* Use Jepsen worklow directly in PR workflow [#37599](https://github.com/ClickHouse/ClickHouse/pull/37599) ([Antonio Andelic](https://github.com/antonio2368)). +* Added LpNorm and LpDistance functions for arrays [#37601](https://github.com/ClickHouse/ClickHouse/pull/37601) ([Alexander Gololobov](https://github.com/davenger)). +* Turn on s3 tests to red mode [#37604](https://github.com/ClickHouse/ClickHouse/pull/37604) ([alesapin](https://github.com/alesapin)). +* Update FileCache.cpp [#37606](https://github.com/ClickHouse/ClickHouse/pull/37606) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix Clang-Tidy: remove std::move() from trivially-copyable object [#37609](https://github.com/ClickHouse/ClickHouse/pull/37609) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix clang-tidy-14, part 1 [#37612](https://github.com/ClickHouse/ClickHouse/pull/37612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove margin in test reports and change the font [#37614](https://github.com/ClickHouse/ClickHouse/pull/37614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bugs in WindowView when using table identifier [#37619](https://github.com/ClickHouse/ClickHouse/pull/37619) ([vxider](https://github.com/Vxider)). +* Try fix tests [#37622](https://github.com/ClickHouse/ClickHouse/pull/37622) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix refactoring issue [#37629](https://github.com/ClickHouse/ClickHouse/pull/37629) ([alesapin](https://github.com/alesapin)). +* fix typo [#37637](https://github.com/ClickHouse/ClickHouse/pull/37637) ([flynn](https://github.com/ucasfl)). +* Fix excessive LIST requests to coordinator for transactions [#37640](https://github.com/ClickHouse/ClickHouse/pull/37640) ([Azat Khuzhin](https://github.com/azat)). +* Less flaky jbod rebalancer test [#37642](https://github.com/ClickHouse/ClickHouse/pull/37642) ([Amos Bird](https://github.com/amosbird)). +* Avoid useless context copy when building query interpreters [#37643](https://github.com/ClickHouse/ClickHouse/pull/37643) ([Amos Bird](https://github.com/amosbird)). +* Disable amqp-cpp and cassandra build if libuv is disabled [#37644](https://github.com/ClickHouse/ClickHouse/pull/37644) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix errors of CheckTriviallyCopyableMove type [#37647](https://github.com/ClickHouse/ClickHouse/pull/37647) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix hung check [#37655](https://github.com/ClickHouse/ClickHouse/pull/37655) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flaky test test_row_policy [#37658](https://github.com/ClickHouse/ClickHouse/pull/37658) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to remove prewhere_info.alias_actions as they seem to always be included in prewhere_actions [#37661](https://github.com/ClickHouse/ClickHouse/pull/37661) ([Alexander Gololobov](https://github.com/davenger)). +* test for [#36995](https://github.com/ClickHouse/ClickHouse/issues/36995) [#37668](https://github.com/ClickHouse/ClickHouse/pull/37668) ([Denny Crane](https://github.com/den-crane)). +* Cleanup unused file [#37674](https://github.com/ClickHouse/ClickHouse/pull/37674) ([hongbin](https://github.com/xlwh)). +* more verbose sanity checks [#37676](https://github.com/ClickHouse/ClickHouse/pull/37676) ([Sergei Trifonov](https://github.com/serxa)). +* Fix `test_keeper_force_recovery*` tests [#37679](https://github.com/ClickHouse/ClickHouse/pull/37679) ([Antonio Andelic](https://github.com/antonio2368)). +* Display entires for failed tests at the top of report by default [#37704](https://github.com/ClickHouse/ClickHouse/pull/37704) ([Vladimir C](https://github.com/vdimir)). +* Make GROUPING function skip constant folding [#37710](https://github.com/ClickHouse/ClickHouse/pull/37710) ([Dmitry Novik](https://github.com/novikd)). +* Get rid of duplicate download_previous_release [#37712](https://github.com/ClickHouse/ClickHouse/pull/37712) ([Vladimir C](https://github.com/vdimir)). +* Fix possible flaky 00814_replicated_minimalistic_part_header_zookeeper [#37714](https://github.com/ClickHouse/ClickHouse/pull/37714) ([Vladimir C](https://github.com/vdimir)). +* Fix build with -DENABLE_LIBRARIES=0 [#37719](https://github.com/ClickHouse/ClickHouse/pull/37719) ([Robert Schulze](https://github.com/rschu1ze)). +* Update grpc submodule to PR 9 [#37721](https://github.com/ClickHouse/ClickHouse/pull/37721) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Initialize `ParallelReadBuffer` after construction [#37726](https://github.com/ClickHouse/ClickHouse/pull/37726) ([Antonio Andelic](https://github.com/antonio2368)). +* Some fixes for tests [#37740](https://github.com/ClickHouse/ClickHouse/pull/37740) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test 02315_pmj_union_ubsan_35857 [#37744](https://github.com/ClickHouse/ClickHouse/pull/37744) ([Vladimir C](https://github.com/vdimir)). +* Upgrade libxml2 to 2.9.14 [#37745](https://github.com/ClickHouse/ClickHouse/pull/37745) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Implement new submodule sync to fix nightly scan [#37750](https://github.com/ClickHouse/ClickHouse/pull/37750) ([Boris Kuschel](https://github.com/bkuschel)). +* Return back [#37266](https://github.com/ClickHouse/ClickHouse/issues/37266) [#37755](https://github.com/ClickHouse/ClickHouse/pull/37755) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky 01154_move_partition_long [#37763](https://github.com/ClickHouse/ClickHouse/pull/37763) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Moved `ClientConfigurationPerRequest` to ClickHouse [#37767](https://github.com/ClickHouse/ClickHouse/pull/37767) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Got rid of `S3AuthSigner` [#37769](https://github.com/ClickHouse/ClickHouse/pull/37769) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix empty target table id in WindowView [#37770](https://github.com/ClickHouse/ClickHouse/pull/37770) ([vxider](https://github.com/Vxider)). +* Typos [#37773](https://github.com/ClickHouse/ClickHouse/pull/37773) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use multitarget framework for numZerosInTail implementation [#37777](https://github.com/ClickHouse/ClickHouse/pull/37777) ([Alexander Gololobov](https://github.com/davenger)). +* Dictionaries improve exception messages [#37778](https://github.com/ClickHouse/ClickHouse/pull/37778) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore PART_IS_TEMPORARILY_LOCKED error in BC check [#37788](https://github.com/ClickHouse/ClickHouse/pull/37788) ([Kruglov Pavel](https://github.com/Avogar)). +* Use clang compiler by default and cleanup cmake preload [#37796](https://github.com/ClickHouse/ClickHouse/pull/37796) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading of empty S3 files [#37801](https://github.com/ClickHouse/ClickHouse/pull/37801) ([Azat Khuzhin](https://github.com/azat)). +* Metadata storage abstraction for Disks [#37804](https://github.com/ClickHouse/ClickHouse/pull/37804) ([alesapin](https://github.com/alesapin)). +* Minor follow-up to cache table: std::{vector-->array} [#37806](https://github.com/ClickHouse/ClickHouse/pull/37806) ([Robert Schulze](https://github.com/rschu1ze)). +* Delay schedule of the cleaning task in WindowView [#37807](https://github.com/ClickHouse/ClickHouse/pull/37807) ([vxider](https://github.com/Vxider)). +* Fix submodule changed label condition [#37825](https://github.com/ClickHouse/ClickHouse/pull/37825) ([Vladimir C](https://github.com/vdimir)). +* Fix build (packager) [#37829](https://github.com/ClickHouse/ClickHouse/pull/37829) ([Nikita Taranov](https://github.com/nickitat)). +* Executable task forward job callback [#37830](https://github.com/ClickHouse/ClickHouse/pull/37830) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix docs check (remove git clean for yarn.lock) [#37836](https://github.com/ClickHouse/ClickHouse/pull/37836) ([Vladimir C](https://github.com/vdimir)). +* Opentracing minimal changes for processors [#37837](https://github.com/ClickHouse/ClickHouse/pull/37837) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not count covered unexpected parts on sanity check [#37839](https://github.com/ClickHouse/ClickHouse/pull/37839) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Print query in one line on fatal errors [#37840](https://github.com/ClickHouse/ClickHouse/pull/37840) ([Azat Khuzhin](https://github.com/azat)). +* Hotfix for minio in functional tests [#37845](https://github.com/ClickHouse/ClickHouse/pull/37845) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Hotfix for minio in functional tests 2 [#37846](https://github.com/ClickHouse/ClickHouse/pull/37846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fire and clean windows in WindowView only when data is inserted [#37853](https://github.com/ClickHouse/ClickHouse/pull/37853) ([vxider](https://github.com/Vxider)). +* Don't try to kill empty list of containers in `integration/runner` [#37854](https://github.com/ClickHouse/ClickHouse/pull/37854) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Minor cleanup [#37866](https://github.com/ClickHouse/ClickHouse/pull/37866) ([Igor Nikonov](https://github.com/devcrafter)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Fix keeper converter test [#37890](https://github.com/ClickHouse/ClickHouse/pull/37890) ([alesapin](https://github.com/alesapin)). +* Fix stress hung because of attached MV with sleep() [#37892](https://github.com/ClickHouse/ClickHouse/pull/37892) ([Azat Khuzhin](https://github.com/azat)). +* Fix build [#37903](https://github.com/ClickHouse/ClickHouse/pull/37903) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable flaky tests with MaterializedPostgreSQL [#37924](https://github.com/ClickHouse/ClickHouse/pull/37924) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProcfsMetricsProvider fix typo [#37927](https://github.com/ClickHouse/ClickHouse/pull/37927) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet check arguments size [#37930](https://github.com/ClickHouse/ClickHouse/pull/37930) ([Maksim Kita](https://github.com/kitaisreal)). +* Hierarchical dictionaries performance test fix [#37953](https://github.com/ClickHouse/ClickHouse/pull/37953) ([Maksim Kita](https://github.com/kitaisreal)). +* Normalize UTF8 performance test fix [#37954](https://github.com/ClickHouse/ClickHouse/pull/37954) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky `test_parts_delete_zookeeper` [#37957](https://github.com/ClickHouse/ClickHouse/pull/37957) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better parsing of `versionId` in `S3::URI::URI` [#37964](https://github.com/ClickHouse/ClickHouse/pull/37964) ([Vladimir Chebotarev](https://github.com/excitoon)). +* More consistent use of platform macros [#37969](https://github.com/ClickHouse/ClickHouse/pull/37969) ([Robert Schulze](https://github.com/rschu1ze)). +* Always disable --color-diagnostics for LLVM [#37970](https://github.com/ClickHouse/ClickHouse/pull/37970) ([Robert Schulze](https://github.com/rschu1ze)). +* Try fix `test_consistent_parts_after_clone_replica` [#37976](https://github.com/ClickHouse/ClickHouse/pull/37976) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Generalize setup_minio.sh [#37979](https://github.com/ClickHouse/ClickHouse/pull/37979) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add checks for numChildren in Keeper [#37980](https://github.com/ClickHouse/ClickHouse/pull/37980) ([alesapin](https://github.com/alesapin)). +* PartialSortingTransform refactoring [#37992](https://github.com/ClickHouse/ClickHouse/pull/37992) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow up on self-extracting-executable [#38011](https://github.com/ClickHouse/ClickHouse/pull/38011) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update docker run client command in docs [#38015](https://github.com/ClickHouse/ClickHouse/pull/38015) ([Vladimir C](https://github.com/vdimir)). +* Add another useful `docker exec` option [#38016](https://github.com/ClickHouse/ClickHouse/pull/38016) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add release commit hash to image labels [#38020](https://github.com/ClickHouse/ClickHouse/pull/38020) ([Pavel Novitskiy](https://github.com/pnovitskiy)). +* Use correct version in `DiskObjectStorageMetadata` [#38021](https://github.com/ClickHouse/ClickHouse/pull/38021) ([Antonio Andelic](https://github.com/antonio2368)). +* Examples coroutines build fix [#38023](https://github.com/ClickHouse/ClickHouse/pull/38023) ([Maksim Kita](https://github.com/kitaisreal)). +* Use pdqsort instead of standard sort [#38025](https://github.com/ClickHouse/ClickHouse/pull/38025) ([Maksim Kita](https://github.com/kitaisreal)). +* Decimal: noexcept move constructor/assignment operator [#38026](https://github.com/ClickHouse/ClickHouse/pull/38026) ([Igor Nikonov](https://github.com/devcrafter)). +* tests: avoid "_csv.Error: field larger than field limit (131072)" error [#38030](https://github.com/ClickHouse/ClickHouse/pull/38030) ([Azat Khuzhin](https://github.com/azat)). +* More consts for disks [#38033](https://github.com/ClickHouse/ClickHouse/pull/38033) ([alesapin](https://github.com/alesapin)). +* Remove link to past event [#38045](https://github.com/ClickHouse/ClickHouse/pull/38045) ([Ivan Blinkov](https://github.com/blinkov)). +* Use workflow URL as "link" ref [#38057](https://github.com/ClickHouse/ClickHouse/pull/38057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix a compile errors in self-extracting-executable (de)compressor [#38087](https://github.com/ClickHouse/ClickHouse/pull/38087) ([Robert Schulze](https://github.com/rschu1ze)). +* Small follow-up for FPC codec [#38089](https://github.com/ClickHouse/ClickHouse/pull/38089) ([Robert Schulze](https://github.com/rschu1ze)). +* Add dispatch to CherryPick action [#38095](https://github.com/ClickHouse/ClickHouse/pull/38095) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix: build error [#38098](https://github.com/ClickHouse/ClickHouse/pull/38098) ([Igor Nikonov](https://github.com/devcrafter)). +* Try random container name in integration tests runner [#38107](https://github.com/ClickHouse/ClickHouse/pull/38107) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to fix fpc codec [#38117](https://github.com/ClickHouse/ClickHouse/pull/38117) ([Anton Popov](https://github.com/CurtizJ)). +* Remove unused class member [#38120](https://github.com/ClickHouse/ClickHouse/pull/38120) ([Daohui Wang](https://github.com/wangdh15)). + From 45e3b9c2f3dab75c2a11bbe8d1f3882d967da253 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 23 Jun 2022 12:12:54 +0200 Subject: [PATCH 095/408] Regenerate v21.* changelogs with headers and `not for changelog` --- docs/changelogs/v21.1.1.5646-prestable.md | 204 +++++++++++++++ docs/changelogs/v21.1.2.15-stable.md | 7 + docs/changelogs/v21.1.3.32-stable.md | 11 + docs/changelogs/v21.1.4.46-stable.md | 12 + docs/changelogs/v21.1.5.4-stable.md | 7 + docs/changelogs/v21.1.6.13-stable.md | 7 + docs/changelogs/v21.1.7.1-stable.md | 7 + docs/changelogs/v21.1.8.30-stable.md | 12 + docs/changelogs/v21.1.9.41-stable.md | 12 + docs/changelogs/v21.10.1.8013-prestable.md | 130 ++++++++++ docs/changelogs/v21.10.2.15-stable.md | 21 ++ docs/changelogs/v21.10.3.9-stable.md | 17 ++ docs/changelogs/v21.10.4.26-stable.md | 11 + docs/changelogs/v21.10.5.3-stable.md | 7 + docs/changelogs/v21.10.6.2-stable.md | 17 ++ docs/changelogs/v21.11.1.8636-prestable.md | 282 ++++++++++++++++++++- docs/changelogs/v21.11.10.1-stable.md | 7 + docs/changelogs/v21.11.11.1-stable.md | 7 + docs/changelogs/v21.11.2.2-stable.md | 11 + docs/changelogs/v21.11.3.6-stable.md | 7 + docs/changelogs/v21.11.4.14-stable.md | 13 + docs/changelogs/v21.11.5.33-stable.md | 11 + docs/changelogs/v21.11.6.7-stable.md | 12 + docs/changelogs/v21.11.7.9-stable.md | 13 + docs/changelogs/v21.11.8.4-stable.md | 12 + docs/changelogs/v21.11.9.1-stable.md | 13 + docs/changelogs/v21.12.1.9017-prestable.md | 224 +++++++++++++++- docs/changelogs/v21.12.2.17-stable.md | 15 ++ docs/changelogs/v21.12.3.32-stable.md | 16 ++ docs/changelogs/v21.12.4.1-stable.md | 7 + docs/changelogs/v21.2.1.5869-prestable.md | 120 +++++++++ docs/changelogs/v21.2.10.48-stable.md | 11 + docs/changelogs/v21.2.2.8-stable.md | 123 ++++++++- docs/changelogs/v21.2.3.15-stable.md | 11 + docs/changelogs/v21.2.4.6-stable.md | 7 + docs/changelogs/v21.2.5.5-stable.md | 7 + docs/changelogs/v21.2.6.1-stable.md | 7 + docs/changelogs/v21.2.7.11-stable.md | 11 + docs/changelogs/v21.2.8.31-stable.md | 11 + docs/changelogs/v21.2.9.41-stable.md | 12 + docs/changelogs/v21.3.1.6185-prestable.md | 166 ++++++++++++ docs/changelogs/v21.3.10.1-lts.md | 11 + docs/changelogs/v21.3.11.5-lts.md | 7 + docs/changelogs/v21.3.12.2-lts.md | 13 + docs/changelogs/v21.3.13.9-lts.md | 12 + docs/changelogs/v21.3.14.1-lts.md | 11 + docs/changelogs/v21.3.15.4-stable.md | 11 + docs/changelogs/v21.3.16.5-lts.md | 19 ++ docs/changelogs/v21.3.17.2-lts.md | 7 + docs/changelogs/v21.3.18.4-lts.md | 14 + docs/changelogs/v21.3.19.1-lts.md | 12 + docs/changelogs/v21.3.2.5-lts.md | 168 ++++++++++++ docs/changelogs/v21.3.20.1-lts.md | 17 ++ docs/changelogs/v21.3.3.14-lts.md | 11 + docs/changelogs/v21.3.4.25-lts.md | 12 + docs/changelogs/v21.3.5.42-lts.md | 11 + docs/changelogs/v21.3.6.55-lts.md | 11 + docs/changelogs/v21.3.7.62-stable.md | 11 + docs/changelogs/v21.3.8.76-lts.md | 11 + docs/changelogs/v21.3.9.83-lts.md | 7 + docs/changelogs/v21.4.1.6422-prestable.md | 118 +++++++++ docs/changelogs/v21.4.2.10-prestable.md | 115 +++++++++ docs/changelogs/v21.4.3.21-stable.md | 11 + docs/changelogs/v21.4.4.30-stable.md | 7 + docs/changelogs/v21.4.5.46-stable.md | 13 + docs/changelogs/v21.4.6.55-stable.md | 7 + docs/changelogs/v21.4.7.3-stable.md | 11 + docs/changelogs/v21.5.1.6601-prestable.md | 102 +++++++- docs/changelogs/v21.5.2.25-prestable.md | 14 + docs/changelogs/v21.5.3.1-prestable.md | 11 + docs/changelogs/v21.5.4.6-prestable.md | 7 + docs/changelogs/v21.5.5.12-stable.md | 7 + docs/changelogs/v21.5.6.6-stable.md | 13 + docs/changelogs/v21.5.7.9-stable.md | 13 + docs/changelogs/v21.5.8.21-stable.md | 12 + docs/changelogs/v21.5.9.4-stable.md | 12 + docs/changelogs/v21.6.1.6891-prestable.md | 170 +++++++++++++ docs/changelogs/v21.6.2.7-prestable.md | 13 + docs/changelogs/v21.6.3.14-stable.md | 7 + docs/changelogs/v21.6.4.26-stable.md | 12 + docs/changelogs/v21.6.5.37-stable.md | 12 + docs/changelogs/v21.6.6.51-stable.md | 11 + docs/changelogs/v21.6.7.57-stable.md | 11 + docs/changelogs/v21.6.8.62-stable.md | 11 + docs/changelogs/v21.6.9.7-stable.md | 23 ++ docs/changelogs/v21.7.1.7283-prestable.md | 237 +++++++++++++++++ docs/changelogs/v21.7.10.4-stable.md | 15 ++ docs/changelogs/v21.7.11.3-stable.md | 13 + docs/changelogs/v21.7.2.7-stable.md | 12 + docs/changelogs/v21.7.3.14-stable.md | 11 + docs/changelogs/v21.7.4.18-stable.md | 7 + docs/changelogs/v21.7.5.29-stable.md | 14 + docs/changelogs/v21.7.6.39-stable.md | 11 + docs/changelogs/v21.7.7.47-stable.md | 16 ++ docs/changelogs/v21.7.8.58-stable.md | 12 + docs/changelogs/v21.7.9.7-stable.md | 19 ++ docs/changelogs/v21.8.1.7409-prestable.md | 66 +++++ docs/changelogs/v21.8.10.19-lts.md | 11 + docs/changelogs/v21.8.11.4-lts.md | 14 + docs/changelogs/v21.8.12.29-lts.md | 11 + docs/changelogs/v21.8.13.6-lts.md | 19 ++ docs/changelogs/v21.8.14.5-lts.md | 7 + docs/changelogs/v21.8.15.7-lts.md | 7 + docs/changelogs/v21.8.2.19-prestable.md | 14 + docs/changelogs/v21.8.3.44-lts.md | 21 ++ docs/changelogs/v21.8.4.51-lts.md | 7 + docs/changelogs/v21.8.5.7-lts.md | 20 ++ docs/changelogs/v21.8.6.15-lts.md | 16 ++ docs/changelogs/v21.8.7.22-lts.md | 13 + docs/changelogs/v21.8.8.29-lts.md | 7 + docs/changelogs/v21.8.9.13-lts.md | 13 + docs/changelogs/v21.9.1.8000-prestable.md | 213 ++++++++++++++++ docs/changelogs/v21.9.2.17-stable.md | 19 ++ docs/changelogs/v21.9.3.30-stable.md | 14 + docs/changelogs/v21.9.4.35-stable.md | 13 + docs/changelogs/v21.9.5.16-stable.md | 13 + docs/changelogs/v21.9.6.24-stable.md | 14 + 117 files changed, 3631 insertions(+), 5 deletions(-) diff --git a/docs/changelogs/v21.1.1.5646-prestable.md b/docs/changelogs/v21.1.1.5646-prestable.md index ec8abc8a05b..97b645c45f9 100644 --- a/docs/changelogs/v21.1.1.5646-prestable.md +++ b/docs/changelogs/v21.1.1.5646-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.1.5646-prestable FIXME as compared to v20.12.1.5236-prestable #### Backward Incompatible Change @@ -257,3 +264,200 @@ * NO CL ENTRY: 'Revert "Add metrics for part number in MergeTree in ClickHouse"'. [#18834](https://github.com/ClickHouse/ClickHouse/pull/18834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Fixed typo in metrics.md'. [#18920](https://github.com/ClickHouse/ClickHouse/pull/18920) ([Mark Frost](https://github.com/frostmark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Minor change in query profiler [#16899](https://github.com/ClickHouse/ClickHouse/pull/16899) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove escaping from toString(std::string) [#17206](https://github.com/ClickHouse/ClickHouse/pull/17206) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix libunwind build for cmake 3.19+ [#17271](https://github.com/ClickHouse/ClickHouse/pull/17271) ([Azat Khuzhin](https://github.com/azat)). +* Fix AST formatting in log messages [#17274](https://github.com/ClickHouse/ClickHouse/pull/17274) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny cleanup [#17275](https://github.com/ClickHouse/ClickHouse/pull/17275) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#13990](https://github.com/ClickHouse/ClickHouse/issues/13990) [#17298](https://github.com/ClickHouse/ClickHouse/pull/17298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed flaky test_storage_s3::test_custom_auth_headers [#17299](https://github.com/ClickHouse/ClickHouse/pull/17299) ([Pavel Kovalenko](https://github.com/Jokser)). +* Minor changes for ODBC storage [#17301](https://github.com/ClickHouse/ClickHouse/pull/17301) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#16309](https://github.com/ClickHouse/ClickHouse/issues/16309) [#17309](https://github.com/ClickHouse/ClickHouse/pull/17309) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix some flaky tests [#17311](https://github.com/ClickHouse/ClickHouse/pull/17311) ([alesapin](https://github.com/alesapin)). +* Remove outdated test [#17361](https://github.com/ClickHouse/ClickHouse/pull/17361) ([Anton Popov](https://github.com/CurtizJ)). +* Added a test for what was always working [#17375](https://github.com/ClickHouse/ClickHouse/pull/17375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for mutation with empty partition [#17410](https://github.com/ClickHouse/ClickHouse/pull/17410) ([alesapin](https://github.com/alesapin)). +* Implement GRPC protocol (corrections) [#17435](https://github.com/ClickHouse/ClickHouse/pull/17435) ([Vitaly Baranov](https://github.com/vitlibar)). +* Drop include of the removed libbtrie in cmake rules [#17454](https://github.com/ClickHouse/ClickHouse/pull/17454) ([Azat Khuzhin](https://github.com/azat)). +* Merge expressions [#17458](https://github.com/ClickHouse/ClickHouse/pull/17458) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fail queries on unknown settings in perf tests [#17460](https://github.com/ClickHouse/ClickHouse/pull/17460) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test thresholds [#17485](https://github.com/ClickHouse/ClickHouse/pull/17485) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix more flaky integration tests [#17486](https://github.com/ClickHouse/ClickHouse/pull/17486) ([alesapin](https://github.com/alesapin)). +* Fix data race on global BlockStreamProfileInfo in PullingAsyncPipelineExecutor [#17498](https://github.com/ClickHouse/ClickHouse/pull/17498) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix CMake generation and build for native Xcode and AppleClang [#17501](https://github.com/ClickHouse/ClickHouse/pull/17501) ([Denis Glazachev](https://github.com/traceon)). +* Fix bad test 01317_no_password_in_command_line.sh [#17506](https://github.com/ClickHouse/ClickHouse/pull/17506) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix spelling errors [#17527](https://github.com/ClickHouse/ClickHouse/pull/17527) ([flynn](https://github.com/ucasfl)). +* Fix toUnixTimestamp(Date()) error (use type name not column type name) [#17536](https://github.com/ClickHouse/ClickHouse/pull/17536) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for StorageJoin and UUID [#17541](https://github.com/ClickHouse/ClickHouse/pull/17541) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra for BoringSSL [#17544](https://github.com/ClickHouse/ClickHouse/pull/17544) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Kafka for BoringSSL [#17545](https://github.com/ClickHouse/ClickHouse/pull/17545) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update MariaDB for BoringSSL [#17546](https://github.com/ClickHouse/ClickHouse/pull/17546) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update OpenLDAP for BoringSSL [#17547](https://github.com/ClickHouse/ClickHouse/pull/17547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update AMQP for BoringSSL [#17548](https://github.com/ClickHouse/ClickHouse/pull/17548) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* getMemoryAmount: make code worse [#17556](https://github.com/ClickHouse/ClickHouse/pull/17556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#12297](https://github.com/ClickHouse/ClickHouse/issues/12297) [#17557](https://github.com/ClickHouse/ClickHouse/pull/17557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9490](https://github.com/ClickHouse/ClickHouse/issues/9490) [#17561](https://github.com/ClickHouse/ClickHouse/pull/17561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11803](https://github.com/ClickHouse/ClickHouse/issues/11803) [#17562](https://github.com/ClickHouse/ClickHouse/pull/17562) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mixed statements in 01018_ip_dictionary.sql [#17570](https://github.com/ClickHouse/ClickHouse/pull/17570) ([Vladimir C](https://github.com/vdimir)). +* Fix GRPC tests [#17597](https://github.com/ClickHouse/ClickHouse/pull/17597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Port ClickHouse code to BoringSSL [#17606](https://github.com/ClickHouse/ClickHouse/pull/17606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the OpenTelemetry test [#17621](https://github.com/ClickHouse/ClickHouse/pull/17621) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Try to fix Yandex Synchronization check [#17634](https://github.com/ClickHouse/ClickHouse/pull/17634) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use feature testing macro to test if char8_t is supported [#17645](https://github.com/ClickHouse/ClickHouse/pull/17645) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Make LockExceptionInThread/BlockerInThread nested [#17658](https://github.com/ClickHouse/ClickHouse/pull/17658) ([Azat Khuzhin](https://github.com/azat)). +* Mark grpc protocol's tests as flaky. [#17662](https://github.com/ClickHouse/ClickHouse/pull/17662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Use feature testing macro once more [#17685](https://github.com/ClickHouse/ClickHouse/pull/17685) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Fix sequential number in TestKeeper [#17700](https://github.com/ClickHouse/ClickHouse/pull/17700) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix arcadia build [#17720](https://github.com/ClickHouse/ClickHouse/pull/17720) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadian build [#17781](https://github.com/ClickHouse/ClickHouse/pull/17781) ([Ivan](https://github.com/abyss7)). +* Fix a typo [#17791](https://github.com/ClickHouse/ClickHouse/pull/17791) ([achimbab](https://github.com/achimbab)). +* Attempt to use IOStream in AWS SDK [#17794](https://github.com/ClickHouse/ClickHouse/pull/17794) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Update StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry to support waiting on foreign shards / tables [#17800](https://github.com/ClickHouse/ClickHouse/pull/17800) ([nvartolomei](https://github.com/nvartolomei)). +* Fix flaky test_ttl_move [#17805](https://github.com/ClickHouse/ClickHouse/pull/17805) ([Azat Khuzhin](https://github.com/azat)). +* Merging data type Map [#15806](https://github.com/ClickHouse/ClickHouse/issues/15806) [#17829](https://github.com/ClickHouse/ClickHouse/pull/17829) ([Anton Popov](https://github.com/CurtizJ)). +* Kill network container with retries in integration tests [#17856](https://github.com/ClickHouse/ClickHouse/pull/17856) ([alesapin](https://github.com/alesapin)). +* Merging [#17750](https://github.com/ClickHouse/ClickHouse/issues/17750) [#17874](https://github.com/ClickHouse/ClickHouse/pull/17874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix bad code [#17878](https://github.com/ClickHouse/ClickHouse/pull/17878) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Retrieve logs from grpc [#17888](https://github.com/ClickHouse/ClickHouse/pull/17888) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed typo in log message format [#17900](https://github.com/ClickHouse/ClickHouse/pull/17900) ([Alexander Kazakov](https://github.com/Akazz)). +* Perf tests fixes [#17907](https://github.com/ClickHouse/ClickHouse/pull/17907) ([Azat Khuzhin](https://github.com/azat)). +* Better exception message for MaterializeMySQL [#17915](https://github.com/ClickHouse/ClickHouse/pull/17915) ([Winter Zhang](https://github.com/zhang2014)). +* Add additional columns size check for MergeTree in debug mode [#17919](https://github.com/ClickHouse/ClickHouse/pull/17919) ([alesapin](https://github.com/alesapin)). +* Small simplification of MergeTreeDataWriter [#17943](https://github.com/ClickHouse/ClickHouse/pull/17943) ([alesapin](https://github.com/alesapin)). +* Use ryu instead of dragonbox in Arcadia [#17963](https://github.com/ClickHouse/ClickHouse/pull/17963) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Tiny build changes [#17982](https://github.com/ClickHouse/ClickHouse/pull/17982) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#17984](https://github.com/ClickHouse/ClickHouse/pull/17984) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* More forward declaration for generic headers [#17986](https://github.com/ClickHouse/ClickHouse/pull/17986) ([Azat Khuzhin](https://github.com/azat)). +* Remove some redundant includes to speed up build [#17988](https://github.com/ClickHouse/ClickHouse/pull/17988) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flappy test_log_family_s3 [#18027](https://github.com/ClickHouse/ClickHouse/pull/18027) ([Pavel Kovalenko](https://github.com/Jokser)). +* encodeXMLComponent: rename files after [#17659](https://github.com/ClickHouse/ClickHouse/issues/17659) [#18033](https://github.com/ClickHouse/ClickHouse/pull/18033) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* HashTable erase added tests [#18047](https://github.com/ClickHouse/ClickHouse/pull/18047) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove obsolete settings [#18054](https://github.com/ClickHouse/ClickHouse/pull/18054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01600_remerge_sort_lowered_memory_bytes_ratio flap [#18057](https://github.com/ClickHouse/ClickHouse/pull/18057) ([Azat Khuzhin](https://github.com/azat)). +* Remove test_keeper_server in perf tests [#18058](https://github.com/ClickHouse/ClickHouse/pull/18058) ([Azat Khuzhin](https://github.com/azat)). +* Add changelog for 20.12 [#18062](https://github.com/ClickHouse/ClickHouse/pull/18062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed unsafe ast rewrite in InterpreterSelectWithUnionQuery [#18064](https://github.com/ClickHouse/ClickHouse/pull/18064) ([Alexander Kazakov](https://github.com/Akazz)). +* Build utils in CI, at least in split build [#18066](https://github.com/ClickHouse/ClickHouse/pull/18066) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Do not pass -fsanitize-blacklist for gcc (it does not support it) for UBSAN [#18081](https://github.com/ClickHouse/ClickHouse/pull/18081) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix Arcadia [#18084](https://github.com/ClickHouse/ClickHouse/pull/18084) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dependencies for docker stateful_with_coverage [#18105](https://github.com/ClickHouse/ClickHouse/pull/18105) ([alesapin](https://github.com/alesapin)). +* Avoid using symlinks for top_level_domains [#18113](https://github.com/ClickHouse/ClickHouse/pull/18113) ([Azat Khuzhin](https://github.com/azat)). +* gcc10 sanitizers support [#18114](https://github.com/ClickHouse/ClickHouse/pull/18114) ([Azat Khuzhin](https://github.com/azat)). +* Port Kerberos to BoringSSL [#18128](https://github.com/ClickHouse/ClickHouse/pull/18128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix integration tests. [#18132](https://github.com/ClickHouse/ClickHouse/pull/18132) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to fix 00620_optimize_on_nonleader_test [#18140](https://github.com/ClickHouse/ClickHouse/pull/18140) ([alesapin](https://github.com/alesapin)). +* Suppress error in 00993_system_parts_race_condition_drop_zookeeper [#18148](https://github.com/ClickHouse/ClickHouse/pull/18148) ([alesapin](https://github.com/alesapin)). +* Better exception message for unknown function [#18168](https://github.com/ClickHouse/ClickHouse/pull/18168) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Test watches for in-memory zookeeper [#18191](https://github.com/ClickHouse/ClickHouse/pull/18191) ([alesapin](https://github.com/alesapin)). +* Add support for LTS branches in backport automation [#18195](https://github.com/ClickHouse/ClickHouse/pull/18195) ([Ivan](https://github.com/abyss7)). +* Enable optimize_on_insert for MaterializeMySQL [#18198](https://github.com/ClickHouse/ClickHouse/pull/18198) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add one more case in zk test util [#18199](https://github.com/ClickHouse/ClickHouse/pull/18199) ([alesapin](https://github.com/alesapin)). +* More logs during quorum insert [#18200](https://github.com/ClickHouse/ClickHouse/pull/18200) ([alesapin](https://github.com/alesapin)). +* Update libc headers [#18202](https://github.com/ClickHouse/ClickHouse/pull/18202) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix watches processing in TestKeeper [#18217](https://github.com/ClickHouse/ClickHouse/pull/18217) ([alesapin](https://github.com/alesapin)). +* Add test for fixed bug with skip indices [#18219](https://github.com/ClickHouse/ClickHouse/pull/18219) ([Anton Popov](https://github.com/CurtizJ)). +* [wip] a prototype for window functions [#18222](https://github.com/ClickHouse/ClickHouse/pull/18222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Return clickhouse-git-import [#18234](https://github.com/ClickHouse/ClickHouse/pull/18234) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* MADV_DONTNEED check in runtime for qemu (via patching jemalloc) [#18238](https://github.com/ClickHouse/ClickHouse/pull/18238) ([Azat Khuzhin](https://github.com/azat)). +* New Year preparations [#18274](https://github.com/ClickHouse/ClickHouse/pull/18274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add powf and powl to glibc-compatibility [#18279](https://github.com/ClickHouse/ClickHouse/pull/18279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#18286](https://github.com/ClickHouse/ClickHouse/pull/18286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01076_parallel_alter test [#18293](https://github.com/ClickHouse/ClickHouse/pull/18293) ([alesapin](https://github.com/alesapin)). +* Make single image for building coverage report [#18312](https://github.com/ClickHouse/ClickHouse/pull/18312) ([alesapin](https://github.com/alesapin)). +* Fixed flaky test [#18313](https://github.com/ClickHouse/ClickHouse/pull/18313) ([Vasily Nemkov](https://github.com/Enmk)). +* Perf test for ColumnMap [#18317](https://github.com/ClickHouse/ClickHouse/pull/18317) ([Vasily Nemkov](https://github.com/Enmk)). +* Add more tests to skip-list [#18318](https://github.com/ClickHouse/ClickHouse/pull/18318) ([Ivan](https://github.com/abyss7)). +* Add support for ANTLR inside clickhouse-test [#18319](https://github.com/ClickHouse/ClickHouse/pull/18319) ([Ivan](https://github.com/abyss7)). +* Fix clickhouse-test [#18330](https://github.com/ClickHouse/ClickHouse/pull/18330) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Provide extra constructor for Async metrics [#18331](https://github.com/ClickHouse/ClickHouse/pull/18331) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* tests: remove unused configs/scripts [#18334](https://github.com/ClickHouse/ClickHouse/pull/18334) ([Azat Khuzhin](https://github.com/azat)). +* Fix log message for memory tracking drift [#18335](https://github.com/ClickHouse/ClickHouse/pull/18335) ([Azat Khuzhin](https://github.com/azat)). +* try to pass ninja flags in deb package [#18348](https://github.com/ClickHouse/ClickHouse/pull/18348) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove useless code [#18350](https://github.com/ClickHouse/ClickHouse/pull/18350) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused code [#18366](https://github.com/ClickHouse/ClickHouse/pull/18366) ([Anton Popov](https://github.com/CurtizJ)). +* Return back some configs, that are used in Arcadia [#18370](https://github.com/ClickHouse/ClickHouse/pull/18370) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Return clickhouse-test-server for Arcadia needs [#18372](https://github.com/ClickHouse/ClickHouse/pull/18372) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Drop libnsl dependency (because of gRPC -> c-ares) [#18375](https://github.com/ClickHouse/ClickHouse/pull/18375) ([Azat Khuzhin](https://github.com/azat)). +* bump compatibility level to 10 for debian manifests [#18376](https://github.com/ClickHouse/ClickHouse/pull/18376) ([Azat Khuzhin](https://github.com/azat)). +* Do not override RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK in rocksdb [#18378](https://github.com/ClickHouse/ClickHouse/pull/18378) ([Azat Khuzhin](https://github.com/azat)). +* Make some perf tests faster on slower machines [#18386](https://github.com/ClickHouse/ClickHouse/pull/18386) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix exception text from Pipe.cpp [#18396](https://github.com/ClickHouse/ClickHouse/pull/18396) ([filimonov](https://github.com/filimonov)). +* Add test for already working code [#18405](https://github.com/ClickHouse/ClickHouse/pull/18405) ([alesapin](https://github.com/alesapin)). +* Try fix ya.make [#18409](https://github.com/ClickHouse/ClickHouse/pull/18409) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use Port::Data instead of Chunk in LazyOutputFormat. [#18411](https://github.com/ClickHouse/ClickHouse/pull/18411) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not use watchdog when server is run from tty [#18433](https://github.com/ClickHouse/ClickHouse/pull/18433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Raise an error if more than one key is provided to ip_dictionary [#18435](https://github.com/ClickHouse/ClickHouse/pull/18435) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#18436](https://github.com/ClickHouse/ClickHouse/pull/18436) ([Anton Popov](https://github.com/CurtizJ)). +* Ignore SOURCE_DATE_EPOCH for newer ccache (4+) [#18441](https://github.com/ClickHouse/ClickHouse/pull/18441) ([Azat Khuzhin](https://github.com/azat)). +* Poco build fixes [#18443](https://github.com/ClickHouse/ClickHouse/pull/18443) ([Azat Khuzhin](https://github.com/azat)). +* [wip] some window function fixes [#18455](https://github.com/ClickHouse/ClickHouse/pull/18455) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Merging [#17858](https://github.com/ClickHouse/ClickHouse/issues/17858) [#18475](https://github.com/ClickHouse/ClickHouse/pull/18475) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow AppleClang builds [#18488](https://github.com/ClickHouse/ClickHouse/pull/18488) ([Denis Glazachev](https://github.com/traceon)). +* Build job pool tiny fixes [#18489](https://github.com/ClickHouse/ClickHouse/pull/18489) ([Azat Khuzhin](https://github.com/azat)). +* Add NuRaft to contrib [#18491](https://github.com/ClickHouse/ClickHouse/pull/18491) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01584_distributed_buffer_cannot_find_column [#18493](https://github.com/ClickHouse/ClickHouse/pull/18493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct error code on incorrect interserver protocol [#18515](https://github.com/ClickHouse/ClickHouse/pull/18515) ([alesapin](https://github.com/alesapin)). +* Merging [#18188](https://github.com/ClickHouse/ClickHouse/issues/18188) [#18516](https://github.com/ClickHouse/ClickHouse/pull/18516) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix usage of concurrent bounded queue in test keeper storage [#18522](https://github.com/ClickHouse/ClickHouse/pull/18522) ([alesapin](https://github.com/alesapin)). +* Fix cast to map from tuple of arrays with unequal sizes [#18523](https://github.com/ClickHouse/ClickHouse/pull/18523) ([Anton Popov](https://github.com/CurtizJ)). +* Sim/Min Hash fixes [#18524](https://github.com/ClickHouse/ClickHouse/pull/18524) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use RLIMIT_DATA/RLIMIT_AS over RLIMIT_RSS for ENABLE_CHECK_HEAVY_BUILDS [#18537](https://github.com/ClickHouse/ClickHouse/pull/18537) ([Azat Khuzhin](https://github.com/azat)). +* Do not throw logical error from IPAddressDictionary ctor [#18548](https://github.com/ClickHouse/ClickHouse/pull/18548) ([Vladimir C](https://github.com/vdimir)). +* Check for CLICKHOUSE_CLIENT_OPT env before setting it [#18574](https://github.com/ClickHouse/ClickHouse/pull/18574) ([Ivan](https://github.com/abyss7)). +* Minor fixes for min/sim hash [#18595](https://github.com/ClickHouse/ClickHouse/pull/18595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change union_default_mode to throw exception [#18615](https://github.com/ClickHouse/ClickHouse/pull/18615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed exit code of watchdog [#18616](https://github.com/ClickHouse/ClickHouse/pull/18616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small utils improvements (check-marks and compressor) [#18619](https://github.com/ClickHouse/ClickHouse/pull/18619) ([Azat Khuzhin](https://github.com/azat)). +* Fix too long perf test [#18634](https://github.com/ClickHouse/ClickHouse/pull/18634) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct words about parser [#18646](https://github.com/ClickHouse/ClickHouse/pull/18646) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless support for symbolic port names [#18647](https://github.com/ClickHouse/ClickHouse/pull/18647) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Send fatal logs in all tests [#18648](https://github.com/ClickHouse/ClickHouse/pull/18648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix AppleClang compilation - Remove auto in function parameters [#18674](https://github.com/ClickHouse/ClickHouse/pull/18674) ([Denis Glazachev](https://github.com/traceon)). +* compressor: remove extra check for seeking of input [#18675](https://github.com/ClickHouse/ClickHouse/pull/18675) ([Azat Khuzhin](https://github.com/azat)). +* Better linker name matcher [#18678](https://github.com/ClickHouse/ClickHouse/pull/18678) ([Amos Bird](https://github.com/amosbird)). +* Remove "harmful" function from mariadbclient [#18682](https://github.com/ClickHouse/ClickHouse/pull/18682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance comparison [#18686](https://github.com/ClickHouse/ClickHouse/pull/18686) ([Azat Khuzhin](https://github.com/azat)). +* Simplify code of function "bar" [#18687](https://github.com/ClickHouse/ClickHouse/pull/18687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge byteSize function [#18688](https://github.com/ClickHouse/ClickHouse/pull/18688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow Replicated tables in Arcadia [#18697](https://github.com/ClickHouse/ClickHouse/pull/18697) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Bump replxx to fix loading of multi-line entries from the history [#18700](https://github.com/ClickHouse/ClickHouse/pull/18700) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for already fixed issue [#18702](https://github.com/ClickHouse/ClickHouse/pull/18702) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report: applying non-zero offset to nullptr [#18703](https://github.com/ClickHouse/ClickHouse/pull/18703) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable more tests and see what will happen [#18704](https://github.com/ClickHouse/ClickHouse/pull/18704) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a test for [#13477](https://github.com/ClickHouse/ClickHouse/issues/13477) [#18708](https://github.com/ClickHouse/ClickHouse/pull/18708) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzz errors in sumMap [#18710](https://github.com/ClickHouse/ClickHouse/pull/18710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use relaxed for flag in RemoteQueryExecutorReadContext. [#18715](https://github.com/ClickHouse/ClickHouse/pull/18715) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not throw from Parser [#18745](https://github.com/ClickHouse/ClickHouse/pull/18745) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pink screen with confusing questions about Kerberos [#18748](https://github.com/ClickHouse/ClickHouse/pull/18748) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow conversion between UUID and numeric types [#18749](https://github.com/ClickHouse/ClickHouse/pull/18749) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove bad code in HashJoin [#18750](https://github.com/ClickHouse/ClickHouse/pull/18750) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More robust stateful test [#18751](https://github.com/ClickHouse/ClickHouse/pull/18751) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test from [#15641](https://github.com/ClickHouse/ClickHouse/issues/15641) [#18753](https://github.com/ClickHouse/ClickHouse/pull/18753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark some TestFlows as flaky [#18757](https://github.com/ClickHouse/ClickHouse/pull/18757) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some headers [#18758](https://github.com/ClickHouse/ClickHouse/pull/18758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a regression test for marks corruption [#18763](https://github.com/ClickHouse/ClickHouse/pull/18763) ([Azat Khuzhin](https://github.com/azat)). +* Use sigdescr_np() over sys_siglist (fixes glibc 2.32+ unbundled build) [#18764](https://github.com/ClickHouse/ClickHouse/pull/18764) ([Azat Khuzhin](https://github.com/azat)). +* Do not materialize block for FetchColumns header. [#18768](https://github.com/ClickHouse/ClickHouse/pull/18768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Query Fuzzer: fix some cultural issues [#18770](https://github.com/ClickHouse/ClickHouse/pull/18770) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* more tests for simple aggregate functions in summingMT aggregatingMT try2 [#18771](https://github.com/ClickHouse/ClickHouse/pull/18771) ([Denny Crane](https://github.com/den-crane)). +* Check if XCODE_IDE is true and avoid enforcing ninja in that case [#18773](https://github.com/ClickHouse/ClickHouse/pull/18773) ([Denis Glazachev](https://github.com/traceon)). +* Respect memory tracker blocker level during deallocations [#18774](https://github.com/ClickHouse/ClickHouse/pull/18774) ([Azat Khuzhin](https://github.com/azat)). +* Do not allow Fuzzer to enable LLVM [#18777](https://github.com/ClickHouse/ClickHouse/pull/18777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add sanity checks for Sim/Min hash arguments [#18803](https://github.com/ClickHouse/ClickHouse/pull/18803) ([Azat Khuzhin](https://github.com/azat)). +* Respect MINSIGSTKSZ for alternative stack to fix under aarch64 [#18832](https://github.com/ClickHouse/ClickHouse/pull/18832) ([Azat Khuzhin](https://github.com/azat)). +* Do not check bit flips for big buffers (since the size can be corrupted) [#18852](https://github.com/ClickHouse/ClickHouse/pull/18852) ([Azat Khuzhin](https://github.com/azat)). +* Correctly override default settings remotely [#18857](https://github.com/ClickHouse/ClickHouse/pull/18857) ([Amos Bird](https://github.com/amosbird)). +* Import strsignal from Musl [#18858](https://github.com/ClickHouse/ClickHouse/pull/18858) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Storage features improvements [#18870](https://github.com/ClickHouse/ClickHouse/pull/18870) ([Azat Khuzhin](https://github.com/azat)). +* Fix integrity check [#18871](https://github.com/ClickHouse/ClickHouse/pull/18871) ([Azat Khuzhin](https://github.com/azat)). +* Minor fix in backport script [#18873](https://github.com/ClickHouse/ClickHouse/pull/18873) ([Ivan](https://github.com/abyss7)). +* Query Fuzzer: return fail fast semantics [#18880](https://github.com/ClickHouse/ClickHouse/pull/18880) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18881](https://github.com/ClickHouse/ClickHouse/pull/18881) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use time_macros over unset SOURCE_DATE_EPOCH in ccache 4.2 (unreleased) [#18885](https://github.com/ClickHouse/ClickHouse/pull/18885) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#18886](https://github.com/ClickHouse/ClickHouse/pull/18886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows due to timeouts [#18888](https://github.com/ClickHouse/ClickHouse/pull/18888) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18892](https://github.com/ClickHouse/ClickHouse/pull/18892) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop RESTART REPLICAS from stateless tests to avoid locking lots of mutexes [#18897](https://github.com/ClickHouse/ClickHouse/pull/18897) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.1.2.15-stable.md b/docs/changelogs/v21.1.2.15-stable.md index 205794b94c2..ac76c79ea4b 100644 --- a/docs/changelogs/v21.1.2.15-stable.md +++ b/docs/changelogs/v21.1.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.2.15-stable FIXME as compared to v21.1.1.5646-prestable #### Improvement diff --git a/docs/changelogs/v21.1.3.32-stable.md b/docs/changelogs/v21.1.3.32-stable.md index ea4c9fd0fe6..85e7162a509 100644 --- a/docs/changelogs/v21.1.3.32-stable.md +++ b/docs/changelogs/v21.1.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.3.32-stable FIXME as compared to v21.1.2.15-stable #### Bug Fix @@ -29,3 +36,7 @@ * Backported in [#19938](https://github.com/ClickHouse/ClickHouse/issues/19938): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#19934](https://github.com/ClickHouse/ClickHouse/issues/19934): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.1.4.46-stable.md b/docs/changelogs/v21.1.4.46-stable.md index 3033c5edd21..cc9010880e4 100644 --- a/docs/changelogs/v21.1.4.46-stable.md +++ b/docs/changelogs/v21.1.4.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.4.46-stable FIXME as compared to v21.1.3.32-stable #### Bug Fix @@ -20,3 +27,8 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.1: Fix access control manager destruction order"'. [#20395](https://github.com/ClickHouse/ClickHouse/pull/20395) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.1.5.4-stable.md b/docs/changelogs/v21.1.5.4-stable.md index c67724a4512..277814b74be 100644 --- a/docs/changelogs/v21.1.5.4-stable.md +++ b/docs/changelogs/v21.1.5.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.5.4-stable FIXME as compared to v21.1.4.46-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.6.13-stable.md b/docs/changelogs/v21.1.6.13-stable.md index 547cd38a06f..5308dc43dea 100644 --- a/docs/changelogs/v21.1.6.13-stable.md +++ b/docs/changelogs/v21.1.6.13-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.6.13-stable FIXME as compared to v21.1.5.4-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.7.1-stable.md b/docs/changelogs/v21.1.7.1-stable.md index 371efb8d5df..0e66e1bc623 100644 --- a/docs/changelogs/v21.1.7.1-stable.md +++ b/docs/changelogs/v21.1.7.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.7.1-stable FIXME as compared to v21.1.6.13-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.8.30-stable.md b/docs/changelogs/v21.1.8.30-stable.md index 0859cc8ccbd..f9776ce4810 100644 --- a/docs/changelogs/v21.1.8.30-stable.md +++ b/docs/changelogs/v21.1.8.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.8.30-stable FIXME as compared to v21.1.7.1-stable #### Bug Fix @@ -27,3 +34,8 @@ * Backported in [#22700](https://github.com/ClickHouse/ClickHouse/issues/22700): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22739](https://github.com/ClickHouse/ClickHouse/issues/22739): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.1 [#22506](https://github.com/ClickHouse/ClickHouse/pull/22506) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.1.9.41-stable.md b/docs/changelogs/v21.1.9.41-stable.md index 2a94073b810..bef4385e2cb 100644 --- a/docs/changelogs/v21.1.9.41-stable.md +++ b/docs/changelogs/v21.1.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.9.41-stable FIXME as compared to v21.1.8.30-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22813](https://github.com/ClickHouse/ClickHouse/issues/22813): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md index d3e06c056cf..506f3642287 100644 --- a/docs/changelogs/v21.10.1.8013-prestable.md +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.1.8013-prestable FIXME as compared to v21.9.1.7770-prestable #### Backward Incompatible Change @@ -136,3 +143,126 @@ * NO CL ENTRY: 'Revert "Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398)"'. [#28274](https://github.com/ClickHouse/ClickHouse/pull/28274) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'fix minor typo'. [#28629](https://github.com/ClickHouse/ClickHouse/pull/28629) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Experiment with asynchronous readers [#26791](https://github.com/ClickHouse/ClickHouse/pull/26791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* FILTER clause for aggregate functions [#27036](https://github.com/ClickHouse/ClickHouse/pull/27036) ([Nikita Taranov](https://github.com/nickitat)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* S3 disk unstable reads test [#27176](https://github.com/ClickHouse/ClickHouse/pull/27176) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Refactor NotJoined [#27299](https://github.com/ClickHouse/ClickHouse/pull/27299) ([Vladimir C](https://github.com/vdimir)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Break some tests [#27529](https://github.com/ClickHouse/ClickHouse/pull/27529) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better code around decompression [2] [#27743](https://github.com/ClickHouse/ClickHouse/pull/27743) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* try to collect some core dumps in perf tests [#27752](https://github.com/ClickHouse/ClickHouse/pull/27752) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Merging [#20089](https://github.com/ClickHouse/ClickHouse/issues/20089) [#27755](https://github.com/ClickHouse/ClickHouse/pull/27755) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Use sessions more [#27817](https://github.com/ClickHouse/ClickHouse/pull/27817) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for clickhouse-keeper start after conversion [#27818](https://github.com/ClickHouse/ClickHouse/pull/27818) ([alesapin](https://github.com/alesapin)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix test 00443_preferred_block_size_bytes.sh [#27846](https://github.com/ClickHouse/ClickHouse/pull/27846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Mute some integration tests until failures are fixed [#27862](https://github.com/ClickHouse/ClickHouse/pull/27862) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix bad cast in insertPostgreSQLValue [#27869](https://github.com/ClickHouse/ClickHouse/pull/27869) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add fuzzers for codecs [#27872](https://github.com/ClickHouse/ClickHouse/pull/27872) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove tmp folders from tests [#27878](https://github.com/ClickHouse/ClickHouse/pull/27878) ([Kruglov Pavel](https://github.com/Avogar)). +* blog article about perf tests [#27879](https://github.com/ClickHouse/ClickHouse/pull/27879) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* make the sql-standard window functions case insensitive [#27880](https://github.com/ClickHouse/ClickHouse/pull/27880) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Bump librdkafka (fixes use of an invalid/destroyed mutex) [#27883](https://github.com/ClickHouse/ClickHouse/pull/27883) ([Azat Khuzhin](https://github.com/azat)). +* fix decimal formatting settings in perf test [#27884](https://github.com/ClickHouse/ClickHouse/pull/27884) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add separate constants for interfaces LOCAL and TCP_INTERSERVER. [#27886](https://github.com/ClickHouse/ClickHouse/pull/27886) ([Vitaly Baranov](https://github.com/vitlibar)). +* Build fuzzers with clang-tidy [#27895](https://github.com/ClickHouse/ClickHouse/pull/27895) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow implicit cast bool to Field [#27921](https://github.com/ClickHouse/ClickHouse/pull/27921) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve server logs checking in integration tests [#27934](https://github.com/ClickHouse/ClickHouse/pull/27934) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of mutable value in FunctionGetSetting. [#27982](https://github.com/ClickHouse/ClickHouse/pull/27982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable fuzzers build with clang-tidy [#27985](https://github.com/ClickHouse/ClickHouse/pull/27985) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Build fuzzers in CI [#27990](https://github.com/ClickHouse/ClickHouse/pull/27990) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix file progress for local [#27991](https://github.com/ClickHouse/ClickHouse/pull/27991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update libunwind [#27993](https://github.com/ClickHouse/ClickHouse/pull/27993) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix style typos [#28020](https://github.com/ClickHouse/ClickHouse/pull/28020) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Bump replxx [#28039](https://github.com/ClickHouse/ClickHouse/pull/28039) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398) [#28054](https://github.com/ClickHouse/ClickHouse/pull/28054) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix zookeeper secure client test [#28066](https://github.com/ClickHouse/ClickHouse/pull/28066) ([alesapin](https://github.com/alesapin)). +* Fix typo in docs [#28077](https://github.com/ClickHouse/ClickHouse/pull/28077) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed a typo in comments to `SinkToStorage` [#28078](https://github.com/ClickHouse/ClickHouse/pull/28078) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use jinja template tests in fuzzer [#28079](https://github.com/ClickHouse/ClickHouse/pull/28079) ([Vladimir C](https://github.com/vdimir)). +* Clickhouse-keeper: renames and comments [#28080](https://github.com/ClickHouse/ClickHouse/pull/28080) ([alesapin](https://github.com/alesapin)). +* Update nanodbc [#28084](https://github.com/ClickHouse/ClickHouse/pull/28084) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve 01730_distributed_group_by_no_merge_order_by_long [#28123](https://github.com/ClickHouse/ClickHouse/pull/28123) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of useless projection columns during merge [#28135](https://github.com/ClickHouse/ClickHouse/pull/28135) ([Amos Bird](https://github.com/amosbird)). +* Fix style [#28140](https://github.com/ClickHouse/ClickHouse/pull/28140) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix clickhouse keeper jepsen tests [#28143](https://github.com/ClickHouse/ClickHouse/pull/28143) ([alesapin](https://github.com/alesapin)). +* Updated ya.make files [#28157](https://github.com/ClickHouse/ClickHouse/pull/28157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Better detection of the default interface in replicated fetches tests [#28184](https://github.com/ClickHouse/ClickHouse/pull/28184) ([alesapin](https://github.com/alesapin)). +* Reserve protocol number for ALTER PRIMARY KEY. [#28193](https://github.com/ClickHouse/ClickHouse/pull/28193) ([Amos Bird](https://github.com/amosbird)). +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Guard UDF container with a lock [#28211](https://github.com/ClickHouse/ClickHouse/pull/28211) ([Azat Khuzhin](https://github.com/azat)). +* Fix error codes [#28234](https://github.com/ClickHouse/ClickHouse/pull/28234) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries small fixes [#28249](https://github.com/ClickHouse/ClickHouse/pull/28249) ([Maksim Kita](https://github.com/kitaisreal)). +* Better nullable primary key implementation [#28269](https://github.com/ClickHouse/ClickHouse/pull/28269) ([Amos Bird](https://github.com/amosbird)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* test/stress: fix patterns for filtering out Raft messages [#28303](https://github.com/ClickHouse/ClickHouse/pull/28303) ([Azat Khuzhin](https://github.com/azat)). +* Rename system.views columns [#28319](https://github.com/ClickHouse/ClickHouse/pull/28319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix arcadia build (generate ya.make) [#28326](https://github.com/ClickHouse/ClickHouse/pull/28326) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build [#28333](https://github.com/ClickHouse/ClickHouse/pull/28333) ([DimasKovas](https://github.com/DimasKovas)). +* Fix test_storage_s3/test_put_get_with_globs (cleanup after test) [#28336](https://github.com/ClickHouse/ClickHouse/pull/28336) ([ianton-ru](https://github.com/ianton-ru)). +* Fix sed argument in test/fuzzer/run-fuzzer.sh [#28350](https://github.com/ClickHouse/ClickHouse/pull/28350) ([Vladimir C](https://github.com/vdimir)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rework kafka topic creation. [#28354](https://github.com/ClickHouse/ClickHouse/pull/28354) ([Ilya Yatsishin](https://github.com/qoega)). +* Add compat between SinkToStorage and BlockOutputStream [#28361](https://github.com/ClickHouse/ClickHouse/pull/28361) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build (generate ya.make) [#28382](https://github.com/ClickHouse/ClickHouse/pull/28382) ([DimasKovas](https://github.com/DimasKovas)). +* Add a test for a friend [#28396](https://github.com/ClickHouse/ClickHouse/pull/28396) ([alesapin](https://github.com/alesapin)). +* Update ya.make [#28403](https://github.com/ClickHouse/ClickHouse/pull/28403) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix disk with static files a little [#28411](https://github.com/ClickHouse/ClickHouse/pull/28411) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27980](https://github.com/ClickHouse/ClickHouse/issues/27980) [#28413](https://github.com/ClickHouse/ClickHouse/pull/28413) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01457_create_as_table_function_structure [#28428](https://github.com/ClickHouse/ClickHouse/pull/28428) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some rename tests [#28437](https://github.com/ClickHouse/ClickHouse/pull/28437) ([alesapin](https://github.com/alesapin)). +* Fix race in UDF (follow up) [#28438](https://github.com/ClickHouse/ClickHouse/pull/28438) ([Azat Khuzhin](https://github.com/azat)). +* Executable multiple pipes added test [#28503](https://github.com/ClickHouse/ClickHouse/pull/28503) ([Maksim Kita](https://github.com/kitaisreal)). +* UserDefinedFunctionFactory added comments [#28516](https://github.com/ClickHouse/ClickHouse/pull/28516) ([Maksim Kita](https://github.com/kitaisreal)). +* BorrowedObjectPool fix style [#28523](https://github.com/ClickHouse/ClickHouse/pull/28523) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for keeper 2 node configuration [#28526](https://github.com/ClickHouse/ClickHouse/pull/28526) ([alesapin](https://github.com/alesapin)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix race in zlib [#28534](https://github.com/ClickHouse/ClickHouse/pull/28534) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Bump poco to remove getpid() calls [#28537](https://github.com/ClickHouse/ClickHouse/pull/28537) ([Azat Khuzhin](https://github.com/azat)). +* Fix broken kafka test [#28542](https://github.com/ClickHouse/ClickHouse/pull/28542) ([alesapin](https://github.com/alesapin)). +* Fix format names in docs [#28557](https://github.com/ClickHouse/ClickHouse/pull/28557) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix wrong header of minmax_count projection [#28560](https://github.com/ClickHouse/ClickHouse/pull/28560) ([Amos Bird](https://github.com/amosbird)). +* remove recursion in ZstdInflatingReadBuffer [#28561](https://github.com/ClickHouse/ClickHouse/pull/28561) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Improve [C|T]SV errors [#28579](https://github.com/ClickHouse/ClickHouse/pull/28579) ([Raúl Marín](https://github.com/Algunenano)). +* Function dictGet small fix [#28615](https://github.com/ClickHouse/ClickHouse/pull/28615) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix arcadia build [#28640](https://github.com/ClickHouse/ClickHouse/pull/28640) ([DimasKovas](https://github.com/DimasKovas)). +* Add missed log level into TextLog [#28648](https://github.com/ClickHouse/ClickHouse/pull/28648) ([alesapin](https://github.com/alesapin)). +* Revert [#28082](https://github.com/ClickHouse/ClickHouse/issues/28082) [#28665](https://github.com/ClickHouse/ClickHouse/pull/28665) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert [#28397](https://github.com/ClickHouse/ClickHouse/issues/28397) [#28667](https://github.com/ClickHouse/ClickHouse/pull/28667) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md index 05e278f03ae..d23ff95307c 100644 --- a/docs/changelogs/v21.10.2.15-stable.md +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.2.15-stable FIXME as compared to v21.10.1.8013-prestable #### Improvement @@ -63,3 +70,17 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30182](https://github.com/ClickHouse/ClickHouse/pull/30182) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arcadia (pre)stable 21.10 build [#29250](https://github.com/ClickHouse/ClickHouse/pull/29250) ([DimasKovas](https://github.com/DimasKovas)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unnecessary PEERDIR to libcxx-filesystem [#29798](https://github.com/ClickHouse/ClickHouse/pull/29798) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md index 78240367d55..8cccd01e77a 100644 --- a/docs/changelogs/v21.10.3.9-stable.md +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.3.9-stable FIXME as compared to v21.10.2.15-stable #### New Feature @@ -43,3 +50,13 @@ * Backported in [#31255](https://github.com/ClickHouse/ClickHouse/issues/31255): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). * Backported in [#31436](https://github.com/ClickHouse/ClickHouse/issues/31436): Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md index b2089fad0c4..06eab149e6c 100644 --- a/docs/changelogs/v21.10.4.26-stable.md +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.4.26-stable FIXME as compared to v21.10.3.9-stable #### Performance Improvement @@ -23,3 +30,7 @@ * Backported in [#31939](https://github.com/ClickHouse/ClickHouse/issues/31939): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). * Backported in [#31909](https://github.com/ClickHouse/ClickHouse/issues/31909): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md index a591c4a07a8..7a87b349324 100644 --- a/docs/changelogs/v21.10.5.3-stable.md +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.5.3-stable FIXME as compared to v21.10.4.26-stable #### Bug Fix diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md index da146ee364d..c9afcfae75e 100644 --- a/docs/changelogs/v21.10.6.2-stable.md +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.6.2-stable FIXME as compared to v21.10.5.3-stable #### Bug Fix @@ -20,3 +27,13 @@ * Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md index 2aab0293223..95c982a2a2e 100644 --- a/docs/changelogs/v21.11.1.8636-prestable.md +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.1.8636-prestable FIXME as compared to v21.10.1.8013-prestable #### Backward Incompatible Change @@ -13,7 +20,7 @@ * Users now can set comments to database in `CREATE DATABASE` statement ... [#29429](https://github.com/ClickHouse/ClickHouse/pull/29429) ([Vasily Nemkov](https://github.com/Enmk)). * New function` mapContainsKeyLike` to get the map that key matches a simple regular expression. [#29471](https://github.com/ClickHouse/ClickHouse/pull/29471) ([凌涛](https://github.com/lingtaolf)). * Huawei OBS Storage support. Closes [#24294](https://github.com/ClickHouse/ClickHouse/issues/24294). [#29511](https://github.com/ClickHouse/ClickHouse/pull/29511) ([kevin wan](https://github.com/MaxWk)). -* ClickHouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). +* Clickhouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). * - Added MD4 and SHA384 functions. [#29602](https://github.com/ClickHouse/ClickHouse/pull/29602) ([Nikita Tikhomirov](https://github.com/NSTikhomirov)). * Support EXISTS(subquery). Closes [#6852](https://github.com/ClickHouse/ClickHouse/issues/6852). [#29731](https://github.com/ClickHouse/ClickHouse/pull/29731) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added function `ngram`. Closes [#29699](https://github.com/ClickHouse/ClickHouse/issues/29699). [#29738](https://github.com/ClickHouse/ClickHouse/pull/29738) ([Maksim Kita](https://github.com/kitaisreal)). @@ -190,6 +197,279 @@ * NO CL ENTRY: 'Revert "Revert "Improve usability of `remote_url_allow_hosts`""'. [#30708](https://github.com/ClickHouse/ClickHouse/pull/30708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'remove some unneeded header files'. [#30722](https://github.com/ClickHouse/ClickHouse/pull/30722) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix race in StorageSystemRocksDB [#29289](https://github.com/ClickHouse/ClickHouse/pull/29289) ([Vladimir C](https://github.com/vdimir)). +* Fixed logging level for message in `S3Common.cpp` [#29308](https://github.com/ClickHouse/ClickHouse/pull/29308) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Revert parse number binary literal [#29309](https://github.com/ClickHouse/ClickHouse/pull/29309) ([Maksim Kita](https://github.com/kitaisreal)). +* Parser number binary literal update [#29310](https://github.com/ClickHouse/ClickHouse/pull/29310) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix assertion in DDLDependencyVisitor [#29323](https://github.com/ClickHouse/ClickHouse/pull/29323) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Function h3GetFaces optimization [#29335](https://github.com/ClickHouse/ClickHouse/pull/29335) ([Maksim Kita](https://github.com/kitaisreal)). +* Less sleeps in integration tests. [#29338](https://github.com/ClickHouse/ClickHouse/pull/29338) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix trivial mistake in DiskWebServer [#29340](https://github.com/ClickHouse/ClickHouse/pull/29340) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test for RocksDB [#29341](https://github.com/ClickHouse/ClickHouse/pull/29341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in comment [#29342](https://github.com/ClickHouse/ClickHouse/pull/29342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add lld into Build-Depends for debian package [#29343](https://github.com/ClickHouse/ClickHouse/pull/29343) ([Azat Khuzhin](https://github.com/azat)). +* Apply a patch from Azat [#29344](https://github.com/ClickHouse/ClickHouse/pull/29344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update libhdfs3 [#29345](https://github.com/ClickHouse/ClickHouse/pull/29345) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove ya.make [#29346](https://github.com/ClickHouse/ClickHouse/pull/29346) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless files related to pytest [#29361](https://github.com/ClickHouse/ClickHouse/pull/29361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete non-automated tests [#29362](https://github.com/ClickHouse/ClickHouse/pull/29362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test [#29365](https://github.com/ClickHouse/ClickHouse/pull/29365) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix H3 function name (h3ResIsClassIII to h3IsResClassIII) [#29368](https://github.com/ClickHouse/ClickHouse/pull/29368) ([Bharat Nallan](https://github.com/bharatnc)). +* RFC: tests: purge unused configs [#29375](https://github.com/ClickHouse/ClickHouse/pull/29375) ([Azat Khuzhin](https://github.com/azat)). +* Fix ugly typo [#29379](https://github.com/ClickHouse/ClickHouse/pull/29379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments in AggregateFunctionFactory [#29380](https://github.com/ClickHouse/ClickHouse/pull/29380) ([Bharat Nallan](https://github.com/bharatnc)). +* Add GraphCDN in adopter list [#29387](https://github.com/ClickHouse/ClickHouse/pull/29387) ([Mohamad Fadhil](https://github.com/sdil)). +* Fix 02015_async_inserts_2 flakiness [#29390](https://github.com/ClickHouse/ClickHouse/pull/29390) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) ( nullable PK and negate cond ) [#29392](https://github.com/ClickHouse/ClickHouse/pull/29392) ([Denny Crane](https://github.com/den-crane)). +* Merge EmbeddedRocksDBBlockInputStream and EmbeddedRocksDBSource [#29428](https://github.com/ClickHouse/ClickHouse/pull/29428) ([Vladimir C](https://github.com/vdimir)). +* Map bloom filter index mapValues equals function support [#29431](https://github.com/ClickHouse/ClickHouse/pull/29431) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary Array nested Map added test [#29432](https://github.com/ClickHouse/ClickHouse/pull/29432) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary view different database added test [#29443](https://github.com/ClickHouse/ClickHouse/pull/29443) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix deadlock in concurrent async inserts and truncates [#29444](https://github.com/ClickHouse/ClickHouse/pull/29444) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky test 01158_zookeeper_log_long [#29445](https://github.com/ClickHouse/ClickHouse/pull/29445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: fix long tag check for flaky check (--test-runs > 1) [#29449](https://github.com/ClickHouse/ClickHouse/pull/29449) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: fix shared list object (by fixing manager lifetime) [#29452](https://github.com/ClickHouse/ClickHouse/pull/29452) ([Azat Khuzhin](https://github.com/azat)). +* Enable memory profiler on CI [#29453](https://github.com/ClickHouse/ClickHouse/pull/29453) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) added tests for tuples [#29460](https://github.com/ClickHouse/ClickHouse/pull/29460) ([Denny Crane](https://github.com/den-crane)). +* Fix 2024_merge_regexp_assert [#29461](https://github.com/ClickHouse/ClickHouse/pull/29461) ([Azat Khuzhin](https://github.com/azat)). +* Add std::cerr/std::cout style check [#29464](https://github.com/ClickHouse/ClickHouse/pull/29464) ([Azat Khuzhin](https://github.com/azat)). +* [github] we're switching to CLA based on Apache CLA [#29466](https://github.com/ClickHouse/ClickHouse/pull/29466) ([ClickHouse Admin](https://github.com/clickhouse-admin)). +* Bloom filter indexes updated tests [#29474](https://github.com/ClickHouse/ClickHouse/pull/29474) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove more data streams [#29491](https://github.com/ClickHouse/ClickHouse/pull/29491) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible hang in PushingAsyncPipelineExecutor. [#29494](https://github.com/ClickHouse/ClickHouse/pull/29494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use `max_parse_depth` in fuzzers [#29497](https://github.com/ClickHouse/ClickHouse/pull/29497) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Introducing Fn concept for function signature checking, simplifying SimpleCache [#29500](https://github.com/ClickHouse/ClickHouse/pull/29500) ([Mike Kot](https://github.com/myrrc)). +* Fix LOGICAL_ERROR for INSERT with concurrent ALTER [#29502](https://github.com/ClickHouse/ClickHouse/pull/29502) ([Azat Khuzhin](https://github.com/azat)). +* Log queries to external databases (since they may be rewritten) [#29503](https://github.com/ClickHouse/ClickHouse/pull/29503) ([Azat Khuzhin](https://github.com/azat)). +* Fix server pid (hence exit code and attaching with gdb) in fuzzer tests [#29513](https://github.com/ClickHouse/ClickHouse/pull/29513) ([Azat Khuzhin](https://github.com/azat)). +* Bump cmake minimum required version to 3.14 [#29515](https://github.com/ClickHouse/ClickHouse/pull/29515) ([Azat Khuzhin](https://github.com/azat)). +* Changelog heredoc added backward incompatible change [#29530](https://github.com/ClickHouse/ClickHouse/pull/29530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix test names [#29539](https://github.com/ClickHouse/ClickHouse/pull/29539) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Making Monotonicity an aggregate to use with designated initializers [#29540](https://github.com/ClickHouse/ClickHouse/pull/29540) ([Mike Kot](https://github.com/myrrc)). +* Generalize code in IColumn::permute [#29545](https://github.com/ClickHouse/ClickHouse/pull/29545) ([Anton Popov](https://github.com/CurtizJ)). +* Add error for multiple keys without current_key_id [#29546](https://github.com/ClickHouse/ClickHouse/pull/29546) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* support conversion between date32 and datetime64 [#29552](https://github.com/ClickHouse/ClickHouse/pull/29552) ([kevin wan](https://github.com/MaxWk)). +* Fixed test with dictGet without database name as column default value [#29568](https://github.com/ClickHouse/ClickHouse/pull/29568) ([Maksim Kita](https://github.com/kitaisreal)). +* Add FAIL message to test_results.tsv [#29583](https://github.com/ClickHouse/ClickHouse/pull/29583) ([Dmitry Novik](https://github.com/novikd)). +* Generalize code in `IColumn::updatePermutation` [#29595](https://github.com/ClickHouse/ClickHouse/pull/29595) ([Anton Popov](https://github.com/CurtizJ)). +* Add fuzzer for `executeQuery` function [#29596](https://github.com/ClickHouse/ClickHouse/pull/29596) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix docker/test/util declaration [#29597](https://github.com/ClickHouse/ClickHouse/pull/29597) ([Dmitry Novik](https://github.com/novikd)). +* Skip test for executable table function under MSan [#29600](https://github.com/ClickHouse/ClickHouse/pull/29600) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve some more integration tests [#29603](https://github.com/ClickHouse/ClickHouse/pull/29603) ([Ilya Yatsishin](https://github.com/qoega)). +* Improve CompressedReadBuffer [#29605](https://github.com/ClickHouse/ClickHouse/pull/29605) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not send many signals at server restart (integration tests). [#29608](https://github.com/ClickHouse/ClickHouse/pull/29608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between MergeTask and storage destruction [#29614](https://github.com/ClickHouse/ClickHouse/pull/29614) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* BloomFilter indexes map data type support fix unsafe identifier cast [#29623](https://github.com/ClickHouse/ClickHouse/pull/29623) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow-up for [#26231](https://github.com/ClickHouse/ClickHouse/issues/26231) [#29626](https://github.com/ClickHouse/ClickHouse/pull/29626) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Map get non const key added test [#29636](https://github.com/ClickHouse/ClickHouse/pull/29636) ([Maksim Kita](https://github.com/kitaisreal)). +* Add columns in columns.sql for INFORMATION_SCHEMA [#29637](https://github.com/ClickHouse/ClickHouse/pull/29637) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Merging [#29376](https://github.com/ClickHouse/ClickHouse/issues/29376) [#29649](https://github.com/ClickHouse/ClickHouse/pull/29649) ([Anton Popov](https://github.com/CurtizJ)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not try to fuzz `USE`/`SET` queries and print stacktrace [#29660](https://github.com/ClickHouse/ClickHouse/pull/29660) ([Azat Khuzhin](https://github.com/azat)). +* Rename `common` to `base` [#29661](https://github.com/ClickHouse/ClickHouse/pull/29661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move modelEvaluate function to its own file [#29662](https://github.com/ClickHouse/ClickHouse/pull/29662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Perf fixes [#29663](https://github.com/ClickHouse/ClickHouse/pull/29663) ([Azat Khuzhin](https://github.com/azat)). +* UserDefinedExecutableFunctions added implicit cast [#29666](https://github.com/ClickHouse/ClickHouse/pull/29666) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup common defines [#29680](https://github.com/ClickHouse/ClickHouse/pull/29680) ([Azat Khuzhin](https://github.com/azat)). +* Fix Xcode 13 build [#29682](https://github.com/ClickHouse/ClickHouse/pull/29682) ([Denis Glazachev](https://github.com/traceon)). +* Non-recursive implementation for type list and its functions [#29683](https://github.com/ClickHouse/ClickHouse/pull/29683) ([Mike Kot](https://github.com/myrrc)). +* Reorganize contrib/ IDE folders [#29684](https://github.com/ClickHouse/ClickHouse/pull/29684) ([Denis Glazachev](https://github.com/traceon)). +* Cleanup unbundled image [#29689](https://github.com/ClickHouse/ClickHouse/pull/29689) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory tracking for merges and mutations [#29691](https://github.com/ClickHouse/ClickHouse/pull/29691) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race in WriteIndirectBuffer (used in DiskMemory) [#29692](https://github.com/ClickHouse/ClickHouse/pull/29692) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BorrowedObjectPool condition variable notify fix [#29722](https://github.com/ClickHouse/ClickHouse/pull/29722) ([Maksim Kita](https://github.com/kitaisreal)). +* Better exception message for local interactive [#29737](https://github.com/ClickHouse/ClickHouse/pull/29737) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix --stage for clickhouse-local [#29745](https://github.com/ClickHouse/ClickHouse/pull/29745) ([Azat Khuzhin](https://github.com/azat)). +* Forbid Nullable for JSONExtract* (JSONExtract() still supports Nullable), leads to SIGSEGV before [#29746](https://github.com/ClickHouse/ClickHouse/pull/29746) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionDictionary fix comment [#29747](https://github.com/ClickHouse/ClickHouse/pull/29747) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some merging streams [#29768](https://github.com/ClickHouse/ClickHouse/pull/29768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Parse query from metadata exception throw fix [#29779](https://github.com/ClickHouse/ClickHouse/pull/29779) ([Maksim Kita](https://github.com/kitaisreal)). +* Simplify obtaining of server pid in fuzzer tests [#29781](https://github.com/ClickHouse/ClickHouse/pull/29781) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race between fatal error handler and progress packets [#29783](https://github.com/ClickHouse/ClickHouse/pull/29783) ([Azat Khuzhin](https://github.com/azat)). +* Fix parallel_view_processing [#29786](https://github.com/ClickHouse/ClickHouse/pull/29786) ([Azat Khuzhin](https://github.com/azat)). +* Function reinterpretAs improve readability [#29796](https://github.com/ClickHouse/ClickHouse/pull/29796) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor ConcurrentBoundedQueue [#29801](https://github.com/ClickHouse/ClickHouse/pull/29801) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve usability of error messages when error is caused by sophisticated interventions [#29803](https://github.com/ClickHouse/ClickHouse/pull/29803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvement for [#28373](https://github.com/ClickHouse/ClickHouse/issues/28373) [#29804](https://github.com/ClickHouse/ClickHouse/pull/29804) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better diagnostic for OPTIMIZE [#29812](https://github.com/ClickHouse/ClickHouse/pull/29812) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add listen_backlog to documentation [#29813](https://github.com/ClickHouse/ClickHouse/pull/29813) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary attributes updated documentation [#29816](https://github.com/ClickHouse/ClickHouse/pull/29816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix compilation with glibc 2.34 (MINSIGSTKSZ defined as sysconf(_SC_SIGSTKSZ)) [#29820](https://github.com/ClickHouse/ClickHouse/pull/29820) ([Azat Khuzhin](https://github.com/azat)). +* Make memory_profiler_step API cleaner [#29825](https://github.com/ClickHouse/ClickHouse/pull/29825) ([Azat Khuzhin](https://github.com/azat)). +* Add coroutines example. [#29827](https://github.com/ClickHouse/ClickHouse/pull/29827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add coroutines example. [#29841](https://github.com/ClickHouse/ClickHouse/pull/29841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update changelog to include caveats about Nullable types in data skipping indexes [#29855](https://github.com/ClickHouse/ClickHouse/pull/29855) ([Azat Khuzhin](https://github.com/azat)). +* Rewrite clickhouse-test to use python clickhouse_driver [#29856](https://github.com/ClickHouse/ClickHouse/pull/29856) ([Azat Khuzhin](https://github.com/azat)). +* Fix client [#29864](https://github.com/ClickHouse/ClickHouse/pull/29864) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some more streams. [#29898](https://github.com/ClickHouse/ClickHouse/pull/29898) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add logging in ZooKeeper client [#29901](https://github.com/ClickHouse/ClickHouse/pull/29901) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some flacky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Grep server log even if it contains binary data [#29903](https://github.com/ClickHouse/ClickHouse/pull/29903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cosmetic refactoring of server constants. [#29913](https://github.com/ClickHouse/ClickHouse/pull/29913) ([Amos Bird](https://github.com/amosbird)). +* Format improvement of AlterQuery [#29916](https://github.com/ClickHouse/ClickHouse/pull/29916) ([flynn](https://github.com/ucasfl)). +* Fix flaky integration tests (test_backup_restore/test_input_format_parallel_parsing_memory_tracking) [#29919](https://github.com/ClickHouse/ClickHouse/pull/29919) ([Azat Khuzhin](https://github.com/azat)). +* Fix some flaky tests [#29923](https://github.com/ClickHouse/ClickHouse/pull/29923) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix build due to conflicts in serverConstants [#29931](https://github.com/ClickHouse/ClickHouse/pull/29931) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes [#29932](https://github.com/ClickHouse/ClickHouse/pull/29932) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash from SentryWriter [#29933](https://github.com/ClickHouse/ClickHouse/pull/29933) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `printf` function usage. [#29935](https://github.com/ClickHouse/ClickHouse/pull/29935) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Stateless flaky tests fixes [#29947](https://github.com/ClickHouse/ClickHouse/pull/29947) ([Azat Khuzhin](https://github.com/azat)). +* Fix filtering by tuple (some conditions was lost during analyzing) [#29956](https://github.com/ClickHouse/ClickHouse/pull/29956) ([Azat Khuzhin](https://github.com/azat)). +* Allow memory profiler under sanitizers (ASan/UBsan/MSan only) [#29979](https://github.com/ClickHouse/ClickHouse/pull/29979) ([Azat Khuzhin](https://github.com/azat)). +* Tests naming fix [#29982](https://github.com/ClickHouse/ClickHouse/pull/29982) ([Maksim Kita](https://github.com/kitaisreal)). +* More timeouts in test scripts [#29992](https://github.com/ClickHouse/ClickHouse/pull/29992) ([alesapin](https://github.com/alesapin)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Remove streams from formats. [#30001](https://github.com/ClickHouse/ClickHouse/pull/30001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#29901](https://github.com/ClickHouse/ClickHouse/issues/29901) [#30003](https://github.com/ClickHouse/ClickHouse/pull/30003) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More strict check for intersecting parts [#30005](https://github.com/ClickHouse/ClickHouse/pull/30005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Ignore parallel removing warning in 00992_system_parts_race_condition_zookeeper_long [#30007](https://github.com/ClickHouse/ClickHouse/pull/30007) ([Azat Khuzhin](https://github.com/azat)). +* Try remove excessive logging [#30008](https://github.com/ClickHouse/ClickHouse/pull/30008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* remove redundant dot in exception message [#30017](https://github.com/ClickHouse/ClickHouse/pull/30017) ([flynn](https://github.com/ucasfl)). +* Fix build [#30028](https://github.com/ClickHouse/ClickHouse/pull/30028) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Attach trace_log to stateless tests report as is [#30030](https://github.com/ClickHouse/ClickHouse/pull/30030) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove unused headers and handle exception 'unrecognised option' in clickhouse-local [#30053](https://github.com/ClickHouse/ClickHouse/pull/30053) ([Kruglov Pavel](https://github.com/Avogar)). +* clickhouse-test: replace clickhouse-driver with http interface (via http.client) [#30065](https://github.com/ClickHouse/ClickHouse/pull/30065) ([Azat Khuzhin](https://github.com/azat)). +* Fix hardware utilization info printing in client [#30072](https://github.com/ClickHouse/ClickHouse/pull/30072) ([Dmitry Novik](https://github.com/novikd)). +* cmake generator: unlink before creating a link [#30073](https://github.com/ClickHouse/ClickHouse/pull/30073) ([lehasm](https://github.com/lehasm)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Greenhouse careers page to website [#30077](https://github.com/ClickHouse/ClickHouse/pull/30077) ([Cody Baker](https://github.com/codyrobert)). +* Update team photos on website [#30078](https://github.com/ClickHouse/ClickHouse/pull/30078) ([Cody Baker](https://github.com/codyrobert)). +* Add webinar signup promo to website homepage [#30079](https://github.com/ClickHouse/ClickHouse/pull/30079) ([Cody Baker](https://github.com/codyrobert)). +* test for rename atomic hanging [#30080](https://github.com/ClickHouse/ClickHouse/pull/30080) ([Denny Crane](https://github.com/den-crane)). +* Modify ConnectionPoolWithFailover get_priority ROUND_ROBIN comments [#30092](https://github.com/ClickHouse/ClickHouse/pull/30092) ([小路](https://github.com/nicelulu)). +* Fix flaky test 01939_network_send_bytes_metrics [#30134](https://github.com/ClickHouse/ClickHouse/pull/30134) ([Dmitry Novik](https://github.com/novikd)). +* System data skipping indices size fix test names [#30141](https://github.com/ClickHouse/ClickHouse/pull/30141) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsJSON avoid copying object element during iteration [#30145](https://github.com/ClickHouse/ClickHouse/pull/30145) ([Maksim Kita](https://github.com/kitaisreal)). +* Disable fsync_metadata on CI [#30149](https://github.com/ClickHouse/ClickHouse/pull/30149) ([Azat Khuzhin](https://github.com/azat)). +* Make test_MemoryTracking::test_http not flaky [#30150](https://github.com/ClickHouse/ClickHouse/pull/30150) ([Azat Khuzhin](https://github.com/azat)). +* Add careers menu item to navigation [#30151](https://github.com/ClickHouse/ClickHouse/pull/30151) ([Cody Baker](https://github.com/codyrobert)). +* Update yandex logo on homepage [#30152](https://github.com/ClickHouse/ClickHouse/pull/30152) ([Cody Baker](https://github.com/codyrobert)). +* Remove stream interfaces [#30171](https://github.com/ClickHouse/ClickHouse/pull/30171) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better interfaces for `IDataType` and `ISerialization` [#30174](https://github.com/ClickHouse/ClickHouse/pull/30174) ([Anton Popov](https://github.com/CurtizJ)). +* Add blog post for v21.10 release [#30186](https://github.com/ClickHouse/ClickHouse/pull/30186) ([Cody Baker](https://github.com/codyrobert)). +* Smaller smoothing window in throttler. [#30193](https://github.com/ClickHouse/ClickHouse/pull/30193) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clang-tidy warnings in FunctionsJSON code [#30217](https://github.com/ClickHouse/ClickHouse/pull/30217) ([Azat Khuzhin](https://github.com/azat)). +* Fix --hung-check in clickhouse-test [#30218](https://github.com/ClickHouse/ClickHouse/pull/30218) ([Azat Khuzhin](https://github.com/azat)). +* FunctionsJSON updated [#30228](https://github.com/ClickHouse/ClickHouse/pull/30228) ([Maksim Kita](https://github.com/kitaisreal)). +* PolygonDictionary fix bytes_allocated [#30239](https://github.com/ClickHouse/ClickHouse/pull/30239) ([Maksim Kita](https://github.com/kitaisreal)). +* ComplexKeyHashedDictionary fix keys copy [#30241](https://github.com/ClickHouse/ClickHouse/pull/30241) ([Maksim Kita](https://github.com/kitaisreal)). +* Removing data streams folder [#30247](https://github.com/ClickHouse/ClickHouse/pull/30247) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* --stage for clickhouse-local [#30275](https://github.com/ClickHouse/ClickHouse/pull/30275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use forward declaration for Buffer<> in generic headers [#30276](https://github.com/ClickHouse/ClickHouse/pull/30276) ([Azat Khuzhin](https://github.com/azat)). +* Less threads in clickhouse-local, fix Ok. printing [#30282](https://github.com/ClickHouse/ClickHouse/pull/30282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Adjust resource heavy tests [#30284](https://github.com/ClickHouse/ClickHouse/pull/30284) ([Raúl Marín](https://github.com/Algunenano)). +* Fix printing stacktraces for clickhouse-local [#30285](https://github.com/ClickHouse/ClickHouse/pull/30285) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throwing syntax exception in clickhouse-local [#30288](https://github.com/ClickHouse/ClickHouse/pull/30288) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Various fixes to install procedure [#30295](https://github.com/ClickHouse/ClickHouse/pull/30295) ([Denis Glazachev](https://github.com/traceon)). +* Fix clickhouse-local break on timeout [#30297](https://github.com/ClickHouse/ClickHouse/pull/30297) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add RISC-V build [#30298](https://github.com/ClickHouse/ClickHouse/pull/30298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move SquashingTransform to Interpreters (to fix split build) [#30318](https://github.com/ClickHouse/ClickHouse/pull/30318) ([Azat Khuzhin](https://github.com/azat)). +* Increase default wait of the server start in clickhouse-test [#30320](https://github.com/ClickHouse/ClickHouse/pull/30320) ([Azat Khuzhin](https://github.com/azat)). +* Update memory optimisation for MergingSorted. [#30322](https://github.com/ClickHouse/ClickHouse/pull/30322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Single sorting step [#30335](https://github.com/ClickHouse/ClickHouse/pull/30335) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Send table columns in clickhouse-local [#30336](https://github.com/ClickHouse/ClickHouse/pull/30336) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix consumed memory indication in clickhouse-client [#30337](https://github.com/ClickHouse/ClickHouse/pull/30337) ([Dmitry Novik](https://github.com/novikd)). +* Fix fuzzer build [#30344](https://github.com/ClickHouse/ClickHouse/pull/30344) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* ExecutablePool dictionary source fix borrow timeout milliseconds [#30345](https://github.com/ClickHouse/ClickHouse/pull/30345) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* Update obsolete comments. [#30359](https://github.com/ClickHouse/ClickHouse/pull/30359) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rename readWkt* functions to readWKT*, added an alias SVG for svg function [#30361](https://github.com/ClickHouse/ClickHouse/pull/30361) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash when minmax_count projection is used with primary key in partition expr [#30384](https://github.com/ClickHouse/ClickHouse/pull/30384) ([Amos Bird](https://github.com/amosbird)). +* Follow-up for [#30282](https://github.com/ClickHouse/ClickHouse/issues/30282) [#30412](https://github.com/ClickHouse/ClickHouse/pull/30412) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExecutableUDF example [#30436](https://github.com/ClickHouse/ClickHouse/pull/30436) ([Maksim Kita](https://github.com/kitaisreal)). +* Use robot token in actions for statuses [#30439](https://github.com/ClickHouse/ClickHouse/pull/30439) ([alesapin](https://github.com/alesapin)). +* Remove statuses from actions [#30444](https://github.com/ClickHouse/ClickHouse/pull/30444) ([alesapin](https://github.com/alesapin)). +* Fix s3 for github actions [#30447](https://github.com/ClickHouse/ClickHouse/pull/30447) ([alesapin](https://github.com/alesapin)). +* ExecutableUDF example fix style check [#30451](https://github.com/ClickHouse/ClickHouse/pull/30451) ([Maksim Kita](https://github.com/kitaisreal)). +* Support VALUES format in async inserts [#30456](https://github.com/ClickHouse/ClickHouse/pull/30456) ([Anton Popov](https://github.com/CurtizJ)). +* Update release date and add training link [#30475](https://github.com/ClickHouse/ClickHouse/pull/30475) ([Cody Baker](https://github.com/codyrobert)). +* Fix horizontal scroll bar [#30476](https://github.com/ClickHouse/ClickHouse/pull/30476) ([Cody Baker](https://github.com/codyrobert)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying builds on github actions [#30493](https://github.com/ClickHouse/ClickHouse/pull/30493) ([alesapin](https://github.com/alesapin)). +* HashedArrayDictionary optimize read multiple attributes [#30501](https://github.com/ClickHouse/ClickHouse/pull/30501) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix style regressions on benchmark page [#30506](https://github.com/ClickHouse/ClickHouse/pull/30506) ([Cody Baker](https://github.com/codyrobert)). +* Change link in nav from community -> learn [#30508](https://github.com/ClickHouse/ClickHouse/pull/30508) ([Cody Baker](https://github.com/codyrobert)). +* Add placeholder to play.html inputs [#30509](https://github.com/ClickHouse/ClickHouse/pull/30509) ([Vitaly Orlov](https://github.com/orloffv)). +* Add bytes to stats and human readable rows to play.html [#30511](https://github.com/ClickHouse/ClickHouse/pull/30511) ([Vitaly Orlov](https://github.com/orloffv)). +* clickhouse-local: fix block lost in interactive mode [#30521](https://github.com/ClickHouse/ClickHouse/pull/30521) ([Azat Khuzhin](https://github.com/azat)). +* Remove check_columns argument from MergeTree code (false was never passed) [#30522](https://github.com/ClickHouse/ClickHouse/pull/30522) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo and update NuRaft [#30550](https://github.com/ClickHouse/ClickHouse/pull/30550) ([alesapin](https://github.com/alesapin)). +* Refactoring in codec encrypted [#30564](https://github.com/ClickHouse/ClickHouse/pull/30564) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Try to fix [#30397](https://github.com/ClickHouse/ClickHouse/issues/30397) [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support Nullable(String) type in arrayStringConcat [#30566](https://github.com/ClickHouse/ClickHouse/pull/30566) ([Nikita Taranov](https://github.com/nickitat)). +* Function ngrams fixed tests [#30567](https://github.com/ClickHouse/ClickHouse/pull/30567) ([Maksim Kita](https://github.com/kitaisreal)). +* Test clickhouse local columns description fix number [#30568](https://github.com/ClickHouse/ClickHouse/pull/30568) ([Maksim Kita](https://github.com/kitaisreal)). +* Update documentation for distributed_push_down_limit [#30577](https://github.com/ClickHouse/ClickHouse/pull/30577) ([Azat Khuzhin](https://github.com/azat)). +* Fix tests that relies on checking stack size under TSan [#30579](https://github.com/ClickHouse/ClickHouse/pull/30579) ([Azat Khuzhin](https://github.com/azat)). +* Add metadata for *_log into tests artifacts [#30589](https://github.com/ClickHouse/ClickHouse/pull/30589) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR on connection draining in case of ECONNRESET [#30594](https://github.com/ClickHouse/ClickHouse/pull/30594) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test for simdjson [#30596](https://github.com/ClickHouse/ClickHouse/pull/30596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update minimum allowed compiler versions [#30597](https://github.com/ClickHouse/ClickHouse/pull/30597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace estimated_size with optional for readFile/createReadBufferFromFileBase [#30611](https://github.com/ClickHouse/ClickHouse/pull/30611) ([Azat Khuzhin](https://github.com/azat)). +* RFC: Relax minimal clang version (set it to 12, was 13) [#30613](https://github.com/ClickHouse/ClickHouse/pull/30613) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: increase delay for initial SELECT 1 check [#30619](https://github.com/ClickHouse/ClickHouse/pull/30619) ([Azat Khuzhin](https://github.com/azat)). +* Update Client.cpp [#30636](https://github.com/ClickHouse/ClickHouse/pull/30636) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update LocalServer.cpp [#30637](https://github.com/ClickHouse/ClickHouse/pull/30637) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Typo fix in play.html [#30638](https://github.com/ClickHouse/ClickHouse/pull/30638) ([Vitaly Orlov](https://github.com/orloffv)). +* Fix argument types for now and now64 [#30639](https://github.com/ClickHouse/ClickHouse/pull/30639) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix intersecting parts in `parts_to_do` [#30651](https://github.com/ClickHouse/ClickHouse/pull/30651) ([Alexander Tokmakov](https://github.com/tavplubix)). +* HashedDictionaries added read performance tests [#30653](https://github.com/ClickHouse/ClickHouse/pull/30653) ([Maksim Kita](https://github.com/kitaisreal)). +* Change `alter_lock` from `RWLock` to `std::timed_mutex` [#30658](https://github.com/ClickHouse/ClickHouse/pull/30658) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Initialize custom TLDs in clickhouse-local [#30660](https://github.com/ClickHouse/ClickHouse/pull/30660) ([Azat Khuzhin](https://github.com/azat)). +* Add QueryProfilerRuns profile event [#30661](https://github.com/ClickHouse/ClickHouse/pull/30661) ([Azat Khuzhin](https://github.com/azat)). +* Switch everything left from `` to `` [#30662](https://github.com/ClickHouse/ClickHouse/pull/30662) ([Azat Khuzhin](https://github.com/azat)). +* Improve usability of `remote_url_allow_hosts` [#30673](https://github.com/ClickHouse/ClickHouse/pull/30673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecency [#30337](https://github.com/ClickHouse/ClickHouse/issues/30337) [#30674](https://github.com/ClickHouse/ClickHouse/pull/30674) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better wait for server start in integration tests [#30677](https://github.com/ClickHouse/ClickHouse/pull/30677) ([alesapin](https://github.com/alesapin)). +* Fix keeper zookeeper converter test [#30678](https://github.com/ClickHouse/ClickHouse/pull/30678) ([alesapin](https://github.com/alesapin)). +* Don't run build checks for documentation [#30681](https://github.com/ClickHouse/ClickHouse/pull/30681) ([alesapin](https://github.com/alesapin)). +* Refactoring of Log family [#30689](https://github.com/ClickHouse/ClickHouse/pull/30689) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update hardware page colors [#30719](https://github.com/ClickHouse/ClickHouse/pull/30719) ([Cody Baker](https://github.com/codyrobert)). +* Fix test_part_uuid::test_part_uuid_wal [#30723](https://github.com/ClickHouse/ClickHouse/pull/30723) ([Azat Khuzhin](https://github.com/azat)). +* Fix gtest_disk_encrypted (for new readFile/createReadBufferFromFileBase() interfaces) [#30725](https://github.com/ClickHouse/ClickHouse/pull/30725) ([Azat Khuzhin](https://github.com/azat)). +* tests/ci/docker_images_check: add missing time import [#30727](https://github.com/ClickHouse/ClickHouse/pull/30727) ([Azat Khuzhin](https://github.com/azat)). +* Add functional tests to github actions [#30729](https://github.com/ClickHouse/ClickHouse/pull/30729) ([alesapin](https://github.com/alesapin)). +* Run pylint over python scripts for github actions [#30733](https://github.com/ClickHouse/ClickHouse/pull/30733) ([Azat Khuzhin](https://github.com/azat)). +* DictionarySource unknown column name in dictionary fix [#30736](https://github.com/ClickHouse/ClickHouse/pull/30736) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01083_expressions_in_engine_arguments.sql [#30741](https://github.com/ClickHouse/ClickHouse/pull/30741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Performance tests tuning [#30773](https://github.com/ClickHouse/ClickHouse/pull/30773) ([Azat Khuzhin](https://github.com/azat)). +* Remove cruft [#30780](https://github.com/ClickHouse/ClickHouse/pull/30780) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test 02022 [#30786](https://github.com/ClickHouse/ClickHouse/pull/30786) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Trying to make fuzzer check less hardcoded [#30795](https://github.com/ClickHouse/ClickHouse/pull/30795) ([alesapin](https://github.com/alesapin)). +* DictionarySource unknown column name fix [#30800](https://github.com/ClickHouse/ClickHouse/pull/30800) ([Maksim Kita](https://github.com/kitaisreal)). +* Add funding press release [#30806](https://github.com/ClickHouse/ClickHouse/pull/30806) ([Cody Baker](https://github.com/codyrobert)). +* Update installation success message [#30672](https://github.com/ClickHouse/ClickHouse/issues/30672) [#30830](https://github.com/ClickHouse/ClickHouse/pull/30830) ([Teja](https://github.com/tejasrivastav)). +* filelog engine tests improve [#30832](https://github.com/ClickHouse/ClickHouse/pull/30832) ([flynn](https://github.com/ucasfl)). +* Remove redundant from http buffer [#30837](https://github.com/ClickHouse/ClickHouse/pull/30837) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Docs check on github actions [#30839](https://github.com/ClickHouse/ClickHouse/pull/30839) ([alesapin](https://github.com/alesapin)). +* Fix fuzzer on master [#30841](https://github.com/ClickHouse/ClickHouse/pull/30841) ([alesapin](https://github.com/alesapin)). +* Disable check for ZooKeeper session uptime by default [#30847](https://github.com/ClickHouse/ClickHouse/pull/30847) ([Alexander Tokmakov](https://github.com/tavplubix)). +* enable modify table comment of rest table engine [#30852](https://github.com/ClickHouse/ClickHouse/pull/30852) ([flynn](https://github.com/ucasfl)). +* Fix typo [#30853](https://github.com/ClickHouse/ClickHouse/pull/30853) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed `--disable-net-host` in `runner` [#30863](https://github.com/ClickHouse/ClickHouse/pull/30863) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix protocol revision. [#30864](https://github.com/ClickHouse/ClickHouse/pull/30864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add docker cleanup after actions [#30868](https://github.com/ClickHouse/ClickHouse/pull/30868) ([alesapin](https://github.com/alesapin)). +* Fix error message in Keeper handler [#30880](https://github.com/ClickHouse/ClickHouse/pull/30880) ([alesapin](https://github.com/alesapin)). +* Better handling of `xtables.lock` in `runner` [#30892](https://github.com/ClickHouse/ClickHouse/pull/30892) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Reimplement [#28639](https://github.com/ClickHouse/ClickHouse/issues/28639) [#30903](https://github.com/ClickHouse/ClickHouse/pull/30903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Miscellaneous [#30917](https://github.com/ClickHouse/ClickHouse/pull/30917) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix yml in docs release [#30928](https://github.com/ClickHouse/ClickHouse/pull/30928) ([alesapin](https://github.com/alesapin)). +* Debug github event [#30929](https://github.com/ClickHouse/ClickHouse/pull/30929) ([alesapin](https://github.com/alesapin)). +* Fix docs release [#30933](https://github.com/ClickHouse/ClickHouse/pull/30933) ([alesapin](https://github.com/alesapin)). +* Fix style check [#30937](https://github.com/ClickHouse/ClickHouse/pull/30937) ([alesapin](https://github.com/alesapin)). +* Fix file progress for clickhouse-local [#30938](https://github.com/ClickHouse/ClickHouse/pull/30938) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix reading from TinyLog [#30941](https://github.com/ClickHouse/ClickHouse/pull/30941) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add github to known hosts in docs release [#30947](https://github.com/ClickHouse/ClickHouse/pull/30947) ([alesapin](https://github.com/alesapin)). +* Parse json from response in ci checks [#30948](https://github.com/ClickHouse/ClickHouse/pull/30948) ([alesapin](https://github.com/alesapin)). + #### Testing Improvement * Implemented structure-aware fuzzing approach in ClickHouse for select statement parser. [#30012](https://github.com/ClickHouse/ClickHouse/pull/30012) ([Paul](https://github.com/PaulCher)). diff --git a/docs/changelogs/v21.11.10.1-stable.md b/docs/changelogs/v21.11.10.1-stable.md index f07eccd30c5..c180918bb07 100644 --- a/docs/changelogs/v21.11.10.1-stable.md +++ b/docs/changelogs/v21.11.10.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.10.1-stable FIXME as compared to v21.11.9.1-stable diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md index e46f43c53e0..922245d031e 100644 --- a/docs/changelogs/v21.11.11.1-stable.md +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md index f48f91a9b13..9a11dfd03c5 100644 --- a/docs/changelogs/v21.11.2.2-stable.md +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,7 @@ * Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31026](https://github.com/ClickHouse/ClickHouse/issues/31026): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md index bf46ecec590..5db65d42c6d 100644 --- a/docs/changelogs/v21.11.3.6-stable.md +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.3.6-stable FIXME as compared to v21.11.2.2-stable #### Bug Fix diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md index b3d44b8c193..be661ad244d 100644 --- a/docs/changelogs/v21.11.4.14-stable.md +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.4.14-stable FIXME as compared to v21.11.3.6-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#31283](https://github.com/ClickHouse/ClickHouse/issues/31283): Rename setting value `read_threadpool` to `threadpool` for setting `remote_filesystem_read_method`. [#31224](https://github.com/ClickHouse/ClickHouse/pull/31224) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md index 973c443d9f8..2895505715f 100644 --- a/docs/changelogs/v21.11.5.33-stable.md +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.5.33-stable FIXME as compared to v21.11.4.14-stable #### Performance Improvement @@ -31,3 +38,7 @@ * Backported in [#32091](https://github.com/ClickHouse/ClickHouse/issues/32091): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#32020](https://github.com/ClickHouse/ClickHouse/issues/32020): Fix FileLog engine unnesessary create meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md index 1f3df589466..c3127723a68 100644 --- a/docs/changelogs/v21.11.6.7-stable.md +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.6.7-stable FIXME as compared to v21.11.5.33-stable #### Bug Fix @@ -22,3 +29,8 @@ * NO CL ENTRY: 'Manual backport of [#31766](https://github.com/ClickHouse/ClickHouse/issues/31766) into 21.11'. [#32202](https://github.com/ClickHouse/ClickHouse/pull/32202) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md index 1d907ad0ce1..5595e7bc0a9 100644 --- a/docs/changelogs/v21.11.7.9-stable.md +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.7.9-stable FIXME as compared to v21.11.6.7-stable #### Bug Fix @@ -13,3 +20,9 @@ * Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md index 0826b473758..c670180a52e 100644 --- a/docs/changelogs/v21.11.8.4-stable.md +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -10,3 +17,8 @@ * Backported in [#32891](https://github.com/ClickHouse/ClickHouse/issues/32891): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#32792](https://github.com/ClickHouse/ClickHouse/issues/32792): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md index c1754614c3c..bca99350b47 100644 --- a/docs/changelogs/v21.11.9.1-stable.md +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md index f5e036c9c52..7ca0cbc3605 100644 --- a/docs/changelogs/v21.12.1.9017-prestable.md +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.1.9017-prestable FIXME as compared to v21.11.1.8636-prestable #### Backward Incompatible Change @@ -21,7 +28,7 @@ * Exposes all GlobalThreadPool configurations to the configuration files. [#31285](https://github.com/ClickHouse/ClickHouse/pull/31285) ([Tomáš Hromada](https://github.com/gyfis)). * Aliyun OSS Storage support. [#31286](https://github.com/ClickHouse/ClickHouse/pull/31286) ([cfcz48](https://github.com/cfcz48)). * Allow to print/parse names and types of colums in CustomSeparated input/output format. Add formats CustomSeparatedWithNames/WithNamesAndTypes similar to TSVWithNames/WithNamesAndTypes. [#31434](https://github.com/ClickHouse/ClickHouse/pull/31434) ([Kruglov Pavel](https://github.com/Avogar)). -* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([michael1589](https://github.com/michael1589)). +* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([Peng Liu](https://github.com/michael1589)). #### Performance Improvement * ... Allow to split GraphiteMergeTree rollup rules for plain/tagged metrics (optional rule_type field). [#25122](https://github.com/ClickHouse/ClickHouse/pull/25122) ([Michail Safronov](https://github.com/msaf1980)). @@ -54,7 +61,7 @@ * Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)). * Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([Alexander Tokmakov](https://github.com/tavplubix)). * Only grab AlterLock when we do alter command. Let's see if the assumption is correct. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)). -* The local session inside a ClickHouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). +* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). * Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)). * Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query, If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)). * Function name normalization for ALTER queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)). @@ -204,3 +211,216 @@ * NO CL ENTRY: 'Revert "Add a test with 20000 mutations in one query"'. [#32326](https://github.com/ClickHouse/ClickHouse/pull/32326) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'Revert "Revert "Add a test with 20000 mutations in one query""'. [#32327](https://github.com/ClickHouse/ClickHouse/pull/32327) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Refactor pipeline executor [#19587](https://github.com/ClickHouse/ClickHouse/pull/19587) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support NULLS FIRST in KeyCondition [#29528](https://github.com/ClickHouse/ClickHouse/pull/29528) ([DimasKovas](https://github.com/DimasKovas)). +* Integration test improvements [#29806](https://github.com/ClickHouse/ClickHouse/pull/29806) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not allow zero-length reads [#30190](https://github.com/ClickHouse/ClickHouse/pull/30190) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_backward_compatibility [#30950](https://github.com/ClickHouse/ClickHouse/pull/30950) ([Ilya Yatsishin](https://github.com/qoega)). +* Add stress test to github actions [#30952](https://github.com/ClickHouse/ClickHouse/pull/30952) ([alesapin](https://github.com/alesapin)). +* Try smaller blacklist of non parallel integration tests [#30963](https://github.com/ClickHouse/ClickHouse/pull/30963) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Move access-rights source code [#30973](https://github.com/ClickHouse/ClickHouse/pull/30973) ([Vitaly Baranov](https://github.com/vitlibar)). +* Set output_format_avro_rows_in_file default to 1 [#30990](https://github.com/ClickHouse/ClickHouse/pull/30990) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove remaining usages of Y_IGNORE [#30993](https://github.com/ClickHouse/ClickHouse/pull/30993) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Rename AccessControlManager [#30998](https://github.com/ClickHouse/ClickHouse/pull/30998) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add fuzzer check to actions [#31002](https://github.com/ClickHouse/ClickHouse/pull/31002) ([alesapin](https://github.com/alesapin)). +* Small refactoring in formats [#31004](https://github.com/ClickHouse/ClickHouse/pull/31004) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog for version 21.11 [#31014](https://github.com/ClickHouse/ClickHouse/pull/31014) ([Rich Raposa](https://github.com/rfraposa)). +* perf: add missing DROP TABLE queries [#31028](https://github.com/ClickHouse/ClickHouse/pull/31028) ([Azat Khuzhin](https://github.com/azat)). +* Tune perf tests configs/scripts [#31029](https://github.com/ClickHouse/ClickHouse/pull/31029) ([Azat Khuzhin](https://github.com/azat)). +* Remove metric_log/part_log overrides in tests (enabled by default) [#31030](https://github.com/ClickHouse/ClickHouse/pull/31030) ([Azat Khuzhin](https://github.com/azat)). +* improve error message while using OFFSET FETCH clause without ORDER BY [#31031](https://github.com/ClickHouse/ClickHouse/pull/31031) ([SuperDJY](https://github.com/cmsxbc)). +* Log size of remapped memory (remap_executable) [#31033](https://github.com/ClickHouse/ClickHouse/pull/31033) ([Azat Khuzhin](https://github.com/azat)). +* Separate option for enabling fuse syntax for sum, avg, count [#31035](https://github.com/ClickHouse/ClickHouse/pull/31035) ([Vladimir C](https://github.com/vdimir)). +* Add integration test on top of github actions [#31045](https://github.com/ClickHouse/ClickHouse/pull/31045) ([alesapin](https://github.com/alesapin)). +* Fix intersecting parts in `parts_to_do` 2 [#31060](https://github.com/ClickHouse/ClickHouse/pull/31060) ([Alexander Tokmakov](https://github.com/tavplubix)). +* perf: switch *_log tables to Memory engine (attempt to reduce cache misses) [#31063](https://github.com/ClickHouse/ClickHouse/pull/31063) ([Azat Khuzhin](https://github.com/azat)). +* Add new employee photos to team [#31084](https://github.com/ClickHouse/ClickHouse/pull/31084) ([Cody Baker](https://github.com/codyrobert)). +* Update button widths for responsive sizing [#31085](https://github.com/ClickHouse/ClickHouse/pull/31085) ([Cody Baker](https://github.com/codyrobert)). +* Add overflow to benchmark tables [#31086](https://github.com/ClickHouse/ClickHouse/pull/31086) ([Cody Baker](https://github.com/codyrobert)). +* Remove padding below greenhouse iframe [#31087](https://github.com/ClickHouse/ClickHouse/pull/31087) ([Cody Baker](https://github.com/codyrobert)). +* Crb update case study cards [#31088](https://github.com/ClickHouse/ClickHouse/pull/31088) ([Cody Baker](https://github.com/codyrobert)). +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Doc build: remove single.md [#31118](https://github.com/ClickHouse/ClickHouse/pull/31118) ([lehasm](https://github.com/lehasm)). +* remove unnecessary assert in StorageFileLog [#31119](https://github.com/ClickHouse/ClickHouse/pull/31119) ([nauta](https://github.com/nautaa)). +* Add lambda for approve [#31139](https://github.com/ClickHouse/ClickHouse/pull/31139) ([alesapin](https://github.com/alesapin)). +* Do not include unnecessary experimental/type_traits [#31147](https://github.com/ClickHouse/ClickHouse/pull/31147) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Disable optimize_syntax_fuse_functions by default [#31149](https://github.com/ClickHouse/ClickHouse/pull/31149) ([Vladimir C](https://github.com/vdimir)). +* Add pvs studio to actions [#31156](https://github.com/ClickHouse/ClickHouse/pull/31156) ([alesapin](https://github.com/alesapin)). +* Add cherry-pick on github actions [#31158](https://github.com/ClickHouse/ClickHouse/pull/31158) ([alesapin](https://github.com/alesapin)). +* correct disk space calculations [#31159](https://github.com/ClickHouse/ClickHouse/pull/31159) ([Alexandre Snarskii](https://github.com/snar)). +* Fix typo [#31164](https://github.com/ClickHouse/ClickHouse/pull/31164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update text on thank you page [#31166](https://github.com/ClickHouse/ClickHouse/pull/31166) ([Cody Baker](https://github.com/codyrobert)). +* Add unit tests to CI [#31175](https://github.com/ClickHouse/ClickHouse/pull/31175) ([alesapin](https://github.com/alesapin)). +* Debug cherry-pick CI [#31177](https://github.com/ClickHouse/ClickHouse/pull/31177) ([alesapin](https://github.com/alesapin)). +* Update docker_compose_postgres.yml [#31179](https://github.com/ClickHouse/ClickHouse/pull/31179) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests [#31223](https://github.com/ClickHouse/ClickHouse/pull/31223) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Relax test 02026_storage_filelog_largefile.sh [#31225](https://github.com/ClickHouse/ClickHouse/pull/31225) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Download ccache if not exists in builds [#31227](https://github.com/ClickHouse/ClickHouse/pull/31227) ([alesapin](https://github.com/alesapin)). +* Disable fuzzer builds in CI [#31244](https://github.com/ClickHouse/ClickHouse/pull/31244) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add flaky check to new CI [#31248](https://github.com/ClickHouse/ClickHouse/pull/31248) ([alesapin](https://github.com/alesapin)). +* Fix test [#31250](https://github.com/ClickHouse/ClickHouse/pull/31250) ([Kseniia Sumarokova](https://github.com/kssenii)). +* move InputCreatorFunc to InputCreator [#31258](https://github.com/ClickHouse/ClickHouse/pull/31258) ([Alex Cao](https://github.com/cccgp)). +* Print warning during old directories cleanup in MergeTree only if it is old [#31259](https://github.com/ClickHouse/ClickHouse/pull/31259) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup extern ProfileEvents/CurrentMetrics and add a style check [#31260](https://github.com/ClickHouse/ClickHouse/pull/31260) ([Azat Khuzhin](https://github.com/azat)). +* Fix and refactor WriteBiffer-s a little [#31265](https://github.com/ClickHouse/ClickHouse/pull/31265) ([Kruglov Pavel](https://github.com/Avogar)). +* Followup to ccache build [#31287](https://github.com/ClickHouse/ClickHouse/pull/31287) ([alesapin](https://github.com/alesapin)). +* Add compatibility check [#31294](https://github.com/ClickHouse/ClickHouse/pull/31294) ([alesapin](https://github.com/alesapin)). +* Add assertions to ZooKeeperLock [#31295](https://github.com/ClickHouse/ClickHouse/pull/31295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add split build check [#31299](https://github.com/ClickHouse/ClickHouse/pull/31299) ([alesapin](https://github.com/alesapin)). +* Remove strange code from mutations [#31300](https://github.com/ClickHouse/ClickHouse/pull/31300) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable integration tests in new CI [#31301](https://github.com/ClickHouse/ClickHouse/pull/31301) ([alesapin](https://github.com/alesapin)). +* Merging [#31081](https://github.com/ClickHouse/ClickHouse/issues/31081) [#31305](https://github.com/ClickHouse/ClickHouse/pull/31305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Debug rabbitmq tests [#31316](https://github.com/ClickHouse/ClickHouse/pull/31316) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible data-race in case of query cancellation with async_socket_for_remote [#31317](https://github.com/ClickHouse/ClickHouse/pull/31317) ([Azat Khuzhin](https://github.com/azat)). +* Improve fuzzer report in case of fuzzer killed [#31324](https://github.com/ClickHouse/ClickHouse/pull/31324) ([Azat Khuzhin](https://github.com/azat)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add Blog Post for 21.11 Release Update [#31326](https://github.com/ClickHouse/ClickHouse/pull/31326) ([Cody Baker](https://github.com/codyrobert)). +* Add blog post for Moscow meetup [#31327](https://github.com/ClickHouse/ClickHouse/pull/31327) ([Cody Baker](https://github.com/codyrobert)). +* Do not try to resolve temporary tables from global context [#31333](https://github.com/ClickHouse/ClickHouse/pull/31333) ([Azat Khuzhin](https://github.com/azat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove some copypaste from CI [#31340](https://github.com/ClickHouse/ClickHouse/pull/31340) ([alesapin](https://github.com/alesapin)). +* Fix test_kafka_insert_avro by pinning avro version [#31387](https://github.com/ClickHouse/ClickHouse/pull/31387) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryScope in MaterializedMySQLSyncThread [#31392](https://github.com/ClickHouse/ClickHouse/pull/31392) ([Azat Khuzhin](https://github.com/azat)). +* Check stderr is writable before reopining it (to avoid losing errors) [#31393](https://github.com/ClickHouse/ClickHouse/pull/31393) ([Azat Khuzhin](https://github.com/azat)). +* Remove thread_local std::string [#31400](https://github.com/ClickHouse/ClickHouse/pull/31400) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix client [#31403](https://github.com/ClickHouse/ClickHouse/pull/31403) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove excessive debug info from the log message in DDLWorker [#31406](https://github.com/ClickHouse/ClickHouse/pull/31406) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Turn on more CI checks [#31413](https://github.com/ClickHouse/ClickHouse/pull/31413) ([alesapin](https://github.com/alesapin)). +* Update description for webinar calendar links [#31433](https://github.com/ClickHouse/ClickHouse/pull/31433) ([Cody Baker](https://github.com/codyrobert)). +* Trying to debug integration tests [#31443](https://github.com/ClickHouse/ClickHouse/pull/31443) ([alesapin](https://github.com/alesapin)). +* Try increase `snapshot_distance` for functional tests [#31448](https://github.com/ClickHouse/ClickHouse/pull/31448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Minor improvement for test_replicated_fetches_bandwidth [#31451](https://github.com/ClickHouse/ClickHouse/pull/31451) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18787](https://github.com/ClickHouse/ClickHouse/issues/18787) (Constraints) [#31476](https://github.com/ClickHouse/ClickHouse/pull/31476) ([Anton Popov](https://github.com/CurtizJ)). +* Crb support page [#31490](https://github.com/ClickHouse/ClickHouse/pull/31490) ([Cody Baker](https://github.com/codyrobert)). +* Add new team members to company page [#31491](https://github.com/ClickHouse/ClickHouse/pull/31491) ([Cody Baker](https://github.com/codyrobert)). +* [ci] whitelist codyrobert [#31492](https://github.com/ClickHouse/ClickHouse/pull/31492) ([Ivan Blinkov](https://github.com/blinkov)). +* Reapply style changes to hardware page [#31506](https://github.com/ClickHouse/ClickHouse/pull/31506) ([Cody Baker](https://github.com/codyrobert)). +* Split row policy and quota headers [#31509](https://github.com/ClickHouse/ClickHouse/pull/31509) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not clean iptables rules in session-scope fixture [#31527](https://github.com/ClickHouse/ClickHouse/pull/31527) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push tests results to CI database [#31540](https://github.com/ClickHouse/ClickHouse/pull/31540) ([alesapin](https://github.com/alesapin)). +* Remove strange multimap for mutations in StorageMergeTree [#31542](https://github.com/ClickHouse/ClickHouse/pull/31542) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove duplicated not compressed logs from CI (actions) [#31544](https://github.com/ClickHouse/ClickHouse/pull/31544) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02010_lc_native flakiness (Query with id = 123456 is already running) [#31556](https://github.com/ClickHouse/ClickHouse/pull/31556) ([Azat Khuzhin](https://github.com/azat)). +* Make 01114_database_atomic more stable in debug builds [#31564](https://github.com/ClickHouse/ClickHouse/pull/31564) ([Azat Khuzhin](https://github.com/azat)). +* Fix MySQLWire format (this will also fix performance tests) [#31565](https://github.com/ClickHouse/ClickHouse/pull/31565) ([Azat Khuzhin](https://github.com/azat)). +* get Build ID via Section headers first [#31566](https://github.com/ClickHouse/ClickHouse/pull/31566) ([Ilya Golshtein](https://github.com/ilejn)). +* Try to debug expired sessions [#31584](https://github.com/ClickHouse/ClickHouse/pull/31584) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix cleanup in integration tests [#31605](https://github.com/ClickHouse/ClickHouse/pull/31605) ([Vitaly Baranov](https://github.com/vitlibar)). +* Stop all periodic reloading of all the configuration files on shutdown earlier [#31607](https://github.com/ClickHouse/ClickHouse/pull/31607) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix build requirements for unit tests [#31617](https://github.com/ClickHouse/ClickHouse/pull/31617) ([alesapin](https://github.com/alesapin)). +* Add workflow run for release branches [#31618](https://github.com/ClickHouse/ClickHouse/pull/31618) ([alesapin](https://github.com/alesapin)). +* Other event for release PR [#31619](https://github.com/ClickHouse/ClickHouse/pull/31619) ([alesapin](https://github.com/alesapin)). +* Trying push event again [#31623](https://github.com/ClickHouse/ClickHouse/pull/31623) ([alesapin](https://github.com/alesapin)). +* Add github actions for master [#31629](https://github.com/ClickHouse/ClickHouse/pull/31629) ([alesapin](https://github.com/alesapin)). +* Fix master yml [#31630](https://github.com/ClickHouse/ClickHouse/pull/31630) ([alesapin](https://github.com/alesapin)). +* fix kerberized_hadoop image [#31637](https://github.com/ClickHouse/ClickHouse/pull/31637) ([Constantine Peresypkin](https://github.com/pkit)). +* [ci] add flickerbox-tom to whitelist [#31651](https://github.com/ClickHouse/ClickHouse/pull/31651) ([Ivan Blinkov](https://github.com/blinkov)). +* Try to fix possible data race in RemoteQueryExecutorReadContext [#31652](https://github.com/ClickHouse/ClickHouse/pull/31652) ([Kruglov Pavel](https://github.com/Avogar)). +* Integration tests flaky check and small fixes [#31654](https://github.com/ClickHouse/ClickHouse/pull/31654) ([alesapin](https://github.com/alesapin)). +* Update base64 library [#31677](https://github.com/ClickHouse/ClickHouse/pull/31677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#31678](https://github.com/ClickHouse/ClickHouse/pull/31678) ([flynn](https://github.com/ucasfl)). +* Try fix OOMs with TSAN [#31685](https://github.com/ClickHouse/ClickHouse/pull/31685) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix workflow in master [#31688](https://github.com/ClickHouse/ClickHouse/pull/31688) ([alesapin](https://github.com/alesapin)). +* Add perf test for writing valid UTF8 [#31695](https://github.com/ClickHouse/ClickHouse/pull/31695) ([Kruglov Pavel](https://github.com/Avogar)). +* hdfs disable stderr logging [#31703](https://github.com/ClickHouse/ClickHouse/pull/31703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible Logical error: Cannot write to finalized buffer [#31712](https://github.com/ClickHouse/ClickHouse/pull/31712) ([Kruglov Pavel](https://github.com/Avogar)). +* 02050: Use CLICKHOUSE_TMP and delete files when finished [#31713](https://github.com/ClickHouse/ClickHouse/pull/31713) ([Raúl Marín](https://github.com/Algunenano)). +* Make 02112_with_fill_interval independent of the server timezone [#31714](https://github.com/ClickHouse/ClickHouse/pull/31714) ([Raúl Marín](https://github.com/Algunenano)). +* 02010_lc_native: Generate a new id for each query [#31720](https://github.com/ClickHouse/ClickHouse/pull/31720) ([Raúl Marín](https://github.com/Algunenano)). +* 00623_replicated_truncate_table_zookeeper_long: Wait for truncate in replicas [#31721](https://github.com/ClickHouse/ClickHouse/pull/31721) ([Raúl Marín](https://github.com/Algunenano)). +* Try to push data into another ci database. [#31724](https://github.com/ClickHouse/ClickHouse/pull/31724) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove OpenCL completely [#31744](https://github.com/ClickHouse/ClickHouse/pull/31744) ([Timur Magomedov](https://github.com/tmagomedov)). +* Fix diff for backports. [#31765](https://github.com/ClickHouse/ClickHouse/pull/31765) ([alesapin](https://github.com/alesapin)). +* Fail fasttest, builds and functional checks if some tests was not successful [#31767](https://github.com/ClickHouse/ClickHouse/pull/31767) ([alesapin](https://github.com/alesapin)). +* Improve how queries are output in the performance dashboard [#31780](https://github.com/ClickHouse/ClickHouse/pull/31780) ([Raúl Marín](https://github.com/Algunenano)). +* Use version from git describe in builds [#31782](https://github.com/ClickHouse/ClickHouse/pull/31782) ([alesapin](https://github.com/alesapin)). +* Fix stylecheck for tests/ci/push_to_artifactory.py [#31798](https://github.com/ClickHouse/ClickHouse/pull/31798) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Shorter stress and fuzzer tests [#31803](https://github.com/ClickHouse/ClickHouse/pull/31803) ([alesapin](https://github.com/alesapin)). +* Fix oss-fuzz build [#31818](https://github.com/ClickHouse/ClickHouse/pull/31818) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix check in approve lambda [#31821](https://github.com/ClickHouse/ClickHouse/pull/31821) ([alesapin](https://github.com/alesapin)). +* Cover query_views_log [#31825](https://github.com/ClickHouse/ClickHouse/pull/31825) ([Azat Khuzhin](https://github.com/azat)). +* Forbid files that differ only by character case [#31834](https://github.com/ClickHouse/ClickHouse/pull/31834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added ubsan suppression for libprotobuf-mutator [#31835](https://github.com/ClickHouse/ClickHouse/pull/31835) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update 01155_rename_move_materialized_view.sql [#31849](https://github.com/ClickHouse/ClickHouse/pull/31849) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix style. [#31850](https://github.com/ClickHouse/ClickHouse/pull/31850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Minor improvements to DUMP macro [#31858](https://github.com/ClickHouse/ClickHouse/pull/31858) ([Vasily Nemkov](https://github.com/Enmk)). +* Get rid of build numbers and simplify builds paths in S3 [#31861](https://github.com/ClickHouse/ClickHouse/pull/31861) ([alesapin](https://github.com/alesapin)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added -no-sanitize=unsigned-integer-overflow build flag [#31881](https://github.com/ClickHouse/ClickHouse/pull/31881) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix typos [#31886](https://github.com/ClickHouse/ClickHouse/pull/31886) ([Anton Popov](https://github.com/CurtizJ)). +* Try to fix flacky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce the files that depend on parser headers [#31896](https://github.com/ClickHouse/ClickHouse/pull/31896) ([Raúl Marín](https://github.com/Algunenano)). +* Fix magic_enum for debug helpers (fixes build w/ USE_DEBUG_HELPERS) [#31922](https://github.com/ClickHouse/ClickHouse/pull/31922) ([Azat Khuzhin](https://github.com/azat)). +* Remove some trash from build [#31923](https://github.com/ClickHouse/ClickHouse/pull/31923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add json type to changes in documentation [#31926](https://github.com/ClickHouse/ClickHouse/pull/31926) ([alesapin](https://github.com/alesapin)). +* fix some broken links [#31948](https://github.com/ClickHouse/ClickHouse/pull/31948) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Kill container in integration tests if it's already running [#31950](https://github.com/ClickHouse/ClickHouse/pull/31950) ([alesapin](https://github.com/alesapin)). +* Drop libc-headers [#31951](https://github.com/ClickHouse/ClickHouse/pull/31951) ([Raúl Marín](https://github.com/Algunenano)). +* Give some love to macOS platform [#31957](https://github.com/ClickHouse/ClickHouse/pull/31957) ([Denis Glazachev](https://github.com/traceon)). +* Fix segfault in MaterializedMySQL [#31960](https://github.com/ClickHouse/ClickHouse/pull/31960) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix docs check [#31980](https://github.com/ClickHouse/ClickHouse/pull/31980) ([alesapin](https://github.com/alesapin)). +* Make stress tests slightly more stable [#31985](https://github.com/ClickHouse/ClickHouse/pull/31985) ([alesapin](https://github.com/alesapin)). +* Add rest functional tests to CI [#31987](https://github.com/ClickHouse/ClickHouse/pull/31987) ([alesapin](https://github.com/alesapin)). +* Add special builds to CI [#31991](https://github.com/ClickHouse/ClickHouse/pull/31991) ([alesapin](https://github.com/alesapin)). +* Run less tests for backport branches [#31992](https://github.com/ClickHouse/ClickHouse/pull/31992) ([alesapin](https://github.com/alesapin)). +* Fix race in ParallelFormattingOutputFormat constructor [#32004](https://github.com/ClickHouse/ClickHouse/pull/32004) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix build on master [#32009](https://github.com/ClickHouse/ClickHouse/pull/32009) ([alesapin](https://github.com/alesapin)). +* Fix style-check for ProfileEvents checking [#32033](https://github.com/ClickHouse/ClickHouse/pull/32033) ([Azat Khuzhin](https://github.com/azat)). +* Provide clickhouse binary w/o debug symbols (stripped) in fasttest [#32036](https://github.com/ClickHouse/ClickHouse/pull/32036) ([Azat Khuzhin](https://github.com/azat)). +* Minor fixes for `StorageMergeTree` [#32037](https://github.com/ClickHouse/ClickHouse/pull/32037) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Remove most of old checks [#32041](https://github.com/ClickHouse/ClickHouse/pull/32041) ([alesapin](https://github.com/alesapin)). +* Remove fast test from master [#32042](https://github.com/ClickHouse/ClickHouse/pull/32042) ([alesapin](https://github.com/alesapin)). +* Better scripts for runners [#32043](https://github.com/ClickHouse/ClickHouse/pull/32043) ([alesapin](https://github.com/alesapin)). +* Fix force tests label [#32044](https://github.com/ClickHouse/ClickHouse/pull/32044) ([alesapin](https://github.com/alesapin)). +* Don't run checks for label event [#32046](https://github.com/ClickHouse/ClickHouse/pull/32046) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 00925 [#32050](https://github.com/ClickHouse/ClickHouse/pull/32050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cancel redundant checks with lambda [#32051](https://github.com/ClickHouse/ClickHouse/pull/32051) ([alesapin](https://github.com/alesapin)). +* Fix flaky integration test for MaterializedMySQL CREATE TABLE LIKE [#32052](https://github.com/ClickHouse/ClickHouse/pull/32052) ([Stig Bakken](https://github.com/stigsb)). +* Use functional test group for tests with thread sanitizer [#32062](https://github.com/ClickHouse/ClickHouse/pull/32062) ([alesapin](https://github.com/alesapin)). +* Add ability for lightweight checks rerun [#32064](https://github.com/ClickHouse/ClickHouse/pull/32064) ([alesapin](https://github.com/alesapin)). +* Increase length of random database in clickhouse-test [#32094](https://github.com/ClickHouse/ClickHouse/pull/32094) ([Azat Khuzhin](https://github.com/azat)). +* make looping in H3 funcs uniform [#32110](https://github.com/ClickHouse/ClickHouse/pull/32110) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove PVS check from master [#32114](https://github.com/ClickHouse/ClickHouse/pull/32114) ([alesapin](https://github.com/alesapin)). +* Fix flaky keeper whitelist test [#32115](https://github.com/ClickHouse/ClickHouse/pull/32115) ([alesapin](https://github.com/alesapin)). +* Fix flacky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move fuzzers and unit tests to another group [#32120](https://github.com/ClickHouse/ClickHouse/pull/32120) ([alesapin](https://github.com/alesapin)). +* Add a test with 20000 mutations in one query [#32122](https://github.com/ClickHouse/ClickHouse/pull/32122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Change test 02117_custom_separated_with_names_and_types [#32123](https://github.com/ClickHouse/ClickHouse/pull/32123) ([Kruglov Pavel](https://github.com/Avogar)). +* Use seq_cst semantic for MergeTreeBackgroundExecutor mertic. [#32125](https://github.com/ClickHouse/ClickHouse/pull/32125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove 02116_global_in_time_limit. [#32126](https://github.com/ClickHouse/ClickHouse/pull/32126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fixing postgres tests [#32129](https://github.com/ClickHouse/ClickHouse/pull/32129) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Small improvements in lambda code [#32155](https://github.com/ClickHouse/ClickHouse/pull/32155) ([alesapin](https://github.com/alesapin)). +* Update featured image for 21.11 release blog post [#32156](https://github.com/ClickHouse/ClickHouse/pull/32156) ([Cody Baker](https://github.com/codyrobert)). +* Fix CI [#32159](https://github.com/ClickHouse/ClickHouse/pull/32159) ([alesapin](https://github.com/alesapin)). +* tests/ci: do not compress logs twice [#32162](https://github.com/ClickHouse/ClickHouse/pull/32162) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: do not use random generator with shared state [#32163](https://github.com/ClickHouse/ClickHouse/pull/32163) ([Azat Khuzhin](https://github.com/azat)). +* Fix stress tests [#32164](https://github.com/ClickHouse/ClickHouse/pull/32164) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryProfiler (query_profiler_{cpu,real}_time_period_ns) reset [#32165](https://github.com/ClickHouse/ClickHouse/pull/32165) ([Azat Khuzhin](https://github.com/azat)). +* MaterializedMySQL support VARBINARY type [#32173](https://github.com/ClickHouse/ClickHouse/pull/32173) ([zzsmdfj](https://github.com/zzsmdfj)). +* perf: fix waiting of the server after running tests [#32174](https://github.com/ClickHouse/ClickHouse/pull/32174) ([Azat Khuzhin](https://github.com/azat)). +* Better output for some actions [#32175](https://github.com/ClickHouse/ClickHouse/pull/32175) ([alesapin](https://github.com/alesapin)). +* Use ccache in fast test [#32177](https://github.com/ClickHouse/ClickHouse/pull/32177) ([alesapin](https://github.com/alesapin)). +* Fix window view tests [#32178](https://github.com/ClickHouse/ClickHouse/pull/32178) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Function accurateCastOrDefault remove separate branch [#32184](https://github.com/ClickHouse/ClickHouse/pull/32184) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for [#32186](https://github.com/ClickHouse/ClickHouse/issues/32186) [#32203](https://github.com/ClickHouse/ClickHouse/pull/32203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix uncaught exception in DatabaseLazy [#32206](https://github.com/ClickHouse/ClickHouse/pull/32206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update ASTCreateQuery.cpp [#32208](https://github.com/ClickHouse/ClickHouse/pull/32208) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix jemalloc under osx [#32219](https://github.com/ClickHouse/ClickHouse/pull/32219) ([Azat Khuzhin](https://github.com/azat)). +* Add missing timezones to some tests [#32222](https://github.com/ClickHouse/ClickHouse/pull/32222) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix versioning of aggregate functions (fixes performance tests) [#32236](https://github.com/ClickHouse/ClickHouse/pull/32236) ([Azat Khuzhin](https://github.com/azat)). +* Disable window view tests temporarily because still flacky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in tupleToNameValuePairs doc [#32262](https://github.com/ClickHouse/ClickHouse/pull/32262) ([Vladimir C](https://github.com/vdimir)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible crash in DataTypeAggregateFunction [#32287](https://github.com/ClickHouse/ClickHouse/pull/32287) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update backport.py [#32323](https://github.com/ClickHouse/ClickHouse/pull/32323) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix graphite-bench build [#32351](https://github.com/ClickHouse/ClickHouse/pull/32351) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Revert "graphite: split tagged/plain rollup rules (for merges perfoma… [#32376](https://github.com/ClickHouse/ClickHouse/pull/32376) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Another attempt to fix unit test Executor::RemoveTasksStress [#32390](https://github.com/ClickHouse/ClickHouse/pull/32390) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md index 909bc7917c7..a35006d9c57 100644 --- a/docs/changelogs/v21.12.2.17-stable.md +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.2.17-stable FIXME as compared to v21.12.1.9017-prestable #### Bug Fix @@ -20,3 +27,11 @@ * Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md index b650f62dd34..f14057981a2 100644 --- a/docs/changelogs/v21.12.3.32-stable.md +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.3.32-stable FIXME as compared to v21.12.2.17-stable #### Bug Fix @@ -15,3 +22,12 @@ * Backported in [#33100](https://github.com/ClickHouse/ClickHouse/issues/33100): Fix Context leak in case of cancel_http_readonly_queries_on_client_close (i.e. leaking of external tables that had been uploaded the the server and other resources). [#32982](https://github.com/ClickHouse/ClickHouse/pull/32982) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#33123](https://github.com/ClickHouse/ClickHouse/issues/33123): Fix error `Invalid version for SerializationLowCardinality key column` in case of reading from `LowCardinality` column with `local_filesystem_read_prefetch` or `remote_filesystem_read_prefetch` enabled. [#33046](https://github.com/ClickHouse/ClickHouse/pull/33046) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md index 7c028592876..d08997378af 100644 --- a/docs/changelogs/v21.12.4.1-stable.md +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.4.1-stable FIXME as compared to v21.12.3.32-stable #### Improvement diff --git a/docs/changelogs/v21.2.1.5869-prestable.md b/docs/changelogs/v21.2.1.5869-prestable.md index 43703bc13b9..ae5efdcf14f 100644 --- a/docs/changelogs/v21.2.1.5869-prestable.md +++ b/docs/changelogs/v21.2.1.5869-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.1.5869-prestable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -154,3 +161,116 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.10.48-stable.md b/docs/changelogs/v21.2.10.48-stable.md index 11eea960931..f9d29374e91 100644 --- a/docs/changelogs/v21.2.10.48-stable.md +++ b/docs/changelogs/v21.2.10.48-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.10.48-stable FIXME as compared to v21.2.9.41-stable #### Improvement @@ -9,3 +16,7 @@ * Backported in [#23032](https://github.com/ClickHouse/ClickHouse/issues/23032): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23171](https://github.com/ClickHouse/ClickHouse/issues/23171): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.2.8-stable.md b/docs/changelogs/v21.2.2.8-stable.md index 73baea91547..d3b1f43bcc3 100644 --- a/docs/changelogs/v21.2.2.8-stable.md +++ b/docs/changelogs/v21.2.2.8-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.2.8-stable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -68,7 +75,7 @@ * Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)). * In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)). -* ClickHouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). +* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). * Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)). * Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -164,3 +171,117 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.2.3.15-stable.md b/docs/changelogs/v21.2.3.15-stable.md index 26653e780bb..0471c499748 100644 --- a/docs/changelogs/v21.2.3.15-stable.md +++ b/docs/changelogs/v21.2.3.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.3.15-stable FIXME as compared to v21.2.2.8-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.2: Fix access control manager destruction order"'. [#20397](https://github.com/ClickHouse/ClickHouse/pull/20397) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.2.4.6-stable.md b/docs/changelogs/v21.2.4.6-stable.md index 1605186701d..709abbcf730 100644 --- a/docs/changelogs/v21.2.4.6-stable.md +++ b/docs/changelogs/v21.2.4.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.4.6-stable FIXME as compared to v21.2.3.15-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.5.5-stable.md b/docs/changelogs/v21.2.5.5-stable.md index b5275e89519..c4b16d8123e 100644 --- a/docs/changelogs/v21.2.5.5-stable.md +++ b/docs/changelogs/v21.2.5.5-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.5.5-stable FIXME as compared to v21.2.4.6-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.6.1-stable.md b/docs/changelogs/v21.2.6.1-stable.md index 1f28c14c485..ef7b13e260e 100644 --- a/docs/changelogs/v21.2.6.1-stable.md +++ b/docs/changelogs/v21.2.6.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.6.1-stable FIXME as compared to v21.2.5.5-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.7.11-stable.md b/docs/changelogs/v21.2.7.11-stable.md index 1f0f94ee0bf..277ddb9fcb3 100644 --- a/docs/changelogs/v21.2.7.11-stable.md +++ b/docs/changelogs/v21.2.7.11-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.7.11-stable FIXME as compared to v21.2.6.1-stable #### Bug Fix @@ -10,3 +17,7 @@ * Backported in [#21979](https://github.com/ClickHouse/ClickHouse/issues/21979): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#22140](https://github.com/ClickHouse/ClickHouse/issues/22140): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.8.31-stable.md b/docs/changelogs/v21.2.8.31-stable.md index 884dcb5a649..d7f7e5a7099 100644 --- a/docs/changelogs/v21.2.8.31-stable.md +++ b/docs/changelogs/v21.2.8.31-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.8.31-stable FIXME as compared to v21.2.7.11-stable #### Bug Fix @@ -28,3 +35,7 @@ * Backported in [#22699](https://github.com/ClickHouse/ClickHouse/issues/22699): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22740](https://github.com/ClickHouse/ClickHouse/issues/22740): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.2 [#22505](https://github.com/ClickHouse/ClickHouse/pull/22505) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.2.9.41-stable.md b/docs/changelogs/v21.2.9.41-stable.md index ab4303aaa2a..d3a8644a879 100644 --- a/docs/changelogs/v21.2.9.41-stable.md +++ b/docs/changelogs/v21.2.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.9.41-stable FIXME as compared to v21.2.8.31-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22814](https://github.com/ClickHouse/ClickHouse/issues/22814): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.1.6185-prestable.md b/docs/changelogs/v21.3.1.6185-prestable.md index dabfe3cfeb3..8d2a222f8aa 100644 --- a/docs/changelogs/v21.3.1.6185-prestable.md +++ b/docs/changelogs/v21.3.1.6185-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.1.6185-prestable FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -157,3 +164,162 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.10.1-lts.md b/docs/changelogs/v21.3.10.1-lts.md index 49ece009ad1..d43237428e2 100644 --- a/docs/changelogs/v21.3.10.1-lts.md +++ b/docs/changelogs/v21.3.10.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.10.1-lts FIXME as compared to v21.3.9.83-lts #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#23817](https://github.com/ClickHouse/ClickHouse/issues/23817): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). * Backported in [#23814](https://github.com/ClickHouse/ClickHouse/issues/23814): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.11.5-lts.md b/docs/changelogs/v21.3.11.5-lts.md index 61aa8a54688..2918ab860a6 100644 --- a/docs/changelogs/v21.3.11.5-lts.md +++ b/docs/changelogs/v21.3.11.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.11.5-lts FIXME as compared to v21.3.10.1-lts #### Improvement diff --git a/docs/changelogs/v21.3.12.2-lts.md b/docs/changelogs/v21.3.12.2-lts.md index cfddf8c9cdd..02c649284cc 100644 --- a/docs/changelogs/v21.3.12.2-lts.md +++ b/docs/changelogs/v21.3.12.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.12.2-lts FIXME as compared to v21.3.11.5-lts #### Bug Fix @@ -10,3 +17,9 @@ * Backported in [#24241](https://github.com/ClickHouse/ClickHouse/issues/24241): Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). * Backported in [#24353](https://github.com/ClickHouse/ClickHouse/issues/24353): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.13.9-lts.md b/docs/changelogs/v21.3.13.9-lts.md index dccb9c53162..f78725e4408 100644 --- a/docs/changelogs/v21.3.13.9-lts.md +++ b/docs/changelogs/v21.3.13.9-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.13.9-lts FIXME as compared to v21.3.12.2-lts #### Improvement @@ -41,3 +48,8 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.3: Remove endless `wait` from ZooKeeper client"'. [#24799](https://github.com/ClickHouse/ClickHouse/pull/24799) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.14.1-lts.md b/docs/changelogs/v21.3.14.1-lts.md index b2408471ccd..eb2ce73b01f 100644 --- a/docs/changelogs/v21.3.14.1-lts.md +++ b/docs/changelogs/v21.3.14.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.14.1-lts FIXME as compared to v21.3.13.9-lts #### Bug Fix @@ -8,3 +15,7 @@ * Backported in [#25716](https://github.com/ClickHouse/ClickHouse/issues/25716): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25712](https://github.com/ClickHouse/ClickHouse/issues/25712): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25681](https://github.com/ClickHouse/ClickHouse/pull/25681) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.3.15.4-stable.md b/docs/changelogs/v21.3.15.4-stable.md index c6b7a15aa4d..ca5a25850a1 100644 --- a/docs/changelogs/v21.3.15.4-stable.md +++ b/docs/changelogs/v21.3.15.4-stable.md @@ -1,6 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.15.4-stable FIXME as compared to v21.3.14.1-lts #### Bug Fix * Backported in [#25956](https://github.com/ClickHouse/ClickHouse/issues/25956): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26141](https://github.com/ClickHouse/ClickHouse/issues/26141): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.3 Docker unbundled fix [#26153](https://github.com/ClickHouse/ClickHouse/pull/26153) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md index b5b31b64488..a1e73b7c019 100644 --- a/docs/changelogs/v21.3.16.5-lts.md +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.16.5-lts FIXME as compared to v21.3.15.4-stable #### Bug Fix @@ -23,3 +30,15 @@ * Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). * Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md index b6b48ddab61..41007a45634 100644 --- a/docs/changelogs/v21.3.17.2-lts.md +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.17.2-lts FIXME as compared to v21.3.16.5-lts #### Bug Fix diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md index 612b3660e3b..5bb29b08df8 100644 --- a/docs/changelogs/v21.3.18.4-lts.md +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.18.4-lts FIXME as compared to v21.3.17.2-lts #### Improvement @@ -20,3 +27,10 @@ * Backported in [#30332](https://github.com/ClickHouse/ClickHouse/issues/30332): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30379](https://github.com/ClickHouse/ClickHouse/issues/30379): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md index 6a2be8d6dcb..fa8e94fa7a1 100644 --- a/docs/changelogs/v21.3.19.1-lts.md +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.19.1-lts FIXME as compared to v21.3.18.4-lts #### Performance Improvement @@ -24,3 +31,8 @@ * Backported in [#31894](https://github.com/ClickHouse/ClickHouse/issues/31894): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#32214](https://github.com/ClickHouse/ClickHouse/issues/32214): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.2.5-lts.md b/docs/changelogs/v21.3.2.5-lts.md index 5135f909b7f..0eb32439c6b 100644 --- a/docs/changelogs/v21.3.2.5-lts.md +++ b/docs/changelogs/v21.3.2.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.2.5-lts FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -159,3 +166,164 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable hedged requests for release 21.3 [#21534](https://github.com/ClickHouse/ClickHouse/pull/21534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md index ac8c7d2ece2..19cca4babf4 100644 --- a/docs/changelogs/v21.3.20.1-lts.md +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.20.1-lts FIXME as compared to v21.3.19.1-lts #### Bug Fix @@ -8,3 +15,13 @@ * Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cherry pick [#33065](https://github.com/ClickHouse/ClickHouse/issues/33065) to 21.3: Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33079](https://github.com/ClickHouse/ClickHouse/issues/33079) [#33578](https://github.com/ClickHouse/ClickHouse/pull/33578) ([Heena Bansal](https://github.com/HeenaBansal2009)). + diff --git a/docs/changelogs/v21.3.3.14-lts.md b/docs/changelogs/v21.3.3.14-lts.md index 3dbfd7f0a04..9b99c6c1b5c 100644 --- a/docs/changelogs/v21.3.3.14-lts.md +++ b/docs/changelogs/v21.3.3.14-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.3.14-lts FIXME as compared to v21.3.2.5-lts #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#21883](https://github.com/ClickHouse/ClickHouse/issues/21883): Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). * Backported in [#21874](https://github.com/ClickHouse/ClickHouse/issues/21874): Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.4.25-lts.md b/docs/changelogs/v21.3.4.25-lts.md index 431c498dbed..dee9e6a48b3 100644 --- a/docs/changelogs/v21.3.4.25-lts.md +++ b/docs/changelogs/v21.3.4.25-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.4.25-lts FIXME as compared to v21.3.3.14-lts #### Bug Fix @@ -16,3 +23,8 @@ * Backported in [#22185](https://github.com/ClickHouse/ClickHouse/issues/22185): Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#22205](https://github.com/ClickHouse/ClickHouse/issues/22205): Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.3: quick fix for broken resolution of apt.llvm.org on Yandex infra [#22047](https://github.com/ClickHouse/ClickHouse/pull/22047) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.5.42-lts.md b/docs/changelogs/v21.3.5.42-lts.md index d8616efd0ff..bc68841062c 100644 --- a/docs/changelogs/v21.3.5.42-lts.md +++ b/docs/changelogs/v21.3.5.42-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.5.42-lts FIXME as compared to v21.3.4.25-lts #### Bug Fix @@ -25,3 +32,7 @@ * Backported in [#22653](https://github.com/ClickHouse/ClickHouse/issues/22653): Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#22681](https://github.com/ClickHouse/ClickHouse/issues/22681): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.3 [#22504](https://github.com/ClickHouse/ClickHouse/pull/22504) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.3.6.55-lts.md b/docs/changelogs/v21.3.6.55-lts.md index 68980af6264..3a517f7386f 100644 --- a/docs/changelogs/v21.3.6.55-lts.md +++ b/docs/changelogs/v21.3.6.55-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.6.55-lts FIXME as compared to v21.3.5.42-lts #### Improvement @@ -17,3 +24,7 @@ * Backported in [#22916](https://github.com/ClickHouse/ClickHouse/issues/22916): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#22920](https://github.com/ClickHouse/ClickHouse/issues/22920): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.7.62-stable.md b/docs/changelogs/v21.3.7.62-stable.md index df919be0f42..24f759ca3c0 100644 --- a/docs/changelogs/v21.3.7.62-stable.md +++ b/docs/changelogs/v21.3.7.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.7.62-stable FIXME as compared to v21.3.6.55-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#23075](https://github.com/ClickHouse/ClickHouse/issues/23075): Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#23170](https://github.com/ClickHouse/ClickHouse/issues/23170): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.8.76-lts.md b/docs/changelogs/v21.3.8.76-lts.md index 2002d9e3e1f..acb0b99bae5 100644 --- a/docs/changelogs/v21.3.8.76-lts.md +++ b/docs/changelogs/v21.3.8.76-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.8.76-lts FIXME as compared to v21.3.7.62-stable #### Improvement @@ -19,3 +26,7 @@ * Backported in [#23536](https://github.com/ClickHouse/ClickHouse/issues/23536): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23533](https://github.com/ClickHouse/ClickHouse/issues/23533): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.3.9.83-lts.md b/docs/changelogs/v21.3.9.83-lts.md index e437ee4800f..450faeea3f2 100644 --- a/docs/changelogs/v21.3.9.83-lts.md +++ b/docs/changelogs/v21.3.9.83-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.9.83-lts FIXME as compared to v21.3.8.76-lts #### Improvement diff --git a/docs/changelogs/v21.4.1.6422-prestable.md b/docs/changelogs/v21.4.1.6422-prestable.md index 8cd10834fae..684fde2e3b7 100644 --- a/docs/changelogs/v21.4.1.6422-prestable.md +++ b/docs/changelogs/v21.4.1.6422-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.1.6422-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -167,6 +174,117 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flapping test_s3_zero_copy_replication [#22440](https://github.com/ClickHouse/ClickHouse/pull/22440) ([ianton-ru](https://github.com/ianton-ru)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust Keeper settings for CI [#22470](https://github.com/ClickHouse/ClickHouse/pull/22470) ([Alexander Tokmakov](https://github.com/tavplubix)). +* try clang 11 in fast test [#22472](https://github.com/ClickHouse/ClickHouse/pull/22472) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.2.10-prestable.md b/docs/changelogs/v21.4.2.10-prestable.md index 1bc440d126c..11767613551 100644 --- a/docs/changelogs/v21.4.2.10-prestable.md +++ b/docs/changelogs/v21.4.2.10-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.2.10-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -170,6 +177,114 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.3.21-stable.md b/docs/changelogs/v21.4.3.21-stable.md index dc3d7b7005b..a320e16f06a 100644 --- a/docs/changelogs/v21.4.3.21-stable.md +++ b/docs/changelogs/v21.4.3.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.3.21-stable FIXME as compared to v21.4.2.10-prestable #### Improvement @@ -16,3 +23,7 @@ * Backported in [#22921](https://github.com/ClickHouse/ClickHouse/issues/22921): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Backported in [#22957](https://github.com/ClickHouse/ClickHouse/issues/22957): Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.4.30-stable.md b/docs/changelogs/v21.4.4.30-stable.md index f029d334fc7..755bbede8f3 100644 --- a/docs/changelogs/v21.4.4.30-stable.md +++ b/docs/changelogs/v21.4.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.4.30-stable FIXME as compared to v21.4.3.21-stable #### Backward Incompatible Change diff --git a/docs/changelogs/v21.4.5.46-stable.md b/docs/changelogs/v21.4.5.46-stable.md index 664037ba596..9ca0553a154 100644 --- a/docs/changelogs/v21.4.5.46-stable.md +++ b/docs/changelogs/v21.4.5.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.5.46-stable FIXME as compared to v21.4.4.30-stable #### Improvement @@ -19,3 +26,9 @@ * Backported in [#23534](https://github.com/ClickHouse/ClickHouse/issues/23534): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23531](https://github.com/ClickHouse/ClickHouse/issues/23531): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.6.55-stable.md b/docs/changelogs/v21.4.6.55-stable.md index ea3e413ea0c..daca28d596c 100644 --- a/docs/changelogs/v21.4.6.55-stable.md +++ b/docs/changelogs/v21.4.6.55-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.6.55-stable FIXME as compared to v21.4.5.46-stable #### Improvement diff --git a/docs/changelogs/v21.4.7.3-stable.md b/docs/changelogs/v21.4.7.3-stable.md index 0dad6cfcb2b..612201a73ee 100644 --- a/docs/changelogs/v21.4.7.3-stable.md +++ b/docs/changelogs/v21.4.7.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.7.3-stable FIXME as compared to v21.4.6.55-stable #### Bug Fix @@ -15,3 +22,7 @@ * Backported in [#24215](https://github.com/ClickHouse/ClickHouse/issues/24215): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#24243](https://github.com/ClickHouse/ClickHouse/issues/24243): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.1.6601-prestable.md b/docs/changelogs/v21.5.1.6601-prestable.md index d64936fefce..dbe75de93d4 100644 --- a/docs/changelogs/v21.5.1.6601-prestable.md +++ b/docs/changelogs/v21.5.1.6601-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.1.6601-prestable FIXME as compared to v21.4.1.6422-prestable #### Backward Incompatible Change @@ -34,7 +41,7 @@ * Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)). * Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)). * Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)). -* MaterializeMySQL (experimental feature). Make ClickHouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). +* MaterializeMySQL (experimental feature). Make Clickhouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). * `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) ... [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)). * Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)). @@ -107,3 +114,96 @@ * NO CL ENTRY: 'Error message reads better'. [#22983](https://github.com/ClickHouse/ClickHouse/pull/22983) ([Igor O'sten](https://github.com/borodark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix SIGSEGV by waiting servers thread pool [#21318](https://github.com/ClickHouse/ClickHouse/pull/21318) ([Azat Khuzhin](https://github.com/azat)). +* Better filter push down [#22087](https://github.com/ClickHouse/ClickHouse/pull/22087) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better tests for finalize in nested writers [#22110](https://github.com/ClickHouse/ClickHouse/pull/22110) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Replace all Context references with std::weak_ptr [#22297](https://github.com/ClickHouse/ClickHouse/pull/22297) ([Ivan](https://github.com/abyss7)). +* make some perf test queries more stable [#22324](https://github.com/ClickHouse/ClickHouse/pull/22324) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ccache broken by prlimit [#22356](https://github.com/ClickHouse/ClickHouse/pull/22356) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove old MSan suppressions (part 5) [#22359](https://github.com/ClickHouse/ClickHouse/pull/22359) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for copier [#22441](https://github.com/ClickHouse/ClickHouse/pull/22441) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix window frame offset check and add more tests [#22459](https://github.com/ClickHouse/ClickHouse/pull/22459) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* FormatSettings null_as_default default value fix [#22528](https://github.com/ClickHouse/ClickHouse/pull/22528) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes in tests for AArch64 [#22534](https://github.com/ClickHouse/ClickHouse/pull/22534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Filter removed/renamed tests from ci-changed-files.txt for fuzzer [#22542](https://github.com/ClickHouse/ClickHouse/pull/22542) ([Azat Khuzhin](https://github.com/azat)). +* Try fix flaky test [#22543](https://github.com/ClickHouse/ClickHouse/pull/22543) ([Alexander Tokmakov](https://github.com/tavplubix)). +* AppleClang compilation fix [#22561](https://github.com/ClickHouse/ClickHouse/pull/22561) ([Denis Glazachev](https://github.com/traceon)). +* Fix assert in Arena when doing GROUP BY Array of Nothing of non-zero size. [#22565](https://github.com/ClickHouse/ClickHouse/pull/22565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor improvement in index deserialization [#22586](https://github.com/ClickHouse/ClickHouse/pull/22586) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test after [#22427](https://github.com/ClickHouse/ClickHouse/issues/22427) [#22587](https://github.com/ClickHouse/ClickHouse/pull/22587) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments [#22589](https://github.com/ClickHouse/ClickHouse/pull/22589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some uncaught exceptions (in SCOPE_EXIT) under memory pressure [#22592](https://github.com/ClickHouse/ClickHouse/pull/22592) ([Azat Khuzhin](https://github.com/azat)). +* Introduce IStorage::distributedWrite method for distributed INSERT SELECTS [#22593](https://github.com/ClickHouse/ClickHouse/pull/22593) ([Max Akhmedov](https://github.com/zlobober)). +* Fix flaky test 00816_long_concurrent_alter_column [#22628](https://github.com/ClickHouse/ClickHouse/pull/22628) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FlatDictionary fix perf test [#22629](https://github.com/ClickHouse/ClickHouse/pull/22629) ([Maksim Kita](https://github.com/kitaisreal)). +* DirectDictionary dictGet multiple columns optimization [#22630](https://github.com/ClickHouse/ClickHouse/pull/22630) ([Maksim Kita](https://github.com/kitaisreal)). +* Better retries on ZK errors in sh tests [#22633](https://github.com/ClickHouse/ClickHouse/pull/22633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add log_comment setting for DROP/CREATE DATABASE in clickhouse-test [#22646](https://github.com/ClickHouse/ClickHouse/pull/22646) ([Azat Khuzhin](https://github.com/azat)). +* Add retires for docker-compose pull in integration tests [#22647](https://github.com/ClickHouse/ClickHouse/pull/22647) ([Azat Khuzhin](https://github.com/azat)). +* Fix impossible invalid-read for system.errors accounting [#22655](https://github.com/ClickHouse/ClickHouse/pull/22655) ([Azat Khuzhin](https://github.com/azat)). +* Skip compiling xz if we're using system xz (unbundled) [#22656](https://github.com/ClickHouse/ClickHouse/pull/22656) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix test 01294_create_settings_profile [#22662](https://github.com/ClickHouse/ClickHouse/pull/22662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test 01039_row_policy_dcl [#22663](https://github.com/ClickHouse/ClickHouse/pull/22663) ([Anton Popov](https://github.com/CurtizJ)). +* Another attempt to enable pytest [#22664](https://github.com/ClickHouse/ClickHouse/pull/22664) ([Ivan](https://github.com/abyss7)). +* Fix random failures of tests that are using query_log [#22666](https://github.com/ClickHouse/ClickHouse/pull/22666) ([Anton Popov](https://github.com/CurtizJ)). +* fix build error 'always_inline' function might not be inlinable [#22667](https://github.com/ClickHouse/ClickHouse/pull/22667) ([flynn](https://github.com/ucasfl)). +* Add bool type in postgres engine [#22668](https://github.com/ClickHouse/ClickHouse/pull/22668) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix mutation killers tests [#22670](https://github.com/ClickHouse/ClickHouse/pull/22670) ([alesapin](https://github.com/alesapin)). +* Change Aggregatingmergetree to AggregatingMergeTree in docs [#22687](https://github.com/ClickHouse/ClickHouse/pull/22687) ([Kruglov Pavel](https://github.com/Avogar)). +* fix window functions with multiple input streams and no sorting [#22704](https://github.com/ClickHouse/ClickHouse/pull/22704) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove redundant fsync on coordination logs rotation [#22707](https://github.com/ClickHouse/ClickHouse/pull/22707) ([alesapin](https://github.com/alesapin)). +* Fix test 01702_system_query_log [#22708](https://github.com/ClickHouse/ClickHouse/pull/22708) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryStorage sync comments and code [#22721](https://github.com/ClickHouse/ClickHouse/pull/22721) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix potential segfault on Keeper startup [#22743](https://github.com/ClickHouse/ClickHouse/pull/22743) ([alesapin](https://github.com/alesapin)). +* Avoid using harmful function rand() [#22744](https://github.com/ClickHouse/ClickHouse/pull/22744) ([Amos Bird](https://github.com/amosbird)). +* Fix flacky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)). +* add more messages when flushing the logs [#22761](https://github.com/ClickHouse/ClickHouse/pull/22761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Moved BorrowedObjectPool to common [#22764](https://github.com/ClickHouse/ClickHouse/pull/22764) ([Maksim Kita](https://github.com/kitaisreal)). +* Functions ExternalDictionaries standardize exception throw [#22821](https://github.com/ClickHouse/ClickHouse/pull/22821) ([Maksim Kita](https://github.com/kitaisreal)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). +* Small change in replicated database tests run [#22826](https://github.com/ClickHouse/ClickHouse/pull/22826) ([alesapin](https://github.com/alesapin)). +* Slightly improve logging messages for Distributed async sends [#22829](https://github.com/ClickHouse/ClickHouse/pull/22829) ([Azat Khuzhin](https://github.com/azat)). +* Fix what looks like a trivial mistake [#22833](https://github.com/ClickHouse/ClickHouse/pull/22833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for already fixed issue [#22855](https://github.com/ClickHouse/ClickHouse/pull/22855) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DataTypeLowCardinality format tsv parsing issue [#22863](https://github.com/ClickHouse/ClickHouse/pull/22863) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated MariaDB connector fix cmake [#22865](https://github.com/ClickHouse/ClickHouse/pull/22865) ([Maksim Kita](https://github.com/kitaisreal)). +* Prettify logs of integration tests [#22868](https://github.com/ClickHouse/ClickHouse/pull/22868) ([Anton Popov](https://github.com/CurtizJ)). +* Util `memcpy-bench` is built only when position independent code is disabled [#22875](https://github.com/ClickHouse/ClickHouse/pull/22875) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix vanilla GCC compilation in macOS [#22885](https://github.com/ClickHouse/ClickHouse/pull/22885) ([Denis Glazachev](https://github.com/traceon)). +* Better diagnostics for OOM in stress test [#22894](https://github.com/ClickHouse/ClickHouse/pull/22894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionaries updated performance tests [#22925](https://github.com/ClickHouse/ClickHouse/pull/22925) ([Maksim Kita](https://github.com/kitaisreal)). +* IAggreagteFunction allocatesMemoryInArena removed default implementation [#22938](https://github.com/ClickHouse/ClickHouse/pull/22938) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flapping test_merge_tree_s3 test [#22942](https://github.com/ClickHouse/ClickHouse/pull/22942) ([ianton-ru](https://github.com/ianton-ru)). +* less flaky test [#22944](https://github.com/ClickHouse/ClickHouse/pull/22944) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Dictionaries standardize exceptions [#22961](https://github.com/ClickHouse/ClickHouse/pull/22961) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExternalDistributed arcadia fix [#22962](https://github.com/ClickHouse/ClickHouse/pull/22962) ([Maksim Kita](https://github.com/kitaisreal)). +* Check out of range values in FieldVisitorConverToNumber [#22964](https://github.com/ClickHouse/ClickHouse/pull/22964) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix combinators with common prefix name (State and SimpleState) with libstdc++ [#22977](https://github.com/ClickHouse/ClickHouse/pull/22977) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#22982](https://github.com/ClickHouse/ClickHouse/pull/22982) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix excessive warning in StorageDistributed with cross-replication [#22990](https://github.com/ClickHouse/ClickHouse/pull/22990) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update MergeTreeData.cpp Better error message. [#22994](https://github.com/ClickHouse/ClickHouse/pull/22994) ([Denny Crane](https://github.com/den-crane)). +* Fix assertion when filtering tables in StorageMerge [#22998](https://github.com/ClickHouse/ClickHouse/pull/22998) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#2719](https://github.com/ClickHouse/ClickHouse/issues/2719) [#23008](https://github.com/ClickHouse/ClickHouse/pull/23008) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix multi response in TestKeeper [#23041](https://github.com/ClickHouse/ClickHouse/pull/23041) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve hung check in Stress tests [#23043](https://github.com/ClickHouse/ClickHouse/pull/23043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* blog article about code review [#23045](https://github.com/ClickHouse/ClickHouse/pull/23045) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* LibraryDictionary bridge library interface [#23048](https://github.com/ClickHouse/ClickHouse/pull/23048) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#23049](https://github.com/ClickHouse/ClickHouse/pull/23049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Upload keeper logs from stateless tests [#23077](https://github.com/ClickHouse/ClickHouse/pull/23077) ([alesapin](https://github.com/alesapin)). +* CI runner intergation tests logs update to tar.gz [#23078](https://github.com/ClickHouse/ClickHouse/pull/23078) ([Maksim Kita](https://github.com/kitaisreal)). +* Tiny logging improvements [#23083](https://github.com/ClickHouse/ClickHouse/pull/23083) ([Azat Khuzhin](https://github.com/azat)). +* Block all memory tracking limits in dtors/SCOPE_EXIT_*SAFE/tryLogCurrentException [#23086](https://github.com/ClickHouse/ClickHouse/pull/23086) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc tuning [#23088](https://github.com/ClickHouse/ClickHouse/pull/23088) ([Azat Khuzhin](https://github.com/azat)). +* Rename strange tests [#23111](https://github.com/ClickHouse/ClickHouse/pull/23111) ([alesapin](https://github.com/alesapin)). +* Fix arcadia build S3 [#23114](https://github.com/ClickHouse/ClickHouse/pull/23114) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated zlib [#23153](https://github.com/ClickHouse/ClickHouse/pull/23153) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Change logging from trace to debug for messages with rows/bytes [#23160](https://github.com/ClickHouse/ClickHouse/pull/23160) ([Azat Khuzhin](https://github.com/azat)). +* More verbose logs for debuging test failures with Replicated and Keeper [#23161](https://github.com/ClickHouse/ClickHouse/pull/23161) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception message for "parts_to_throw_insert" [#23177](https://github.com/ClickHouse/ClickHouse/pull/23177) ([madianjun](https://github.com/mdianjun)). +* Fix flapping tests test_s3_zero_copy_replication [#23184](https://github.com/ClickHouse/ClickHouse/pull/23184) ([ianton-ru](https://github.com/ianton-ru)). +* Add a test for [#14610](https://github.com/ClickHouse/ClickHouse/issues/14610) [#23209](https://github.com/ClickHouse/ClickHouse/pull/23209) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for fixed issue [#9363](https://github.com/ClickHouse/ClickHouse/issues/9363) [#23216](https://github.com/ClickHouse/ClickHouse/pull/23216) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable postgresql_port in perf tests [#23218](https://github.com/ClickHouse/ClickHouse/pull/23218) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.5.2.25-prestable.md b/docs/changelogs/v21.5.2.25-prestable.md index 45e784218da..1d7418a30c2 100644 --- a/docs/changelogs/v21.5.2.25-prestable.md +++ b/docs/changelogs/v21.5.2.25-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.2.25-prestable FIXME as compared to v21.5.1.6601-prestable #### Improvement @@ -38,3 +45,10 @@ * Backported in [#23347](https://github.com/ClickHouse/ClickHouse/issues/23347):. [#23334](https://github.com/ClickHouse/ClickHouse/pull/23334) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.3.1-prestable.md b/docs/changelogs/v21.5.3.1-prestable.md index 24b29d1495e..2e52d8ba6a8 100644 --- a/docs/changelogs/v21.5.3.1-prestable.md +++ b/docs/changelogs/v21.5.3.1-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.3.1-prestable FIXME as compared to v21.5.2.25-prestable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#23832](https://github.com/ClickHouse/ClickHouse/issues/23832): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23927](https://github.com/ClickHouse/ClickHouse/issues/23927): HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.4.6-prestable.md b/docs/changelogs/v21.5.4.6-prestable.md index c9e040309cc..37de3225996 100644 --- a/docs/changelogs/v21.5.4.6-prestable.md +++ b/docs/changelogs/v21.5.4.6-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.4.6-prestable FIXME as compared to v21.5.3.1-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.5.12-stable.md b/docs/changelogs/v21.5.5.12-stable.md index b49e2c60b08..71467dd2ffa 100644 --- a/docs/changelogs/v21.5.5.12-stable.md +++ b/docs/changelogs/v21.5.5.12-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.5.12-stable FIXME as compared to v21.5.4.6-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.6.6-stable.md b/docs/changelogs/v21.5.6.6-stable.md index e6160dfa784..04fe3ec4010 100644 --- a/docs/changelogs/v21.5.6.6-stable.md +++ b/docs/changelogs/v21.5.6.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.6.6-stable FIXME as compared to v21.5.5.12-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#24750](https://github.com/ClickHouse/ClickHouse/issues/24750):. [#23276](https://github.com/ClickHouse/ClickHouse/pull/23276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.5.7.9-stable.md b/docs/changelogs/v21.5.7.9-stable.md index 9586e398547..eaf0cb6255b 100644 --- a/docs/changelogs/v21.5.7.9-stable.md +++ b/docs/changelogs/v21.5.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.7.9-stable FIXME as compared to v21.5.6.6-stable #### Improvement @@ -40,3 +47,9 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.5: Remove endless `wait` from ZooKeeper client"'. [#24798](https://github.com/ClickHouse/ClickHouse/pull/24798) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.5.8.21-stable.md b/docs/changelogs/v21.5.8.21-stable.md index 4d9ffd687eb..fe59efd2255 100644 --- a/docs/changelogs/v21.5.8.21-stable.md +++ b/docs/changelogs/v21.5.8.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.8.21-stable FIXME as compared to v21.5.7.9-stable #### Bug Fix @@ -11,3 +18,8 @@ * Backported in [#25695](https://github.com/ClickHouse/ClickHouse/issues/25695): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25754](https://github.com/ClickHouse/ClickHouse/issues/25754): Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25680](https://github.com/ClickHouse/ClickHouse/pull/25680) ([alesapin](https://github.com/alesapin)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.9.4-stable.md b/docs/changelogs/v21.5.9.4-stable.md index 17ef067194b..b9d643dda09 100644 --- a/docs/changelogs/v21.5.9.4-stable.md +++ b/docs/changelogs/v21.5.9.4-stable.md @@ -1,6 +1,18 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.9.4-stable FIXME as compared to v21.5.8.21-stable #### Bug Fix * Backported in [#25958](https://github.com/ClickHouse/ClickHouse/issues/25958): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26144](https://github.com/ClickHouse/ClickHouse/issues/26144): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.5 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26139](https://github.com/ClickHouse/ClickHouse/pull/26139) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.5 Docker unbundled fix [#26154](https://github.com/ClickHouse/ClickHouse/pull/26154) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.1.6891-prestable.md b/docs/changelogs/v21.6.1.6891-prestable.md index 7750cac32eb..11fcd854989 100644 --- a/docs/changelogs/v21.6.1.6891-prestable.md +++ b/docs/changelogs/v21.6.1.6891-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.1.6891-prestable FIXME as compared to v21.5.1.6601-prestable #### New Feature @@ -152,6 +159,169 @@ * NO CL ENTRY: 'Revert "Fix CI build for gcc-10"'. [#23772](https://github.com/ClickHouse/ClickHouse/pull/23772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Update syntax.md'. [#24267](https://github.com/ClickHouse/ClickHouse/pull/24267) ([lulichao](https://github.com/lulichao)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Apply idle_connnection_timeout/poll_interval after each query [#21938](https://github.com/ClickHouse/ClickHouse/pull/21938) ([Azat Khuzhin](https://github.com/azat)). +* Do not silently catch errors for writing to S3 [#22208](https://github.com/ClickHouse/ClickHouse/pull/22208) ([Azat Khuzhin](https://github.com/azat)). +* Add dockerhub-proxy to runner [#23138](https://github.com/ClickHouse/ClickHouse/pull/23138) ([Ilya Yatsishin](https://github.com/qoega)). +* merging sumCount fusion PR [#21337](https://github.com/ClickHouse/ClickHouse/issues/21337) [#23159](https://github.com/ClickHouse/ClickHouse/pull/23159) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor fixes in ATTACH query [#23189](https://github.com/ClickHouse/ClickHouse/pull/23189) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#22503](https://github.com/ClickHouse/ClickHouse/issues/22503) [#23195](https://github.com/ClickHouse/ClickHouse/pull/23195) ([Anton Popov](https://github.com/CurtizJ)). +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more stable formatting for negate() [#23201](https://github.com/ClickHouse/ClickHouse/pull/23201) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Use double quote identifier in odbc as default in case of error [#23217](https://github.com/ClickHouse/ClickHouse/pull/23217) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add -Wundef for gcc builds [#23258](https://github.com/ClickHouse/ClickHouse/pull/23258) ([Azat Khuzhin](https://github.com/azat)). +* Report an error if jemalloc.background_thread was requested [#23259](https://github.com/ClickHouse/ClickHouse/pull/23259) ([Azat Khuzhin](https://github.com/azat)). +* Add trace_log into stateless/stress test artifacts [#23264](https://github.com/ClickHouse/ClickHouse/pull/23264) ([Azat Khuzhin](https://github.com/azat)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function toDateTime decimal overflow ubsan fix [#23278](https://github.com/ClickHouse/ClickHouse/pull/23278) ([Maksim Kita](https://github.com/kitaisreal)). +* Link keeper-bench to clickhouse_common_zookeeper [#23302](https://github.com/ClickHouse/ClickHouse/pull/23302) ([Raúl Marín](https://github.com/Algunenano)). +* Print errors on db creation in clickhouse-test [#23304](https://github.com/ClickHouse/ClickHouse/pull/23304) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix broken perf test [#23308](https://github.com/ClickHouse/ClickHouse/pull/23308) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix bad test 01602_max_distributed_connections [#23317](https://github.com/ClickHouse/ClickHouse/pull/23317) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hdfs reading from files with spaces [#23318](https://github.com/ClickHouse/ClickHouse/pull/23318) ([Kseniia Sumarokova](https://github.com/kssenii)). +* add check that p.pruning works [#23321](https://github.com/ClickHouse/ClickHouse/pull/23321) ([Denny Crane](https://github.com/den-crane)). +* Add test for [#7815](https://github.com/ClickHouse/ClickHouse/issues/7815) [#23332](https://github.com/ClickHouse/ClickHouse/pull/23332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01666_merge_tree_max_query_limit.sh [#23335](https://github.com/ClickHouse/ClickHouse/pull/23335) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#12077](https://github.com/ClickHouse/ClickHouse/issues/12077) [#23352](https://github.com/ClickHouse/ClickHouse/pull/23352) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable clickhouse-odbc-bridge build when ODBC is disabled [#23357](https://github.com/ClickHouse/ClickHouse/pull/23357) ([Denis Glazachev](https://github.com/traceon)). +* Fix AppleClang build [#23358](https://github.com/ClickHouse/ClickHouse/pull/23358) ([Denis Glazachev](https://github.com/traceon)). +* Use Atomic database for development environment [#23377](https://github.com/ClickHouse/ClickHouse/pull/23377) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#2582](https://github.com/ClickHouse/ClickHouse/issues/2582) [#23378](https://github.com/ClickHouse/ClickHouse/pull/23378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#1647](https://github.com/ClickHouse/ClickHouse/issues/1647) [#23379](https://github.com/ClickHouse/ClickHouse/pull/23379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow rabbitmq vhost in table settings [#23452](https://github.com/ClickHouse/ClickHouse/pull/23452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). +* ISSUES-23310 Try fix MySQL 8.0 address already in use [#23462](https://github.com/ClickHouse/ClickHouse/pull/23462) ([Winter Zhang](https://github.com/zhang2014)). +* Fix error in perf test [#23469](https://github.com/ClickHouse/ClickHouse/pull/23469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: print database name on failures [#23486](https://github.com/ClickHouse/ClickHouse/pull/23486) ([Azat Khuzhin](https://github.com/azat)). +* upload cpu model to perf test db [#23514](https://github.com/ClickHouse/ClickHouse/pull/23514) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix function tests flaps [#23517](https://github.com/ClickHouse/ClickHouse/pull/23517) ([Azat Khuzhin](https://github.com/azat)). +* fix pvs warnings [#23520](https://github.com/ClickHouse/ClickHouse/pull/23520) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ignore empty input chunks generated by joins [#23542](https://github.com/ClickHouse/ClickHouse/pull/23542) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix window functions for Distributed tables [#23546](https://github.com/ClickHouse/ClickHouse/pull/23546) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add more info to perf test report [#23550](https://github.com/ClickHouse/ClickHouse/pull/23550) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* export trace log from stateless tests in flamegraph-friendly format [#23553](https://github.com/ClickHouse/ClickHouse/pull/23553) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add numactl info to perf test run attributes [#23554](https://github.com/ClickHouse/ClickHouse/pull/23554) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Less chance of OOM in stress tests [#23561](https://github.com/ClickHouse/ClickHouse/pull/23561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix inconsistent formatting for tupleElement (for fuzzer) [#23595](https://github.com/ClickHouse/ClickHouse/pull/23595) ([Azat Khuzhin](https://github.com/azat)). +* Remove unneeded code from CMakeLists [#23597](https://github.com/ClickHouse/ClickHouse/pull/23597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Somewhat better handling of paths in CMake (incomplete) [#23598](https://github.com/ClickHouse/ClickHouse/pull/23598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old trash (a little) [#23599](https://github.com/ClickHouse/ClickHouse/pull/23599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some garbage from RocksDB CMakeLists [#23601](https://github.com/ClickHouse/ClickHouse/pull/23601) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add warning about gcc usage [#23603](https://github.com/ClickHouse/ClickHouse/pull/23603) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove garbage from CMakeLists (2) [#23604](https://github.com/ClickHouse/ClickHouse/pull/23604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove rotten parts of release script [#23605](https://github.com/ClickHouse/ClickHouse/pull/23605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#23606](https://github.com/ClickHouse/ClickHouse/pull/23606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Skip CatBoost tests under MSan [#23614](https://github.com/ClickHouse/ClickHouse/pull/23614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test 01641_memory_tracking_insert_optimize_long [#23617](https://github.com/ClickHouse/ClickHouse/pull/23617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pbuilder [#23618](https://github.com/ClickHouse/ClickHouse/pull/23618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Aggregator remove unused code [#23635](https://github.com/ClickHouse/ClickHouse/pull/23635) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#22984](https://github.com/ClickHouse/ClickHouse/issues/22984) [#23637](https://github.com/ClickHouse/ClickHouse/pull/23637) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move non gtest unit tests to /examples folder [#23644](https://github.com/ClickHouse/ClickHouse/pull/23644) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Kerberized HDFS test is fluky - retries and extra output if failure [#23650](https://github.com/ClickHouse/ClickHouse/pull/23650) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix documentation for DETACH ON CLUSTER PERMANENTLY [#23653](https://github.com/ClickHouse/ClickHouse/pull/23653) ([Azat Khuzhin](https://github.com/azat)). +* Skip integration test for library bridge under MSan [#23662](https://github.com/ClickHouse/ClickHouse/pull/23662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a few PVS-Studio warnings [#23663](https://github.com/ClickHouse/ClickHouse/pull/23663) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable use-after-destruction detection in MSan [#23664](https://github.com/ClickHouse/ClickHouse/pull/23664) ([Azat Khuzhin](https://github.com/azat)). +* Fix cyrus-sasl msan warning [#23672](https://github.com/ClickHouse/ClickHouse/pull/23672) ([Ilya Yatsishin](https://github.com/qoega)). +* A little bit faster merge of aggregating states. [#23681](https://github.com/ClickHouse/ClickHouse/pull/23681) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* stable query indexes in perf test [#23707](https://github.com/ClickHouse/ClickHouse/pull/23707) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor changes in code [#14254](https://github.com/ClickHouse/ClickHouse/issues/14254) [#23709](https://github.com/ClickHouse/ClickHouse/pull/23709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix illiterate error message [#23700](https://github.com/ClickHouse/ClickHouse/issues/23700) [#23710](https://github.com/ClickHouse/ClickHouse/pull/23710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete compilers [#23711](https://github.com/ClickHouse/ClickHouse/pull/23711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#23644](https://github.com/ClickHouse/ClickHouse/issues/23644) [#23712](https://github.com/ClickHouse/ClickHouse/pull/23712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable test_odbc_interaction for MSan build [#23722](https://github.com/ClickHouse/ClickHouse/pull/23722) ([Vladimir C](https://github.com/vdimir)). +* fix a typo in query formatting check in fuzzer [#23726](https://github.com/ClickHouse/ClickHouse/pull/23726) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ExternalLoaderRepository fix arcadia [#23732](https://github.com/ClickHouse/ClickHouse/pull/23732) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore cmake-in-clickhouse [#23740](https://github.com/ClickHouse/ClickHouse/pull/23740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refactor join step [#23743](https://github.com/ClickHouse/ClickHouse/pull/23743) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* clickhouse-test: send TERM to all childs (to avoid hung check triggering) [#23750](https://github.com/ClickHouse/ClickHouse/pull/23750) ([Azat Khuzhin](https://github.com/azat)). +* Improve test_insert_into_distributed [#23751](https://github.com/ClickHouse/ClickHouse/pull/23751) ([Azat Khuzhin](https://github.com/azat)). +* Fix CI build for gcc-10 [#23760](https://github.com/ClickHouse/ClickHouse/pull/23760) ([Maksim Kita](https://github.com/kitaisreal)). +* Update array-functions.md [#23762](https://github.com/ClickHouse/ClickHouse/pull/23762) ([fancno](https://github.com/fancno)). +* Remove unused compilers (fixed for the troublesome "unbundled" build) [#23766](https://github.com/ClickHouse/ClickHouse/pull/23766) ([Maksim Kita](https://github.com/kitaisreal)). +* Slightly better hardening for intersecting parts [#23767](https://github.com/ClickHouse/ClickHouse/pull/23767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Workaround for PVS-Studio [#23776](https://github.com/ClickHouse/ClickHouse/pull/23776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add examples folder filter to ya.make.in [#23786](https://github.com/ClickHouse/ClickHouse/pull/23786) ([Maksim Kita](https://github.com/kitaisreal)). +* Function default implementation for nulls small optimization [#23799](https://github.com/ClickHouse/ClickHouse/pull/23799) ([Maksim Kita](https://github.com/kitaisreal)). +* autodetect arch of gosu in server dockerfile [#23802](https://github.com/ClickHouse/ClickHouse/pull/23802) ([filimonov](https://github.com/filimonov)). +* Add test for [#18170](https://github.com/ClickHouse/ClickHouse/issues/18170) [#23813](https://github.com/ClickHouse/ClickHouse/pull/23813) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* clickhouse-test: add missing whitespace before printing database on error [#23820](https://github.com/ClickHouse/ClickHouse/pull/23820) ([Azat Khuzhin](https://github.com/azat)). +* Improve 00840_long_concurrent_select_and_drop_deadlock [#23823](https://github.com/ClickHouse/ClickHouse/pull/23823) ([Azat Khuzhin](https://github.com/azat)). +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). +* XDBCBridgeHelper use global context [#23836](https://github.com/ClickHouse/ClickHouse/pull/23836) ([Maksim Kita](https://github.com/kitaisreal)). +* gcc-10 installation no-install-reccomends option fix [#23840](https://github.com/ClickHouse/ClickHouse/pull/23840) ([Maksim Kita](https://github.com/kitaisreal)). +* replxx readline compatibility [#23855](https://github.com/ClickHouse/ClickHouse/pull/23855) ([Azat Khuzhin](https://github.com/azat)). +* Add file paths into logs on failed distributed async sends [#23856](https://github.com/ClickHouse/ClickHouse/pull/23856) ([Azat Khuzhin](https://github.com/azat)). +* Reduce the amount of logs that StorageMergeTree::selectPartsToMutate outputs in busy systems. [#23863](https://github.com/ClickHouse/ClickHouse/pull/23863) ([Raúl Marín](https://github.com/Algunenano)). +* Add DiskRestartProxy.cpp to ya.make [#23868](https://github.com/ClickHouse/ClickHouse/pull/23868) ([Vladimir C](https://github.com/vdimir)). +* Fix some warnings by PVS-Studio [#23877](https://github.com/ClickHouse/ClickHouse/pull/23877) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Read/WriteBufferFromFileDecorator.cpp to ya.make [#23879](https://github.com/ClickHouse/ClickHouse/pull/23879) ([Vladimir C](https://github.com/vdimir)). +* @CurtizJ convinced me that this test has to be deleted [#23883](https://github.com/ClickHouse/ClickHouse/pull/23883) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* bash completion improvements [#23884](https://github.com/ClickHouse/ClickHouse/pull/23884) ([Azat Khuzhin](https://github.com/azat)). +* Remove obsolete code [#23886](https://github.com/ClickHouse/ClickHouse/pull/23886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code simplification (implemented TODO) [#23896](https://github.com/ClickHouse/ClickHouse/pull/23896) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add run options with default config path according to issues/23875 [#23898](https://github.com/ClickHouse/ClickHouse/pull/23898) ([ice1x](https://github.com/ice1x)). +* Fix test_insert_into_distributed flaps [#23903](https://github.com/ClickHouse/ClickHouse/pull/23903) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test about compression codecs [#23908](https://github.com/ClickHouse/ClickHouse/pull/23908) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cleanup IDatabase.h from extra headers [#23912](https://github.com/ClickHouse/ClickHouse/pull/23912) ([Azat Khuzhin](https://github.com/azat)). +* Minor simplification [#23923](https://github.com/ClickHouse/ClickHouse/pull/23923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop unnecessary ports where applicable [#23928](https://github.com/ClickHouse/ClickHouse/pull/23928) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Check MIN/MAX attributes in the list of dictionary attributes [#23948](https://github.com/ClickHouse/ClickHouse/pull/23948) ([Azat Khuzhin](https://github.com/azat)). +* typo: fix a typo in Compression/CodecT64 [#23952](https://github.com/ClickHouse/ClickHouse/pull/23952) ([mwish](https://github.com/mapleFU)). +* Don't try GLIBC_COMPATIBILITY for i686 Linux [#23959](https://github.com/ClickHouse/ClickHouse/pull/23959) ([divanorama](https://github.com/divanorama)). +* Function arrayDifference decimal math overflow [#23961](https://github.com/ClickHouse/ClickHouse/pull/23961) ([Maksim Kita](https://github.com/kitaisreal)). +* Use 0 over nan for hit_rate in case of 0 queries to the cache dictionary [#23963](https://github.com/ClickHouse/ClickHouse/pull/23963) ([Azat Khuzhin](https://github.com/azat)). +* Round floats in Aggregator log messages [#23965](https://github.com/ClickHouse/ClickHouse/pull/23965) ([Azat Khuzhin](https://github.com/azat)). +* support longer query ids in trace log for perf tests [#23969](https://github.com/ClickHouse/ClickHouse/pull/23969) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* PVS-Studio fixes, part 6 [#23970](https://github.com/ClickHouse/ClickHouse/pull/23970) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Functional stateless tests fix numbers [#23974](https://github.com/ClickHouse/ClickHouse/pull/23974) ([Maksim Kita](https://github.com/kitaisreal)). +* use LowCardinality for AsynchronousMetricLog name column [#23981](https://github.com/ClickHouse/ClickHouse/pull/23981) ([flynn](https://github.com/ucasfl)). +* Function dictGetOrNull handle empty rows execute [#23990](https://github.com/ClickHouse/ClickHouse/pull/23990) ([Maksim Kita](https://github.com/kitaisreal)). +* CompileDAG fix Sip hash [#24004](https://github.com/ClickHouse/ClickHouse/pull/24004) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix bad code [#24007](https://github.com/ClickHouse/ClickHouse/pull/24007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better error codes in Keeper when no leader alive [#24017](https://github.com/ClickHouse/ClickHouse/pull/24017) ([alesapin](https://github.com/alesapin)). +* Fix ArenaWithFreeLists test [#24021](https://github.com/ClickHouse/ClickHouse/pull/24021) ([Maksim Kita](https://github.com/kitaisreal)). +* Run check_*_compiler_flag earlier [#24037](https://github.com/ClickHouse/ClickHouse/pull/24037) ([Amos Bird](https://github.com/amosbird)). +* Performance tests disable compile expressions [#24043](https://github.com/ClickHouse/ClickHouse/pull/24043) ([Maksim Kita](https://github.com/kitaisreal)). +* for trivial INSERT SELECT, adjust block size in bytes as well [#24048](https://github.com/ClickHouse/ClickHouse/pull/24048) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable thread_local in Arcadia build [#24051](https://github.com/ClickHouse/ClickHouse/pull/24051) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* lower two-level aggregation threshold for uniq test to avoid jitter [#24058](https://github.com/ClickHouse/ClickHouse/pull/24058) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix empty key projection query analysis [#24062](https://github.com/ClickHouse/ClickHouse/pull/24062) ([Amos Bird](https://github.com/amosbird)). +* Generate ya.make for missing UUID.cpp [#24064](https://github.com/ClickHouse/ClickHouse/pull/24064) ([Alexander Gololobov](https://github.com/davenger)). +* bash-completion: complete available formats [#24065](https://github.com/ClickHouse/ClickHouse/pull/24065) ([Azat Khuzhin](https://github.com/azat)). +* Fix concurrent snapshot read/write [#24073](https://github.com/ClickHouse/ClickHouse/pull/24073) ([alesapin](https://github.com/alesapin)). +* Also retry database creation in `clickhouse-test` [#24088](https://github.com/ClickHouse/ClickHouse/pull/24088) ([alesapin](https://github.com/alesapin)). +* Calculate header from ActionsDAG [#24108](https://github.com/ClickHouse/ClickHouse/pull/24108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24133](https://github.com/ClickHouse/ClickHouse/pull/24133) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* TestReadAfterAIO: Use the current path instead of /tmp for temporal files [#24139](https://github.com/ClickHouse/ClickHouse/pull/24139) ([Raúl Marín](https://github.com/Algunenano)). +* Fix a few trailing whitespaces in output [#24150](https://github.com/ClickHouse/ClickHouse/pull/24150) ([Azat Khuzhin](https://github.com/azat)). +* IFunction refactoring [#24155](https://github.com/ClickHouse/ClickHouse/pull/24155) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix add projection to replicated mergetree [#24162](https://github.com/ClickHouse/ClickHouse/pull/24162) ([Amos Bird](https://github.com/amosbird)). +* Fix distributed processing when using projection [#24168](https://github.com/ClickHouse/ClickHouse/pull/24168) ([Amos Bird](https://github.com/amosbird)). +* fix tiny code style [#24169](https://github.com/ClickHouse/ClickHouse/pull/24169) ([flynn](https://github.com/ucasfl)). +* fix reinterpretAsFixedString for UUID [#24177](https://github.com/ClickHouse/ClickHouse/pull/24177) ([flynn](https://github.com/ucasfl)). +* Function move file [#24178](https://github.com/ClickHouse/ClickHouse/pull/24178) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated LRUHashMap benchmarks [#24182](https://github.com/ClickHouse/ClickHouse/pull/24182) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove temporary files [#24186](https://github.com/ClickHouse/ClickHouse/pull/24186) ([Kruglov Pavel](https://github.com/Avogar)). +* Try mute grpc msan [#24197](https://github.com/ClickHouse/ClickHouse/pull/24197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update slack link [#24200](https://github.com/ClickHouse/ClickHouse/pull/24200) ([Ilya Yatsishin](https://github.com/qoega)). +* Simplier isLocalAddress [#24203](https://github.com/ClickHouse/ClickHouse/pull/24203) ([alesapin](https://github.com/alesapin)). +* Increase timeout for server restart in integration tests. [#24205](https://github.com/ClickHouse/ClickHouse/pull/24205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Addition to [#23997](https://github.com/ClickHouse/ClickHouse/issues/23997) [#24208](https://github.com/ClickHouse/ClickHouse/pull/24208) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Small code simplification [#24210](https://github.com/ClickHouse/ClickHouse/pull/24210) ([Maksim Kita](https://github.com/kitaisreal)). +* Speedup test [test_jbod_balancer/test.py] [#24247](https://github.com/ClickHouse/ClickHouse/pull/24247) ([alesapin](https://github.com/alesapin)). +* remove useless code [#24248](https://github.com/ClickHouse/ClickHouse/pull/24248) ([flynn](https://github.com/ucasfl)). +* One more attempt to fix retries in clickhouse-test [#24249](https://github.com/ClickHouse/ClickHouse/pull/24249) ([alesapin](https://github.com/alesapin)). +* Log exception in Allocator::free [#24256](https://github.com/ClickHouse/ClickHouse/pull/24256) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix broken HTML markup on website [#24265](https://github.com/ClickHouse/ClickHouse/pull/24265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix arcadia [#24282](https://github.com/ClickHouse/ClickHouse/pull/24282) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed Arcadia after IFunctionOverloadResolver interface refactoring [#24284](https://github.com/ClickHouse/ClickHouse/pull/24284) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix isLocalAddress() (ifa_addr maybe NULL) [#24308](https://github.com/ClickHouse/ClickHouse/pull/24308) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test for Enum hints [#24313](https://github.com/ClickHouse/ClickHouse/pull/24313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature #14893 * - Add uniqThetaSketch to support [Theta Sketch](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html) in ClickHouse. [#22609](https://github.com/ClickHouse/ClickHouse/pull/22609) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.6.2.7-prestable.md b/docs/changelogs/v21.6.2.7-prestable.md index c5dec251786..55032e0258e 100644 --- a/docs/changelogs/v21.6.2.7-prestable.md +++ b/docs/changelogs/v21.6.2.7-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.2.7-prestable FIXME as compared to v21.6.1.6891-prestable #### Bug Fix @@ -11,3 +18,9 @@ * Backported in [#24597](https://github.com/ClickHouse/ClickHouse/issues/24597): Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#24600](https://github.com/ClickHouse/ClickHouse/issues/24600): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.6.3.14-stable.md b/docs/changelogs/v21.6.3.14-stable.md index 8d2e2a03320..551f959aefd 100644 --- a/docs/changelogs/v21.6.3.14-stable.md +++ b/docs/changelogs/v21.6.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.3.14-stable FIXME as compared to v21.6.2.7-prestable #### Improvement diff --git a/docs/changelogs/v21.6.4.26-stable.md b/docs/changelogs/v21.6.4.26-stable.md index 9bf7d0e68cb..dc235d91e6d 100644 --- a/docs/changelogs/v21.6.4.26-stable.md +++ b/docs/changelogs/v21.6.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.4.26-stable FIXME as compared to v21.6.3.14-stable #### Bug Fix @@ -12,3 +19,8 @@ * Backported in [#25157](https://github.com/ClickHouse/ClickHouse/issues/25157): Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25182](https://github.com/ClickHouse/ClickHouse/issues/25182): Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.5.37-stable.md b/docs/changelogs/v21.6.5.37-stable.md index 3d03c31c7e8..45a43199c28 100644 --- a/docs/changelogs/v21.6.5.37-stable.md +++ b/docs/changelogs/v21.6.5.37-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.5.37-stable FIXME as compared to v21.6.4.26-stable #### Improvement @@ -19,3 +26,8 @@ * Backported in [#25448](https://github.com/ClickHouse/ClickHouse/issues/25448): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.6.6.51-stable.md b/docs/changelogs/v21.6.6.51-stable.md index 55f1fd46119..39441adaa9e 100644 --- a/docs/changelogs/v21.6.6.51-stable.md +++ b/docs/changelogs/v21.6.6.51-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.6.51-stable FIXME as compared to v21.6.5.37-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Partial backport [#24061](https://github.com/ClickHouse/ClickHouse/issues/24061) to 21.6'. [#25621](https://github.com/ClickHouse/ClickHouse/pull/25621) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.7.57-stable.md b/docs/changelogs/v21.6.7.57-stable.md index 5ef9026794b..26887fac120 100644 --- a/docs/changelogs/v21.6.7.57-stable.md +++ b/docs/changelogs/v21.6.7.57-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.7.57-stable FIXME as compared to v21.6.6.51-stable #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#26096](https://github.com/ClickHouse/ClickHouse/issues/26096): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). * Backported in [#26143](https://github.com/ClickHouse/ClickHouse/issues/26143): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.6 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26138](https://github.com/ClickHouse/ClickHouse/pull/26138) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.8.62-stable.md b/docs/changelogs/v21.6.8.62-stable.md index 1357c762181..0dc35d41344 100644 --- a/docs/changelogs/v21.6.8.62-stable.md +++ b/docs/changelogs/v21.6.8.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.8.62-stable FIXME as compared to v21.6.7.57-stable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#26205](https://github.com/ClickHouse/ClickHouse/issues/26205): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#26227](https://github.com/ClickHouse/ClickHouse/issues/26227): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#25531](https://github.com/ClickHouse/ClickHouse/issues/25531) in 21.6 [#25776](https://github.com/ClickHouse/ClickHouse/pull/25776) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md index ecac9dd75ef..fae1c8d2fef 100644 --- a/docs/changelogs/v21.6.9.7-stable.md +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.9.7-stable FIXME as compared to v21.6.8.62-stable #### Improvement @@ -41,3 +48,19 @@ * Backported in [#28264](https://github.com/ClickHouse/ClickHouse/issues/28264): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#28291](https://github.com/ClickHouse/ClickHouse/issues/28291): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.1.7283-prestable.md b/docs/changelogs/v21.7.1.7283-prestable.md index 52de493c4ab..a862e6c4750 100644 --- a/docs/changelogs/v21.7.1.7283-prestable.md +++ b/docs/changelogs/v21.7.1.7283-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.1.7283-prestable FIXME as compared to v21.6.1.6891-prestable #### Backward Incompatible Change @@ -172,6 +179,236 @@ * NO CL ENTRY: 'Revert "Add run-id option to integration tests"'. [#25526](https://github.com/ClickHouse/ClickHouse/pull/25526) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Revert "Implement h3ToGeo function"'. [#25593](https://github.com/ClickHouse/ClickHouse/pull/25593) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Part movement between shards [#17871](https://github.com/ClickHouse/ClickHouse/pull/17871) ([nvartolomei](https://github.com/nvartolomei)). +* Pass Settings to aggregate function creator [#22762](https://github.com/ClickHouse/ClickHouse/pull/22762) ([Vladimir C](https://github.com/vdimir)). +* Add uniqTheta in performance test [#23311](https://github.com/ClickHouse/ClickHouse/pull/23311) ([Kruglov Pavel](https://github.com/Avogar)). +* More pytest fixes [#23538](https://github.com/ClickHouse/ClickHouse/pull/23538) ([Ivan](https://github.com/abyss7)). +* Improved `test_storage_s3_get_unstable` [#23976](https://github.com/ClickHouse/ClickHouse/pull/23976) ([Vladimir Chebotarev](https://github.com/excitoon)). +* CompileExpressions comparison function constant case fix [#24023](https://github.com/ClickHouse/ClickHouse/pull/24023) ([Maksim Kita](https://github.com/kitaisreal)). +* Added llvm-project submodule [#24030](https://github.com/ClickHouse/ClickHouse/pull/24030) ([Maksim Kita](https://github.com/kitaisreal)). +* complain about unstable perf test queries [#24049](https://github.com/ClickHouse/ClickHouse/pull/24049) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.5 [#24098](https://github.com/ClickHouse/ClickHouse/pull/24098) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Implement antlr parser for projections [#24245](https://github.com/ClickHouse/ClickHouse/pull/24245) ([Amos Bird](https://github.com/amosbird)). +* Try to fix GROUP BY _shard_num in a different way [#24252](https://github.com/ClickHouse/ClickHouse/pull/24252) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* consolidate connection loss handling in fuzzer [#24290](https://github.com/ClickHouse/ClickHouse/pull/24290) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Skip data finalization when doing projection materialization. [#24296](https://github.com/ClickHouse/ClickHouse/pull/24296) ([Amos Bird](https://github.com/amosbird)). +* One more error to retry in clickhouse-test [#24307](https://github.com/ClickHouse/ClickHouse/pull/24307) ([alesapin](https://github.com/alesapin)). +* Merging [#23461](https://github.com/ClickHouse/ClickHouse/issues/23461) [#24311](https://github.com/ClickHouse/ClickHouse/pull/24311) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove AutoArray [#24320](https://github.com/ClickHouse/ClickHouse/pull/24320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Compile expressions added partition by test [#24323](https://github.com/ClickHouse/ClickHouse/pull/24323) ([Maksim Kita](https://github.com/kitaisreal)). +* PODArray insert in the middle tests [#24333](https://github.com/ClickHouse/ClickHouse/pull/24333) ([Maksim Kita](https://github.com/kitaisreal)). +* remove retries from fast test [#24338](https://github.com/ClickHouse/ClickHouse/pull/24338) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Better parallelism for functional tests [#24349](https://github.com/ClickHouse/ClickHouse/pull/24349) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* AARCH64 hash tables benchmark [#24364](https://github.com/ClickHouse/ClickHouse/pull/24364) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix diagnostics in documentation script [#24379](https://github.com/ClickHouse/ClickHouse/pull/24379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01033_quota_dcl [#24393](https://github.com/ClickHouse/ClickHouse/pull/24393) ([alesapin](https://github.com/alesapin)). +* Fix logical error AggregateFunctionFactory returned nullptr [#24398](https://github.com/ClickHouse/ClickHouse/pull/24398) ([Kruglov Pavel](https://github.com/Avogar)). +* Add log record for removed from AWS S3 keys [#24400](https://github.com/ClickHouse/ClickHouse/pull/24400) ([ianton-ru](https://github.com/ianton-ru)). +* Added AggregateFunctionSegmentLengthSum to ya.make [#24408](https://github.com/ClickHouse/ClickHouse/pull/24408) ([Maksim Kita](https://github.com/kitaisreal)). +* Stateless tests fixes [#24411](https://github.com/ClickHouse/ClickHouse/pull/24411) ([Azat Khuzhin](https://github.com/azat)). +* Function constant result with nonconstant arguments [#24417](https://github.com/ClickHouse/ClickHouse/pull/24417) ([Maksim Kita](https://github.com/kitaisreal)). +* Libunwind update version [#24419](https://github.com/ClickHouse/ClickHouse/pull/24419) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not built clickhouse-keeper w/o NuRaft [#24421](https://github.com/ClickHouse/ClickHouse/pull/24421) ([Azat Khuzhin](https://github.com/azat)). +* Switch message level to WARNING for FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION=OFF [#24423](https://github.com/ClickHouse/ClickHouse/pull/24423) ([Azat Khuzhin](https://github.com/azat)). +* Merging "experimental compression codecs" [#24424](https://github.com/ClickHouse/ClickHouse/pull/24424) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix formatting of negative values [#24427](https://github.com/ClickHouse/ClickHouse/pull/24427) ([Azat Khuzhin](https://github.com/azat)). +* Fix dictionary functions documentation [#24432](https://github.com/ClickHouse/ClickHouse/pull/24432) ([Maksim Kita](https://github.com/kitaisreal)). +* Mark false positives for PVS-Studio [#24434](https://github.com/ClickHouse/ClickHouse/pull/24434) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ActionsDAG compile expressions update [#24442](https://github.com/ClickHouse/ClickHouse/pull/24442) ([Maksim Kita](https://github.com/kitaisreal)). +* LLVM remove orc jit library [#24443](https://github.com/ClickHouse/ClickHouse/pull/24443) ([Maksim Kita](https://github.com/kitaisreal)). +* Add YAMLParser to ya.make [#24447](https://github.com/ClickHouse/ClickHouse/pull/24447) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix fast test [#24454](https://github.com/ClickHouse/ClickHouse/pull/24454) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test 01085_max_distributed_connections [#24455](https://github.com/ClickHouse/ClickHouse/pull/24455) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Manually disable LLVM parts submodule update [#24460](https://github.com/ClickHouse/ClickHouse/pull/24460) ([Maksim Kita](https://github.com/kitaisreal)). +* Account total_rows_approx as soon as possible [#24462](https://github.com/ClickHouse/ClickHouse/pull/24462) ([Azat Khuzhin](https://github.com/azat)). +* Fix header mismatch for UNION. [#24463](https://github.com/ClickHouse/ClickHouse/pull/24463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to improve kafka flaky test [#24465](https://github.com/ClickHouse/ClickHouse/pull/24465) ([filimonov](https://github.com/filimonov)). +* CompileExpression cached functions with context fix [#24468](https://github.com/ClickHouse/ClickHouse/pull/24468) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed IFunction then typos [#24469](https://github.com/ClickHouse/ClickHouse/pull/24469) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryTracker new no throw [#24483](https://github.com/ClickHouse/ClickHouse/pull/24483) ([Maksim Kita](https://github.com/kitaisreal)). +* Revent libunwind [#24485](https://github.com/ClickHouse/ClickHouse/pull/24485) ([Maksim Kita](https://github.com/kitaisreal)). +* Log file name on error in FileChecker ctor [#24489](https://github.com/ClickHouse/ClickHouse/pull/24489) ([Vladimir C](https://github.com/vdimir)). +* Remove containers in intergational runner entrypoint [#24492](https://github.com/ClickHouse/ClickHouse/pull/24492) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix benign race (detected by clang-12) in Keeper snapshots [#24499](https://github.com/ClickHouse/ClickHouse/pull/24499) ([alesapin](https://github.com/alesapin)). +* [backport] Find only opened PRs by label [#24501](https://github.com/ClickHouse/ClickHouse/pull/24501) ([Ivan](https://github.com/abyss7)). +* shellcheck fix [#24512](https://github.com/ClickHouse/ClickHouse/pull/24512) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not peerdir enabled-by-default libcxxabi-parts [#24518](https://github.com/ClickHouse/ClickHouse/pull/24518) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Merging [#22762](https://github.com/ClickHouse/ClickHouse/issues/22762) [#24525](https://github.com/ClickHouse/ClickHouse/pull/24525) ([Vladimir C](https://github.com/vdimir)). +* Fix Arcadia [#24527](https://github.com/ClickHouse/ClickHouse/pull/24527) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make `get/check/read/list` requests always read in Keeper [#24533](https://github.com/ClickHouse/ClickHouse/pull/24533) ([alesapin](https://github.com/alesapin)). +* calculate perf test precision thresholds from historical data [#24534](https://github.com/ClickHouse/ClickHouse/pull/24534) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update jit_example [#24550](https://github.com/ClickHouse/ClickHouse/pull/24550) ([Maksim Kita](https://github.com/kitaisreal)). +* Update test_multiple_disks [#24560](https://github.com/ClickHouse/ClickHouse/pull/24560) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test [#24561](https://github.com/ClickHouse/ClickHouse/pull/24561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24563](https://github.com/ClickHouse/ClickHouse/pull/24563) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix avro [#24569](https://github.com/ClickHouse/ClickHouse/pull/24569) ([Ilya Yatsishin](https://github.com/qoega)). +* Refactor MergeTreeDataSelectExecutor [#24574](https://github.com/ClickHouse/ClickHouse/pull/24574) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* allow inheriting from a named window in window definition [#24576](https://github.com/ClickHouse/ClickHouse/pull/24576) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* do not auto-apply -OrNull combinator to pure window functions [#24579](https://github.com/ClickHouse/ClickHouse/pull/24579) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ORDER BY after window fuctions over Distributed [#24580](https://github.com/ClickHouse/ClickHouse/pull/24580) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ast fuzzer: determine server death more robustly [#24584](https://github.com/ClickHouse/ClickHouse/pull/24584) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable tests, utils and examples in builds with clang-tidy [#24587](https://github.com/ClickHouse/ClickHouse/pull/24587) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more fuzzer fixes [#24690](https://github.com/ClickHouse/ClickHouse/pull/24690) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flaky integration tests [#24691](https://github.com/ClickHouse/ClickHouse/pull/24691) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix alter table drop projection if exists [#24692](https://github.com/ClickHouse/ClickHouse/pull/24692) ([Amos Bird](https://github.com/amosbird)). +* Block memory tracker earlier in `tryLogCurrentException` [#24722](https://github.com/ClickHouse/ClickHouse/pull/24722) ([alesapin](https://github.com/alesapin)). +* Minor fixes in AggregateFunctionSegmentLengthSumData [#24729](https://github.com/ClickHouse/ClickHouse/pull/24729) ([Vladimir C](https://github.com/vdimir)). +* remove mutable references to Context from IFunction interface [#24732](https://github.com/ClickHouse/ClickHouse/pull/24732) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* MemoryTracker enable throw logical error [#24733](https://github.com/ClickHouse/ClickHouse/pull/24733) ([Maksim Kita](https://github.com/kitaisreal)). +* support expressions in window frame [#24734](https://github.com/ClickHouse/ClickHouse/pull/24734) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Updated libunwind [#24735](https://github.com/ClickHouse/ClickHouse/pull/24735) ([Maksim Kita](https://github.com/kitaisreal)). +* ExecuteScalarSubqueriesVisitor fix error code style check [#24736](https://github.com/ClickHouse/ClickHouse/pull/24736) ([Maksim Kita](https://github.com/kitaisreal)). +* Process config w/o extensions as XML format [#24763](https://github.com/ClickHouse/ClickHouse/pull/24763) ([Azat Khuzhin](https://github.com/azat)). +* Update NuRaft [#24775](https://github.com/ClickHouse/ClickHouse/pull/24775) ([alesapin](https://github.com/alesapin)). +* Fix empty part set with force_use_projection = 1 [#24782](https://github.com/ClickHouse/ClickHouse/pull/24782) ([Amos Bird](https://github.com/amosbird)). +* Better exception for invalid projection creation [#24785](https://github.com/ClickHouse/ClickHouse/pull/24785) ([Amos Bird](https://github.com/amosbird)). +* Improve "Cannot schedule a task" error message [#24786](https://github.com/ClickHouse/ClickHouse/pull/24786) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00953_zookeeper_suetin_deduplication_bug [#24801](https://github.com/ClickHouse/ClickHouse/pull/24801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't capture temporary references in ZooKeeper client callbacks. [#24803](https://github.com/ClickHouse/ClickHouse/pull/24803) ([alesapin](https://github.com/alesapin)). +* Small improvement for StorageMaterializedView::getActionLock(...) [#24806](https://github.com/ClickHouse/ClickHouse/pull/24806) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make ContextPtr const by default. [#24808](https://github.com/ClickHouse/ClickHouse/pull/24808) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix typo. Remove the "allow_experimental_bigint_types" setting. [#24812](https://github.com/ClickHouse/ClickHouse/pull/24812) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ExpressionJIT remove unncecessary logging [#24813](https://github.com/ClickHouse/ClickHouse/pull/24813) ([Maksim Kita](https://github.com/kitaisreal)). +* ExpressionJIT simplify loop [#24814](https://github.com/ClickHouse/ClickHouse/pull/24814) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed clang tidy [#24821](https://github.com/ClickHouse/ClickHouse/pull/24821) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#23260](https://github.com/ClickHouse/ClickHouse/issues/23260) [#24822](https://github.com/ClickHouse/ClickHouse/pull/24822) ([Anton Popov](https://github.com/CurtizJ)). +* Simplify code around TraceCollector, alternative to [#24829](https://github.com/ClickHouse/ClickHouse/issues/24829) [#24833](https://github.com/ClickHouse/ClickHouse/pull/24833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some improvements in stress test [#24835](https://github.com/ClickHouse/ClickHouse/pull/24835) ([alesapin](https://github.com/alesapin)). +* Rename ContextConstPtr to ContextPtr. [#24855](https://github.com/ClickHouse/ClickHouse/pull/24855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reverted libunwind from upstream [#24860](https://github.com/ClickHouse/ClickHouse/pull/24860) ([Maksim Kita](https://github.com/kitaisreal)). +* Testflows Extended Precision Data Type testing - Adding snapshots for output comparison. [#24861](https://github.com/ClickHouse/ClickHouse/pull/24861) ([MyroTk](https://github.com/MyroTk)). +* clickhouse-client: echo hint improvements [#24863](https://github.com/ClickHouse/ClickHouse/pull/24863) ([Azat Khuzhin](https://github.com/azat)). +* docs: update requests (to fix conflicts with urllib3) [#24865](https://github.com/ClickHouse/ClickHouse/pull/24865) ([Azat Khuzhin](https://github.com/azat)). +* Trying to resurrect woboq [#24875](https://github.com/ClickHouse/ClickHouse/pull/24875) ([alesapin](https://github.com/alesapin)). +* Get rid of std::stringstream in Suggest [#24882](https://github.com/ClickHouse/ClickHouse/pull/24882) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Suppress RocksDB error [#24886](https://github.com/ClickHouse/ClickHouse/pull/24886) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove some outdated integration tests [#24911](https://github.com/ClickHouse/ClickHouse/pull/24911) ([alesapin](https://github.com/alesapin)). +* Mute test_memory_consumption before [#24784](https://github.com/ClickHouse/ClickHouse/issues/24784) [#24919](https://github.com/ClickHouse/ClickHouse/pull/24919) ([Ilya Yatsishin](https://github.com/qoega)). +* Update something in KeyCondition [#24920](https://github.com/ClickHouse/ClickHouse/pull/24920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Relax hung check [#24929](https://github.com/ClickHouse/ClickHouse/pull/24929) ([alesapin](https://github.com/alesapin)). +* some perf test script improvements [#24938](https://github.com/ClickHouse/ClickHouse/pull/24938) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* RFC: clickhouse-test: do not substitude db name in stderr by default [#24939](https://github.com/ClickHouse/ClickHouse/pull/24939) ([Azat Khuzhin](https://github.com/azat)). +* Add test issue [#23430](https://github.com/ClickHouse/ClickHouse/issues/23430) [#24941](https://github.com/ClickHouse/ClickHouse/pull/24941) ([filimonov](https://github.com/filimonov)). +* Followup fixes for integration tests [#24954](https://github.com/ClickHouse/ClickHouse/pull/24954) ([alesapin](https://github.com/alesapin)). +* Fix bad error message in docker entrypoint [#24955](https://github.com/ClickHouse/ClickHouse/pull/24955) ([filimonov](https://github.com/filimonov)). +* Fix endless wait in replica clone [#24957](https://github.com/ClickHouse/ClickHouse/pull/24957) ([alesapin](https://github.com/alesapin)). +* Remove subprocess_call from cluster.py [#24959](https://github.com/ClickHouse/ClickHouse/pull/24959) ([Ilya Yatsishin](https://github.com/qoega)). +* Delete support for waiting on queue- entries, is this dead code? [#24960](https://github.com/ClickHouse/ClickHouse/pull/24960) ([nvartolomei](https://github.com/nvartolomei)). +* Fix the test after [#20393](https://github.com/ClickHouse/ClickHouse/issues/20393) [#24967](https://github.com/ClickHouse/ClickHouse/pull/24967) ([filimonov](https://github.com/filimonov)). +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix typo in usage section in s3 documentation [#24970](https://github.com/ClickHouse/ClickHouse/pull/24970) ([presto53](https://github.com/presto53)). +* Cleanup iptables and containers on session start [#24973](https://github.com/ClickHouse/ClickHouse/pull/24973) ([Ilya Yatsishin](https://github.com/qoega)). +* Part movement between shards ACL [#24979](https://github.com/ClickHouse/ClickHouse/pull/24979) ([nvartolomei](https://github.com/nvartolomei)). +* Cleanup changelog script [#24987](https://github.com/ClickHouse/ClickHouse/pull/24987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.6 [#24989](https://github.com/ClickHouse/ClickHouse/pull/24989) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: fix log_comment for .sql tests [#24999](https://github.com/ClickHouse/ClickHouse/pull/24999) ([Azat Khuzhin](https://github.com/azat)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* Compression codecs refactoring [#25002](https://github.com/ClickHouse/ClickHouse/pull/25002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe Minio starts for too long in tests [#25007](https://github.com/ClickHouse/ClickHouse/pull/25007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test [#25012](https://github.com/ClickHouse/ClickHouse/pull/25012) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disks fix ya.make [#25031](https://github.com/ClickHouse/ClickHouse/pull/25031) ([Maksim Kita](https://github.com/kitaisreal)). +* tests: disable suggestions for expect tests that does not requires it [#25033](https://github.com/ClickHouse/ClickHouse/pull/25033) ([Azat Khuzhin](https://github.com/azat)). +* fix jemalloc build on OS_DARWIN [#25034](https://github.com/ClickHouse/ClickHouse/pull/25034) ([sdk2](https://github.com/sdk2)). +* Update waitForTableReplicaToProcessLogEntry comments [#25037](https://github.com/ClickHouse/ClickHouse/pull/25037) ([nvartolomei](https://github.com/nvartolomei)). +* Remove database before model name in docs [#25038](https://github.com/ClickHouse/ClickHouse/pull/25038) ([Kruglov Pavel](https://github.com/Avogar)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). +* test for attach partition from [#25060](https://github.com/ClickHouse/ClickHouse/pull/25060) ([Denny Crane](https://github.com/den-crane)). +* AggregateFunctionAnyHeavyData use fixed size type [#25066](https://github.com/ClickHouse/ClickHouse/pull/25066) ([Maksim Kita](https://github.com/kitaisreal)). +* Update Dockerfile [#25078](https://github.com/ClickHouse/ClickHouse/pull/25078) ([Ivan](https://github.com/abyss7)). +* Trying to debug fetches bandwith test failures [#25079](https://github.com/ClickHouse/ClickHouse/pull/25079) ([alesapin](https://github.com/alesapin)). +* Remove strange timeout in test [#25084](https://github.com/ClickHouse/ClickHouse/pull/25084) ([alesapin](https://github.com/alesapin)). +* Remove copypaste from StorageReplicatedMergeTree [#25087](https://github.com/ClickHouse/ClickHouse/pull/25087) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Less timeouts in integration tests [#25111](https://github.com/ClickHouse/ClickHouse/pull/25111) ([alesapin](https://github.com/alesapin)). +* Better odbc integration test [#25113](https://github.com/ClickHouse/ClickHouse/pull/25113) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flaky check report in CI runner [#25126](https://github.com/ClickHouse/ClickHouse/pull/25126) ([alesapin](https://github.com/alesapin)). +* Better hdfs tests [#25128](https://github.com/ClickHouse/ClickHouse/pull/25128) ([Ilya Yatsishin](https://github.com/qoega)). +* Use zstd/include in ya.make [#25145](https://github.com/ClickHouse/ClickHouse/pull/25145) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Adds a better way to include binary resources [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)). +* check for row_policy defined using user() function [#25147](https://github.com/ClickHouse/ClickHouse/pull/25147) ([Denny Crane](https://github.com/den-crane)). +* Update Roaring Bitmaps just in case [#25151](https://github.com/ClickHouse/ClickHouse/pull/25151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move code to more appropriate place [#25152](https://github.com/ClickHouse/ClickHouse/pull/25152) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionary sources add update lag option [#25161](https://github.com/ClickHouse/ClickHouse/pull/25161) ([Maksim Kita](https://github.com/kitaisreal)). +* RewriteFunctionToSubcolumnVisitor add into ya.make [#25162](https://github.com/ClickHouse/ClickHouse/pull/25162) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable back the ANTLR in .sql [#25170](https://github.com/ClickHouse/ClickHouse/pull/25170) ([Ivan](https://github.com/abyss7)). +* Added ExecutablePool documentation [#25196](https://github.com/ClickHouse/ClickHouse/pull/25196) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not optimize query plan for mutations. [#25197](https://github.com/ClickHouse/ClickHouse/pull/25197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Dictionary added update field documentation [#25198](https://github.com/ClickHouse/ClickHouse/pull/25198) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries attribute support default nullable type [#25203](https://github.com/ClickHouse/ClickHouse/pull/25203) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix tests [#25204](https://github.com/ClickHouse/ClickHouse/pull/25204) ([Ivan](https://github.com/abyss7)). +* Try run tests with Replicated database in parallel [#25210](https://github.com/ClickHouse/ClickHouse/pull/25210) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More integration tests improvements [#25212](https://github.com/ClickHouse/ClickHouse/pull/25212) ([Ilya Yatsishin](https://github.com/qoega)). +* Add function toJSONString to ya.make [#25246](https://github.com/ClickHouse/ClickHouse/pull/25246) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor change [#25247](https://github.com/ClickHouse/ClickHouse/pull/25247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25248](https://github.com/ClickHouse/ClickHouse/pull/25248) ([Matwey V. Kornilov](https://github.com/matwey)). +* Add test for [#4113](https://github.com/ClickHouse/ClickHouse/issues/4113) [#25249](https://github.com/ClickHouse/ClickHouse/pull/25249) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#11535](https://github.com/ClickHouse/ClickHouse/issues/11535) [#25250](https://github.com/ClickHouse/ClickHouse/pull/25250) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9932](https://github.com/ClickHouse/ClickHouse/issues/9932) [#25253](https://github.com/ClickHouse/ClickHouse/pull/25253) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Copy-paste some code [#25264](https://github.com/ClickHouse/ClickHouse/pull/25264) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor change [#25265](https://github.com/ClickHouse/ClickHouse/pull/25265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315) [#25266](https://github.com/ClickHouse/ClickHouse/pull/25266) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25267](https://github.com/ClickHouse/ClickHouse/pull/25267) ([Matwey V. Kornilov](https://github.com/matwey)). +* test for `PARTITION BY 0 * id` [#25274](https://github.com/ClickHouse/ClickHouse/pull/25274) ([Denny Crane](https://github.com/den-crane)). +* Enable TestFlows LDAP tests [#25278](https://github.com/ClickHouse/ClickHouse/pull/25278) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a test for [#17964](https://github.com/ClickHouse/ClickHouse/issues/17964) [#25285](https://github.com/ClickHouse/ClickHouse/pull/25285) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#17367](https://github.com/ClickHouse/ClickHouse/issues/17367) [#25286](https://github.com/ClickHouse/ClickHouse/pull/25286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code from KeyCondition. [#25295](https://github.com/ClickHouse/ClickHouse/pull/25295) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CMake dictionaries disable debug info fix [#25301](https://github.com/ClickHouse/ClickHouse/pull/25301) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge ext into common [#25303](https://github.com/ClickHouse/ClickHouse/pull/25303) ([Maksim Kita](https://github.com/kitaisreal)). +* test for null array orc load [#25304](https://github.com/ClickHouse/ClickHouse/pull/25304) ([Denny Crane](https://github.com/den-crane)). +* Improve Replicated database tests [#25305](https://github.com/ClickHouse/ClickHouse/pull/25305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* SimpleCache key constructor improvement [#25307](https://github.com/ClickHouse/ClickHouse/pull/25307) ([Maksim Kita](https://github.com/kitaisreal)). +* Catch ErrnoException during parts cleaning [#25309](https://github.com/ClickHouse/ClickHouse/pull/25309) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test 01520_client_print_query_id and others. [#25311](https://github.com/ClickHouse/ClickHouse/pull/25311) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileTiming [#25314](https://github.com/ClickHouse/ClickHouse/pull/25314) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update test_ttl_replicated [#25317](https://github.com/ClickHouse/ClickHouse/pull/25317) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `00601_kill_running_query` [#25318](https://github.com/ClickHouse/ClickHouse/pull/25318) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use signal 15 in restart_with_latest_version [#25323](https://github.com/ClickHouse/ClickHouse/pull/25323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Setting min_count_to_compile_expression fix [#25332](https://github.com/ClickHouse/ClickHouse/pull/25332) ([Maksim Kita](https://github.com/kitaisreal)). +* Function formatDateTime fix code comments [#25334](https://github.com/ClickHouse/ClickHouse/pull/25334) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve Replicated database tests 2 [#25373](https://github.com/ClickHouse/ClickHouse/pull/25373) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#23163](https://github.com/ClickHouse/ClickHouse/issues/23163) [#25390](https://github.com/ClickHouse/ClickHouse/pull/25390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rename & reimport murmurhash sources from smhasher repo [#25400](https://github.com/ClickHouse/ClickHouse/pull/25400) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add test from issue [#20624](https://github.com/ClickHouse/ClickHouse/issues/20624) [#25409](https://github.com/ClickHouse/ClickHouse/pull/25409) ([Vladimir C](https://github.com/vdimir)). +* Turn off WITH_COVERAGE in build with clang-tidy [#25417](https://github.com/ClickHouse/ClickHouse/pull/25417) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix container-overflow in replxx during incremental search (Ctrl-R) [#25427](https://github.com/ClickHouse/ClickHouse/pull/25427) ([Azat Khuzhin](https://github.com/azat)). +* DatabaseMySQL rename [#25430](https://github.com/ClickHouse/ClickHouse/pull/25430) ([Maksim Kita](https://github.com/kitaisreal)). +* Support REPLACE DICTIONARY, CREATE OR REPLACE DICTIONARY queries [#25444](https://github.com/ClickHouse/ClickHouse/pull/25444) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8417](https://github.com/ClickHouse/ClickHouse/issues/8417) [#25446](https://github.com/ClickHouse/ClickHouse/pull/25446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove assumeMutable from removeColumnNullability [#25454](https://github.com/ClickHouse/ClickHouse/pull/25454) ([Vladimir C](https://github.com/vdimir)). +* DataTypeLowCardinality support DataTypeInterval tests [#25458](https://github.com/ClickHouse/ClickHouse/pull/25458) ([Maksim Kita](https://github.com/kitaisreal)). +* Add run-id option to integration tests [#25459](https://github.com/ClickHouse/ClickHouse/pull/25459) ([alesapin](https://github.com/alesapin)). +* Stable NOT chain formatting [#25494](https://github.com/ClickHouse/ClickHouse/pull/25494) ([Azat Khuzhin](https://github.com/azat)). +* Fix alternative stack for SIGSEGV handling [#25509](https://github.com/ClickHouse/ClickHouse/pull/25509) ([Azat Khuzhin](https://github.com/azat)). +* Catch "Maximum parse depth" error in fuzzer [#25510](https://github.com/ClickHouse/ClickHouse/pull/25510) ([Azat Khuzhin](https://github.com/azat)). +* Fix NOT parsing [#25520](https://github.com/ClickHouse/ClickHouse/pull/25520) ([Azat Khuzhin](https://github.com/azat)). +* Catch TOO_DEEP_RECURSION in fuzzer for formatted query too [#25521](https://github.com/ClickHouse/ClickHouse/pull/25521) ([Azat Khuzhin](https://github.com/azat)). +* Fix some bugs in integration tests [#25525](https://github.com/ClickHouse/ClickHouse/pull/25525) ([alesapin](https://github.com/alesapin)). +* Fix query progress [#25545](https://github.com/ClickHouse/ClickHouse/pull/25545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add test for progress bar [#25551](https://github.com/ClickHouse/ClickHouse/pull/25551) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix some tests [#25564](https://github.com/ClickHouse/ClickHouse/pull/25564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 01249_flush_interactive.sh [#25565](https://github.com/ClickHouse/ClickHouse/pull/25565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* METR-41529 [#25568](https://github.com/ClickHouse/ClickHouse/pull/25568) ([egatov](https://github.com/egatov)). +* clickhouse-client: fix NULL dereference for --param w/o value [#25579](https://github.com/ClickHouse/ClickHouse/pull/25579) ([Azat Khuzhin](https://github.com/azat)). +* Remove only symlinks during force_restore_data of Atomic engine [#25582](https://github.com/ClickHouse/ClickHouse/pull/25582) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: use basename (instead of full path) for log_comment [#25583](https://github.com/ClickHouse/ClickHouse/pull/25583) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo in hardware failure error message [#25584](https://github.com/ClickHouse/ClickHouse/pull/25584) ([Stas Kelvich](https://github.com/kelvich)). +* Small change in Roaring Bitmaps [#25604](https://github.com/ClickHouse/ClickHouse/pull/25604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ANTLR parser and enable it back in CI [#25638](https://github.com/ClickHouse/ClickHouse/pull/25638) ([Ivan](https://github.com/abyss7)). +* Fix alternative stack (MINSIGSTKSZ) and stack size check under osx [#25654](https://github.com/ClickHouse/ClickHouse/pull/25654) ([Azat Khuzhin](https://github.com/azat)). +* Enable MurmurHash in ArcadiaBuild [#25666](https://github.com/ClickHouse/ClickHouse/pull/25666) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add working test [#25720](https://github.com/ClickHouse/ClickHouse/pull/25720) ([alesapin](https://github.com/alesapin)). +* Compile expressions updated documentation [#25725](https://github.com/ClickHouse/ClickHouse/pull/25725) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some more missed includes [#25732](https://github.com/ClickHouse/ClickHouse/pull/25732) ([Matwey V. Kornilov](https://github.com/matwey)). +* Fix native macOS (Xcode) builds [#25736](https://github.com/ClickHouse/ClickHouse/pull/25736) ([Denis Glazachev](https://github.com/traceon)). +* Fix arcadia [#25738](https://github.com/ClickHouse/ClickHouse/pull/25738) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Change error code in LIVE VIEW [#25739](https://github.com/ClickHouse/ClickHouse/pull/25739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### Testing Improvement * * Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md index 9056da8ac89..963ac6d6ac7 100644 --- a/docs/changelogs/v21.7.10.4-stable.md +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.10.4-stable FIXME as compared to v21.7.9.7-stable #### Improvement @@ -17,3 +24,11 @@ * Backported in [#28947](https://github.com/ClickHouse/ClickHouse/issues/28947): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28931](https://github.com/ClickHouse/ClickHouse/issues/28931): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md index 66672204713..e130550fb9c 100644 --- a/docs/changelogs/v21.7.11.3-stable.md +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -5,3 +12,9 @@ * Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.2.7-stable.md b/docs/changelogs/v21.7.2.7-stable.md index a7af0fed667..5c772eab664 100644 --- a/docs/changelogs/v21.7.2.7-stable.md +++ b/docs/changelogs/v21.7.2.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.2.7-stable FIXME as compared to v21.7.1.7283-prestable #### Improvement @@ -15,3 +22,8 @@ * Backported in [#26010](https://github.com/ClickHouse/ClickHouse/issues/26010): Fix formatting of type `Map` with integer keys to `JSON`. [#25982](https://github.com/ClickHouse/ClickHouse/pull/25982) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#26097](https://github.com/ClickHouse/ClickHouse/issues/26097): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.7.3.14-stable.md b/docs/changelogs/v21.7.3.14-stable.md index d24b7bcbf39..0ad15f765a1 100644 --- a/docs/changelogs/v21.7.3.14-stable.md +++ b/docs/changelogs/v21.7.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.3.14-stable FIXME as compared to v21.7.2.7-stable #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#26229](https://github.com/ClickHouse/ClickHouse/issues/26229): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix throwing exception when iterate over non existing remote directory. [#26296](https://github.com/ClickHouse/ClickHouse/pull/26296) ([ianton-ru](https://github.com/ianton-ru)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.4.18-stable.md b/docs/changelogs/v21.7.4.18-stable.md index 7bc08e2a0e3..34d457015cb 100644 --- a/docs/changelogs/v21.7.4.18-stable.md +++ b/docs/changelogs/v21.7.4.18-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.4.18-stable FIXME as compared to v21.7.3.14-stable #### Bug Fix diff --git a/docs/changelogs/v21.7.5.29-stable.md b/docs/changelogs/v21.7.5.29-stable.md index 3f24e3eded9..6b01d912e17 100644 --- a/docs/changelogs/v21.7.5.29-stable.md +++ b/docs/changelogs/v21.7.5.29-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.5.29-stable FIXME as compared to v21.7.4.18-stable #### Performance Improvement @@ -14,3 +21,10 @@ * Backported in [#26772](https://github.com/ClickHouse/ClickHouse/issues/26772): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#26907](https://github.com/ClickHouse/ClickHouse/issues/26907): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.6.39-stable.md b/docs/changelogs/v21.7.6.39-stable.md index a7913aca193..03afc95515e 100644 --- a/docs/changelogs/v21.7.6.39-stable.md +++ b/docs/changelogs/v21.7.6.39-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.6.39-stable FIXME as compared to v21.7.5.29-stable #### Bug Fix @@ -12,3 +19,7 @@ * Backported in [#27156](https://github.com/ClickHouse/ClickHouse/issues/27156): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#27261](https://github.com/ClickHouse/ClickHouse/issues/27261): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.7.47-stable.md b/docs/changelogs/v21.7.7.47-stable.md index f81abca2600..3c7bf09433e 100644 --- a/docs/changelogs/v21.7.7.47-stable.md +++ b/docs/changelogs/v21.7.7.47-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.7.47-stable FIXME as compared to v21.7.6.39-stable #### Bug Fix @@ -6,3 +13,12 @@ * Backported in [#27418](https://github.com/ClickHouse/ClickHouse/issues/27418): Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#27416](https://github.com/ClickHouse/ClickHouse/issues/27416): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.8.58-stable.md b/docs/changelogs/v21.7.8.58-stable.md index aea1ae083f0..4f99e5439a4 100644 --- a/docs/changelogs/v21.7.8.58-stable.md +++ b/docs/changelogs/v21.7.8.58-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.8.58-stable FIXME as compared to v21.7.7.47-stable #### Bug Fix @@ -10,3 +17,8 @@ * Backported in [#27697](https://github.com/ClickHouse/ClickHouse/issues/27697): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#27748](https://github.com/ClickHouse/ClickHouse/issues/27748): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md index 0d1a2a521e7..1d859b08da5 100644 --- a/docs/changelogs/v21.7.9.7-stable.md +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.9.7-stable FIXME as compared to v21.7.8.58-stable #### Improvement @@ -25,3 +32,15 @@ * Backported in [#28288](https://github.com/ClickHouse/ClickHouse/issues/28288): Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#28295](https://github.com/ClickHouse/ClickHouse/issues/28295): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.1.7409-prestable.md b/docs/changelogs/v21.8.1.7409-prestable.md index e703d227603..96f13713984 100644 --- a/docs/changelogs/v21.8.1.7409-prestable.md +++ b/docs/changelogs/v21.8.1.7409-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.1.7409-prestable FIXME as compared to v21.7.1.7283-prestable #### Backward Incompatible Change @@ -86,3 +93,62 @@ * NO CL ENTRY: '[ImgBot] Optimize images'. [#26054](https://github.com/ClickHouse/ClickHouse/pull/26054) ([imgbot[bot]](https://github.com/apps/imgbot)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix hang and incorrect exit code returned from clickhouse-test [#25537](https://github.com/ClickHouse/ClickHouse/pull/25537) ([nvartolomei](https://github.com/nvartolomei)). +* Remove PrewhereDAGInfo. [#25719](https://github.com/ClickHouse/ClickHouse/pull/25719) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix 01641_memory_tracking_insert_optimize [#25731](https://github.com/ClickHouse/ClickHouse/pull/25731) ([Azat Khuzhin](https://github.com/azat)). +* Separate log files for separate runs in stress test [#25741](https://github.com/ClickHouse/ClickHouse/pull/25741) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow performance test [#25742](https://github.com/ClickHouse/ClickHouse/pull/25742) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DatabaseAtomic EXCHANGE DICTIONARIES fix test [#25753](https://github.com/ClickHouse/ClickHouse/pull/25753) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flacky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#13993](https://github.com/ClickHouse/ClickHouse/issues/13993) [#25758](https://github.com/ClickHouse/ClickHouse/pull/25758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set follow-fork-mode child for gdb in stress/fasttest/fuzzer [#25769](https://github.com/ClickHouse/ClickHouse/pull/25769) ([Azat Khuzhin](https://github.com/azat)). +* Ignore TOO_DEEP_RECURSION server exception during fuzzing [#25770](https://github.com/ClickHouse/ClickHouse/pull/25770) ([Azat Khuzhin](https://github.com/azat)). +* Add comments for VERSION_REVISION vs DBMS_TCP_PROTOCOL_VERSION [#25771](https://github.com/ClickHouse/ClickHouse/pull/25771) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test and wrong message [#25772](https://github.com/ClickHouse/ClickHouse/pull/25772) ([alesapin](https://github.com/alesapin)). +* MaterializeMySQL: Improved column comments support [#25781](https://github.com/ClickHouse/ClickHouse/pull/25781) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Fix ANTRL merge_prewhere_table test [#25782](https://github.com/ClickHouse/ClickHouse/pull/25782) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove dead code from VirtualColumnUtils.cpp [#25787](https://github.com/ClickHouse/ClickHouse/pull/25787) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some typos in Storage classes [#25796](https://github.com/ClickHouse/ClickHouse/pull/25796) ([Raúl Marín](https://github.com/Algunenano)). +* Fix DateLUT on Darwin [#25803](https://github.com/ClickHouse/ClickHouse/pull/25803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better test_version_update_after_mutation [#25810](https://github.com/ClickHouse/ClickHouse/pull/25810) ([alesapin](https://github.com/alesapin)). +* Collect stderr.log and stdout.log in all integration tests by default. [#25816](https://github.com/ClickHouse/ClickHouse/pull/25816) ([Vitaly Baranov](https://github.com/vitlibar)). +* METR-41529 [#25819](https://github.com/ClickHouse/ClickHouse/pull/25819) ([egatov](https://github.com/egatov)). +* tests/integration: use iptables --wait [#25823](https://github.com/ClickHouse/ClickHouse/pull/25823) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#25611](https://github.com/ClickHouse/ClickHouse/issues/25611) [#25831](https://github.com/ClickHouse/ClickHouse/pull/25831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix odbc test [#25834](https://github.com/ClickHouse/ClickHouse/pull/25834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix performance tests after converting ProfileEvents to Map type [#25837](https://github.com/ClickHouse/ClickHouse/pull/25837) ([Azat Khuzhin](https://github.com/azat)). +* Correct messages in integration tests. [#25861](https://github.com/ClickHouse/ClickHouse/pull/25861) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix typo in the test 00900_orc_arrow_parquet_maps [#25874](https://github.com/ClickHouse/ClickHouse/pull/25874) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve logging in integration tests. [#25899](https://github.com/ClickHouse/ClickHouse/pull/25899) ([Vitaly Baranov](https://github.com/vitlibar)). +* kerberized HDFS test fix if run in parallel [#25908](https://github.com/ClickHouse/ClickHouse/pull/25908) ([Ilya Golshtein](https://github.com/ilejn)). +* fix special build on clang 11 [#25912](https://github.com/ClickHouse/ClickHouse/pull/25912) ([flynn](https://github.com/ucasfl)). +* Remove obsolete code from init script [#25920](https://github.com/ClickHouse/ClickHouse/pull/25920) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionInitializeAggregation build fix [#25922](https://github.com/ClickHouse/ClickHouse/pull/25922) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clang tidy build check [#25939](https://github.com/ClickHouse/ClickHouse/pull/25939) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build subquery in replaceJoinedTable without parsing [#25941](https://github.com/ClickHouse/ClickHouse/pull/25941) ([Vladimir C](https://github.com/vdimir)). +* Remove experimental ANTLR parser [#25942](https://github.com/ClickHouse/ClickHouse/pull/25942) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#20470](https://github.com/ClickHouse/ClickHouse/issues/20470) [#25975](https://github.com/ClickHouse/ClickHouse/pull/25975) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Correct test [#25984](https://github.com/ClickHouse/ClickHouse/pull/25984) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Compile aggregate functions perf tests fix [#25989](https://github.com/ClickHouse/ClickHouse/pull/25989) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix alter of settings in `MergeTree` tables [#25995](https://github.com/ClickHouse/ClickHouse/pull/25995) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia [#26002](https://github.com/ClickHouse/ClickHouse/pull/26002) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove old code [#26014](https://github.com/ClickHouse/ClickHouse/pull/26014) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionsLogical const result for non const arguments fix [#26018](https://github.com/ClickHouse/ClickHouse/pull/26018) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionSQLJSON ContextPtr build fix [#26022](https://github.com/ClickHouse/ClickHouse/pull/26022) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace print() with logging.debug() in integration tests. [#26023](https://github.com/ClickHouse/ClickHouse/pull/26023) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try to fix some flaky tests [#26032](https://github.com/ClickHouse/ClickHouse/pull/26032) ([Anton Popov](https://github.com/CurtizJ)). +* Fix for ZK watch metric drift in rare conditions [#26034](https://github.com/ClickHouse/ClickHouse/pull/26034) ([nvartolomei](https://github.com/nvartolomei)). +* AsynchronousMetrics: Don't assume temperature is always positive [#26045](https://github.com/ClickHouse/ClickHouse/pull/26045) ([Raúl Marín](https://github.com/Algunenano)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). +* ClickHouse dictionary source secure setting added documentation [#26055](https://github.com/ClickHouse/ClickHouse/pull/26055) ([Maksim Kita](https://github.com/kitaisreal)). +* Add changelog for 21.7 [#26057](https://github.com/ClickHouse/ClickHouse/pull/26057) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01791_dist_INSERT_block_structure_mismatch flakiness [#26058](https://github.com/ClickHouse/ClickHouse/pull/26058) ([Azat Khuzhin](https://github.com/azat)). +* Fix logical error with signed and unsigned offset in WindowFrame::checkValid [#26072](https://github.com/ClickHouse/ClickHouse/pull/26072) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#26077](https://github.com/ClickHouse/ClickHouse/pull/26077) ([Anton Popov](https://github.com/CurtizJ)). +* Disabling annoying copier tests [#26099](https://github.com/ClickHouse/ClickHouse/pull/26099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make graph pipeline rendering compatible with Dagre.JS [#26114](https://github.com/ClickHouse/ClickHouse/pull/26114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md index 92347642ffa..3e73e7892ec 100644 --- a/docs/changelogs/v21.8.10.19-lts.md +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.10.19-lts FIXME as compared to v21.8.9.13-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#30333](https://github.com/ClickHouse/ClickHouse/issues/30333): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30377](https://github.com/ClickHouse/ClickHouse/issues/30377): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md index e36d75c32ea..644a9ece633 100644 --- a/docs/changelogs/v21.8.11.4-lts.md +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.11.4-lts FIXME as compared to v21.8.10.19-lts #### New Feature @@ -38,3 +45,10 @@ * Backported in [#31132](https://github.com/ClickHouse/ClickHouse/issues/31132): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#31372](https://github.com/ClickHouse/ClickHouse/issues/31372): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md index 63b34c367f3..3dea113485d 100644 --- a/docs/changelogs/v21.8.12.29-lts.md +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.12.29-lts FIXME as compared to v21.8.11.4-lts #### Performance Improvement @@ -22,3 +29,7 @@ * Backported in [#31890](https://github.com/ClickHouse/ClickHouse/issues/31890): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31910](https://github.com/ClickHouse/ClickHouse/issues/31910): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md index 06e8c366ff2..1911de83009 100644 --- a/docs/changelogs/v21.8.13.6-lts.md +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.13.6-lts FIXME as compared to v21.8.12.29-lts #### Bug Fix @@ -27,3 +34,15 @@ * NO CL ENTRY: 'fix json error after downgrade'. [#33166](https://github.com/ClickHouse/ClickHouse/pull/33166) ([bullet1337](https://github.com/bullet1337)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix build [#32501](https://github.com/ClickHouse/ClickHouse/pull/32501) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix [#32752](https://github.com/ClickHouse/ClickHouse/pull/32752) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md index 481327a35c9..8310573f94c 100644 --- a/docs/changelogs/v21.8.14.5-lts.md +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md index 7411fbff9ae..64da1824bcf 100644 --- a/docs/changelogs/v21.8.15.7-lts.md +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.2.19-prestable.md b/docs/changelogs/v21.8.2.19-prestable.md index 15726747e65..8b2bdd4e185 100644 --- a/docs/changelogs/v21.8.2.19-prestable.md +++ b/docs/changelogs/v21.8.2.19-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.2.19-prestable FIXME as compared to v21.8.1.7409-prestable #### Performance Improvement @@ -27,3 +34,10 @@ * Backported in [#26705](https://github.com/ClickHouse/ClickHouse/issues/26705): Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). * Backported in [#26771](https://github.com/ClickHouse/ClickHouse/issues/26771): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.3.44-lts.md b/docs/changelogs/v21.8.3.44-lts.md index 21fe655870a..12ecd521c15 100644 --- a/docs/changelogs/v21.8.3.44-lts.md +++ b/docs/changelogs/v21.8.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.3.44-lts FIXME as compared to v21.8.2.19-prestable #### Improvement @@ -26,3 +33,17 @@ * Backported in [#27419](https://github.com/ClickHouse/ClickHouse/issues/27419): /proc/info contains metrics like. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). * Backported in [#27472](https://github.com/ClickHouse/ClickHouse/issues/27472): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.4.51-lts.md b/docs/changelogs/v21.8.4.51-lts.md index a8494ebb1d1..9b0dba786e5 100644 --- a/docs/changelogs/v21.8.4.51-lts.md +++ b/docs/changelogs/v21.8.4.51-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.4.51-lts FIXME as compared to v21.8.3.44-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md index d78eb98b472..782c93ce9b1 100644 --- a/docs/changelogs/v21.8.5.7-lts.md +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.5.7-lts FIXME as compared to v21.8.4.51-lts #### Improvement @@ -29,3 +36,16 @@ * Backported in [#28292](https://github.com/ClickHouse/ClickHouse/issues/28292): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28402](https://github.com/ClickHouse/ClickHouse/issues/28402): Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix several bugs in ZooKeeper snapshots deserialization [#26127](https://github.com/ClickHouse/ClickHouse/pull/26127) ([alesapin](https://github.com/alesapin)). +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md index de38572d94a..95d349dcacb 100644 --- a/docs/changelogs/v21.8.6.15-lts.md +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.6.15-lts FIXME as compared to v21.8.5.7-lts #### Improvement @@ -23,3 +30,12 @@ * Backported in [#28948](https://github.com/ClickHouse/ClickHouse/issues/28948): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28930](https://github.com/ClickHouse/ClickHouse/issues/28930): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some images in release PRs [#29059](https://github.com/ClickHouse/ClickHouse/pull/29059) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md index 92ba59f13cd..31d9c15debb 100644 --- a/docs/changelogs/v21.8.7.22-lts.md +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,9 @@ * Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29193](https://github.com/ClickHouse/ClickHouse/issues/29193): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md index 199be63424c..b6645362139 100644 --- a/docs/changelogs/v21.8.8.29-lts.md +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.8.29-lts FIXME as compared to v21.8.7.22-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md index e4cd5f45b9b..1d9b436f302 100644 --- a/docs/changelogs/v21.8.9.13-lts.md +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.9.13-lts FIXME as compared to v21.8.8.29-lts #### Improvement @@ -29,3 +36,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30187](https://github.com/ClickHouse/ClickHouse/pull/30187) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.1.8000-prestable.md b/docs/changelogs/v21.9.1.8000-prestable.md index cee357658d2..31fb71948d0 100644 --- a/docs/changelogs/v21.9.1.8000-prestable.md +++ b/docs/changelogs/v21.9.1.8000-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.1.8000-prestable FIXME as compared to v21.8.1.7409-prestable #### Backward Incompatible Change @@ -188,3 +195,209 @@ * NO CL ENTRY: 'Revert "less sys calls #2: make vdso work again"'. [#27829](https://github.com/ClickHouse/ClickHouse/pull/27829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Do not miss exceptions from the ThreadPool"'. [#27844](https://github.com/ClickHouse/ClickHouse/pull/27844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Add comments for the implementations of the pad functions [#26147](https://github.com/ClickHouse/ClickHouse/pull/26147) ([Vitaly Baranov](https://github.com/vitlibar)). +* Change color in client for double colon [#26152](https://github.com/ClickHouse/ClickHouse/pull/26152) ([Anton Popov](https://github.com/CurtizJ)). +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Merging [#26041](https://github.com/ClickHouse/ClickHouse/issues/26041). [#26180](https://github.com/ClickHouse/ClickHouse/pull/26180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad code (default function argument) [#26213](https://github.com/ClickHouse/ClickHouse/pull/26213) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support for `pread` in `ReadBufferFromFile` [#26232](https://github.com/ClickHouse/ClickHouse/pull/26232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ReadBufferFromS3 [#26249](https://github.com/ClickHouse/ClickHouse/pull/26249) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix output of TSV in integration tests [#26257](https://github.com/ClickHouse/ClickHouse/pull/26257) ([Anton Popov](https://github.com/CurtizJ)). +* Enum type additional support for compilation [#26258](https://github.com/ClickHouse/ClickHouse/pull/26258) ([Maksim Kita](https://github.com/kitaisreal)). +* Bump poco (now poco fork has CI via github actions) [#26262](https://github.com/ClickHouse/ClickHouse/pull/26262) ([Azat Khuzhin](https://github.com/azat)). +* Add check for sqlite database path [#26266](https://github.com/ClickHouse/ClickHouse/pull/26266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix arcadia [#26285](https://github.com/ClickHouse/ClickHouse/pull/26285) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* yandex/clickhouse-test-base Dockerfile fix (mysql connector moved) [#26294](https://github.com/ClickHouse/ClickHouse/pull/26294) ([Ilya Golshtein](https://github.com/ilejn)). +* Improve read_file_to_stringcolumn test compatibility [#26317](https://github.com/ClickHouse/ClickHouse/pull/26317) ([Raúl Marín](https://github.com/Algunenano)). +* Make socket poll() 7x faster (by replacing epoll() with poll()) [#26323](https://github.com/ClickHouse/ClickHouse/pull/26323) ([Azat Khuzhin](https://github.com/azat)). +* Modifications to an obscure Yandex TSKV format [#26326](https://github.com/ClickHouse/ClickHouse/pull/26326) ([egatov](https://github.com/egatov)). +* Fixing RBAC sample by tests in TestFlows. [#26329](https://github.com/ClickHouse/ClickHouse/pull/26329) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix error in stress test script [#26349](https://github.com/ClickHouse/ClickHouse/pull/26349) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration test about "replicated max parallel fetches". [#26362](https://github.com/ClickHouse/ClickHouse/pull/26362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Continuation of [#25774](https://github.com/ClickHouse/ClickHouse/issues/25774) [#26364](https://github.com/ClickHouse/ClickHouse/pull/26364) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enabling all TestFlows modules except LDAP after Kerberos merge. [#26366](https://github.com/ClickHouse/ClickHouse/pull/26366) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix flaky test 01293_client_interactive_vertical_multiline_long [#26367](https://github.com/ClickHouse/ClickHouse/pull/26367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small bugfix in Block [#26373](https://github.com/ClickHouse/ClickHouse/pull/26373) ([Anton Popov](https://github.com/CurtizJ)). +* More integration tests improvements. [#26375](https://github.com/ClickHouse/ClickHouse/pull/26375) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadia [#26378](https://github.com/ClickHouse/ClickHouse/pull/26378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add separate step to read from remote. [#26381](https://github.com/ClickHouse/ClickHouse/pull/26381) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix calculating of intersection of access rights. [#26383](https://github.com/ClickHouse/ClickHouse/pull/26383) ([Vitaly Baranov](https://github.com/vitlibar)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* stress tests report improvements [#26393](https://github.com/ClickHouse/ClickHouse/pull/26393) ([Azat Khuzhin](https://github.com/azat)). +* Fix failed assertion in RocksDB in case of bad_alloc exception during batch write [#26394](https://github.com/ClickHouse/ClickHouse/pull/26394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add integrity for loaded scripts in play.html [#26409](https://github.com/ClickHouse/ClickHouse/pull/26409) ([Vladimir C](https://github.com/vdimir)). +* Relax condition in flaky test [#26425](https://github.com/ClickHouse/ClickHouse/pull/26425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Split FunctionsCoding into several files [#26426](https://github.com/ClickHouse/ClickHouse/pull/26426) ([Vladimir C](https://github.com/vdimir)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix "While sending batch" (on Distributed async send) [#26430](https://github.com/ClickHouse/ClickHouse/pull/26430) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes v21.9.1.7477 [#26431](https://github.com/ClickHouse/ClickHouse/pull/26431) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test_replicated_mutations (due to lack of threads in pool) [#26461](https://github.com/ClickHouse/ClickHouse/pull/26461) ([Azat Khuzhin](https://github.com/azat)). +* Fix undefined-behavior in DirectoryMonitor (for exponential back off) [#26464](https://github.com/ClickHouse/ClickHouse/pull/26464) ([Azat Khuzhin](https://github.com/azat)). +* Remove some code [#26468](https://github.com/ClickHouse/ClickHouse/pull/26468) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rewrite distributed DDL to Processors [#26469](https://github.com/ClickHouse/ClickHouse/pull/26469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test `distributed_ddl_output_mode` [#26470](https://github.com/ClickHouse/ClickHouse/pull/26470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* SELECT String from ClickHouse as Avro string - PartialMatch [#26499](https://github.com/ClickHouse/ClickHouse/pull/26499) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix arcadia [#26505](https://github.com/ClickHouse/ClickHouse/pull/26505) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build under AppleClang 12 [#26509](https://github.com/ClickHouse/ClickHouse/pull/26509) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix lagInFrame for nullable types [#26521](https://github.com/ClickHouse/ClickHouse/pull/26521) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Handle empty testset in 'Functional stateless tests flaky check' [#26552](https://github.com/ClickHouse/ClickHouse/pull/26552) ([Vladimir C](https://github.com/vdimir)). +* Remove some streams. [#26590](https://github.com/ClickHouse/ClickHouse/pull/26590) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test 01622_defaults_for_url_engine [#26617](https://github.com/ClickHouse/ClickHouse/pull/26617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01509_check_many_parallel_quorum_inserts [#26618](https://github.com/ClickHouse/ClickHouse/pull/26618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix one possible cause of tests flakiness [#26619](https://github.com/ClickHouse/ClickHouse/pull/26619) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code, more C++ way [#26620](https://github.com/ClickHouse/ClickHouse/pull/26620) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Minor bugfix [#26678](https://github.com/ClickHouse/ClickHouse/pull/26678) ([Anton Popov](https://github.com/CurtizJ)). +* copypaste error [#26679](https://github.com/ClickHouse/ClickHouse/pull/26679) ([Denny Crane](https://github.com/den-crane)). +* Fix flaky test `mutation_stuck_after_replace_partition` [#26684](https://github.com/ClickHouse/ClickHouse/pull/26684) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Log errors on integration test listing error in ci-runner [#26691](https://github.com/ClickHouse/ClickHouse/pull/26691) ([Vladimir C](https://github.com/vdimir)). +* more debug checks for window functions [#26701](https://github.com/ClickHouse/ClickHouse/pull/26701) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* record server exit code in fuzzer [#26706](https://github.com/ClickHouse/ClickHouse/pull/26706) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove more streams. [#26713](https://github.com/ClickHouse/ClickHouse/pull/26713) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01946_test_wrong_host_name_access: Clear DNS in the end [#26715](https://github.com/ClickHouse/ClickHouse/pull/26715) ([Raúl Marín](https://github.com/Algunenano)). +* Setting min_count_to_compile_aggregate_expression fix [#26718](https://github.com/ClickHouse/ClickHouse/pull/26718) ([Maksim Kita](https://github.com/kitaisreal)). +* Compile aggregate functions profile events fix [#26719](https://github.com/ClickHouse/ClickHouse/pull/26719) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix use after free in AsyncDrain connection from S3Cluster [#26731](https://github.com/ClickHouse/ClickHouse/pull/26731) ([Vladimir C](https://github.com/vdimir)). +* Fixed wrong error message in `S3Common` [#26738](https://github.com/ClickHouse/ClickHouse/pull/26738) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Lock mutex before access to std::cerr in clickhouse-benchmark [#26742](https://github.com/ClickHouse/ClickHouse/pull/26742) ([Vladimir C](https://github.com/vdimir)). +* Remove some output streams [#26758](https://github.com/ClickHouse/ClickHouse/pull/26758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#25960](https://github.com/ClickHouse/ClickHouse/issues/25960) (Bolonini/read_from_file) [#26777](https://github.com/ClickHouse/ClickHouse/pull/26777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Heredoc updated tests [#26784](https://github.com/ClickHouse/ClickHouse/pull/26784) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix double unlock in RocksDB [#26786](https://github.com/ClickHouse/ClickHouse/pull/26786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix sqlite engine attach [#26795](https://github.com/ClickHouse/ClickHouse/pull/26795) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unneeded mutex during connection draining [#26807](https://github.com/ClickHouse/ClickHouse/pull/26807) ([Amos Bird](https://github.com/amosbird)). +* Make sure table is readonly when restarting fails. [#26808](https://github.com/ClickHouse/ClickHouse/pull/26808) ([Amos Bird](https://github.com/amosbird)). +* Check stdout for messages to retry in clickhouse-test [#26817](https://github.com/ClickHouse/ClickHouse/pull/26817) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not start new hedged connection if query was already canceled. [#26820](https://github.com/ClickHouse/ClickHouse/pull/26820) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Flush LazyOutputFormat on query cancel. [#26828](https://github.com/ClickHouse/ClickHouse/pull/26828) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix timeout in functional tests with pytest [#26830](https://github.com/ClickHouse/ClickHouse/pull/26830) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Compile aggregate functions without key [#26845](https://github.com/ClickHouse/ClickHouse/pull/26845) ([Maksim Kita](https://github.com/kitaisreal)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rabbitmq sink [#26871](https://github.com/ClickHouse/ClickHouse/pull/26871) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix keeper bench compilation [#26874](https://github.com/ClickHouse/ClickHouse/pull/26874) ([Raúl Marín](https://github.com/Algunenano)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Maybe fix extremely rare `intersecting parts`. [#26896](https://github.com/ClickHouse/ClickHouse/pull/26896) ([alesapin](https://github.com/alesapin)). +* Try increase diff upper bound [#26897](https://github.com/ClickHouse/ClickHouse/pull/26897) ([Ilya Yatsishin](https://github.com/qoega)). +* Enable Arrow format in Arcadia [#26898](https://github.com/ClickHouse/ClickHouse/pull/26898) ([Vitaly Stoyan](https://github.com/vitstn)). +* Improve test compatibility (00646_url_engine and 01854_HTTP_dict_decompression) [#26915](https://github.com/ClickHouse/ClickHouse/pull/26915) ([Raúl Marín](https://github.com/Algunenano)). +* Update NuRaft [#26916](https://github.com/ClickHouse/ClickHouse/pull/26916) ([alesapin](https://github.com/alesapin)). +* 01921_datatype_date32: Adapt it to work under Pacific/Fiji [#26918](https://github.com/ClickHouse/ClickHouse/pull/26918) ([Raúl Marín](https://github.com/Algunenano)). +* Remove unused files that confuse developers [#26913](https://github.com/ClickHouse/ClickHouse/issues/26913) [#26947](https://github.com/ClickHouse/ClickHouse/pull/26947) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set allow_remote_fs_zero_copy_replication to true by default [#26951](https://github.com/ClickHouse/ClickHouse/pull/26951) ([ianton-ru](https://github.com/ianton-ru)). +* Hold context in HedgedConnections to prevent use-after-free on settings. [#26953](https://github.com/ClickHouse/ClickHouse/pull/26953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix client options in stress test [#26959](https://github.com/ClickHouse/ClickHouse/pull/26959) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix window function partition boundary search [#26960](https://github.com/ClickHouse/ClickHouse/pull/26960) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Print trace from std::terminate exception line-by-line to make it grep easier [#26962](https://github.com/ClickHouse/ClickHouse/pull/26962) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix system.zookeeper_log initialization [#26972](https://github.com/ClickHouse/ClickHouse/pull/26972) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Benchmark script fix [#26974](https://github.com/ClickHouse/ClickHouse/pull/26974) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01600_quota_by_forwarded_ip [#26975](https://github.com/ClickHouse/ClickHouse/pull/26975) ([Raúl Marín](https://github.com/Algunenano)). +* 01674_executable_dictionary_implicit_key: executable_dictionary: Use printf [#26978](https://github.com/ClickHouse/ClickHouse/pull/26978) ([Raúl Marín](https://github.com/Algunenano)). +* Remove test_keeper_server usage (for {operation/session}_timeout_ms) [#26984](https://github.com/ClickHouse/ClickHouse/pull/26984) ([Azat Khuzhin](https://github.com/azat)). +* Set insert_quorum_timeout to 1 minute for tests. [#27007](https://github.com/ClickHouse/ClickHouse/pull/27007) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Adjust 00537_quarters to be timezone independent [#27008](https://github.com/ClickHouse/ClickHouse/pull/27008) ([Raúl Marín](https://github.com/Algunenano)). +* Help with [#26424](https://github.com/ClickHouse/ClickHouse/issues/26424) [#27009](https://github.com/ClickHouse/ClickHouse/pull/27009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Attempt to fix flaky 00705_drop_create_merge_tree [#27023](https://github.com/ClickHouse/ClickHouse/pull/27023) ([Raúl Marín](https://github.com/Algunenano)). +* Improved `runner` to use `pytest` keyword expressions [#27026](https://github.com/ClickHouse/ClickHouse/pull/27026) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Improve 01006_simpod_empty_part_single_column_write [#27028](https://github.com/ClickHouse/ClickHouse/pull/27028) ([Raúl Marín](https://github.com/Algunenano)). +* Improved logging of `hwmon` sensor errors in `AsynchronousMetrics` [#27031](https://github.com/ClickHouse/ClickHouse/pull/27031) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make test 01852_cast_operator independent of timezone [#27037](https://github.com/ClickHouse/ClickHouse/pull/27037) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moving to TestFlows 1.7.20 that has native support for parallel tests. [#27040](https://github.com/ClickHouse/ClickHouse/pull/27040) ([vzakaznikov](https://github.com/vzakaznikov)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix synchronization while updating from the config of an encrypted disk. [#27065](https://github.com/ClickHouse/ClickHouse/pull/27065) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix excessive logging in NuRaft on server shutdown [#27081](https://github.com/ClickHouse/ClickHouse/pull/27081) ([alesapin](https://github.com/alesapin)). +* Fix test_merge_tree_s3_failover with debug build [#27089](https://github.com/ClickHouse/ClickHouse/pull/27089) ([ianton-ru](https://github.com/ianton-ru)). +* Stateless tests: Keep an DNS error free log [#27092](https://github.com/ClickHouse/ClickHouse/pull/27092) ([Raúl Marín](https://github.com/Algunenano)). +* Normalize hostname in stateless tests [#27093](https://github.com/ClickHouse/ClickHouse/pull/27093) ([Amos Bird](https://github.com/amosbird)). +* Try update AMQP-CPP [#27095](https://github.com/ClickHouse/ClickHouse/pull/27095) ([Ilya Yatsishin](https://github.com/qoega)). +* Try update arrow [#27097](https://github.com/ClickHouse/ClickHouse/pull/27097) ([Ilya Yatsishin](https://github.com/qoega)). +* GlobalSubqueriesVisitor external storage check fix [#27131](https://github.com/ClickHouse/ClickHouse/pull/27131) ([Maksim Kita](https://github.com/kitaisreal)). +* Better code around decompression [#27136](https://github.com/ClickHouse/ClickHouse/pull/27136) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia src/Access gtest [#27152](https://github.com/ClickHouse/ClickHouse/pull/27152) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Updated readIntTextUnsafe [#27155](https://github.com/ClickHouse/ClickHouse/pull/27155) ([Maksim Kita](https://github.com/kitaisreal)). +* Safer `ReadBufferFromS3` for merges and backports [#27168](https://github.com/ClickHouse/ClickHouse/pull/27168) ([Vladimir Chebotarev](https://github.com/excitoon)). +* properly check the settings in perf test [#27190](https://github.com/ClickHouse/ClickHouse/pull/27190) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Save information about used functions/tables/... into query_log on error [#27194](https://github.com/ClickHouse/ClickHouse/pull/27194) ([Azat Khuzhin](https://github.com/azat)). +* Fix polling of /sys/block [#27195](https://github.com/ClickHouse/ClickHouse/pull/27195) ([Azat Khuzhin](https://github.com/azat)). +* Allow parallel execution of *.sql tests with ReplicatedMergeTree (by using {database} macro) [#27214](https://github.com/ClickHouse/ClickHouse/pull/27214) ([Azat Khuzhin](https://github.com/azat)). +* Fix NLP performance test [#27219](https://github.com/ClickHouse/ClickHouse/pull/27219) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update changelog/README.md [#27221](https://github.com/ClickHouse/ClickHouse/pull/27221) ([filimonov](https://github.com/filimonov)). +* more careful handling of reconnects in fuzzer [#27222](https://github.com/ClickHouse/ClickHouse/pull/27222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ADDINCL proper fast_float directory [#27224](https://github.com/ClickHouse/ClickHouse/pull/27224) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* DatabaseReplicatedWorker logs_to_keep race fix [#27225](https://github.com/ClickHouse/ClickHouse/pull/27225) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Using formatted string literals in clickhouse-test, extracted sort key functions and stacktraces printer [#27228](https://github.com/ClickHouse/ClickHouse/pull/27228) ([Mike Kot](https://github.com/myrrc)). +* Fix polling of /sys/block in case of block devices reopened on error [#27266](https://github.com/ClickHouse/ClickHouse/pull/27266) ([Azat Khuzhin](https://github.com/azat)). +* Remove streams from dicts [#27273](https://github.com/ClickHouse/ClickHouse/pull/27273) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01099_operators_date_and_timestamp: Use dates that work with all available timezones [#27275](https://github.com/ClickHouse/ClickHouse/pull/27275) ([Raúl Marín](https://github.com/Algunenano)). +* Improve kafka integration test error messages [#27294](https://github.com/ClickHouse/ClickHouse/pull/27294) ([Raúl Marín](https://github.com/Algunenano)). +* Improve 00738_lock_for_inner_table stability [#27300](https://github.com/ClickHouse/ClickHouse/pull/27300) ([Raúl Marín](https://github.com/Algunenano)). +* Add and check system.projection_parts for database filter [#27303](https://github.com/ClickHouse/ClickHouse/pull/27303) ([Azat Khuzhin](https://github.com/azat)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01300_client_save_history_when_terminated_long [#27324](https://github.com/ClickHouse/ClickHouse/pull/27324) ([Raúl Marín](https://github.com/Algunenano)). +* Try update contrib/zlib-ng [#27327](https://github.com/ClickHouse/ClickHouse/pull/27327) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add and check system.mutations for database filter [#27384](https://github.com/ClickHouse/ClickHouse/pull/27384) ([Azat Khuzhin](https://github.com/azat)). +* Correct the key data type used in mapContains [#27423](https://github.com/ClickHouse/ClickHouse/pull/27423) ([Fuwang Hu](https://github.com/fuwhu)). +* Fix tests for WithMergeableStateAfterAggregationAndLimit [#27424](https://github.com/ClickHouse/ClickHouse/pull/27424) ([Azat Khuzhin](https://github.com/azat)). +* Do not miss exceptions from the ThreadPool [#27428](https://github.com/ClickHouse/ClickHouse/pull/27428) ([Azat Khuzhin](https://github.com/azat)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Added reserve method to Block [#27483](https://github.com/ClickHouse/ClickHouse/pull/27483) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* make it possible to cancel window functions on ctrl+c [#27487](https://github.com/ClickHouse/ClickHouse/pull/27487) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix Nullable const columns in JOIN [#27516](https://github.com/ClickHouse/ClickHouse/pull/27516) ([Vladimir C](https://github.com/vdimir)). +* Fix Logical error: 'Table UUID is not specified in DDL log' [#27521](https://github.com/ClickHouse/ClickHouse/pull/27521) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01236_graphite_mt for random timezones [#27525](https://github.com/ClickHouse/ClickHouse/pull/27525) ([Raúl Marín](https://github.com/Algunenano)). +* Add timeout for integration tests runner [#27535](https://github.com/ClickHouse/ClickHouse/pull/27535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix polling of /sys/class on errors [#27554](https://github.com/ClickHouse/ClickHouse/pull/27554) ([Azat Khuzhin](https://github.com/azat)). +* fix recalculate QueryMemoryLimitExceeded event error [#27556](https://github.com/ClickHouse/ClickHouse/pull/27556) ([Ben](https://github.com/benbiti)). +* Fix 01961_roaring_memory_tracking for split builds [#27557](https://github.com/ClickHouse/ClickHouse/pull/27557) ([Raúl Marín](https://github.com/Algunenano)). +* Improve the experience of running stateless tests locally [#27561](https://github.com/ClickHouse/ClickHouse/pull/27561) ([Raúl Marín](https://github.com/Algunenano)). +* Fix integration tests [#27562](https://github.com/ClickHouse/ClickHouse/pull/27562) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Dictionaries refactor [#27566](https://github.com/ClickHouse/ClickHouse/pull/27566) ([Maksim Kita](https://github.com/kitaisreal)). +* Less Stopwatch.h [#27569](https://github.com/ClickHouse/ClickHouse/pull/27569) ([filimonov](https://github.com/filimonov)). +* Aggregation temporary disable compilation without key [#27574](https://github.com/ClickHouse/ClickHouse/pull/27574) ([Maksim Kita](https://github.com/kitaisreal)). +* Removed some data streams [#27575](https://github.com/ClickHouse/ClickHouse/pull/27575) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove streams from lv [#27577](https://github.com/ClickHouse/ClickHouse/pull/27577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix ProfileEvents::CompileFunction [#27606](https://github.com/ClickHouse/ClickHouse/pull/27606) ([Ben](https://github.com/benbiti)). +* Refactor mysql format check [#27609](https://github.com/ClickHouse/ClickHouse/pull/27609) ([Raúl Marín](https://github.com/Algunenano)). +* enable part_log by default [#27631](https://github.com/ClickHouse/ClickHouse/pull/27631) ([Denny Crane](https://github.com/den-crane)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* MV: Improve text logs when doing parallel processing [#27639](https://github.com/ClickHouse/ClickHouse/pull/27639) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test_sqlite_odbc_hashed_dictionary [#27647](https://github.com/ClickHouse/ClickHouse/pull/27647) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#10735](https://github.com/ClickHouse/ClickHouse/issues/10735) [#27677](https://github.com/ClickHouse/ClickHouse/pull/27677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md index 3f132b983c0..c125fab9ed6 100644 --- a/docs/changelogs/v21.9.2.17-stable.md +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.2.17-stable FIXME as compared to v21.9.1.8000-prestable #### Improvement @@ -43,3 +50,15 @@ * Backported in [#28715](https://github.com/ClickHouse/ClickHouse/issues/28715): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). * Backported in [#28744](https://github.com/ClickHouse/ClickHouse/issues/28744): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md index 3b665365668..5318f654f88 100644 --- a/docs/changelogs/v21.9.3.30-stable.md +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.3.30-stable FIXME as compared to v21.9.2.17-stable #### Improvement @@ -14,3 +21,10 @@ * Backported in [#28927](https://github.com/ClickHouse/ClickHouse/issues/28927): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#29025](https://github.com/ClickHouse/ClickHouse/issues/29025): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md index 8b919ecb268..65ba54de6e8 100644 --- a/docs/changelogs/v21.9.4.35-stable.md +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md index 7287c58064d..305e49a8633 100644 --- a/docs/changelogs/v21.9.5.16-stable.md +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.5.16-stable FIXME as compared to v21.9.4.35-stable #### Improvement @@ -46,3 +53,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30185](https://github.com/ClickHouse/ClickHouse/pull/30185) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md index f1d097ab646..dbecd4b3634 100644 --- a/docs/changelogs/v21.9.6.24-stable.md +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.6.24-stable FIXME as compared to v21.9.5.16-stable #### New Feature @@ -55,3 +62,10 @@ * Backported in [#32078](https://github.com/ClickHouse/ClickHouse/issues/32078): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). * Backported in [#31907](https://github.com/ClickHouse/ClickHouse/issues/31907): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + From c068593a0d7e34e4b4660a6e3d1f08cc0ee76e37 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 23 Jun 2022 12:58:34 +0200 Subject: [PATCH 096/408] fix tests --- .../00148_summing_merge_tree_aggregate_function.sql | 1 + tests/queries/0_stateless/00214_primary_key_order.sql | 2 ++ tests/queries/0_stateless/00229_prewhere_column_missing.sql | 1 + .../0_stateless/00261_storage_aliases_and_array_join.sql | 1 + tests/queries/0_stateless/00262_alter_alias.sql | 1 + tests/queries/0_stateless/00276_sample.sql | 1 + tests/queries/0_stateless/00282_merging.sql | 2 ++ tests/queries/0_stateless/00294_shard_enums.sql | 1 + tests/queries/0_stateless/00311_array_primary_key.sql | 1 + .../0_stateless/00314_sample_factor_virtual_column.sql | 1 + tests/queries/0_stateless/00318_pk_tuple_order.sql | 1 + tests/queries/0_stateless/00319_index_for_like.sql | 1 + tests/queries/0_stateless/00321_pk_set.sql | 1 + .../queries/0_stateless/00327_summing_composite_nested.sql | 1 + tests/queries/0_stateless/00331_final_and_prewhere.sql | 1 + .../00361_shared_array_offsets_and_squash_blocks.sql | 1 + tests/queries/0_stateless/00363_defaults.sql | 1 + tests/queries/0_stateless/00386_enum_in_pk.sql | 1 + tests/queries/0_stateless/00392_enum_nested_alter.sql | 1 + .../0_stateless/00411_merge_tree_where_const_in_set.sql | 1 + .../0_stateless/00412_logical_expressions_optimizer.sql | 1 + .../00432_aggregate_function_scalars_and_constants.sql | 1 + .../0_stateless/00443_optimize_final_vertical_merge.sh | 2 +- .../queries/0_stateless/00443_preferred_block_size_bytes.sh | 6 +++--- tests/queries/0_stateless/00456_alter_nullable.sql | 2 ++ .../queries/0_stateless/00481_reading_from_last_granula.sql | 1 + .../0_stateless/00483_reading_from_array_structure.sql | 2 ++ tests/queries/0_stateless/00489_pk_subexpression.sql | 1 + tests/queries/0_stateless/00504_insert_miss_columns.sh | 4 ++-- .../00542_materialized_view_and_time_zone_tag.sql | 1 + .../0_stateless/00564_versioned_collapsing_merge_tree.sql | 1 + tests/queries/0_stateless/00565_enum_order.sh | 2 +- tests/queries/0_stateless/00579_virtual_column_and_lazy.sql | 1 + tests/queries/0_stateless/00614_array_nullable.sql | 1 + tests/queries/0_stateless/00616_final_single_part.sql | 1 + tests/queries/0_stateless/00623_truncate_table.sql | 2 ++ .../queries/0_stateless/00625_summing_merge_tree_merge.sql | 1 + .../0_stateless/00709_virtual_column_partition_id.sql | 1 + .../queries/0_stateless/00974_final_predicate_push_down.sql | 1 + tests/queries/0_stateless/01109_exchange_tables.sql | 1 + tests/queries/0_stateless/01128_generate_random_nested.sql | 2 ++ tests/queries/0_stateless/01190_full_attach_syntax.sql | 1 + tests/queries/0_stateless/01297_alter_distributed.sql | 1 + tests/queries/0_stateless/01299_alter_merge_tree.sql | 1 + tests/queries/0_stateless/01652_ttl_old_syntax.sql | 1 + tests/queries/0_stateless/01907_multiple_aliases.sql | 1 + .../0_stateless/02021_create_database_with_comment.sh | 2 +- tests/queries/0_stateless/02135_local_create_db.sh | 2 +- .../0_stateless/02265_rename_join_ordinary_to_atomic.sql | 1 + 49 files changed, 58 insertions(+), 9 deletions(-) diff --git a/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql b/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql index 04edf709bde..649c09dbbf1 100644 --- a/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql +++ b/tests/queries/0_stateless/00148_summing_merge_tree_aggregate_function.sql @@ -2,6 +2,7 @@ drop table if exists summing_merge_tree_aggregate_function; drop table if exists summing_merge_tree_null; ---- partition merge +set allow_deprecated_syntax_for_merge_tree=1; create table summing_merge_tree_aggregate_function ( d Date, k UInt64, diff --git a/tests/queries/0_stateless/00214_primary_key_order.sql b/tests/queries/0_stateless/00214_primary_key_order.sql index 3c751e63e6d..e8a3be5f8dc 100644 --- a/tests/queries/0_stateless/00214_primary_key_order.sql +++ b/tests/queries/0_stateless/00214_primary_key_order.sql @@ -1,4 +1,6 @@ DROP TABLE IF EXISTS primary_key; + +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE primary_key (d Date DEFAULT today(), x Int8) ENGINE = MergeTree(d, -x, 1); INSERT INTO primary_key (x) VALUES (1), (2), (3); diff --git a/tests/queries/0_stateless/00229_prewhere_column_missing.sql b/tests/queries/0_stateless/00229_prewhere_column_missing.sql index 324e37bfce7..1fb74b04af0 100644 --- a/tests/queries/0_stateless/00229_prewhere_column_missing.sql +++ b/tests/queries/0_stateless/00229_prewhere_column_missing.sql @@ -1,5 +1,6 @@ drop table if exists prewhere_column_missing; +set allow_deprecated_syntax_for_merge_tree=1; create table prewhere_column_missing (d Date default '2015-01-01', x UInt64) engine=MergeTree(d, x, 1); insert into prewhere_column_missing (x) values (0); diff --git a/tests/queries/0_stateless/00261_storage_aliases_and_array_join.sql b/tests/queries/0_stateless/00261_storage_aliases_and_array_join.sql index 71562ffd6fa..bb3376a9e61 100644 --- a/tests/queries/0_stateless/00261_storage_aliases_and_array_join.sql +++ b/tests/queries/0_stateless/00261_storage_aliases_and_array_join.sql @@ -1,5 +1,6 @@ drop table if exists aliases_test; +set allow_deprecated_syntax_for_merge_tree=1; create table aliases_test ( date Date, id UInt64, array default ['zero','one','two'], diff --git a/tests/queries/0_stateless/00262_alter_alias.sql b/tests/queries/0_stateless/00262_alter_alias.sql index 56dbda65be7..1c19f8636d1 100644 --- a/tests/queries/0_stateless/00262_alter_alias.sql +++ b/tests/queries/0_stateless/00262_alter_alias.sql @@ -1,5 +1,6 @@ drop table if exists aliases_test; +set allow_deprecated_syntax_for_merge_tree=1; create table aliases_test (date default today(), id default rand(), array default [0, 1, 2]) engine=MergeTree(date, id, 1); insert into aliases_test (id) values (0); diff --git a/tests/queries/0_stateless/00276_sample.sql b/tests/queries/0_stateless/00276_sample.sql index cd28f18b2b8..b75ed188ec4 100644 --- a/tests/queries/0_stateless/00276_sample.sql +++ b/tests/queries/0_stateless/00276_sample.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS sample_00276; +set allow_deprecated_syntax_for_merge_tree=1; SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; SET max_block_size = 10; diff --git a/tests/queries/0_stateless/00282_merging.sql b/tests/queries/0_stateless/00282_merging.sql index a49cde87134..f4a3708eedf 100644 --- a/tests/queries/0_stateless/00282_merging.sql +++ b/tests/queries/0_stateless/00282_merging.sql @@ -1,4 +1,6 @@ DROP TABLE IF EXISTS merge; + +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE IF NOT EXISTS merge (d Date DEFAULT '2000-01-01', x UInt64) ENGINE = MergeTree(d, x, 5); INSERT INTO merge (x) VALUES (1), (2), (3); diff --git a/tests/queries/0_stateless/00294_shard_enums.sql b/tests/queries/0_stateless/00294_shard_enums.sql index 414b9cfad03..dcd74ac3e3a 100644 --- a/tests/queries/0_stateless/00294_shard_enums.sql +++ b/tests/queries/0_stateless/00294_shard_enums.sql @@ -3,6 +3,7 @@ set max_threads = 1; drop table if exists enums; +set allow_deprecated_syntax_for_merge_tree=1; create table enums ( d Date default '2015-12-29', k default 0, e Enum8('world' = 2, 'hello' = 1), sign Enum8('minus' = -1, 'plus' = 1), diff --git a/tests/queries/0_stateless/00311_array_primary_key.sql b/tests/queries/0_stateless/00311_array_primary_key.sql index f6e21beab9b..348ef2d1c6f 100644 --- a/tests/queries/0_stateless/00311_array_primary_key.sql +++ b/tests/queries/0_stateless/00311_array_primary_key.sql @@ -1,5 +1,6 @@ -- Tags: no-parallel +set allow_deprecated_syntax_for_merge_tree=1; DROP TABLE IF EXISTS array_pk; CREATE TABLE array_pk (key Array(UInt8), s String, n UInt64, d Date MATERIALIZED '2000-01-01') ENGINE = MergeTree(d, (key, s, n), 1); diff --git a/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql b/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql index b0ed4fdedcb..6e3dc019069 100644 --- a/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql +++ b/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS sample_00314_1; DROP TABLE IF EXISTS sample_00314_2; DROP TABLE IF EXISTS sample_merge_00314; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE sample_00314_1 (x UInt64, d Date DEFAULT today()) ENGINE = MergeTree(d, intHash64(x), intHash64(x), 10); CREATE TABLE sample_00314_2 (x UInt64, d Date DEFAULT today()) ENGINE = MergeTree(d, intHash64(x), intHash64(x), 10); diff --git a/tests/queries/0_stateless/00318_pk_tuple_order.sql b/tests/queries/0_stateless/00318_pk_tuple_order.sql index 8e2992167ee..585f35d2f3c 100644 --- a/tests/queries/0_stateless/00318_pk_tuple_order.sql +++ b/tests/queries/0_stateless/00318_pk_tuple_order.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS pk; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE pk (d Date DEFAULT '2000-01-01', x UInt64, y UInt64, z UInt64) ENGINE = MergeTree(d, (x, y, z), 1); INSERT INTO pk (x, y, z) VALUES (1, 11, 1235), (1, 11, 4395), (1, 22, 3545), (1, 22, 6984), (1, 33, 4596), (2, 11, 4563), (2, 11, 4578), (2, 11, 3572), (2, 22, 5786), (2, 22, 5786), (2, 22, 2791), (2, 22, 2791), (3, 33, 2791), (3, 33, 2791), (3, 33, 1235), (3, 44, 4935), (3, 44, 4578), (3, 55, 5786), (3, 55, 2791), (3, 55, 1235); diff --git a/tests/queries/0_stateless/00319_index_for_like.sql b/tests/queries/0_stateless/00319_index_for_like.sql index 57ebce439f3..e490e595142 100644 --- a/tests/queries/0_stateless/00319_index_for_like.sql +++ b/tests/queries/0_stateless/00319_index_for_like.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS index_for_like; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE index_for_like (s String, d Date DEFAULT today()) ENGINE = MergeTree(d, (s, d), 1); INSERT INTO index_for_like (s) VALUES ('Hello'), ('Hello, World'), ('Hello, World 1'), ('Hello 1'), ('Goodbye'), ('Goodbye, World'), ('Goodbye 1'), ('Goodbye, World 1'); diff --git a/tests/queries/0_stateless/00321_pk_set.sql b/tests/queries/0_stateless/00321_pk_set.sql index 073a87a6e13..bf61a684ac7 100644 --- a/tests/queries/0_stateless/00321_pk_set.sql +++ b/tests/queries/0_stateless/00321_pk_set.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS pk_set; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE pk_set (d Date, n UInt64, host String, code UInt64) ENGINE = MergeTree(d, (n, host, code), 1); INSERT INTO pk_set (n, host, code) VALUES (1, 'market', 100), (11, 'news', 100); diff --git a/tests/queries/0_stateless/00327_summing_composite_nested.sql b/tests/queries/0_stateless/00327_summing_composite_nested.sql index f9b251ebd8f..701735a7168 100644 --- a/tests/queries/0_stateless/00327_summing_composite_nested.sql +++ b/tests/queries/0_stateless/00327_summing_composite_nested.sql @@ -1,6 +1,7 @@ SET optimize_on_insert = 0; DROP TABLE IF EXISTS summing_composite_key; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE summing_composite_key (d Date, k UInt64, FirstMap Nested(k1 UInt32, k2ID Int8, s Float64), SecondMap Nested(k1ID UInt64, k2Key String, k3Type Int32, s Int64)) ENGINE = SummingMergeTree(d, k, 1); INSERT INTO summing_composite_key VALUES ('2000-01-01', 1, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 1, [2,1], ['4','3'], [20,22], [2,2,1], [5,5,0], [-3,-3,-33], [10,100,1000]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 2, [2,1,1], ['4','3','3'], [20,22,33], [2,2], [5,5], [-3,-3], [10,100]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]); diff --git a/tests/queries/0_stateless/00331_final_and_prewhere.sql b/tests/queries/0_stateless/00331_final_and_prewhere.sql index 02af6d9b450..5d0b80d6363 100644 --- a/tests/queries/0_stateless/00331_final_and_prewhere.sql +++ b/tests/queries/0_stateless/00331_final_and_prewhere.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS replace; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE replace ( EventDate Date, Id UInt64, Data String, Version UInt32) ENGINE = ReplacingMergeTree(EventDate, Id, 8192, Version); INSERT INTO replace VALUES ('2016-06-02', 1, 'version 1', 1); INSERT INTO replace VALUES ('2016-06-02', 2, 'version 1', 1); diff --git a/tests/queries/0_stateless/00361_shared_array_offsets_and_squash_blocks.sql b/tests/queries/0_stateless/00361_shared_array_offsets_and_squash_blocks.sql index 6160edd05b7..7815a35259a 100644 --- a/tests/queries/0_stateless/00361_shared_array_offsets_and_squash_blocks.sql +++ b/tests/queries/0_stateless/00361_shared_array_offsets_and_squash_blocks.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS nested1; DROP TABLE IF EXISTS nested2; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE nested1 (d Date DEFAULT '2000-01-01', x UInt64, n Nested(a String, b String)) ENGINE = MergeTree(d, x, 1); INSERT INTO nested1 (x, n.a, n.b) VALUES (1, ['Hello', 'World'], ['abc', 'def']), (2, [], []); diff --git a/tests/queries/0_stateless/00363_defaults.sql b/tests/queries/0_stateless/00363_defaults.sql index 4ebcc7b0f61..1ec3b13a130 100644 --- a/tests/queries/0_stateless/00363_defaults.sql +++ b/tests/queries/0_stateless/00363_defaults.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS prewhere_defaults; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE prewhere_defaults (d Date DEFAULT '2000-01-01', k UInt64 DEFAULT 0, x UInt16) ENGINE = MergeTree(d, k, 1); INSERT INTO prewhere_defaults (x) VALUES (1); diff --git a/tests/queries/0_stateless/00386_enum_in_pk.sql b/tests/queries/0_stateless/00386_enum_in_pk.sql index 75b8a166523..4fc79b5ef1b 100644 --- a/tests/queries/0_stateless/00386_enum_in_pk.sql +++ b/tests/queries/0_stateless/00386_enum_in_pk.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS enum_pk; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE enum_pk (date Date DEFAULT '0000-00-00', x Enum8('0' = 0, '1' = 1, '2' = 2), d Enum8('0' = 0, '1' = 1, '2' = 2)) ENGINE = MergeTree(date, x, 1); INSERT INTO enum_pk (x, d) VALUES ('0', '0')('1', '1')('0', '0')('1', '1')('1', '1')('0', '0')('0', '0')('2', '2')('0', '0')('1', '1')('1', '1')('1', '1')('1', '1')('0', '0'); diff --git a/tests/queries/0_stateless/00392_enum_nested_alter.sql b/tests/queries/0_stateless/00392_enum_nested_alter.sql index 205b9a7fec0..b5989885de4 100644 --- a/tests/queries/0_stateless/00392_enum_nested_alter.sql +++ b/tests/queries/0_stateless/00392_enum_nested_alter.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS enum_nested_alter; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE enum_nested_alter (d Date DEFAULT '2000-01-01', x UInt64, n Nested(a String, e Enum8('Hello' = 1), b UInt8)) ENGINE = MergeTree(d, x, 1); diff --git a/tests/queries/0_stateless/00411_merge_tree_where_const_in_set.sql b/tests/queries/0_stateless/00411_merge_tree_where_const_in_set.sql index 614f838c2eb..22779509a3d 100644 --- a/tests/queries/0_stateless/00411_merge_tree_where_const_in_set.sql +++ b/tests/queries/0_stateless/00411_merge_tree_where_const_in_set.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS const_in_const; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE const_in_const (id UInt64, date Date, uid UInt32, name String, Sign Int8) ENGINE = CollapsingMergeTree(date, intHash32(uid), (id, date, intHash32(uid)), 8192, Sign); INSERT INTO const_in_const VALUES(1, now(), 1, 'test1', 1); INSERT INTO const_in_const VALUES(2, now(), 1, 'test2', 1); diff --git a/tests/queries/0_stateless/00412_logical_expressions_optimizer.sql b/tests/queries/0_stateless/00412_logical_expressions_optimizer.sql index 5f8b15e980a..c4fad7d5064 100644 --- a/tests/queries/0_stateless/00412_logical_expressions_optimizer.sql +++ b/tests/queries/0_stateless/00412_logical_expressions_optimizer.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS merge_tree; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE merge_tree (x UInt64, date Date) ENGINE = MergeTree(date, x, 1); INSERT INTO merge_tree VALUES (1, '2000-01-01'); diff --git a/tests/queries/0_stateless/00432_aggregate_function_scalars_and_constants.sql b/tests/queries/0_stateless/00432_aggregate_function_scalars_and_constants.sql index c74b4f03371..a6f31b9357c 100644 --- a/tests/queries/0_stateless/00432_aggregate_function_scalars_and_constants.sql +++ b/tests/queries/0_stateless/00432_aggregate_function_scalars_and_constants.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS agg_func_col; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE agg_func_col (p Date, k UInt8, d AggregateFunction(sum, UInt64) DEFAULT arrayReduce('sumState', [toUInt64(200)])) ENGINE = AggregatingMergeTree(p, k, 1); INSERT INTO agg_func_col (k) VALUES (0); INSERT INTO agg_func_col (k, d) SELECT 1 AS k, arrayReduce('sumState', [toUInt64(100)]) AS d; diff --git a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh index a32dde58164..3bdc5892ced 100755 --- a/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh +++ b/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh @@ -16,7 +16,7 @@ function get_num_parts { $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS $name" -$CLICKHOUSE_CLIENT -q "CREATE TABLE $name ( +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE $name ( date Date, Sign Int8, ki UInt64, diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 399a4677a44..5dcc2f3c181 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" -$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO preferred_block_size_bytes (s) SELECT '16_bytes_-_-_-_' AS s FROM system.numbers LIMIT 10, 90" $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE preferred_block_size_bytes" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=26 -q "SELECT DISTINCT blockSize(), ignore(p, s) FROM preferred_block_size_bytes" @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" # PREWHERE using empty column $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS pbs" -$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO pbs (p, i, sa) SELECT toDate(i % 30) AS p, number AS i, ['a'] AS sa FROM system.numbers LIMIT 1000" $CLICKHOUSE_CLIENT -q "ALTER TABLE pbs ADD COLUMN s UInt8 DEFAULT 0" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=100000 -q "SELECT count() FROM pbs PREWHERE s = 0" @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE pbs" # Nullable PREWHERE $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" -$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1-q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO nullable_prewhere SELECT toDate(0) AS p, if(number % 2 = 0, CAST(number AS Nullable(UInt64)), CAST(NULL AS Nullable(UInt64))) AS f, number as d FROM system.numbers LIMIT 1001" $CLICKHOUSE_CLIENT -q "SELECT sum(d), sum(f), max(d) FROM nullable_prewhere PREWHERE NOT isNull(f)" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" diff --git a/tests/queries/0_stateless/00456_alter_nullable.sql b/tests/queries/0_stateless/00456_alter_nullable.sql index 703d1a551a7..0fa3837767d 100644 --- a/tests/queries/0_stateless/00456_alter_nullable.sql +++ b/tests/queries/0_stateless/00456_alter_nullable.sql @@ -1,4 +1,6 @@ DROP TABLE IF EXISTS nullable_alter; + +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE nullable_alter (d Date DEFAULT '2000-01-01', x String) ENGINE = MergeTree(d, d, 1); INSERT INTO nullable_alter (x) VALUES ('Hello'), ('World'); diff --git a/tests/queries/0_stateless/00481_reading_from_last_granula.sql b/tests/queries/0_stateless/00481_reading_from_last_granula.sql index 29d42e41e14..c98068e466b 100644 --- a/tests/queries/0_stateless/00481_reading_from_last_granula.sql +++ b/tests/queries/0_stateless/00481_reading_from_last_granula.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS tab_00481; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE tab_00481 (date Date, value UInt64, s String, m FixedString(16)) ENGINE = MergeTree(date, (date, value), 8); INSERT INTO tab_00481 SELECT today() as date, number as value, '' as s, toFixedString('', 16) as m from system.numbers limit 42; SET preferred_max_column_in_block_size_bytes = 32; diff --git a/tests/queries/0_stateless/00483_reading_from_array_structure.sql b/tests/queries/0_stateless/00483_reading_from_array_structure.sql index 5ba152ef9b6..bab0dcd3707 100644 --- a/tests/queries/0_stateless/00483_reading_from_array_structure.sql +++ b/tests/queries/0_stateless/00483_reading_from_array_structure.sql @@ -1,4 +1,6 @@ drop table if exists `table_00483`; + +set allow_deprecated_syntax_for_merge_tree=1; create table `table_00483` (date Date, `Struct.Key1` Array(UInt64), `Struct.Key2` Array(UInt64), padding FixedString(16)) engine = MergeTree(date, (date), 16); insert into `table_00483` select today() as date, [number], [number + 1], toFixedString('', 16) from system.numbers limit 100; set preferred_max_column_in_block_size_bytes = 96; diff --git a/tests/queries/0_stateless/00489_pk_subexpression.sql b/tests/queries/0_stateless/00489_pk_subexpression.sql index 41499f0bd1b..6f76a13609c 100644 --- a/tests/queries/0_stateless/00489_pk_subexpression.sql +++ b/tests/queries/0_stateless/00489_pk_subexpression.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS pk; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE pk (d Date DEFAULT '2000-01-01', x DateTime, y UInt64, z UInt64) ENGINE = MergeTree(d, (toStartOfMinute(x), y, z), 1); INSERT INTO pk (x, y, z) VALUES (1, 11, 1235), (2, 11, 4395), (3, 22, 3545), (4, 22, 6984), (5, 33, 4596), (61, 11, 4563), (62, 11, 4578), (63, 11, 3572), (64, 22, 5786), (65, 22, 5786), (66, 22, 2791), (67, 22, 2791), (121, 33, 2791), (122, 33, 2791), (123, 33, 1235), (124, 44, 4935), (125, 44, 4578), (126, 55, 5786), (127, 55, 2791), (128, 55, 1235); diff --git a/tests/queries/0_stateless/00504_insert_miss_columns.sh b/tests/queries/0_stateless/00504_insert_miss_columns.sh index ea699ab58a5..fa2cc9d0b22 100755 --- a/tests/queries/0_stateless/00504_insert_miss_columns.sh +++ b/tests/queries/0_stateless/00504_insert_miss_columns.sh @@ -8,8 +8,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS advertiser"; $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS advertiser_test"; -$CLICKHOUSE_CLIENT -q "CREATE TABLE advertiser ( action_date Date, adblock UInt8, imps Int64 ) Engine = SummingMergeTree( action_date, ( adblock ), 8192, ( imps ) )"; -$CLICKHOUSE_CLIENT -q "CREATE TABLE advertiser_test ( action_date Date, adblock UInt8, imps Int64, Hash UInt64 ) Engine = SummingMergeTree( action_date, ( adblock, Hash ), 8192, ( imps ) )"; +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE advertiser ( action_date Date, adblock UInt8, imps Int64 ) Engine = SummingMergeTree( action_date, ( adblock ), 8192, ( imps ) )"; +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE advertiser_test ( action_date Date, adblock UInt8, imps Int64, Hash UInt64 ) Engine = SummingMergeTree( action_date, ( adblock, Hash ), 8192, ( imps ) )"; # This test will fail. It's ok. $CLICKHOUSE_CLIENT -q "INSERT INTO advertiser_test SELECT *, sipHash64( CAST(adblock AS String) ), CAST(1 AS Int8) FROM advertiser;" 2>/dev/null diff --git a/tests/queries/0_stateless/00542_materialized_view_and_time_zone_tag.sql b/tests/queries/0_stateless/00542_materialized_view_and_time_zone_tag.sql index 5e9277c90b6..88808ac20f9 100644 --- a/tests/queries/0_stateless/00542_materialized_view_and_time_zone_tag.sql +++ b/tests/queries/0_stateless/00542_materialized_view_and_time_zone_tag.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS m3; DROP TABLE IF EXISTS m1; DROP TABLE IF EXISTS x; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE x (d Date, t DateTime) ENGINE = MergeTree(d, (d, t), 1); CREATE MATERIALIZED VIEW m1 (d Date, t DateTime, c UInt64) ENGINE = SummingMergeTree(d, (d, t), 1) AS SELECT d, toStartOfMinute(x.t) as t, count() as c FROM x GROUP BY d, t; diff --git a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql index 66bbb02183c..fdee9390642 100644 --- a/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00564_versioned_collapsing_merge_tree.sql @@ -1,5 +1,6 @@ -- Tags: no-parallel +set allow_deprecated_syntax_for_merge_tree=1; set optimize_on_insert = 0; drop table if exists mult_tab; diff --git a/tests/queries/0_stateless/00565_enum_order.sh b/tests/queries/0_stateless/00565_enum_order.sh index 6958a403246..fedd81b91f8 100755 --- a/tests/queries/0_stateless/00565_enum_order.sh +++ b/tests/queries/0_stateless/00565_enum_order.sh @@ -10,7 +10,7 @@ $CLICKHOUSE_CLIENT <<"EOF" DROP TABLE IF EXISTS `test_log` EOF -$CLICKHOUSE_CLIENT <<"EOF" +$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 <<"EOF" CREATE TABLE `test_log` ( date Date, datetime DateTime, diff --git a/tests/queries/0_stateless/00579_virtual_column_and_lazy.sql b/tests/queries/0_stateless/00579_virtual_column_and_lazy.sql index b1f1ec4cfb2..ca58a5fc93b 100644 --- a/tests/queries/0_stateless/00579_virtual_column_and_lazy.sql +++ b/tests/queries/0_stateless/00579_virtual_column_and_lazy.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS sample_00579_1; DROP TABLE IF EXISTS sample_00579_2; DROP TABLE IF EXISTS sample_merge_00579; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE sample_00579_1 (x UInt64, d Date DEFAULT today()) ENGINE = MergeTree(d, intHash64(x), intHash64(x), 10); CREATE TABLE sample_00579_2 (x UInt64, d Date DEFAULT today()) ENGINE = MergeTree(d, intHash64(x), intHash64(x), 10); diff --git a/tests/queries/0_stateless/00614_array_nullable.sql b/tests/queries/0_stateless/00614_array_nullable.sql index d62ddb8242c..1cbfbf128cb 100644 --- a/tests/queries/0_stateless/00614_array_nullable.sql +++ b/tests/queries/0_stateless/00614_array_nullable.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS test; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test(date Date, keys Array(Nullable(UInt8))) ENGINE = MergeTree(date, date, 1); INSERT INTO test VALUES ('2017-09-10', [1, 2, 3, 4, 5, 6, 7, NULL]); SELECT * FROM test LIMIT 1; diff --git a/tests/queries/0_stateless/00616_final_single_part.sql b/tests/queries/0_stateless/00616_final_single_part.sql index 6618d0b1252..605f2c8b216 100644 --- a/tests/queries/0_stateless/00616_final_single_part.sql +++ b/tests/queries/0_stateless/00616_final_single_part.sql @@ -12,6 +12,7 @@ CREATE TABLE test_00616 ENGINE = MergeTree(date, x, 4096); INSERT INTO test_00616 VALUES ('2018-03-21', 1, 1), ('2018-03-21', 1, 2); +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE replacing_00616 ENGINE = ReplacingMergeTree(date, x, 4096, ver) AS SELECT * FROM test_00616; SELECT * FROM test_00616 ORDER BY ver; diff --git a/tests/queries/0_stateless/00623_truncate_table.sql b/tests/queries/0_stateless/00623_truncate_table.sql index 80369ff2bbc..16bc87ea213 100644 --- a/tests/queries/0_stateless/00623_truncate_table.sql +++ b/tests/queries/0_stateless/00623_truncate_table.sql @@ -1,5 +1,7 @@ -- Tags: no-parallel +set allow_deprecated_syntax_for_merge_tree=1; + DROP DATABASE IF EXISTS truncate_test; DROP TABLE IF EXISTS truncate_test_log; DROP TABLE IF EXISTS truncate_test_memory; diff --git a/tests/queries/0_stateless/00625_summing_merge_tree_merge.sql b/tests/queries/0_stateless/00625_summing_merge_tree_merge.sql index a214347a3a7..235048ad18b 100644 --- a/tests/queries/0_stateless/00625_summing_merge_tree_merge.sql +++ b/tests/queries/0_stateless/00625_summing_merge_tree_merge.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS tab_00625; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE tab_00625 ( date Date, diff --git a/tests/queries/0_stateless/00709_virtual_column_partition_id.sql b/tests/queries/0_stateless/00709_virtual_column_partition_id.sql index 084ab904d87..48a3a3fad6a 100644 --- a/tests/queries/0_stateless/00709_virtual_column_partition_id.sql +++ b/tests/queries/0_stateless/00709_virtual_column_partition_id.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS partition_id; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE IF NOT EXISTS partition_id (d Date DEFAULT '2000-01-01', x UInt64) ENGINE = MergeTree(d, x, 5); INSERT INTO partition_id VALUES (100, 1), (200, 2), (300, 3); diff --git a/tests/queries/0_stateless/00974_final_predicate_push_down.sql b/tests/queries/0_stateless/00974_final_predicate_push_down.sql index 96bcbf9aae6..7a6378692f2 100644 --- a/tests/queries/0_stateless/00974_final_predicate_push_down.sql +++ b/tests/queries/0_stateless/00974_final_predicate_push_down.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS test_00974; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_00974 ( date Date, diff --git a/tests/queries/0_stateless/01109_exchange_tables.sql b/tests/queries/0_stateless/01109_exchange_tables.sql index e6900745fed..c118945887b 100644 --- a/tests/queries/0_stateless/01109_exchange_tables.sql +++ b/tests/queries/0_stateless/01109_exchange_tables.sql @@ -30,6 +30,7 @@ SELECT * FROM t2; DROP DATABASE IF EXISTS test_01109_other_atomic; DROP DATABASE IF EXISTS test_01109_ordinary; CREATE DATABASE test_01109_other_atomic; +set allow_deprecated_database_ordinary=1; CREATE DATABASE test_01109_ordinary ENGINE=Ordinary; CREATE TABLE test_01109_other_atomic.t3 ENGINE=MergeTree() ORDER BY tuple() diff --git a/tests/queries/0_stateless/01128_generate_random_nested.sql b/tests/queries/0_stateless/01128_generate_random_nested.sql index 2af52e69893..8098db89491 100644 --- a/tests/queries/0_stateless/01128_generate_random_nested.sql +++ b/tests/queries/0_stateless/01128_generate_random_nested.sql @@ -1,4 +1,6 @@ DROP TABLE IF EXISTS mass_table_312; + +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE mass_table_312 (d Date DEFAULT '2000-01-01', x UInt64, n Nested(a String, b String)) ENGINE = MergeTree(d, x, 1); INSERT INTO mass_table_312 SELECT * FROM generateRandom('`d` Date,`x` UInt64,`n.a` Array(String),`n.b` Array(String)', 1, 10, 2) LIMIT 100; diff --git a/tests/queries/0_stateless/01190_full_attach_syntax.sql b/tests/queries/0_stateless/01190_full_attach_syntax.sql index e0ffe7ede66..e66978e22e1 100644 --- a/tests/queries/0_stateless/01190_full_attach_syntax.sql +++ b/tests/queries/0_stateless/01190_full_attach_syntax.sql @@ -2,6 +2,7 @@ DROP DATABASE IF EXISTS test_01190; set allow_deprecated_database_ordinary=1; +set allow_deprecated_syntax_for_merge_tree=1; CREATE DATABASE test_01190 ENGINE=Ordinary; -- Full ATTACH requires UUID with Atomic USE test_01190; diff --git a/tests/queries/0_stateless/01297_alter_distributed.sql b/tests/queries/0_stateless/01297_alter_distributed.sql index cec64278d47..c79d98b7b3b 100644 --- a/tests/queries/0_stateless/01297_alter_distributed.sql +++ b/tests/queries/0_stateless/01297_alter_distributed.sql @@ -3,6 +3,7 @@ drop table if exists merge_distributed; drop table if exists merge_distributed1; +set allow_deprecated_syntax_for_merge_tree=1; create table merge_distributed1 ( CounterID UInt32, StartDate Date, Sign Int8, VisitID UInt64, UserID UInt64, StartTime DateTime, ClickLogID UInt64) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), tuple(CounterID, StartDate, intHash32(UserID), VisitID, ClickLogID), 8192, Sign); insert into merge_distributed1 values (1, '2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3); diff --git a/tests/queries/0_stateless/01299_alter_merge_tree.sql b/tests/queries/0_stateless/01299_alter_merge_tree.sql index 87608e6d15a..3c4467926f8 100644 --- a/tests/queries/0_stateless/01299_alter_merge_tree.sql +++ b/tests/queries/0_stateless/01299_alter_merge_tree.sql @@ -1,5 +1,6 @@ drop table if exists merge_tree; +set allow_deprecated_syntax_for_merge_tree=1; create table merge_tree ( CounterID UInt32, StartDate Date, Sign Int8, VisitID UInt64, UserID UInt64, StartTime DateTime, ClickLogID UInt64) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), tuple(CounterID, StartDate, intHash32(UserID), VisitID, ClickLogID), 8192, Sign); insert into merge_tree values (1, '2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3) diff --git a/tests/queries/0_stateless/01652_ttl_old_syntax.sql b/tests/queries/0_stateless/01652_ttl_old_syntax.sql index 05c391b85e5..7b11247d968 100644 --- a/tests/queries/0_stateless/01652_ttl_old_syntax.sql +++ b/tests/queries/0_stateless/01652_ttl_old_syntax.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS ttl_old_syntax; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE ttl_old_syntax (d Date, i Int) ENGINE = MergeTree(d, i, 8291); ALTER TABLE ttl_old_syntax MODIFY TTL toDate('2020-01-01'); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01907_multiple_aliases.sql b/tests/queries/0_stateless/01907_multiple_aliases.sql index 611960a5205..5e8efba7ab7 100644 --- a/tests/queries/0_stateless/01907_multiple_aliases.sql +++ b/tests/queries/0_stateless/01907_multiple_aliases.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS t; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE t (d Date, z UInt32) ENGINE = MergeTree(d, (z), 1); INSERT INTO t VALUES ('2017-01-01', 1); diff --git a/tests/queries/0_stateless/02021_create_database_with_comment.sh b/tests/queries/0_stateless/02021_create_database_with_comment.sh index ce7a7bef066..11e62e790b7 100755 --- a/tests/queries/0_stateless/02021_create_database_with_comment.sh +++ b/tests/queries/0_stateless/02021_create_database_with_comment.sh @@ -18,7 +18,7 @@ function test_db_comments() local ENGINE_NAME="$1" echo "engine : ${ENGINE_NAME}" - $CLICKHOUSE_CLIENT -nm < Date: Thu, 23 Jun 2022 11:26:06 +0000 Subject: [PATCH 097/408] fix build --- src/Columns/tests/gtest_column_object.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/tests/gtest_column_object.cpp b/src/Columns/tests/gtest_column_object.cpp index d5e58e5fce2..e1ad949f6a8 100644 --- a/src/Columns/tests/gtest_column_object.cpp +++ b/src/Columns/tests/gtest_column_object.cpp @@ -89,7 +89,7 @@ TEST(ColumnObject, InsertRangeFrom) const auto & type_dst = subcolumn_dst.getLeastCommonType(); const auto & type_src = subcolumn_src.getLeastCommonType(); - auto type_res = getLeastSupertype(DataTypes{type_dst, type_src}, true); + auto type_res = getLeastSupertypeOrString(DataTypes{type_dst, type_src}); size_t from = rng() % subcolumn_src.size(); size_t to = rng() % subcolumn_src.size(); From 3e62d0fb8c3c74017725a82083678713fa648a6a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 23 Jun 2022 11:31:39 +0000 Subject: [PATCH 098/408] fix test --- tests/performance/json_type.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/json_type.xml b/tests/performance/json_type.xml index 29e52f1e53f..ef11856df0b 100644 --- a/tests/performance/json_type.xml +++ b/tests/performance/json_type.xml @@ -1,6 +1,6 @@ - 1 + 1 CREATE TABLE t_json_1(data JSON) ENGINE = MergeTree ORDER BY tuple() From 5af70d66e3dadd01b7318a2d5073f7d56a89f315 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 23 Jun 2022 11:52:28 +0000 Subject: [PATCH 099/408] Add plan tests for final distinct --- ...7_distinct_in_order_optimization.reference | 114 +++++++++++++----- .../02317_distinct_in_order_optimization.sql | 60 +++++---- 2 files changed, 121 insertions(+), 53 deletions(-) diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index 8ab6966665a..0c1236bbfcd 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -1,5 +1,5 @@ -enable optimize_distinct_in_order -distinct pipeline on empty table -> no optimization, source is ReadFromPreparedSource instead of ReadFromMergeTree +-- enable optimize_distinct_in_order +-- distinct pipeline on empty table -> no optimization, table is empty (Expression) ExpressionTransform (Distinct) @@ -10,9 +10,9 @@ ExpressionTransform ExpressionTransform (ReadFromPreparedSource) NullSource 0 → 1 -insert into table to use ReadFromMergeTree source -disable optimize_distinct_in_order -pipeline does _not_ contain the optimization +-- insert into table +-- disable optimize_distinct_in_order +-- distinct all primary key columns -> no optimization (Expression) ExpressionTransform (Distinct) @@ -23,8 +23,8 @@ ExpressionTransform ExpressionTransform (ReadFromMergeTree) MergeTreeInOrder 0 → 1 -enable optimize_distinct_in_order -distinct with all primary key columns -> optimization applied +-- enable optimize_distinct_in_order +-- distinct with all primary key columns -> pre-distinct optimization only (Expression) ExpressionTransform (Distinct) @@ -34,9 +34,8 @@ ExpressionTransform (Expression) ExpressionTransform (ReadFromMergeTree) - Concat 2 → 1 - MergeTreeInOrder × 2 0 → 1 -distinct with primary key prefix -> optimization applied + MergeTreeInOrder 0 → 1 +-- distinct with primary key prefix -> pre-distinct optimization only (Expression) ExpressionTransform (Distinct) @@ -46,9 +45,38 @@ ExpressionTransform (Expression) ExpressionTransform (ReadFromMergeTree) - Concat 2 → 1 - MergeTreeInOrder × 2 0 → 1 -distinct with non-primary key prefix -> no optimization + MergeTreeInOrder 0 → 1 +-- distinct with primary key prefix and order by on column in distinct -> pre-distinct and final distinct optimization +(Expression) +ExpressionTransform + (Distinct) + DistinctSortedTransform + (Sorting) + MergeSortingTransform + LimitsCheckingTransform + PartialSortingTransform + (Distinct) + DistinctSortedChunkTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +-- distinct with primary key prefix and order by on column _not_ in distinct -> pre-distinct optimization only +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Sorting) + MergeSortingTransform + LimitsCheckingTransform + PartialSortingTransform + (Distinct) + DistinctSortedChunkTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +-- distinct with non-primary key prefix -> no optimization (Expression) ExpressionTransform (Distinct) @@ -58,15 +86,41 @@ ExpressionTransform (Expression) ExpressionTransform (ReadFromMergeTree) - Concat 2 → 1 - MergeTreeInOrder × 2 0 → 1 -the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one -single-threaded distinct + MergeTreeInOrder 0 → 1 +-- distinct with non-primary key prefix and order by on column in distinct -> final distinct optimization only +(Expression) +ExpressionTransform + (Distinct) + DistinctSortedTransform + (Sorting) + MergeSortingTransform + LimitsCheckingTransform + PartialSortingTransform + (Distinct) + DistinctTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +-- distinct with non-primary key prefix and order by on column _not_ in distinct -> no optimization +(Expression) +ExpressionTransform + (Distinct) + DistinctTransform + (Sorting) + (Distinct) + DistinctTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +-- the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one +-- single-threaded distinct 0 -multi-threaded distinct +-- multi-threaded distinct 0 -skip part of chunk since it contains values from previous one -single-threaded distinct +-- skip part of chunk since it contains values from previous one +-- single-threaded distinct 0 1 2 @@ -77,7 +131,7 @@ single-threaded distinct 7 8 9 -multi-threaded distinct +-- multi-threaded distinct 0 1 2 @@ -88,10 +142,16 @@ multi-threaded distinct 7 8 9 -table with not only primary key columns -distinct with key-prefix only +-- create table with not only primary key columns +-- distinct with primary key prefix only 0 -distinct with full key +-- distinct with full key +0 0 +0 1 +0 2 +0 3 +0 4 +-- distinct with key prefix and non-sorted column 0 0 0 1 0 2 @@ -102,9 +162,3 @@ distinct with full key 0 7 0 8 0 9 -distinct with key prefix and non-sorted column -0 0 -0 1 -0 2 -0 3 -0 4 diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index 519b511dddb..f1e8da88017 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -1,57 +1,71 @@ drop table if exists distinct_in_order sync; create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by (a, b, c); -select 'enable optimize_distinct_in_order'; +select '-- enable optimize_distinct_in_order'; set optimize_distinct_in_order=1; -select 'distinct pipeline on empty table -> no optimization, source is ReadFromPreparedSource instead of ReadFromMergeTree'; +select '-- distinct pipeline on empty table -> no optimization, table is empty'; explain pipeline select distinct * from distinct_in_order settings max_threads=1; -select 'insert into table to use ReadFromMergeTree source'; -insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,10); +select '-- insert into table'; +insert into distinct_in_order select number % number, number % 5, number % 10 from numbers(1,10); -select 'disable optimize_distinct_in_order'; +select '-- disable optimize_distinct_in_order'; set optimize_distinct_in_order=0; -select 'pipeline does _not_ contain the optimization'; +select '-- distinct all primary key columns -> no optimization'; explain pipeline select distinct * from distinct_in_order settings max_threads=1; -select 'enable optimize_distinct_in_order'; +select '-- enable optimize_distinct_in_order'; set optimize_distinct_in_order=1; -select 'distinct with all primary key columns -> optimization applied'; -insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,10); + +select '-- distinct with all primary key columns -> pre-distinct optimization only'; explain pipeline select distinct * from distinct_in_order settings max_threads=1; -select 'distinct with primary key prefix -> optimization applied'; + +select '-- distinct with primary key prefix -> pre-distinct optimization only'; explain pipeline select distinct a, c from distinct_in_order settings max_threads=1; -select 'distinct with non-primary key prefix -> no optimization'; + +select '-- distinct with primary key prefix and order by on column in distinct -> pre-distinct and final distinct optimization'; +explain pipeline select distinct a, c from distinct_in_order order by c settings max_threads=1; + +select '-- distinct with primary key prefix and order by on column _not_ in distinct -> pre-distinct optimization only'; +explain pipeline select distinct a, c from distinct_in_order order by b settings max_threads=1; + +select '-- distinct with non-primary key prefix -> no optimization'; explain pipeline select distinct b, c from distinct_in_order settings max_threads=1; -select 'the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one'; +select '-- distinct with non-primary key prefix and order by on column in distinct -> final distinct optimization only'; +explain pipeline select distinct b, c from distinct_in_order order by b settings max_threads=1; + +select '-- distinct with non-primary key prefix and order by on column _not_ in distinct -> no optimization'; +explain pipeline select distinct b, c from distinct_in_order order by a settings max_threads=1; + +select '-- the same values in every chunk, distinct in order should skip entire chunks with the same key as previous one'; drop table if exists distinct_in_order sync; create table distinct_in_order (a int) engine=MergeTree() order by a settings index_granularity=10; -insert into distinct_in_order (a) select * from zeros(30); -select 'single-threaded distinct'; +insert into distinct_in_order (a) select * from zeros(100); +select '-- single-threaded distinct'; select distinct * from distinct_in_order settings max_block_size=10, max_threads=1; -select 'multi-threaded distinct'; +select '-- multi-threaded distinct'; select distinct * from distinct_in_order settings max_block_size=10; -select 'skip part of chunk since it contains values from previous one'; +select '-- skip part of chunk since it contains values from previous one'; drop table if exists distinct_in_order sync; create table distinct_in_order (a int) engine=MergeTree() order by a settings index_granularity=10; insert into distinct_in_order (a) select * from zeros(10); insert into distinct_in_order select * from numbers(10); -select 'single-threaded distinct'; +select '-- single-threaded distinct'; select distinct a from distinct_in_order settings max_block_size=10, max_threads=1; -select 'multi-threaded distinct'; +select '-- multi-threaded distinct'; select distinct a from distinct_in_order settings max_block_size=10; -select 'table with not only primary key columns'; +select '-- create table with not only primary key columns'; drop table if exists distinct_in_order sync; create table distinct_in_order (a int, b int, c int) engine=MergeTree() order by (a, b); -insert into distinct_in_order select number % number, number % 10, number % 5 from numbers(1,1000000); -select 'distinct with key-prefix only'; +insert into distinct_in_order select number % number, number % 5, number % 10 from numbers(1,1000000); +select '-- distinct with primary key prefix only'; select distinct a from distinct_in_order; -select 'distinct with full key'; +select '-- distinct with full key'; select distinct a,b from distinct_in_order order by b; -select 'distinct with key prefix and non-sorted column'; +select '-- distinct with key prefix and non-sorted column'; select distinct a,c from distinct_in_order order by c; -- drop table if exists distinct_in_order sync; From 5661280ef6ed8ae53467d6273aecb4b6b2297211 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 23 Jun 2022 14:01:26 +0200 Subject: [PATCH 100/408] Remove some read methods --- .../MergeTree/DataPartStorageOnDisk.cpp | 20 +++++-------------- .../MergeTree/DataPartStorageOnDisk.h | 8 +------- src/Storages/MergeTree/IDataPartStorage.h | 8 +------- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 8 +++++--- src/Storages/MergeTree/MergeTreeDataWriter.h | 5 ++++- .../MergedColumnOnlyOutputStream.cpp | 4 +--- 6 files changed, 17 insertions(+), 36 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 083cbc90cb1..9b4e7654c98 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -773,15 +773,6 @@ DataPartStorageBuilderOnDisk::DataPartStorageBuilderOnDisk(VolumePtr volume_, st { } -std::unique_ptr DataPartStorageBuilderOnDisk::readFile( - const std::string & name, - const ReadSettings & settings, - std::optional read_hint, - std::optional file_size) const -{ - return volume->getDisk()->readFile(fs::path(root_path) / part_dir / name, settings, read_hint, file_size); -} - std::unique_ptr DataPartStorageBuilderOnDisk::writeFile( const String & name, size_t buf_size, @@ -795,6 +786,11 @@ void DataPartStorageBuilderOnDisk::removeFile(const String & name) return volume->getDisk()->removeFile(fs::path(root_path) / part_dir / name); } +void DataPartStorageBuilderOnDisk::removeFileIfExists(const String & name) +{ + return volume->getDisk()->removeFileIfExists(fs::path(root_path) / part_dir / name); +} + void DataPartStorageBuilderOnDisk::removeRecursive() { volume->getDisk()->removeRecursive(fs::path(root_path) / part_dir); @@ -829,12 +825,6 @@ bool DataPartStorageBuilderOnDisk::exists() const return volume->getDisk()->exists(fs::path(root_path) / part_dir); } - -bool DataPartStorageBuilderOnDisk::exists(const std::string & name) const -{ - return volume->getDisk()->exists(fs::path(root_path) / part_dir / name); -} - std::string DataPartStorageBuilderOnDisk::getFullPath() const { return fs::path(volume->getDisk()->getPath()) / root_path / part_dir; diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index d6fcb2f1442..11fc2cd2f6d 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -130,7 +130,6 @@ public: void setRelativePath(const std::string & path) override; bool exists() const override; - bool exists(const std::string & name) const override; void createDirectories() override; void createProjection(const std::string & name) override; @@ -139,18 +138,13 @@ public: std::string getFullPath() const override; std::string getRelativePath() const override; - std::unique_ptr readFile( - const std::string & name, - const ReadSettings & settings, - std::optional read_hint, - std::optional file_size) const override; - std::unique_ptr writeFile( const String & name, size_t buf_size, const WriteSettings & settings) override; void removeFile(const String & name) override; + void removeFileIfExists(const String & name) override; void removeRecursive() override; void removeSharedRecursive(bool keep_in_remote_fs) override; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 0e165e74ed0..89037e4a45a 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -223,20 +223,14 @@ public: virtual std::string getRelativePath() const = 0; virtual bool exists() const = 0; - virtual bool exists(const std::string & name) const = 0; virtual void createDirectories() = 0; virtual void createProjection(const std::string & name) = 0; - virtual std::unique_ptr readFile( - const std::string & name, - const ReadSettings & settings, - std::optional read_hint, - std::optional file_size) const = 0; - virtual std::unique_ptr writeFile(const String & name, size_t buf_size, const WriteSettings & settings) = 0; virtual void removeFile(const String & name) = 0; + virtual void removeFileIfExists(const String & name) = 0; virtual void removeRecursive() = 0; virtual void removeSharedRecursive(bool keep_in_remote_fs) = 0; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index db3580e1f86..e3925940553 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -414,12 +414,14 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai String mrk_path = escaped_name + marks_file_extension; String bin_path = escaped_name + DATA_FILE_EXTENSION; + auto data_part_storage = data_part_storage_builder->getStorage(); + /// Some columns may be removed because of ttl. Skip them. - if (!data_part_storage_builder->exists(mrk_path)) + if (!data_part_storage->exists(mrk_path)) return; - auto mrk_in = data_part_storage_builder->readFile(mrk_path, {}, std::nullopt, std::nullopt); - DB::CompressedReadBufferFromFile bin_in(data_part_storage_builder->readFile(bin_path, {}, std::nullopt, std::nullopt)); + auto mrk_in = data_part_storage->readFile(mrk_path, {}, std::nullopt, std::nullopt); + DB::CompressedReadBufferFromFile bin_in(data_part_storage->readFile(bin_path, {}, std::nullopt, std::nullopt)); bool must_be_last = false; UInt64 offset_in_compressed_file = 0; UInt64 offset_in_decompressed_block = 0; diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index 147b38e828a..c7680888bc0 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -34,7 +34,10 @@ using BlocksWithPartition = std::vector; class MergeTreeDataWriter { public: - explicit MergeTreeDataWriter(MergeTreeData & data_) : data(data_), log(&Poco::Logger::get(data.getLogName() + " (Writer)")) {} + explicit MergeTreeDataWriter(MergeTreeData & data_) + : data(data_) + , log(&Poco::Logger::get(data.getLogName() + " (Writer)")) + {} /** Split the block to blocks, each of them must be written as separate part. * (split rows by partition) diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 740e57a136e..d48a8b90646 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -81,9 +81,7 @@ MergedColumnOnlyOutputStream::fillChecksums( for (const String & removed_file : removed_files) { - /// Can be called multiple times, don't need to remove file twice - if (data_part_storage_builder->exists(removed_file)) - data_part_storage_builder->removeFile(removed_file); + data_part_storage_builder->removeFileIfExists(removed_file); if (all_checksums.files.contains(removed_file)) all_checksums.files.erase(removed_file); From fb249aca621e009c0bc283ac4231933b7df8bfcd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 23 Jun 2022 14:11:16 +0200 Subject: [PATCH 101/408] fix tests --- .../queries/0_stateless/00443_preferred_block_size_bytes.sh | 6 +++--- tests/queries/0_stateless/00616_final_single_part.sql | 1 + tests/queries/0_stateless/01162_strange_mutations.sh | 2 +- tests/queries/0_stateless/01298_alter_merge.sql | 1 + .../0_stateless/01528_clickhouse_local_prepare_parts.sh | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 5dcc2f3c181..399a4677a44 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" -$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO preferred_block_size_bytes (s) SELECT '16_bytes_-_-_-_' AS s FROM system.numbers LIMIT 10, 90" $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE preferred_block_size_bytes" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=26 -q "SELECT DISTINCT blockSize(), ignore(p, s) FROM preferred_block_size_bytes" @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" # PREWHERE using empty column $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS pbs" -$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1 -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO pbs (p, i, sa) SELECT toDate(i % 30) AS p, number AS i, ['a'] AS sa FROM system.numbers LIMIT 1000" $CLICKHOUSE_CLIENT -q "ALTER TABLE pbs ADD COLUMN s UInt8 DEFAULT 0" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=100000 -q "SELECT count() FROM pbs PREWHERE s = 0" @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE pbs" # Nullable PREWHERE $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" -$CLICKHOUSE_CLIENT --allow_deprecated_syntax_for_merge_tree=1-q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO nullable_prewhere SELECT toDate(0) AS p, if(number % 2 = 0, CAST(number AS Nullable(UInt64)), CAST(NULL AS Nullable(UInt64))) AS f, number as d FROM system.numbers LIMIT 1001" $CLICKHOUSE_CLIENT -q "SELECT sum(d), sum(f), max(d) FROM nullable_prewhere PREWHERE NOT isNull(f)" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" diff --git a/tests/queries/0_stateless/00616_final_single_part.sql b/tests/queries/0_stateless/00616_final_single_part.sql index 605f2c8b216..8c7720f8960 100644 --- a/tests/queries/0_stateless/00616_final_single_part.sql +++ b/tests/queries/0_stateless/00616_final_single_part.sql @@ -3,6 +3,7 @@ SET optimize_on_insert = 0; DROP TABLE IF EXISTS test_00616; DROP TABLE IF EXISTS replacing_00616; +set allow_deprecated_syntax_for_merge_tree=1; CREATE TABLE test_00616 ( date Date, diff --git a/tests/queries/0_stateless/01162_strange_mutations.sh b/tests/queries/0_stateless/01162_strange_mutations.sh index 504b8bac0e0..eea9ea5f7e5 100755 --- a/tests/queries/0_stateless/01162_strange_mutations.sh +++ b/tests/queries/0_stateless/01162_strange_mutations.sh @@ -28,7 +28,7 @@ do $CLICKHOUSE_CLIENT -q "CREATE TABLE test ENGINE=$engine AS SELECT number + 100 AS n, 0 AS test FROM numbers(50)" 2>&1| grep -Ev "Removing leftovers from table|removed by another replica" $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" if [[ $engine == *"ReplicatedMergeTree"* ]]; then - $CLICKHOUSE_CLIENT 0 -q "ALTER TABLE test + $CLICKHOUSE_CLIENT -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1 GROUP BY 'dummy')[n - 99] WHERE 1" 2>&1| grep -Fa "DB::Exception: " | grep -Fv "statement with subquery may be nondeterministic" $CLICKHOUSE_CLIENT --allow_nondeterministic_mutations=1 --mutations_sync=1 -q "ALTER TABLE test UPDATE test = (SELECT groupArray(id) FROM t1)[n - 99] WHERE 1" diff --git a/tests/queries/0_stateless/01298_alter_merge.sql b/tests/queries/0_stateless/01298_alter_merge.sql index 86c89c38c8c..24547086e0b 100644 --- a/tests/queries/0_stateless/01298_alter_merge.sql +++ b/tests/queries/0_stateless/01298_alter_merge.sql @@ -2,6 +2,7 @@ drop table if exists merge; drop table if exists merge1; drop table if exists merge2; +set allow_deprecated_syntax_for_merge_tree=1; create table merge1 ( CounterID UInt32, StartDate Date, Sign Int8, VisitID UInt64, UserID UInt64, StartTime DateTime, ClickLogID UInt64) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), tuple(CounterID, StartDate, intHash32(UserID), VisitID, ClickLogID), 8192, Sign); insert into merge1 values (1, '2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3); diff --git a/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh b/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh index 95ecbf09cf5..538d712ad9c 100755 --- a/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh +++ b/tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh @@ -8,7 +8,7 @@ WORKING_FOLDER_01528="${CLICKHOUSE_TMP}/01528_clickhouse_local_prepare_parts" rm -rf "${WORKING_FOLDER_01528}" mkdir -p "${WORKING_FOLDER_01528}/metadata/local" - +echo "ATTACH DATABASE local ENGINE=Ordinary" > "${WORKING_FOLDER_01528}/metadata/local.sql" ## Checks scenario of preparing parts offline by clickhouse-local ## that is the metadata for the table we want to fill From cb748cd8ec46b1962bfd28c41eeaab4b53e90c75 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 23 Jun 2022 16:11:48 +0300 Subject: [PATCH 102/408] Fix code style in KerberosInit --- src/Access/KerberosInit.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 9ca45b12531..ace03a5e0b5 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -47,12 +47,12 @@ private: krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; - String fmtError(krb5_error_code code); + String fmtError(krb5_error_code code) const; }; } -String KerberosInit::fmtError(krb5_error_code code) +String KerberosInit::fmtError(krb5_error_code code) const { const char *msg; msg = krb5_get_error_message(k5.ctx, code); @@ -75,7 +75,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co ret = krb5_init_context(&k5.ctx); if (ret) - throw Exception(fmt::format("Error while initializing Kerberos 5 library ({})", ret), ErrorCodes::KERBEROS_ERROR); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while initializing Kerberos 5 library ({})", ret); if (!cache_name.empty()) { @@ -160,7 +160,8 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); + LOG_TRACE(log,"Renew failed {}", fmtError(ret)); + LOG_TRACE(log,"Trying to get initial credentials"); // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) From 7f7d082fb30e120ddd0cf94ec8fccd01b2442ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 23 Jun 2022 15:23:37 +0200 Subject: [PATCH 103/408] Add implicit_transaction setting --- src/Core/Settings.h | 1 + .../InterpreterTransactionControlQuery.h | 1 - src/Interpreters/executeQuery.cpp | 94 ++++++++++++++----- .../02345_implicit_transaction.reference | 14 +++ .../02345_implicit_transaction.sql | 92 ++++++++++++++++++ 5 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/02345_implicit_transaction.reference create mode 100644 tests/queries/0_stateless/02345_implicit_transaction.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f1fd9d20f00..72ba2c0c13b 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -601,6 +601,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ + M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ diff --git a/src/Interpreters/InterpreterTransactionControlQuery.h b/src/Interpreters/InterpreterTransactionControlQuery.h index bf2dc7891a7..a66a740ce0c 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.h +++ b/src/Interpreters/InterpreterTransactionControlQuery.h @@ -20,7 +20,6 @@ public: bool ignoreLimits() const override { return true; } bool supportsTransactions() const override { return true; } -private: BlockIO executeBegin(ContextMutablePtr session_context); BlockIO executeCommit(ContextMutablePtr session_context); static BlockIO executeRollback(ContextMutablePtr session_context); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 24649128cee..ae622e5e1f0 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -42,18 +42,19 @@ #include #include #include -#include #include +#include +#include #include #include #include -#include #include +#include #include -#include -#include -#include #include +#include +#include +#include #include #include @@ -68,6 +69,7 @@ #include #include +#include #include @@ -416,7 +418,9 @@ static std::tuple executeQueryImpl( chassert(txn->getState() != MergeTreeTransaction::COMMITTING); chassert(txn->getState() != MergeTreeTransaction::COMMITTED); if (txn->getState() == MergeTreeTransaction::ROLLED_BACK && !ast->as() && !ast->as()) - throw Exception(ErrorCodes::INVALID_TRANSACTION, "Cannot execute query because current transaction failed. Expecting ROLLBACK statement."); + throw Exception( + ErrorCodes::INVALID_TRANSACTION, + "Cannot execute query because current transaction failed. Expecting ROLLBACK statement"); } /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), @@ -498,6 +502,7 @@ static std::tuple executeQueryImpl( BlockIO res; String query_for_logging; + std::shared_ptr implicit_txn_control{}; try { @@ -626,6 +631,27 @@ static std::tuple executeQueryImpl( } else { + /// We need to start the (implicit) transaction before getting the interpreter as this will get links to the latest snapshots + if (!context->getCurrentTransaction() && settings.implicit_transaction && !ast->as()) + { + try + { + /// If there is no session (which is the default for the HTTP Handler), set up one just for this as it is necessary + /// to control the transaction lifetime + if (!context->hasSessionContext()) + context->makeSessionContext(); + + auto tc = std::make_shared(ast, context); + tc->executeBegin(context->getSessionContext()); + implicit_txn_control = std::move(tc); + } + catch (Exception & e) + { + e.addMessage("while starting a transaction with 'implicit_transaction'"); + throw; + } + } + interpreter = InterpreterFactory::get(ast, context, SelectQueryOptions(stage).setInternal(internal)); if (context->getCurrentTransaction() && !interpreter->supportsTransactions() && @@ -813,15 +839,16 @@ static std::tuple executeQueryImpl( }; /// Also make possible for caller to log successful query finish and exception during execution. - auto finish_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - log_processors_profiles = settings.log_processors_profiles, - status_info_to_query_log, - pulling_pipeline = pipeline.pulling() - ] - (QueryPipeline & query_pipeline) mutable + auto finish_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + log_processors_profiles = settings.log_processors_profiles, + status_info_to_query_log, + implicit_txn_control, + pulling_pipeline = pipeline.pulling()](QueryPipeline & query_pipeline) mutable { QueryStatus * process_list_elem = context->getProcessListElement(); @@ -942,15 +969,30 @@ static std::tuple executeQueryImpl( opentelemetry_span_log->add(span); } + + if (implicit_txn_control) + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } }; - auto exception_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - quota(quota), status_info_to_query_log] () mutable + auto exception_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + quota(quota), + status_info_to_query_log, + implicit_txn_control]() mutable { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) txn->onException(); if (quota) @@ -1000,7 +1042,6 @@ static std::tuple executeQueryImpl( { ProfileEvents::increment(ProfileEvents::FailedInsertQuery); } - }; res.finish_callback = std::move(finish_callback); @@ -1009,8 +1050,15 @@ static std::tuple executeQueryImpl( } catch (...) { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) + { txn->onException(); + } if (!internal) { diff --git a/tests/queries/0_stateless/02345_implicit_transaction.reference b/tests/queries/0_stateless/02345_implicit_transaction.reference new file mode 100644 index 00000000000..e4dd35600f7 --- /dev/null +++ b/tests/queries/0_stateless/02345_implicit_transaction.reference @@ -0,0 +1,14 @@ +no_transaction_landing 10000 +no_transaction_target 0 +after_transaction_landing 0 +after_transaction_target 0 +after_implicit_txn_in_query_settings_landing 0 +after_implicit_txn_in_query_settings_target 0 +after_implicit_txn_in_session_landing 0 +after_implicit_txn_in_session_target 0 +inside_txn_and_implicit 1 +inside_txn_and_implicit 1 +in_transaction 10000 +out_transaction 0 +{"'implicit_True'":"implicit_True","all":"2","is_empty":0} +{"'implicit_False'":"implicit_False","all":"2","is_empty":1} diff --git a/tests/queries/0_stateless/02345_implicit_transaction.sql b/tests/queries/0_stateless/02345_implicit_transaction.sql new file mode 100644 index 00000000000..677affeec39 --- /dev/null +++ b/tests/queries/0_stateless/02345_implicit_transaction.sql @@ -0,0 +1,92 @@ +CREATE TABLE landing (n Int64) engine=MergeTree order by n; +CREATE TABLE target (n Int64) engine=MergeTree order by n; +CREATE MATERIALIZED VIEW landing_to_target TO target AS + SELECT n + throwIf(n == 3333) + FROM landing; + +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +SELECT 'no_transaction_landing', count() FROM landing; +SELECT 'no_transaction_target', count() FROM target; + +TRUNCATE TABLE landing; +TRUNCATE TABLE target; + + +BEGIN TRANSACTION; +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +ROLLBACK; +SELECT 'after_transaction_landing', count() FROM landing; +SELECT 'after_transaction_target', count() FROM target; + +-- Same but using implicit_transaction +INSERT INTO landing SETTINGS implicit_transaction=True SELECT * FROM numbers(10000); -- { serverError 395 } +SELECT 'after_implicit_txn_in_query_settings_landing', count() FROM landing; +SELECT 'after_implicit_txn_in_query_settings_target', count() FROM target; + +-- Same but using implicit_transaction in a session +SET implicit_transaction=True; +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +SET implicit_transaction=False; +SELECT 'after_implicit_txn_in_session_landing', count() FROM landing; +SELECT 'after_implicit_txn_in_session_target', count() FROM target; + +-- Reading from incompatible sources with implicit_transaction works the same way as with normal transactions: +-- Currently reading from system tables inside a transaction is Not implemented: +SELECT name, value, changed FROM system.settings where name = 'implicit_transaction' SETTINGS implicit_transaction=True; -- { serverError 48 } + + +-- Verify that you don't have to manually close transactions with implicit_transaction +SET implicit_transaction=True; +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SET implicit_transaction=False; + +-- implicit_transaction is ignored when inside a transaction (no recursive transaction error) +BEGIN TRANSACTION; +SELECT 'inside_txn_and_implicit', 1 SETTINGS implicit_transaction=True; +SELECT throwIf(number == 0) FROM numbers(100) SETTINGS implicit_transaction=True; -- { serverError 395 } +ROLLBACK; + +SELECT 'inside_txn_and_implicit', 1 SETTINGS implicit_transaction=True; + +-- You can work with transactions even if `implicit_transaction=True` is set +SET implicit_transaction=True; +BEGIN TRANSACTION; +INSERT INTO target SELECT * FROM numbers(10000); +SELECT 'in_transaction', count() FROM target; +ROLLBACK; +SELECT 'out_transaction', count() FROM target; +SET implicit_transaction=False; + + +-- Verify that the transaction_id column is populated correctly +SELECT 'Looking_at_transaction_id_True' FORMAT Null SETTINGS implicit_transaction=1; +-- Verify that the transaction_id column is NOT populated without transaction +SELECT 'Looking_at_transaction_id_False' FORMAT Null SETTINGS implicit_transaction=0; +SYSTEM FLUSH LOGS; + +SELECT + 'implicit_True', + count() as all, + transaction_id = (0,0,'00000000-0000-0000-0000-000000000000') as is_empty +FROM system.query_log +WHERE + current_database = currentDatabase() AND + event_date >= yesterday() AND + query LIKE '-- Verify that the transaction_id column is populated correctly%' +GROUP BY transaction_id +FORMAT JSONEachRow; + +SELECT + 'implicit_False', + count() as all, + transaction_id = (0,0,'00000000-0000-0000-0000-000000000000') as is_empty +FROM system.query_log +WHERE + current_database = currentDatabase() AND + event_date >= yesterday() AND + query LIKE '-- Verify that the transaction_id column is NOT populated without transaction%' +GROUP BY transaction_id +FORMAT JSONEachRow; From 2bdedf5e0cd366064d3dc4919e95748eb3a25ccd Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 23 Jun 2022 18:21:46 +0200 Subject: [PATCH 104/408] Some changes --- .../MergeTree/DataPartStorageOnDisk.cpp | 31 +++++++++++++------ .../MergeTree/DataPartStorageOnDisk.h | 4 +++ src/Storages/MergeTree/IDataPartStorage.h | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 20 +++++++----- src/Storages/MergeTree/MergeTreeData.h | 9 ++++-- .../MergeTree/MergeTreeDataWriter.cpp | 3 +- src/Storages/MergeTree/MergeTreeDataWriter.h | 1 + .../MergeTree/ReplicatedMergeTreeSink.cpp | 11 ++++--- .../MergeTree/ReplicatedMergeTreeSink.h | 6 +++- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 10 files changed, 61 insertions(+), 28 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 9b4e7654c98..265964b7dc0 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -768,8 +768,14 @@ void DataPartStorageOnDisk::changeRootPath(const std::string & from_root, const root_path = to_root.substr(0, dst_size) + root_path.substr(prefix_size); } -DataPartStorageBuilderOnDisk::DataPartStorageBuilderOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_) - : volume(std::move(volume_)), root_path(std::move(root_path_)), part_dir(std::move(part_dir_)) +DataPartStorageBuilderOnDisk::DataPartStorageBuilderOnDisk( + VolumePtr volume_, + std::string root_path_, + std::string part_dir_) + : volume(std::move(volume_)) + , root_path(std::move(root_path_)) + , part_dir(std::move(part_dir_)) + , transaction(volume->getDisk()->createTransaction()) { } @@ -778,27 +784,27 @@ std::unique_ptr DataPartStorageBuilderOnDisk::writeFile size_t buf_size, const WriteSettings & settings) { - return volume->getDisk()->writeFile(fs::path(root_path) / part_dir / name, buf_size, WriteMode::Rewrite, settings); + return transaction->writeFile(fs::path(root_path) / part_dir / name, buf_size, WriteMode::Rewrite, settings, /* autocommit = */ false); } void DataPartStorageBuilderOnDisk::removeFile(const String & name) { - return volume->getDisk()->removeFile(fs::path(root_path) / part_dir / name); + transaction->removeFile(fs::path(root_path) / part_dir / name); } void DataPartStorageBuilderOnDisk::removeFileIfExists(const String & name) { - return volume->getDisk()->removeFileIfExists(fs::path(root_path) / part_dir / name); + transaction->removeFileIfExists(fs::path(root_path) / part_dir / name); } void DataPartStorageBuilderOnDisk::removeRecursive() { - volume->getDisk()->removeRecursive(fs::path(root_path) / part_dir); + transaction->removeRecursive(fs::path(root_path) / part_dir); } void DataPartStorageBuilderOnDisk::removeSharedRecursive(bool keep_in_remote_fs) { - volume->getDisk()->removeSharedRecursive(fs::path(root_path) / part_dir, keep_in_remote_fs, {}); + transaction->removeSharedRecursive(fs::path(root_path) / part_dir, keep_in_remote_fs, {}); } SyncGuardPtr DataPartStorageBuilderOnDisk::getDirectorySyncGuard() const @@ -815,7 +821,7 @@ void DataPartStorageBuilderOnDisk::createHardLinkFrom(const IDataPartStorage & s "Cannot create hardlink from different storage. Expected DataPartStorageOnDisk, got {}", typeid(source).name()); - volume->getDisk()->createHardLink( + transaction->createHardLink( fs::path(source_on_disk->getRelativePath()) / from, fs::path(root_path) / part_dir / to); } @@ -837,12 +843,12 @@ std::string DataPartStorageBuilderOnDisk::getRelativePath() const void DataPartStorageBuilderOnDisk::createDirectories() { - return volume->getDisk()->createDirectories(fs::path(root_path) / part_dir); + transaction->createDirectories(fs::path(root_path) / part_dir); } void DataPartStorageBuilderOnDisk::createProjection(const std::string & name) { - return volume->getDisk()->createDirectory(fs::path(root_path) / part_dir / name); + transaction->createDirectory(fs::path(root_path) / part_dir / name); } ReservationPtr DataPartStorageBuilderOnDisk::reserve(UInt64 bytes) @@ -864,4 +870,9 @@ DataPartStoragePtr DataPartStorageBuilderOnDisk::getStorage() const return std::make_shared(volume, root_path, part_dir); } +void DataPartStorageBuilderOnDisk::commit() +{ + transaction->commit(); +} + } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index 11fc2cd2f6d..90dec12b283 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -158,10 +159,13 @@ public: DataPartStoragePtr getStorage() const override; + void commit() override; + private: VolumePtr volume; std::string root_path; std::string part_dir; + DiskTransactionPtr transaction; }; } diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 89037e4a45a..f75fbbbd5a3 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -243,6 +243,8 @@ public: virtual std::shared_ptr getProjection(const std::string & name) const = 0; virtual DataPartStoragePtr getStorage() const = 0; + + virtual void commit() = 0; }; using DataPartStorageBuilderPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c24636a56f8..54a881eda12 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2790,7 +2790,8 @@ bool MergeTreeData::renameTempPartAndAdd( SimpleIncrement * increment, Transaction * out_transaction, MergeTreeDeduplicationLog * deduplication_log, - std::string_view deduplication_token) + std::string_view deduplication_token, + DataPartStorageBuilderPtr part_builder) { if (out_transaction && &out_transaction->data != this) throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", @@ -2799,7 +2800,7 @@ bool MergeTreeData::renameTempPartAndAdd( DataPartsVector covered_parts; { auto lock = lockParts(); - if (!renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log, deduplication_token)) + if (!renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log, deduplication_token, part_builder)) return false; } if (!covered_parts.empty()) @@ -2818,12 +2819,19 @@ bool MergeTreeData::renameTempPartAndReplace( std::unique_lock & lock, DataPartsVector * out_covered_parts, MergeTreeDeduplicationLog * deduplication_log, - std::string_view deduplication_token) + std::string_view deduplication_token, + DataPartStorageBuilderPtr part_builder) { + + LOG_TRACE(log, "Renaming temporary part {} to {}.", part->data_part_storage->getPartDirectory(), part_name); + if (out_transaction && &out_transaction->data != this) throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", ErrorCodes::LOGICAL_ERROR); + if (part_builder) + part_builder->commit(); + if (txn) transactions_enabled.store(true); @@ -2853,8 +2861,6 @@ bool MergeTreeData::renameTempPartAndReplace( else /// Parts from ReplicatedMergeTree already have names part_name = part->name; - LOG_TRACE(log, "Renaming temporary part {} to {}.", part->data_part_storage->getPartDirectory(), part_name); - if (auto it_duplicate = data_parts_by_info.find(part_info); it_duplicate != data_parts_by_info.end()) { String message = "Part " + (*it_duplicate)->getNameWithState() + " already exists"; @@ -2960,7 +2966,7 @@ bool MergeTreeData::renameTempPartAndReplace( MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( MutableDataPartPtr & part, MergeTreeTransaction * txn, SimpleIncrement * increment, - Transaction * out_transaction, MergeTreeDeduplicationLog * deduplication_log) + Transaction * out_transaction, MergeTreeDeduplicationLog * deduplication_log, DataPartStorageBuilderPtr part_builder) { if (out_transaction && &out_transaction->data != this) throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", @@ -2969,7 +2975,7 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( DataPartsVector covered_parts; { auto lock = lockParts(); - renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log); + renameTempPartAndReplace(part, txn, increment, out_transaction, lock, &covered_parts, deduplication_log, "", part_builder); } return covered_parts; } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 00a56de9142..4074769926e 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -560,14 +560,16 @@ public: SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr, - std::string_view deduplication_token = std::string_view()); + std::string_view deduplication_token = std::string_view(), + DataPartStorageBuilderPtr part_builder = nullptr); /// The same as renameTempPartAndAdd but the block range of the part can contain existing parts. /// Returns all parts covered by the added part (in ascending order). /// If out_transaction == nullptr, marks covered parts as Outdated. DataPartsVector renameTempPartAndReplace( MutableDataPartPtr & part, MergeTreeTransaction * txn, SimpleIncrement * increment = nullptr, - Transaction * out_transaction = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr); + Transaction * out_transaction = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr, + DataPartStorageBuilderPtr part_builder = nullptr); /// Low-level version of previous one, doesn't lock mutex /// FIXME Transactions: remove add_to_txn flag, maybe merge MergeTreeTransaction and Transaction @@ -579,7 +581,8 @@ public: DataPartsLock & lock, DataPartsVector * out_covered_parts = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr, - std::string_view deduplication_token = std::string_view()); + std::string_view deduplication_token = std::string_view(), + DataPartStorageBuilderPtr part_builder = nullptr); /// Remove parts from working set immediately (without wait for background /// process). Transfer part state to temporary. Have very limited usage only diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index f7c544132bb..bbd48caabaf 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -585,10 +585,9 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( out->writeWithPermutation(block, perm_ptr); auto finalizer = out->finalizePartAsync(new_data_part, false); temp_part.part = new_data_part; + temp_part.builder = data_part_storage_builder; temp_part.streams.emplace_back(TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); - // out.finish(new_data_part, std::move(written_files), false); - ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterUncompressedBytes, block.bytes()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterCompressedBytes, new_data_part->getBytesOnDisk()); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index c7680888bc0..29c7baaa775 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -54,6 +54,7 @@ public: struct TemporaryPart { MergeTreeData::MutableDataPartPtr part; + DataPartStorageBuilderPtr builder; struct Stream { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index d217e16c830..1b2f29baabf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -254,7 +254,7 @@ void ReplicatedMergeTreeSink::finishDelayedChunk(zkutil::ZooKeeperPtr & zookeepe try { - commitPart(zookeeper, part, partition.block_id); + commitPart(zookeeper, part, partition.block_id, partition.temp_part.builder); last_block_is_duplicate = last_block_is_duplicate || part->is_duplicate; @@ -289,7 +289,7 @@ void ReplicatedMergeTreeSink::writeExistingPart(MergeTreeData::MutableDataPartPt try { part->version.setCreationTID(Tx::PrehistoricTID, nullptr); - commitPart(zookeeper, part, ""); + commitPart(zookeeper, part, "", nullptr); PartLog::addNewPart(storage.getContext(), part, watch.elapsed()); } catch (...) @@ -301,7 +301,10 @@ void ReplicatedMergeTreeSink::writeExistingPart(MergeTreeData::MutableDataPartPt void ReplicatedMergeTreeSink::commitPart( - zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) + zkutil::ZooKeeperPtr & zookeeper, + MergeTreeData::MutableDataPartPtr & part, + const String & block_id, + DataPartStorageBuilderPtr part_builder) { metadata_snapshot->check(part->getColumns()); assertSessionIsNotExpired(zookeeper); @@ -476,7 +479,7 @@ void ReplicatedMergeTreeSink::commitPart( try { - renamed = storage.renameTempPartAndAdd(part, NO_TRANSACTION_RAW, nullptr, &transaction); + renamed = storage.renameTempPartAndAdd(part, NO_TRANSACTION_RAW, nullptr, &transaction, nullptr, "", part_builder); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 41953e034fa..f7504d2f784 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -69,7 +69,11 @@ private: void checkQuorumPrecondition(zkutil::ZooKeeperPtr & zookeeper); /// Rename temporary part and commit to ZooKeeper. - void commitPart(zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id); + void commitPart( + zkutil::ZooKeeperPtr & zookeeper, + MergeTreeData::MutableDataPartPtr & part, + const String & block_id, + DataPartStorageBuilderPtr part_builder); /// Wait for quorum to be satisfied on path (quorum_path) form part (part_name) /// Also checks that replica still alive. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e93399918ef..77ca206eb23 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8020,7 +8020,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP try { MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); - auto replaced_parts = renameTempPartAndReplace(new_data_part, NO_TRANSACTION_RAW, nullptr, &transaction); + auto replaced_parts = renameTempPartAndReplace(new_data_part, NO_TRANSACTION_RAW, nullptr, &transaction, nullptr, data_part_storage_builder); if (!replaced_parts.empty()) { From 20614b36782d8c0dffb0550cd9dcf9df24dc7304 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 15:31:20 +0200 Subject: [PATCH 105/408] Improve runners AMI and init scripts - Install CloudWatch agent and atop - Update runner version, that supports hooks - Provide start and complete job hooks - Will update docker version implicitly to 20.10.17~3 --- tests/ci/worker/init_runner.sh | 55 ++++++++++++++++++++++++++-- tests/ci/worker/ubuntu_ami_for_ci.sh | 11 +++++- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 6838d925500..90466892fc9 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -14,14 +14,60 @@ export RUNNER_HOME=/home/ubuntu/actions-runner export RUNNER_URL="https://github.com/ClickHouse" # Funny fact, but metadata service has fixed IP -INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) +INSTANCE_ID=$(ec2metadata --instance-id) export INSTANCE_ID # combine labels -RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" | jq '.Tags[] | select(."Key" == "github:runner-type") | .Value' -r) +RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text) LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE" export LABELS +# Refresh CloudWatch agent config +aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json +systemctl restart amazon-cloudwatch-agent.service + + +# Create a pre-run script that will restart docker daemon before the job started +mkdir -p /tmp/actions-hooks +cat > /tmp/actions-hooks/pre-run.sh << 'EOF' +#!/bin/bash +set -xuo pipefail + +EOF + +cat > /tmp/actions-hooks/post-run.sh << 'EOF' +#!/bin/bash +set -xuo pipefail + +# Free KiB, free percents +ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}')) +if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then + echo "Going to terminate the runner, it has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /" + INSTANCE_ID=$(ec2metadata --instance-id) + ( sleep 10 && aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" ) & + exit 0 +fi + +# shellcheck disable=SC2046 +docker kill $(docker ps -q) ||: +# shellcheck disable=SC2046 +docker rm -f $(docker ps -a -q) ||: + +# If we have hanged containers after the previous commands, than we have a hanged one +# and should restart the daemon +if [ "$(docker ps -a -q)" ]; then + for i in {1..5}; + do + sudo systemctl restart docker && break || sleep 5 + done + + for i in {1..10} + do + docker info && break || sleep 2 + done +fi +EOF + while true; do runner_pid=$(pgrep run.sh) echo "Got runner pid $runner_pid" @@ -38,7 +84,10 @@ while true; do sudo -u ubuntu ./config.sh --url $RUNNER_URL --token "$RUNNER_TOKEN" --name "$INSTANCE_ID" --runnergroup Default --labels "$LABELS" --work _work echo "Run" - sudo -u ubuntu ./run.sh & + sudo -u ubuntu \ + ACTIONS_RUNNER_HOOK_JOB_STARTED=/tmp/actions-hooks/pre-run.sh \ + ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/tmp/actions-hooks/post-run.sh \ + ./run.sh & sleep 15 else echo "Runner is working with pid $runner_pid, nothing to do" diff --git a/tests/ci/worker/ubuntu_ami_for_ci.sh b/tests/ci/worker/ubuntu_ami_for_ci.sh index 23d3b18c810..2552ce58690 100644 --- a/tests/ci/worker/ubuntu_ami_for_ci.sh +++ b/tests/ci/worker/ubuntu_ami_for_ci.sh @@ -3,7 +3,7 @@ set -xeuo pipefail echo "Running prepare script" export DEBIAN_FRONTEND=noninteractive -export RUNNER_VERSION=2.285.1 +export RUNNER_VERSION=2.293.0 export RUNNER_HOME=/home/ubuntu/actions-runner deb_arch() { @@ -28,6 +28,7 @@ apt-get update apt-get install --yes --no-install-recommends \ apt-transport-https \ + atop \ binfmt-support \ build-essential \ ca-certificates \ @@ -96,3 +97,11 @@ aws lambda invoke --region us-east-1 --function-name team-keys-lambda /tmp/core. jq < /tmp/core.keys -r '.body' > /home/ubuntu/.ssh/authorized_keys2 chown ubuntu: /home/ubuntu/.ssh -R chmod 0700 /home/ubuntu/.ssh + +# Download cloudwatch agent and install config for it +wget --directory-prefix=/tmp https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/"$(deb_arch)"/latest/amazon-cloudwatch-agent.deb{,.sig} +gpg --recv-key --keyserver keyserver.ubuntu.com D58167303B789C72 +gpg --verify /tmp/amazon-cloudwatch-agent.deb.sig +dpkg -i /tmp/amazon-cloudwatch-agent.deb +aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json +systemctl enable amazon-cloudwatch-agent.service From 2339906e2ae5ac7d5df4a8d31ec2b6ec4a9ebf54 Mon Sep 17 00:00:00 2001 From: Larry Luo Date: Thu, 23 Jun 2022 11:30:00 -0700 Subject: [PATCH 106/408] Adding TLS V13 Test --- tests/integration/test_tlsv1_3/__init__.py | 0 .../test_tlsv1_3/certs/ca-cert.pem | 32 +++ .../test_tlsv1_3/certs/ca-cert.srl | 1 + .../integration/test_tlsv1_3/certs/ca-key.pem | 52 ++++ .../test_tlsv1_3/certs/client1-cert.pem | 30 +++ .../test_tlsv1_3/certs/client1-key.pem | 52 ++++ .../test_tlsv1_3/certs/client1-req.pem | 27 ++ .../test_tlsv1_3/certs/client2-cert.pem | 30 +++ .../test_tlsv1_3/certs/client2-key.pem | 52 ++++ .../test_tlsv1_3/certs/client2-req.pem | 27 ++ .../test_tlsv1_3/certs/client3-cert.pem | 30 +++ .../test_tlsv1_3/certs/client3-key.pem | 52 ++++ .../test_tlsv1_3/certs/client3-req.pem | 27 ++ .../test_tlsv1_3/certs/dhparam4096.pem | 13 + .../test_tlsv1_3/certs/generate_certs.sh | 23 ++ .../test_tlsv1_3/certs/server-cert.pem | 31 +++ .../test_tlsv1_3/certs/server-ext.cnf | 1 + .../test_tlsv1_3/certs/server-key.pem | 52 ++++ .../test_tlsv1_3/certs/server-req.pem | 27 ++ .../test_tlsv1_3/certs/wrong-cert.pem | 32 +++ .../test_tlsv1_3/certs/wrong-key.pem | 52 ++++ .../test_tlsv1_3/configs/ssl_config.xml | 73 ++++++ .../configs/users_with_ssl_auth.xml | 22 ++ tests/integration/test_tlsv1_3/test.py | 236 ++++++++++++++++++ 24 files changed, 974 insertions(+) create mode 100644 tests/integration/test_tlsv1_3/__init__.py create mode 100644 tests/integration/test_tlsv1_3/certs/ca-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/ca-cert.srl create mode 100644 tests/integration/test_tlsv1_3/certs/ca-key.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client1-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client1-key.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client1-req.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client2-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client2-key.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client2-req.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client3-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client3-key.pem create mode 100644 tests/integration/test_tlsv1_3/certs/client3-req.pem create mode 100644 tests/integration/test_tlsv1_3/certs/dhparam4096.pem create mode 100755 tests/integration/test_tlsv1_3/certs/generate_certs.sh create mode 100644 tests/integration/test_tlsv1_3/certs/server-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/server-ext.cnf create mode 100644 tests/integration/test_tlsv1_3/certs/server-key.pem create mode 100644 tests/integration/test_tlsv1_3/certs/server-req.pem create mode 100644 tests/integration/test_tlsv1_3/certs/wrong-cert.pem create mode 100644 tests/integration/test_tlsv1_3/certs/wrong-key.pem create mode 100644 tests/integration/test_tlsv1_3/configs/ssl_config.xml create mode 100644 tests/integration/test_tlsv1_3/configs/users_with_ssl_auth.xml create mode 100644 tests/integration/test_tlsv1_3/test.py diff --git a/tests/integration/test_tlsv1_3/__init__.py b/tests/integration/test_tlsv1_3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_tlsv1_3/certs/ca-cert.pem b/tests/integration/test_tlsv1_3/certs/ca-cert.pem new file mode 100644 index 00000000000..293e1c7f564 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/ca-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIUVRNcr0jCH3vSTxg8QYQH6CCtyF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALwojNvu +fXQYQ4tucqNOEDHf2sNgxwxqY6QdtJ+zNfVjsK4I3Vqo8TtzxfDYGolkYem/bYJM +xQar9ehUm9ok/0kJgIo8vDXxxDJtvjz5Fd5oFWJLMxojLE9NSa0A4m18jGfbFNsF +XoU0njiInyzNaU9d4bMpaweseCZdt9Y4LR93FkuhSU/v18lWQPob8SSIij059IZP +sEUxpDOTxclAmG/Knd/6v3ecVFiQgexZM0gCtf7kcw41mxsAaP/mOexodIZDR70Y +LYjL7R2ZGhpClfQc8SO5NSpfEqsfreDX7XoaCTsy7/rqr3Nfiby6sc//awG0Ww/f +FRf2+2BU2xEwOVa3i5wU5raYY6eqFLK9q9c2IWPSqYzAmvhK2pqWQ/iaCU/Q89ow +SbKudJTLK8Y6v9LW4Q8ZLZF+CzS5cI+QEfIYqTLFdInH1BLoxx7cymEv07CDkcTo +2WtV8GdMph2P3U/9NoXQDonjCSj0lQUjgUdcrBPaIIVbIn6/5vfw8LQa8PoGDhIx +AYQkqPR+LHxCqIMzdqKZ+OXD/HPhiigpxLhF7mVRLvvoyrOZVJbcu1qmgCcQw0IE +fWzvWne+9cYC9lgt8+/k6d6B1uhYsIwwhgoj0dffFjc0sF6zfceGK+H1K2JCE0aY +zT1HlvSoZdA7lEs5xbGJnkBHqlOvQ63ynXCzAgMBAAGjUzBRMB0GA1UdDgQWBBTn +AtgFU20JF7kTZCKlY7/hi0kYRzAfBgNVHSMEGDAWgBTnAtgFU20JF7kTZCKlY7/h +i0kYRzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCpiWgJ1XUw +a8Bdeznsa57oy+5mqQZWpRVkzTQRHEGV850OGh7WQ6u9kVAHefaHH9hsVxyggton +6/MDsu4KL5jqKmJaIAepPIOw6DTc2zs044I7W/rxRp+w1hL2TS+EahMrSPwdzCcl +NNAM0dXocGylf6qwwMqiYAR1K3UIrlyq4QTr1oEPIqJBkDg1JDYrt4T2DroPjW20 +5hlCQ/tft5ddGL0EFEaKWwAcPFm7jAwJiz2eUqmT6PcmaZ24qPn5RXVkaBAkrSga +1WgM8r3LGu2EKhdiDc5hRJKjS8RZyLvZNNzlL3+N42nGmGZkND5bV6u82OD+qn17 +LRZOt0Cr70HqszSYk/67ijjaa4n/fuuAqorV+yYB8accRXtoi00nxykT+H+yI1rD +swvcrfDvhUgY5zmunWyQUYh0q/2Hj75GbLup3Cd0B4MrBwqyCqcEugM4OSf6aRMr +e/vjeggTVPN08xE1LUkugalx0B0aoO6qFahJ2CmkAcYLLlS2N+F7TMuPavc0kVxD +I3qA5G9zvNCliSLX2+kM+LzslI8+pP/A98bvh6nW4HtZkI0jq1ks7XR0GeOhCI8E +0l/YuElxxgKhN4INKhhMoDKqPib4z8gbmkenR2CenQCpfLMIrhTXZgtw+gvEgpIE +/QK97G8XPqga6zn471wrYJnuyJli+sP7aw== +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/ca-cert.srl b/tests/integration/test_tlsv1_3/certs/ca-cert.srl new file mode 100644 index 00000000000..c02cd0a4526 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/ca-cert.srl @@ -0,0 +1 @@ +05F10C67567FE30795D77AF2540F6AC8D4CF2461 diff --git a/tests/integration/test_tlsv1_3/certs/ca-key.pem b/tests/integration/test_tlsv1_3/certs/ca-key.pem new file mode 100644 index 00000000000..e85dca8553e --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/ca-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC8KIzb7n10GEOL +bnKjThAx39rDYMcMamOkHbSfszX1Y7CuCN1aqPE7c8Xw2BqJZGHpv22CTMUGq/Xo +VJvaJP9JCYCKPLw18cQybb48+RXeaBViSzMaIyxPTUmtAOJtfIxn2xTbBV6FNJ44 +iJ8szWlPXeGzKWsHrHgmXbfWOC0fdxZLoUlP79fJVkD6G/EkiIo9OfSGT7BFMaQz +k8XJQJhvyp3f+r93nFRYkIHsWTNIArX+5HMONZsbAGj/5jnsaHSGQ0e9GC2Iy+0d +mRoaQpX0HPEjuTUqXxKrH63g1+16Ggk7Mu/66q9zX4m8urHP/2sBtFsP3xUX9vtg +VNsRMDlWt4ucFOa2mGOnqhSyvavXNiFj0qmMwJr4StqalkP4mglP0PPaMEmyrnSU +yyvGOr/S1uEPGS2Rfgs0uXCPkBHyGKkyxXSJx9QS6Mce3MphL9Owg5HE6NlrVfBn +TKYdj91P/TaF0A6J4wko9JUFI4FHXKwT2iCFWyJ+v+b38PC0GvD6Bg4SMQGEJKj0 +fix8QqiDM3aimfjlw/xz4YooKcS4Re5lUS776MqzmVSW3LtapoAnEMNCBH1s71p3 +vvXGAvZYLfPv5OnegdboWLCMMIYKI9HX3xY3NLBes33Hhivh9StiQhNGmM09R5b0 +qGXQO5RLOcWxiZ5AR6pTr0Ot8p1wswIDAQABAoICAQCO/c4Wccb7TFlAhD4wpumd +zX5GDq0WXV+94CldWGdARnOFvwzhkhRJ1zDtWH3KPfQ/HJBPfqIY8OQfnPUYMhej +3MnHxGJQKJyuqkHxumYJMFZX7cg3K9XHqne8NzjcddOKNa9Cx3DOkG9RjVpSRQSs +IS+d5XMGUOa6WWyVKvn3uJvD/B1n12DJDHiy2jtHRVCxOPMAg1z1KMWdwMaFrEZs +ZrHV/ow1jSN4btGd2SgkqJLA08IwYUKvoX8qQj9wzu0G/+hr5wzrsfZQEQMKQ+IL +s1b6jAzAV6IrVBbjEZXSviiXyZ0gteuCJW/acpMg+/3JPNQbWrCAFt1wluwowto/ +JAFIvlh29hfE5c+HEMpQNa0tdj7jepBn/0YEbgwpayMikKiLZXEpgheWCGypAQWp +Hm+N0Ym7HSGe82obxi8EjKRnNwFUtotWzUBKeo9aFwPZHLFlspljd+5ynDvKqXnk +txYZj6K3TtMs30HAG6fqxSPyiZ5W+5yF7nt6qLODs6m4Os+lrk1GnoqC0/uLMzIU +CRJKulrJOK4/Z2tPn9IAhcREbS4oROUeNqqo0Cfs3ssvkV7JTHF4IsKhCmElMmGa +bevOI+pvdjfECShy0Jnbtni6ece/II4/edfUp9kWN45xZLpzDjfqCVD66JS9g6ZU +i/EVll+d5zaI2TzzwZgHUQKCAQEA3d8siwXbq7x0cAB013+tvkvGMJ2EuS1TWdLk +a2P6CAnlZMWvv2cPSd2WpimHjqKxrbn6VE79mOc2l9Y1NOUUWWZATrhN7V8xMapQ +0YiYCHeaMERUAUKdzCgRN2/mRbZCBzpPBbWbb6NtKfRFJsD9zAe2JBwDVh9hvAL8 +YVBoczrEfj1ILnmtPhAJVI6s6rDsA4MgKjLs0Tt7Cc7rQxqNSpHEvwv1yLQmjp0N +L5b1TEt7fqVJ9dirykJquBYEKf55Z1qZhQzmnbu9OPnzeqGDakl5F/UsXDB5Bokp +ilcV+nFbh175Q+gTEhaSacGW8gzRw6j18PuciBjeWVEM5hhxOwKCAQEA2RnRMjv9 +46jQarJTFbIHg1SqrR87GSLnt6672M5TX9frzxMCuVDjKgdecstvLjm6X+/cPQKT +Q3javJnJXT4cx//1J7RLO6ZBVSCZf3//XntdHdFVJf5ySQtK+MJyfxjpzP6KBPfb +WPrva8p29ejbBdtsOT0M6gY5tPfadU2XEaf+BoyX9NUmu1U46Iqi+eCOjR+GVvhP +pJzGgLeOsaRVCfc9I7XPoVu3AEx5Kt55yRYm4fyGPsAd+mRDbIXMXdL0k8CfWWDr +8TT5rqKI+gFPFQCwToBW3DwHIGY+3RmoXFfQ0IJaKwOk4AB7m6HC3mv1crtjTFSM +9p74oQzNX7UG6QKCAQBEs2cygRTdH5SaXbnQRKvC4emzggLn5/4IMUIjcqioNpA+ +XOwngzz7rU6JkxBzfTMxTQYTdwYVg3qnF2AQSeK8L+o3teADYVd1PnyZ9QbGkGpB +CddNMJh17+4s0UxnR6E4Zbi0VuCTd/JEbGvBLT8pHzYqBjaOQ1dbBT2q0GAXVhoj +0Mv6ABlBv2t0MF2gqjnaeI7MIkqsGxPlHJpChAU+EtbuJUDs7cOGo2DC3KaGAlVy +CLJXGslO7rPm3oJZkn97HlWtGiqKquhTrSnUThDIJ4oEfhlHTocbG/ut53tZuiIS +T7k1arYFAtJBRv17Y7bMNBQ7k12L0s9+rpck5GqjAoIBAQCVBPSkj6tZbpII+viu +5rHjguVYyhwtx9jYK1eDnTR7kGGrlPgErjIPslkxYNSjHTsCCUnakv70jGtQlBs1 +JqJo4hesNkSB4D/uJ99VNk3a08D566uP1dUqsFa44/flp/ssG/gvKtbkf/KBwcrg +RwK4RYJG09IefUF1J8BLToQIuZBTfIP9qaXZZskWTbtK28ndsqrq3a0FaBuVVOnc +o9k/avcLoQuxTZwS12tAcs+TqOHtswGO5x5stg/V2Q2LxXbeSJTYq/+oZN2R8r0l +JmrbFsruR4fXylh189jouWjoYdrSlPdBmVG99HbkQCfbtq0XIOsrBMpxqnMtUPVT +4ZWpAoIBAQCrao4XHpRM3KsfPqdkg0vqFDBA+LzKlWu1fl8w5TGpFK8H1tv5kHNv +h0XmeU5cXwiweri3KjQz7h+qVBHZeAvyrEoxJQdImax+5OUy7lysDs+SL02gLZb3 +Z7g+u4Buwx+cv4O7hwUEDDk/5X3NBFk7/iwztKUtM+Fl8jt9K3K24s87xXp9YevI +UEawden9jVcuXQjEUrwz8cjoz/y25vK5pQ6k82PVImkMZK99/PmgmGyOl7hdRA3F +ff0Kb8pRGmV/cWRKzHaC8QchW8jdU2EGxMkrFl1DvoVKLbyDf1glRncKP9iozHAR ++s184IJCUvyMxH83uKKAiBGaDRC+Lbm7 +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/certs/client1-cert.pem b/tests/integration/test_tlsv1_3/certs/client1-cert.pem new file mode 100644 index 00000000000..bd6eea62094 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client1-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRfMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0fao +RrITeF4kpN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnT +pwWYPwJ+pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZ +DSl2YHqNs4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivI +InrUWc4RXXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8 +yg0hCB7Y5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mn +zyHPRk4pcn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHes +yYHoQw1HxU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBF +DX3Zj6nkuJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1 +FQA1ABXx2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4 +L/qXR/IlbkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwz +y770PGkhp1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAHwRpqnpcD3EW588GSDZhZrVf3nS9M06ljQGtDUqNSI4XJp1cVT1sMaa4LjM +cWgHtayFw+jbDLwioXHjMlV+8tERH+0x+qsADG349caDYT/OF13v/jyuboUZ9AqE +KpfOQH7jCLU7rEbEl6kvT3F3xaHJg8mE7msyRFfguB2JrqZkKIj4HANxJUJo4PwB +5bq9nE3AVNAgUeQEwfu0r5SjroNpcHfm7xWqMK2mDMCsy/DvI7n97Q7vZajcTT0x +UXfgx+3CLEvLMpa2myE5OIMOeLzfZwxrxyNH7BdZsROnkGv1cX+9HZpYcue/UDxp +P2OApbTuZKaTJOyMADc17s0seE0DTAHnHAWrJwVhf8wYKKtEs+i+Sw5LNSkh5fgS +hTzGF93yClDYzWEqMSKhKPeimtpz4ZBNuGf471KbpVbUKJJvJmOxqoZ5S0kpFILL +YMALf652uf5or5d0cDNvcJTwvMi6evchIV17d/jH+MxyJQs9VCkMpJxFbMrXb3YB +b57K3Z25P6w3Qfj4zuKQFANari7Gs6qSiaUBiEhEdTQlGspkq+FLndtX818sbMk5 +LAK6JaUH0ywV2jn5XSW0irQLDXqb6Q0bSyw6pdpDjk0o4UW67JCE4kGagRDnfSqL +ZODvO/dEtVLyAsjmOx8MkqLyseI7VESVd8eiJAyL0sifh+/E +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/client1-key.pem b/tests/integration/test_tlsv1_3/certs/client1-key.pem new file mode 100644 index 00000000000..8bc1e656566 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client1-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDAVNH2qEayE3he +JKTfNae6oq1/xnOevgrseru0RdTullPb/leYy8PHIUGfO8+EEqxYk9WZ06cFmD8C +fqQ5OgFhKIirjmnHjodG42QrsQAx7PqkK8nuQ/Bl78hb1hNnE9aQ+STRGQ0pdmB6 +jbOLkxkT/QT/Ne4kQ3Lr3caWp2lsX1xn3oSUTIhFYCg7PkVKaFuGlTYryCJ61FnO +EV15M2iSqoaQlN4wCQVeI0g+WZwPD1H3KAI+s+k9yOqYbg30T+ZXQN26/MoNIQge +2OUKzuZUQIBKCLjZwvlOvwCMFjyN+kY4sINxFMCPwSIi+AFo85I0na9Jp88hz0ZO +KXJ9NU9Bk6UIurX2Wabpgw9y5D7hMvbLzKSfJ7ag5wLCCVY24NUutIR3rMmB6EMN +R8VNL653McHEJFWjy8dLhpYqGi5b7vNwFZ1l28pWV7Ff6S8unRNx+dRgRQ192Y+p +5LiYrZPP4TaeONPy73u0mihT6OTXR/vAAlhnRXc988YD7oQQ6MRHqo+HNRUANQAV +8dmpFt5UOVcUwIOQGDkb1YSU6M6LK/ip9zJUXWY5feLZ1Qcg/DBiyxNAuC/6l0fy +JW5Dgs+Sx2qXQuY+tmRAcIhJZjDv7TBpeDkVfbXUwa0WfQzJinpNhPEMM8u+9Dxp +IadXE82belH2RMpqaMJn47YmXxyKtQIDAQABAoICAAEBsKOg19XgwjWD7ZT5e+o/ +JbdQe5RuHDKGperYnres871oBF9ZWan2I5jIwFpJmrtP8sM+V1ZxKItDzGo8QnuW +sbhsI2OW/GBDmmecIosgWWN4kzL7CgwOiDbq1OkqMmpJ04aAohAAfZrGmRT27R+s +qFUJnDh2XeicHYj2UVfu29XzVTBNgj0StsMwnT45c5ktuL3b60pHSD0K3DlhKn/y +AohJLyyDL5MBjkQ9RdLSWrR3ciOP332iSpAHq20G6ga04TQ0VH5jGN7IddJrqMry +F3nLt+Pz4EgoOcGB8Ekx8SIk0ltKJ4PZF+uk7qT0+WPrG1rAVRYxNoX8M4wyNjr4 +TcAZsV2DnGdnp+2u0SSzMczeop5hPTJKxaLaPw1JOoIk5fqW94MbEHqGnEXEIN+D +OWeUKWZ/B1YubavOeR+c3STZrh2SgmhKk6g5NMFlfnyvolPu47H8NOrewOhVG+TZ +gsQoGxSyOXwZTQ/Jd6Yg9lek8nKJBc4Res7ia/x3H+gjjRoNFI+L2HQnWztx5YMZ +H9M6hcpclZubO/w4iLq9OB2QUHn7aIT3lWRV/xS0Yh2zGCufasaMA1KSKC5zq0Fk +gCzAkYDq/ymrJs3LQQ0wegKd1akL4z5fxmXTn2v2BGoEd52uuxhL0mM/9zzRxdR2 +IsOgAym+siLXMCHTDbdVAoIBAQDuMcea66WKidS+A9frCEsabYccKzrdMEhs6Mle +orFieMC+3ZpzFIBkXPZ522I+M4nIdBKuRw9PnYTE5t30euOj60Oq905j2a+Ho4ki +kW6dC+tNDF49Hqxn9e99xbvTUi97dREcERlHA+AnRektEciyD17bi88aUy9w83Mw +G5Z+ej+9o40w8+TDopE2SIJhUAHR6LOAMq1v5y1lmTn0sbTuxZFLA0qWX9aGLi+T +4RD0MzJAtKJDbr3yPTLHAXmaMSKHhWYYgWTH9iwEhGQAm5VJy3oNJUkM7ej7Yfs7 +aTDOk61egCKhEHdWavP68MqmNOPHgnq4/edmvQnhfKtI8SMnAoIBAQDOtWDi/OnU +ZjZPnmJwwoPuXe6IjYg47bFRGv94xEpSesCAYdXNaNLPl0f/Ut9y3nXr+j+XqJWo +UqtRGFu2i9lUK3cu90GLXEaLbYWGcgL8YnJu0senLxkqxPWcGxoKmbo3xMjqk/pF +EVZ5e1qqVTlrB4q7QWmLKrS8YlcaTnChPeSBRFfryg/xvQ11Hxtq89SKkTH4ps16 +0KtiCxvfQHVASyRLIKLdyabPInB+yP3Fsn4BIx8jGtOQ/OCY01TXq9OyaRu2hJTk +qsjOLnqf6huM2so3X0Tw8AdgNoF96JJvfhwiPI5CSo9UKjhuvus1Ip5ZFFNo4Ngy +n3Zlgp1HxZzDAoIBAQC9ffqmo3sxqI8Hj3UxdIqS/rlyzm1o0+V6RwMT92gYx6nG +7fLWRGQT8+TdcotIoqWlQ7oszTlABDdAkc3XlgANQre1hkLlqqM6y/3n8zzFUVsj +E4jRJNrRZdTeAPV4mzRNCgfPhUbPuSSU+cgT48b+6L10+VeMQMtIF1T226uw+L5G +tps3a3/9pxHQ1oRquESKYo6SmT5i/M2fuvNhWBJxtdjtjTPER4AZhRqykWV0cFo1 +Ib7I2Ivh74+6w9Ciux4WJCjhq+aqMYw5F72awitU5rw1QwlHcOldO0irrfZ3EQLm +YBesfLYDmNh6NR9ydDcVXBcXnl593DvFF/IH+FYXAoIBAQCQZydLCzHy3oC8eEH+ +0fRGljooDO+IDYzcwwaLgF0HZ5eJWE97EuqKeP2kAWn2HjC07Hp2YSBDmZTyrxiK +2wG1CjRVjAeu6oShrJ4mAQnS9JdKkldFlOJ4/WUza79yflgX05IkRcIFdAo8DY+W +BLl66qbhD95CiU//dpew2fFWwx0ZrPvazar7zn1TP6rwuWvWbX5CXYyYaqP/dxE+ +khIXGyc8kI0WcWPlugJqn9CgxoO+GaIL7Ra1Z+MjACd6DyBxt3nTtKUrZZ+oYdHq +Wypp6QJxUk2gH56XeRxXMBz0ZF4VEMa0ys98FY6c1yULVqbWRhvK3aBLJRkZ6vgj +BorvAoIBAASy89mnP7d9jY7pSg/8znsUF8fQwKpRJZKS+8xgbzsZP+zT7CjxCbPL +xcNK0fl6pRBv+gyIM013R7J1uvZJ3W6rspVxlXOvofvwYSuLOjwsZA26RM8s7Do5 +e62Bg7PUHbbaD+C8HzbJlyXeQ++oddWPbIkxJMwhP1Uvy3wA6c7E7w/UACZvv20J +KriU33QmW/o0YpOX8xBVwgsCld+IfUIYm1S1mpU6k3oUfGIA5iyKx1XLTMhlaYUG +dTdExwxQp73Jk585qWSpaiQ05OrgYyzZ8OHA2kRTPK+54HSwRfn6senf3TakZHBi +zjy/DZmOU/a/EiR7MCGg+jS1x9GBxOE= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/certs/client1-req.pem b/tests/integration/test_tlsv1_3/certs/client1-req.pem new file mode 100644 index 00000000000..b821609068b --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client1-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0faoRrITeF4k +pN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnTpwWYPwJ+ +pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZDSl2YHqN +s4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivIInrUWc4R +XXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8yg0hCB7Y +5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mnzyHPRk4p +cn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHesyYHoQw1H +xU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBFDX3Zj6nk +uJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1FQA1ABXx +2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4L/qXR/Il +bkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwzy770PGkh +p1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +fGx/D6rNeaVO/vSUGX5q1iJKd8Gnw+/8NRgbuvCDuDOSy8LyqnLmVntj8q9FHpJM +SRH3LnylMVFZdybso2ZbhR1UDReGvHCtKICG3LLP1uWwy5nS3mkGBHFm9COyFP21 +kWOit1+106gEhg2f/NXh31HFmh+myepLjPEj5KxvnQhQfaQESsDYDZAs6/qT1mqp +A7GixOXh7hIFBJ97cU7fKby0Wtv7GqKAYQkaf26ImoGijtMPIlzvwJboJWmOYzIH +zrOHqspFkJD8YvYOwLIKdahViqXU7POL9uRn0vFyaXVcyXRq83Pz+bPSW9AFYsYG +ukSZiJs1yCINZI/Mk1vlfaZWYPIbBkJZ0Ny0vw112dIEilWAkVdsmJyV95aBddQI +Md64CYWZbV5P7/0QOX+v2ZQpWVnaV0m07K6VVuTL3bw6BQ9fcj7vaql6wl8jl/9l +nEotaZiY1f1pUUko3XzXpZEFB1lGBHupuS/Plz8pfFefN/7sOZoWn1VhD9I1A8uh +b2mg6hyQ7pe2NrHOTY1+L1xxxKKHt01kvDhws09qxRXtNsLrL8tl94i1ndLjHIwD +/VRnVU04E/VoTKaEXuETLZwOZu8pLwdiejrWEAmtsbmmcKq/Bk42wa+Wrmge2Chs +V8EOAtq91AjUcQeh7s2fV6yWweMGm1J6pdkNWckCsUs= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_tlsv1_3/certs/client2-cert.pem b/tests/integration/test_tlsv1_3/certs/client2-cert.pem new file mode 100644 index 00000000000..886cc533fcc --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client2-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRgMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwq +rZCqMT+ePwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIH +usrI5pQxcFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAY +O49T//YqM3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKc +dLbo8ehESUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rI +D/or7wew2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpC +KGWyLibaQyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD +3vtHbD1O07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8X +fYZYx4CaY3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS ++Koj/tbSbBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNB +EK1o7L8OugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBib +toxwGIspTlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBALp4L1aky2jfgk18tney56sUL2Us2aHqyOz9LlowWFdNMtCKo0WKpZ1qXGfQ +92QE+zc/MEdmv3V/H1MmSr7trTq1u7E5vVVI9Lq2lNbRLDQLi1+qd9E7Kdl6Oxw/ +Ecc8oxIbg86p83HhzPfJG64m3x6S6m2c4sNrHRAO/gxxJex6ZSFfQwYJZFlcvvBX +CH70RBtBG/ggasVtwqBuuIRNJ2gAtiWG2RtyGlOjPiAg7nUQiYlXLHVOjvrKDvrI +KTjzRdEUMqKtIrNUBHSbWZlxKZ2Ddavshg/0T0reAN/u5KTDxiGaQxlVEA7xfm+j +etqjzTz7LnKuRsA+Z8UUYaV6mKYfKObDoUs/12IomRCUTQi1K8MP3fGmmk+4Xiyu ++t15EqWJzhjuT2RjCAL47X6ksdOtonX9t29l6ykCvYpK1mlzG+EhqDyMIn62TNfx +OFjWwhIFgyEUWtwkihIKtv3ZVtrJVO/j+HCUfq+6IpjYHdlpdb4OaHgBtpokOtM8 +PmTHJbP2bxmNIMAU1WTfV+e/JkdTKHJclC5DTGF48yRgdKSOTq0G1eJYh4DhlEIM +vOw2rXeWR6VSkvA5vF7HANEptl1tkT3dsKR4BXkSIO16ldWBEHMM4UeXx85GGM0k +TRON4FWBMi6PXX6mrmPXcUW7AyKG2JL9gNlxRgWHVK7xmZyp +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/client2-key.pem b/tests/integration/test_tlsv1_3/certs/client2-key.pem new file mode 100644 index 00000000000..462916c0670 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client2-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDhiGp8Kq2QqjE/ +nj8EZIkJw9IMlbefpMJG/F/n3QSURdJK+9GH8zKSnWT+JeFugtCb/8rCB7rKyOaU +MXBQCHv+wy1tCT5DCyoF7DrTzWTyJHO1mJKt3G8wdGU7CvDtyGIcJ9HQGDuPU//2 +KjN0y0xbBvPQerghIO3UIj/YYem6InPDP1sgLHjpCTX8PsGHE1oZdbaCnHS26PHo +RElLVYQjj1U+tNswVGBZJXqij3hyf4M5d/5k56n3lAZYxiNml8Dpe6+qyA/6K+8H +sNh2fNXUxg6gfsqlIQSFTbHiTxzrim5HthNqllcwJEZ1+G9nobEmkpJ6Qihlsi4m +2kMkdPxrfA7rtVyLGSnhYwsNu8qIEk0OsU1ISwxdWp0MqCV/W7UkGbE2w977R2w9 +TtOykn4htGdafC2kddEQBQO39xy8MMunPRMaRruRA/nKPGjuV6ecMrIPF32GWMeA +mmN5qEDyTw0unjqVF3f+oduxZt+IfVXUo9qeo/SWLJk3hIG9MfiYwEMkUviqI/7W +0mwao27zvhVGpXUZRwh2AkE547EjQX+0sahPZX4BYrXIIhyknQ+0CH8DQRCtaOy/ +DroAv0kLRxnt54FBrjZhyPcOiLRrlaqCWD9x8GugpRpmJU0S6AI/DZAYm7aMcBiL +KU5U85pi16anmUz2gNiuXoqvDuQyrwIDAQABAoICAHZuu3RuuOxB41DEGdWFsczV +7wS6zk1gKME8IGTS1GfEbpT/vd1FYaZKTtGDNOlieoehAGl5w6Zfb24ctBzjB7IV +7lHWy8JLJ4sqrQ2ySzM43yZac5QnMKBiTxJ9QV2sn5CnfG9pekVe2Af9yz2m0Hbw +pLIy72Q+NYXzYlGPwTwEgYPjTkgL8oZ1VssabWgwSl0aSng2DrhKhVXyHgcYZiaC +S0J9mKi9dkb5/ndFHfwKZ++Syp1UZhXjvp15lvd181DoqavmGTXHQmNog5NdJLDy +PJYdXu7t8sDJtwLfhpFOBXFU9MdBIZHfSr0CdAYYi710tMTM3wfgVIoEjcOkRzRx +36O66ehHfcyNsK52Z+DZ6uR4c+MOG0kzTiHQhyxjiu+3nYMGw1XdyE+k+eZDMPd3 +vTaR7kYOQvVvdOVAUuFZG9mK2p0mpofb9cFxFD0vJUqTYXxSdKUNIexR4mWQJw/h +rWOg/42GK4iLY2X6/CsDh6pTsM+HCzwmTGGkL54FvDsB2AhAhXPz/kGiBRTrh9/p +QBxacSPoqN+kF3u2qZRPEmjuimiW2AaXARbTABNSBQJIEmWzWOVdgUBVetGoN/ML +8mcYDmXhAc6F96eqPj0dX8cHfqYPguPhtzLj5V6XGym7hYQyOLBcE7tr2BcdjUfM +V6OFHsPNmsYWZ9F6zCv5AoIBAQD3M6gziCA0G0cG05ef0C3D9OVGWpHqr0yiR3MO +ZKsYbJJn4WOtWWvo8N5oqZBQ8VIoyGd1eiSIDuxXEWniFWjn57QN2nrDNTsEQPgk +HzomgFzuDZ7V4JsjJ9F2nAG5i2HoEwKNHdzfni6mhwGaapd+4GlET0jlC71p+h0X +CPsD6Jwabp6OUyT+xm8XW3mTWskBzKfq0OPbsdv8UB1dPt6jVrkjoe76TlTsWXWi +U9p9/h6kI984R9T10J61c21dokuL/KlHqb6TIQY3RcCgm2bfucmuawIq6vs1PBrK +VCvMX1BuTva9CYg/+hxm9Ky08jFWSCEEtzaORyN+4mmf4maFAoIBAQDpj1NoI7RP +mYqG9vHyXSDUUNbchpLOFKIaeh2DGk0sFmLi/obglsxOKu8K3r/EobNt+vpDTBxI +1EjPWdKuaXNYYjNjrVmPHdHPoHD8JmXzJDbZnXSylV9MVYSMNF+7BWUiPg3/QC7b +1a+ljJH/KEWFb0xrIfNPxVzyq8dyFOxcmLfRVLYlEW+fRYeaZ3QApxGi/BoYK8KN +vG8f/a8jpPwYCVa3JJ7/donEtsbxTkm66aacn8Vo2Y/tdo0nxyqC9PyBU+tV0u4w +aYtEZ28kpC9QheRx8D7WzhvsFc/KsshiB6jddjOVR6VgiUFCo+b/5PqpyZVTVrcs +tj8062A3KvyjAoIBAGRPn/eZS4gZcY8BmcuODKQx4j/UTNXw4KYRXE0A6LT2icqB +mZMkcDeMVpQeCqPt6SsHd4QiVmSnuZvzQwYtLe69BUGB4MMJ/LLTMl5mFZC+Efe/ +qy6bABkZ9VOuJr0GJGqqHCTrc0+CvudwbWQd0O/5XH4NtkTLqMcyaU+Jo2KIp5/K +N6kFcEO6fiX6RrFW665BP/p3XZ8u41fVorTN6EZb0LD26yTDWI64FpYSdN0fm4t7 +yv7ply9QwrZa6oxOaV2a345nASBvDDito2cI6IvstjyCy9RimiGWDEECOuup2deJ +T3KSRanAcnoM23Bpvz+F8XAacJb3ox2//qCUnIkCggEBAJHl2XllTF6pEFLs8giv +SjG26fFKE2yukPCvNb5O8MRIm68mxkSHjsqJoVeN/Act57MdI7ZkVgrcqTr15ljT +QJ2GgomSoS54tzbXB51Ls0XmamkYJezkyGobxbf7g42Fej6guwenJV5oJtfobs8Q +bhVDiF4oECDVrhFdYzKNhXT2ZWVbYIjZUnwQ5/t5Aorh0m+Ywgg1VcxKWLSIOR6w +ElZFhyjStIvqlXcPokjc2cvr5wtR9vRfa7wv4U9m59R0i0OSk6DCKc6OL9QkNNaT +xYasjR7rr6VpjSG2Il6BvhEWrdLh4qku30zlkKG7VzKk7Dyh0ykDM1u34NYC7tCn +hrcCggEBAO+Rnkk5eYYqGk/64+Qy5qA7djvvZ8AgihwJL3+ZUDSOxh0W+Er4NB6n +j0kI22N//D2j6hg93TNj9jI6lISfmY+TSikr/P+bQPGXl8wvekQxpjT5JhCYI93M +LXnSULuy7J1ujkMGdxEvfOTjvmD0ejtnuaGd+jM7hx4QNBbJj4VdV+r5BQOJAlfY +gk6n3RgAnu86szquWM6dObIz9BWtIcMVGlxA7yDmxjVDDHLwGpcwG+MTQRcHoeT6 +2+b7FtVN1NFLazfgPS3bxKs5jaUB+Ibm9BD8B7THviNikqRYqwoJMWpJgdWo/lOQ +X0ueOR40kfa077G7jNfb03qOPUR1mFw= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/certs/client2-req.pem b/tests/integration/test_tlsv1_3/certs/client2-req.pem new file mode 100644 index 00000000000..846f6db84dc --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client2-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwqrZCqMT+e +PwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIHusrI5pQx +cFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAYO49T//Yq +M3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKcdLbo8ehE +SUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rID/or7wew +2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpCKGWyLiba +QyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD3vtHbD1O +07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8XfYZYx4Ca +Y3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS+Koj/tbS +bBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNBEK1o7L8O +ugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBibtoxwGIsp +TlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +3DJlf7AkZklzzswgm487f+y2bB7IYr55JwENASDxQEOdVcdgLC3IWu3hLiFwdqac +0Sw2OHZuETwJiIX3fD+qUT6TgbsP21W7wEQ4jfKg/bsXFMbrvw/ILkOW2JLTH4Cc +9ylCN+46dQ9heATkiF/Co+uASz9IoSDdtoycA3BuKGBZI8VGa56QmJOOsMM5NgxT +RTh2r23tV4E8AGYj3HC+b1rzK1RTlsj/m5nM9Jv0/NqoV1cprS1ONr8CBhN0ttuA +WLrG+DUZTMJYFabqTptlgejQFhiFp5HT5A+eXgZ8uEUX1I3q5jq1BEWtLdmJNZ45 +QViSJOokH/+1kfRSWiAH7pdBz4URLBcsDhAag4J7kV38t7fgdaIizY8R2Ss82iEP +xqa4A0PA065wB44zng/VrPrHoH1YnGRugXEnrqgcipC0FxUl3oQjvwOSR/E7yFU0 +GIr1MpRcyrd0z4p16783qnMpE1Aa0msED2SBKIK13WcNY+CtDF/wO47ZNywl1hBo +VkM+ohPpmonaVXNGdpdoZpeGjkBUbqkn+so4aYkX/WuZ6vY2vwdV0prD1vdAFfD2 +AeJx5ypu5aeKn6nK0eMy6W/VEJx6RLCiYVOCIcssgy31rmk4iLQJP2StYVK2mZKp +5aSR4eTv1/XlMujq+ZqcuUqA1id9wP7908Xr0DzdNdA= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_tlsv1_3/certs/client3-cert.pem b/tests/integration/test_tlsv1_3/certs/client3-cert.pem new file mode 100644 index 00000000000..ce9a472cb9a --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client3-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRhMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv +50J66lQ+l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/ +cxpx+J+xF6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5r +VrNSwJQp3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2 +yAG7pLNcgPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551T +OQmDAUxda041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9r +jpsuuXEUOsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6t +J4NHDiimJM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tI +deZ98iQbguVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6F +z5Jij/1lK2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z +5Ywr61t8AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1 +eGsC80RupMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAD+YMVntBdeq7xJEL7xU4QEHzUGhDWodGMJfmswcxe7gf5Nztcq5YIug+akL +ewg0wzgCA5YGz00J92sKDF16RmYyPfkxmrCYdNGwISjNJyEEcPEVkdAzwILjv2Lq +0shFlSsf+Zp/M4XhHeirmzz/jJ9KHlzEYoCz1WOn+UGF12KgV2oQOamJSWOMCoMh +81oy90V5IlCBqnYfZCYj7cbYLBd5jZMZ+7lsVnxttzPTg1gIoP6vrLT32Ubnzx9N +IoAeiUg7az/fbnuOkJtu0cjz9aSdpjm2h2giyVAFJ8DkQ9C92tdr9DWZKn7rDO16 +TMdv0q8NFjRGhqdmqWUG6o2cUmQsJ/ZiIcHx5X1b7j7PYSS+ae9zi1tcpHAN6kCw +WHguIf5I8MIZxE741ZMBokFSIqd6Bh1EP/TUx1+g2a/nH3ZaNd4/KKADxfUU2Y58 +UwdKeX9YpcRz+NNO+1h3NoE1a/i0dhwiBf4OzBiV0WpAjQHT95IlQxTxfHFp42IH +GrbqIS3qK5DKlNFkBBk1beKxBGKmTH+Pw6fhjkuPYQzjmGo4xluivfeT8SiBT2iO +uIGLd+sitIooom0KEjHuHS9cdZ5XEPIUDAFhmIt7Y5K8J2fs+xtYzhibg3n0Q6qh +xTx7GzhTA1HSUE/467af5J3CSfpGAjZQZo/t2/A6tCumzk9F +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/client3-key.pem b/tests/integration/test_tlsv1_3/certs/client3-key.pem new file mode 100644 index 00000000000..b7464eb2866 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client3-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDfAbfIL+dCeupU +PpfzVFvpaluA3rJki79QZ4w3eUkgPLX3M7ffff5dXqQBnhDfz5eHQMVz/3Macfif +sRekijcUNET/jx0DIg8wlIAauDirNPZA0P7r8nFGZULlj7UetdE1o2Uea1azUsCU +Kd1Rg3TchGyo/8nemIuTQtYi7qjwttH8Of/QVU5tT9sFl+579QFrNt+bNsgBu6Sz +XID4k30pl8sk99WAY2/Tp7p+4L+3sSP9yvZkvmlqunItcQF7cpFQrOedUzkJgwFM +XWtONWIIRE4TuDK2WurkmalcJ6FHF05WvvG4i2G9/vQpyIzOcIQDk0fva46bLrlx +FDrK1/TIG9R00vpLJqwPhJGuzHVYutB6KJ9Pbjb3EzPAkijYJ083DwjOrSeDRw4o +piTOVgP3mPItzTQk2tR1BhHiCudVDnYyBSrL8xtEwiNhTO40CySuUt/LSHXmffIk +G4LlRoaAr0aLSxAMFtTNq0jru28QGVOECfdyLlH5C59gYqTrsrkeeWf+hc+SYo/9 +ZStv6SloZ7VB0yKRvmwRwxfvtCzdGgdvbiMGEZKM/SVIJzc+BQ1Wy7yfs+WMK+tb +fALo2XXzGQ6UWJc31jxno/KZu/xLseY4Mf943C7an6pH2S0rhYwSn5FFNXhrAvNE +bqTCSiHqab+eNVup0CZZiWCr+wwr6wIDAQABAoIB/0I0QFst3XnfA7H+4x1Z7e9d +o8yeUFeJJUK5eub9Grh3TY4VzICM5vbRId9ZDalj95gvom7NZ15yd1zxNhOi9LcK +zXERC4vikJ/bdix4hFpPXsvfP87MKtS7OyDriNmVIIbL+zkMpLCX4JQb2ZhZblgI ++DkztrpejxEoxmmYcI8Ft1Ep5sfyi1XoXx1J/YLPOZyarcdme/oHut2EmMUzA/VV +GvnemYOEAa7UHImOL1xZOlYd6wf9f04wC7Vx1v7PBFTu/9O04TnxqnEBStns/y11 +GbjA9k0ssI8tDxpMqZRxVtBp31jqCBpflhzRbPvca1SkZLavN6baODNZzhpqAkDX +3R4lU5C7wu4jtzydUyEsCFNdtkGKlxpZRbRZk+keUC+HeCmXPED7p9egwF6Zi8VI +oaXl1KvHZO2W5x/BV9I1taEPhmOuRR49KxkU4e+IjqaWYN1qsqYqCs/od22Rah72 +KT+thr0mdxC4lb+pvteafricUQuq/dSbEY/lva7PhPQRKVX/VxOaAxBnhA1LHVgZ +imsW8W3eOQYJbxniTrz9EblWAg4dCcupsjMDUDUyACB/E6isDtYU1J2im6p4gbqw +tXg3bRh7KruIHbPSJyrFm1uqe+v97TLhpwPHKCsxE4HiJgRzaQDRckLJQebqNp3Y +e7kLLjg6uGsjAl6OwKECggEBAP5bLGVrmBmAz8RYPnG1MQWlsFg/eIhMFCqMjT3P +swPUU2VJKC3TC3OwFLxlAr0lkXol+8L8aEvxGjHksleA+1z0lav43b1/2jKgLgI6 +Ym5BxMJa+sUJpI6K7CedJ6wf2ozbpVXazvNBZ3o2l0QbC/KpX886CZH9YJgn7N0M +TfPe9er5zmETdHGTWtA0sDI8fZ8XndKmnWG9KTQCGur6gemF8SKuzGv/BnL+BZnv +bDqSvyN8Wjk35KPNeKVW78ROxRuEdB5brryGk955hX50PRRoofW8GSmLJNKNYvIj +VRkKrDKpz8gW1C2/xa9j5tQkGRFMDAptmk+yvtmDxfZz38UCggEBAOByrXLMTcwR +bz4MYcSmEdLv2VA/bZ+y0kW0frUU5il2fyQseoFbunVbTDiXYf40uueMbOONZktM +w04CXKRaTbnS/s6SGU5VW19jv+xzwrzpB2Shm08APwgFnSw40bKCpN4ZWQbOyFVq +QIMXfA0+Go3zJz37MsSgY+mzhHp4WITobVFpdlhaLvrLPCB78uInZrFsvNN6NP+K +OIbOoTA9u+BP73THHkpQdrRJaJWowpqejz8kzQ/Xu0Xe6AG1EGVp39phKpWH9TPF +8xoxjbdIGPkzCzYO3hgz6PlnWVj8iyTxklnaUblqKkY2mOlMA00ujcdF3d3IHvaM +Xolej+XeZ+8CggEBAKeZDdzaE4Oic8RtXN/xwxZ0gYj0cYhlkNgkeqCi7dL1IepY +VQg0ypP1DwTADhjx2zTAOG7XgCWh/V+o0LaFv5sVclW5iuplhzHah9ZiAB+kaHCk +IB6a5vohoc/MZqqs5oXv6LZ0ke6JRxSpSezPYYUIg5/5Hvs6GF7J1/IjPG4XmLS2 +23zto8l+jdUpEnxXjXK5zf1SWdtgF/kz9ealH9rurd/ri7kRdn9oz+oJb6f8r8ND +GfQf1yDzr65KZXxVZt1l3llukemZR2/NZN/Y2bJL64QO6AmOrLmr/emMzHLOrH5J +lCbEnBR1C14xFpTsIDRchoaMh6RCJC0Q/e0Rlv0CggEAAOIysJsBS2ZeK75cvCtz +MoNjNZ+qTNClZ0TYotncNhmTUo8iRFQaHdAoMqjV5+xJOBQjcZni5zT8J9h2iOca +GzsraaDFnLtVSsDXxpSGFbxNHSZNuDfmB6AOCFiI6sz83Sr4YMB7pWpvqpRzFpJC +BIEKjIHqpz+CZS8hvGGw54UKuSFTJ/Hi8XXPXMlgIWfKTbSB4cs/XiorIsy5cbks +fiuSY8FM6zn53afUU5KAgZ9SLQt2CzPsNtAz1Z3i3KNYEEIFquUIIBYNaPL8/dW4 +03JR/vp8AVhi+Ghhv6nu2kxhKR1k6Pf0Bqa8X16/PJSMVlZ+Extwk8Pls2C97Ee9 +3QKCAQEAgjcbHKBjd7AeyNpPSzNpv81Rry5qqOc+Cxx8LtOHBl1wc5VB5FPxfbuX +MX2skvWPnokDoXcI1a1WQwdjaZUsSoqdeyPtw8pFWiNLJZkYImiP3zMCZXYUEkzk +3EXQZryWEqBYBqxlEvTyjbBmnrAwOPOUKARFi1l9JKJ4QpdELXo9Yl+w2IQEQ5N9 +jrSY7LwS/cb25rhEc6oh/89aY83HPyABh4lC9bsciXki54YIeS+y9ijN8yCRxikr +mVGfQ0Y/qcY9spAj05yr/vnlENBB5ohxwKKsemOnH93E2GFxc1dzmWCGvISjUduB +I68TOg71OfCKgfeixNgcOvQoN+xngA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/certs/client3-req.pem b/tests/integration/test_tlsv1_3/certs/client3-req.pem new file mode 100644 index 00000000000..7b4445b3609 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/client3-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv50J66lQ+ +l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/cxpx+J+x +F6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5rVrNSwJQp +3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2yAG7pLNc +gPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551TOQmDAUxd +a041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9rjpsuuXEU +OsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6tJ4NHDiim +JM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tIdeZ98iQb +guVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6Fz5Jij/1l +K2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z5Ywr61t8 +AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1eGsC80Ru +pMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +Rggrols8hXGEcWeIEGn66kY9IVTzaTUf3oMfEbdf/2Q1QzHzmqp53yamHl5ioMgX +o5UBVxthgh1VOxkvCxIzlKDJprzVFkfwwc7h9c0HGt3No/ERobHDT6YRaGukAL5g +muIGBUseyBAOIfyqc5kbCRWfPrAOttAH4gd8XMBgO8XdfHAvyXBC8Ha55O6oriX9 +IAKL5+3nVJkBle+62OmROnstbcdKyK4UtOeki/6ptYVE0d9I+NfKjuk3eKtICW8Q +Pn3IEcNEZoFG2UQ19ENWwYEZyMZJt0aunqnm7L4RYiZT5w4meeendzXSKLKR6+Ye +ULt1sDRskgKoNRzmeCVzci05BG48jv/E7Az6aV/qhGiU2qIAPMdVXncWUhR3fj+E +CL/uLifOvfC6SnKw/7qQmgjUvEe4Duvi670a5QuImpm/mAIN22cXPc+QquSdR5xy +loz/o3JJQZemPAOM0CMIHZ+cGESxH30QCBNn5HfcOf5fRZVCss4Hl6JxHR2G4yN3 +RKEIUXR03qgSK91WHl3WvqwXgmIAiUuvPjo2i7kSuaUUHilZiXK1ngIqYfUTB5SQ +O8pG0fx3fbhVDA3RQfXeJE6FA2AyLvqOcsseRzvcQjQm4MU7p+RVaY17rI6/EkS8 +ac3E7BPwnXqSAkPSEgoiezv/Z0Hkmrcu6fIsUuf4ETU= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_tlsv1_3/certs/dhparam4096.pem b/tests/integration/test_tlsv1_3/certs/dhparam4096.pem new file mode 100644 index 00000000000..102b8dcc72c --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/dhparam4096.pem @@ -0,0 +1,13 @@ +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEA/yTb5We6gyTktHTlb/PimPgJhvY3Spp9zzBO4I2r/f7p/llPLj7u ++VDW8s4Z9+UUVQKoVoU2NLcgbgIUWrkAKuBCqqxxh+/+0NdP/klkWUX084HBvT5e +Tofnv2JT4EB1ynlNCF1q7frF/ELNyPzOWzh2w14XwoWxb3ojrfwG7N9p7CQbSwjH +f1lDRbOcLX+n/pic4X42KqqXqsg6ehtwORz5kMlT3DTAGC7sfB6rL8Y8/GrPmTNV +wny+UdnTyku8+OJ/xhL1ERiOGMCcP5jhIU1Bq9Uf0ayp+3fJazPAyP5iUprwd3DF +9UvaEqIFeaknq5qX+aVf8G7GpCpIC14db6uEJCH/oMSGakJdC0jWZzN6EeJoUILY +I0K/+DA34/Yh7SAehqc2rAukiquCv59/Lm+FlZyIzjQoOtKI06oIjGr7kbS4lvgF +NbN7AXYaou5cJaffPmfgUuU1hw9gn2kYYMb7el63BBzIKX/ptWR/uJ59h05ivYGX +J5bok81H7gYvwHaXkKdQ2t3FoFJHAekKpraiqIW7qHE4O2lb3JOU9GvAQ1QLdNNw +CKJPFKBVes+YxmncJexxvyVXj1N9XXriOG949RwpLF8d85yx3eN+3cq5XJx65Rog +OknNaTV8uTrpX/WGcVylApshMy9+4LP352ZsmXDuP7yiBqlaxyb/KLMCAQI= +-----END DH PARAMETERS----- diff --git a/tests/integration/test_tlsv1_3/certs/generate_certs.sh b/tests/integration/test_tlsv1_3/certs/generate_certs.sh new file mode 100755 index 00000000000..d6126d361f5 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/generate_certs.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# 1. Generate CA's private key and self-signed certificate +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca" + +# 2. Generate server's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" + +# 3. Use CA's private key to sign server's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem + +# 4. Generate client's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout client1-key.pem -out client1-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client1" +openssl req -newkey rsa:4096 -nodes -batch -keyout client2-key.pem -out client2-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client2" +openssl req -newkey rsa:4096 -nodes -batch -keyout client3-key.pem -out client3-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client3" + +# 5. Use CA's private key to sign client's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in client1-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client1-cert.pem +openssl x509 -req -days 3650 -in client2-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client2-cert.pem +openssl x509 -req -days 3650 -in client3-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client3-cert.pem + +# 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" diff --git a/tests/integration/test_tlsv1_3/certs/server-cert.pem b/tests/integration/test_tlsv1_3/certs/server-cert.pem new file mode 100644 index 00000000000..6f8e5a3c6b1 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/server-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSTCCAzGgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC8 +jV8igQGgCvu/7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC +587nZVhQtHpdbCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSv +GiQ1M8G3fiZNflEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31c +K3i2xA7A3V2jBNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9e +bRmbNlJN3Iky3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+ +b9EkVAzpw7mW1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1 +TEVHv2FiUOiZsVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+q +cXdwglqIDqXKlCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45T +PQKP//RMkOrHOYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3Kl +OckWxNx55bYLdLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N +4ObW2H1YEtE5M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABoxMwETAPBgNVHREE +CDAGhwQKBaxNMA0GCSqGSIb3DQEBCwUAA4ICAQCE2eJVcvsMmJu6xAfoE6/u6BrD +opMicCtlC2qt0BgSIzzDQ/iWjnWKGM1C+pO+2G0WTczj7ugsxjPzhkyBpuEZaWt0 +9/tJTKIrgaRZvEe0ifsJxyqL5LJgfxK7TbDPcUBKr1v+TOxPVRq0FuG16x+yka4C +rwxfBHU43FmtEFfgu13r515F3ggXcdlojkce8ZKtTAGEcN0MpbJ6XS90BHU0sy5A +APTm0fR0vM3kg1nuBLbSGF5KfASdw13gb6QsDbll0IqK8LvXYiX5CaVfkAe/pFkO +/2iIxYW74yC2gV+DcFdRPVfFxSKrdg0tDER35OYg1/vXRjV5BWr1EjE3qjrCcUZy +rlF3fms7Arr20ka2nSa8avn4ALpyJZmKasoxNAAsxivingNVZkql48OqsJ3n0qGk +LI6Yu+UM/pc78a3NHsdsCbnf8qvae4oJa1kyiochJu+gUOzHvs4Ydti9iTQn2Byo +2A2LzyVPBmSOhzdQ7SwpvHA4A2ftao+dZoA/+o4rmBtbmgxjpBPyPJTN0ZfKlpKl +Oyi57ov+cJmZctSUbP3M11gBva7aYu1Rd7/eXeCEl1FHhmKL/Ee+UrNZLiwspb2E +Sa+pOHdJX8VgsIYXku2UKaGT2QFITxO7fnxghioxgsyCKrQ+m1gL9vgXj/gJu+48 +c+5CZ9SobLdMkVOtQQ== +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/server-ext.cnf b/tests/integration/test_tlsv1_3/certs/server-ext.cnf new file mode 100644 index 00000000000..83d9b03ccb7 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/server-ext.cnf @@ -0,0 +1 @@ +subjectAltName=IP:10.5.172.77 diff --git a/tests/integration/test_tlsv1_3/certs/server-key.pem b/tests/integration/test_tlsv1_3/certs/server-key.pem new file mode 100644 index 00000000000..065a2290749 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/server-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC8jV8igQGgCvu/ +7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC587nZVhQtHpd +bCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSvGiQ1M8G3fiZN +flEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31cK3i2xA7A3V2j +BNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9ebRmbNlJN3Iky +3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+b9EkVAzpw7mW +1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1TEVHv2FiUOiZ +sVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+qcXdwglqIDqXK +lCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45TPQKP//RMkOrH +OYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3KlOckWxNx55bYL +dLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N4ObW2H1YEtE5 +M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABAoICAQC11lTwLp/Fm7IL9fvquc9P +CMmkv2DfGi80WO2YJ8ccM8gFyEYoP0rLgYSshAUxlvSr1+iG6grQ0izMGfzctcnZ +c3rTjco9fthNG9kFCFVvh536SqAkr5MCIH3/onZn7DGOmNRgZoikkEkaJP66xgME +tuS72W8iIcoNfw63FDIaJOONGCJ+2Nw3HkOjZVIVHRLlp5rkD5H218Vs6MtWlgY/ +eO9K5SC7sskhgL6HyGe40BCjeFpMh97L4Wj7XslZ3A0xQGAYervHES9TWX5A58EK +QT2yUkIMktzklE+PicKYA08rQa1Z5Pf0YOAELSWBdS7iWi3FLjXB35sE5rbT5puH +9hZXSDWLggbefuaUJyowDEZy2aHP5pvXKBDhEANRbU8VaDyHhlNLqVNquE5Cn4HO +zPeH+KLFbbABUok7yYZmIC9Bfn+rXvNzTX6A13AdJI/HcKA5RBGtpAY/168Pt/Aq +dzuqepu42rFAvS45RNevp72IIavx/QdAA1OTgKxh3c2Mf85pIXJ51aWWLnn+EZ5/ +EsE0crfwkuKJvjubNC4oOwMTFMIBI2WsjvaAw8pQw0Kb0ksExKd0wz9mKcqR/v0I +K9oYsaHkx5je0NOZds385+zCoQHYaw1aKUd7ZLqr5G/Nf/2TEYpMWco4ETA8lzu3 +Ty/8XkNw8jd4p+7bUuz1mQKCAQEA4MNU7GWDPwUKNNSz335nGH2oBvSGbYiwLcRM +D+x2+RTfOAFSSJ+Q5tQ+327ZkAB5dK2mxmDYKB+Ln1UBIneViUflkMyh4fuutIXI +wYo+BL71r89MqhRvvMK9hWnCGtJTJedf2iQENJzVn4J76BvTPRYywBv9pofPOlj1 +MtwwMA4CZAmQpCUaF5NQr4nliYx+slkcKwlm+cOxeZGa8mkNgQdmCcTZkRz6qsiR +vQDEDiS1+5lCJ6nWW4L2tOPejNN//hVlbPGMaA0oiu7I7w4aSxnTlLhDgJzJwmN8 +NFYl+u5AcPq9iRtBnzfPmd87S9bg10zcIiMKxw898sU24Pa0jQKCAQEA1sG5hO3c +4API//k7NEWXsx5Ns2JE/AV1LtmBgqXkn1DAJ+b6V1nIUppTs0zspEWrae9KrsAk +z47qIbPaTLHuptLrvEXk2LVfzcK32a7fXXDOB5KkBhzlJM1J3PTRQFR9lr7qX6vr +EDc4p7p55IDEGnJdXa7x+z56QjpAZaHlzexQxvoWWoLBkDuoT389sdU7CbgTa4A+ +CR6D6qKd6H6tfmv5sPlvp+aje+ObacP9I4WyVjscWkzBHxS3n/fTLjY6OFv+o8PM +TdytN4+HZnu4MDJlF3vx9P6CbnnVCaScXDxPGcoSJPcoEQqoyxuvUQLDUQkzWF14 +02EvXW0dbgiPtwKCAQA0EUwFD2ceHD7HClc4+QFNDR71rYPOsBGQKJ8uOSs+fHVR +dgznwf9BWf3OqNFBqLp6KxgtcJXihZxEpt6Ca416pesqZh1CSpmoPC3LmAjR9KLZ +vX4XEHDqG3roAx3yNLMKXtU3pYxL2+Eo+INXu8ptpkzPcCyMfX2mGKGEzLllCHnJ +TuXxAJ9QwtG4OIuyF5fqHPaHicAPMCRW80If0fJM57fdn3p/QWVYVupcDGdel2aJ +CHHo2lFMFcStFvShTwWhiLdcS4CpQhMYTETEDFJO/4aiNyV8D9Y1b/J/9U0LGlJX +Wd66elPzXGx9StdjtD2V4rpENjXy8zb4nHMgHkapAoIBACvvtmTbxTSPka/M7a/k +DQU4Te1FTZfCBhdvqG9yQTPW8Xk4aD82vyUnLbihJEj3d/pUWpMl/GH6eywp/59x +R8IZpOD/67HqaY9PJw4CGPClA4HJHoWho7/DwDjUXXsrzgXpSUoJgi3vHkgyfn2h +Wn2OqEtiX19niNvDzyj71mgq0Nvkjm42EiPQEL8y6QxY85spbc+wjQCQnayDWIsY +X6ZdsNfkMFPJe+j8x+77ie6ai8HYlhRjX59cPbUcnrf1oDOnnpEincnQPCAB3VG6 +PhSeOtBzKy1UZJr1kgBHDTZRoF1GWi/14NybsazcHSIVzp/lofuSJAYa+/XBPSQl +3EECggEBALSLZPdg13906LEyznYnjgq+nMh88usegvU9qsBAFExClLLfr6Ak77og +boNoOwbaFn+xiz5M8BTJIPizJcm5GjYaqg58zotTtG51h6VgMri+fb/BUpVr7p7n +aSq3kXDZlrwZnmooCT+KcGx++w2N2SYSyZX1TELt/dpfuWJvph+E37PkONEEiHPF +ZtSA/f9lpfP5/nx1pLmv4ksKdXqpz3/kNqaf9zbhQLgOm/VoBHL4NVPYRylGpCJb +R68/7yvHBd2EskZoJB53TlJmwu+fC6ee1UiG6aqTULfEsiGidi6jIt56Gz52ox66 +BHL/JsJ0Be5xM3V4x4PtihQ3Dw546FY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/certs/server-req.pem b/tests/integration/test_tlsv1_3/certs/server-req.pem new file mode 100644 index 00000000000..be2f756cc7b --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/server-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy +dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvI1fIoEBoAr7v+wS +QyOVUJeN1RgBYx9jWvxPT+BObsJEpSVRA9ViryJcUVoTmkDnwufO52VYULR6XWwr +wXSqx5peElW03IbOUjdpGbB3qmkM3Bp3o53MxNEmKOMTcp4krxokNTPBt34mTX5R +CDzP/jm9qCN2J1VhRstGua0HB2VkuOSrhiCzD9GkR8s8of99XCt4tsQOwN1dowTb +g+JBxvEy4Hw3qAnnEyNIlPyyZwzqJV+V3acnqCemMmDhoNxfXm0ZmzZSTdyJMt1c +8nOI2Bu8un+Q6n3m2DFuoQry6X/ZTev4DzTm9ioy94byfoW4/m/RJFQM6cO5ltVR +xMvlm7WDD6CstPCU3O869bb/70Bq91xMcJQvYosWVzGlbdZD9UxFR79hYlDombFW +qraBaUv8AbynkHZMzcvuuqUaRvWy7jpy+IZc/q8B5HQZhxpPqnF3cIJaiA6lypQq +SDEpBTx9d+uCi3SdqgYFlqFgiY1JoG6wPPy1Og1/hgWxIsOOUz0Cj//0TJDqxzmE +Q991k1NIij+tc5+fvUQ+Ti8RFDrQry+wHwfo7eL2KzE09lNypTnJFsTceeW2C3S3 +xirXqRldq+cc3HtFDG1kK10EbpGt2v4+VSVE7lIHMIHehaD+DeDm1th9WBLROTN9 +MaWtZHKidURhFmoGpCEoJ9md08sCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQAb +FDegAoUBz9O4JR1u68IMnGkO5nINGAPQOqf9a2BxGujnSB7Lw6SHukjkUqqgnfQ0 +x/aWOI8JVAi/ptscojgMQUDsVNsij5v+jbJE+ZAobxnTmKP0wTc2ktpf4d8UMVc8 +gyM85jLHZ8caCcuy0D97W81vgIv33dNHWtP+sfbQhX9wJ2YQTahIC8NpuQfLAOUH +EFxWil0mfN+9vRQ1C5naKtvrOPqyM0RPrWiudIJ5QjI4aSXxUCupxxnaQMoI0Y50 +MvVVT3VwWgP+hL4b+yEJFHRpE7BwCZijsLIXkXmVZoveHhiSMYen1HWIP1VMDEHP +CUtG5UQcA78CBS8qg4nyFbDU4hWClAkAt96O8Y2epJYepIoYuBBSEfrgupESMLjS +E9Hfq/H6Ac/Q3zWa320udvA+ysfS8pagkoiH9+TarrsDjhxLjg2h2bGcXKlrsP1R +mRVZwfNOl3/ZNq5HBPb9Z5WXKvcsTCQAlnHJdaSmzdyArB0guwUHg8ZZNZqCdVgL +TPsfE84yI/HlwRfuQILfGxq99p/UYFwnee5CoM/PPvaAT+9z/lykMWZA7osuBcK6 +zP8XneGmZOkmez5+YJgSC0xeaDxr2R52eQXlQEJGDbFDtQap/X+cJDGyqmGnbhSu +6XkGy0l8mAkpcurMcy3wWf6+joskZAN4Joi4ZjKsQA== +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_tlsv1_3/certs/wrong-cert.pem b/tests/integration/test_tlsv1_3/certs/wrong-cert.pem new file mode 100644 index 00000000000..ef95a73deba --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/wrong-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIUL2Y/QpwqqHyi43PwPeA6ygdPYK4wDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MB4XDTIy +MDIxODA5NDMxMFoXDTMyMDIxNjA5NDMxMFowVjELMAkGA1UEBhMCUlUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAxO2PSeaiNFMRRiFXpnMw07u6EIdEc1Jx3cPvZjEUg/pdEmMYkrSxr2MeqRkl +tWH8TcIIoiWDLIcM6IU0mF6a5ULu84hFb9b20qRG3wRNb5yO86HnoyzU99t98++a +9iaY1QAt03k8wq4jRjU2k/eoVSoLT5uVP5KxiNzdS2BTHFSsxrt/xcwdgkfJouHN +p+MYUekk6qaQy5fTqTpqdkgO2v/JoYCi0whBNj205d+WnS7xfeyVSJP1OJWHRZ7K +Y+LU6hz6wHIng4s/ag7VdAk0PArWs50BmH5g2zJfvt7VeTQebaJWUtSEY05odOqt +KZteUmmhxW/2M73wGVF3WAJCnaxypsjcmMZFCpMXpwyTFrqobvC3APl6SOP+Ev1M +LxhhCIDuLFu46P55KKEKjUCsYigd1VsHjjvoajGcqlPlMsVHJc9VChsQDz6agzDP +Fb/LyYbrDTTmsI57/s1jAZyemq2SEYPApJvcdZ/ucl741jI0671EZPlip9iUQgt3 +MHlc3t53/GtF2W6GF5Fogch7c+0c2BhMupAHAXwfLABvv5X8GDyjsNlwB6ea9jeC +Hw+0rEotZzCXId3daFytGNm1jI216kXLSbvz6uz1wMGS6Hrhk87whgvQ58RMNs1K +SGDFw1WFv+QZeTO7wqcn8Y/eqF7q9RBhOpPMJMX8Sx/UXuECAwEAAaNTMFEwHQYD +VR0OBBYEFCI7Iy7tY0D4HPa9BZCZxYuJ51mZMB8GA1UdIwQYMBaAFCI7Iy7tY0D4 +HPa9BZCZxYuJ51mZMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +AIKYtBwTp3yvUGSXorV32dnU0Hp0MOie/itgx/la6b3h2bZSoCigKmcmvMaAaNzA +pxeYSsf5wPnONpWfo9hsGrUDMT4ETnXdzA1dbidIrhJbGsY8CN217Qt3YZWNWkrz +xLwxEwAovQZqnGDvtx+tRE8i6YJO6/kca+GB7liHFvUx8zaQ6gCwfloduG8rOAeq +noeCpW/zqYQSQGK35ntQ8MTTRbi7jMOTCikvRlldS73ODQcAR7jywgBYf/i8ANtz +NoWa4KbWuqKsQKMIGOi1fMLMaNlDSzJyw6UJ2GVCcL1NxkCZi0yudfAAxWlRis9G +zLjm7YdNBiC6RVZudGhvzjlsLZpE9DgiwXqcDv3Y1dpstD5ikrNhlQo6THH1YeFy +B8vjVGZZZu4B2JEo+QWH+zFGJosD66YoaKMVuwRPwoGDQoO0Pfbpq41A4KUhB3cf +X49/rbInqwsN5MuGp4l4+T7k7Wm0Y1Qo4FXDVbFxHvvniyHUsZk9Llzf5wBLl84m +xheUGgCHSflfXuuWi76yoADHCv+Eqi4/aLJmkUewKXJlm+XYs9bdBHUI+Y10KmhA +hgcHXF56L+N4mLRwUuLxa5qwQIqNX32+piQoO9opxnVKKCptpATLE30TOMLEXBlp +J+6b1e4BIasAAEGUhTgPj/SLL0u59Bv0K5SlSn7VZ0gI +-----END CERTIFICATE----- diff --git a/tests/integration/test_tlsv1_3/certs/wrong-key.pem b/tests/integration/test_tlsv1_3/certs/wrong-key.pem new file mode 100644 index 00000000000..b2213cd2675 --- /dev/null +++ b/tests/integration/test_tlsv1_3/certs/wrong-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDE7Y9J5qI0UxFG +IVemczDTu7oQh0RzUnHdw+9mMRSD+l0SYxiStLGvYx6pGSW1YfxNwgiiJYMshwzo +hTSYXprlQu7ziEVv1vbSpEbfBE1vnI7zoeejLNT3233z75r2JpjVAC3TeTzCriNG +NTaT96hVKgtPm5U/krGI3N1LYFMcVKzGu3/FzB2CR8mi4c2n4xhR6STqppDLl9Op +Omp2SA7a/8mhgKLTCEE2PbTl35adLvF97JVIk/U4lYdFnspj4tTqHPrAcieDiz9q +DtV0CTQ8CtaznQGYfmDbMl++3tV5NB5tolZS1IRjTmh06q0pm15SaaHFb/YzvfAZ +UXdYAkKdrHKmyNyYxkUKkxenDJMWuqhu8LcA+XpI4/4S/UwvGGEIgO4sW7jo/nko +oQqNQKxiKB3VWweOO+hqMZyqU+UyxUclz1UKGxAPPpqDMM8Vv8vJhusNNOawjnv+ +zWMBnJ6arZIRg8Ckm9x1n+5yXvjWMjTrvURk+WKn2JRCC3cweVze3nf8a0XZboYX +kWiByHtz7RzYGEy6kAcBfB8sAG+/lfwYPKOw2XAHp5r2N4IfD7SsSi1nMJch3d1o +XK0Y2bWMjbXqRctJu/Pq7PXAwZLoeuGTzvCGC9DnxEw2zUpIYMXDVYW/5Bl5M7vC +pyfxj96oXur1EGE6k8wkxfxLH9Re4QIDAQABAoICAQCjj/CAX/f/X7MsPYtQa8J1 +Sinbio42/pYmrJPNnBw/FhZxrC7/wucGFlyj9IgWZCEr8Go9SsztkeoNwn2RxJoA +q5xOV7PclX4CLIHUv/0VI8Kz5pi/NgBZMUwm7K8Xna041OI7ECqARCR2LsJ7GasN +uVMVttK6r7uXQmLnNUUydb3ffmI8xjEIQVnfWI74z60mc2+/GcOP5jXeC+/a+DSm +fudYpcAXaXbId24ls5SkTxYzEepYEtQNQFzPXXkah49yN8mpR+c74c805scxjmd9 +Kz9yhYiKwQTvaqKNpQVHmxte0gPC3lJrLPejjDtxIGOyLZw4oaqrBSpDzR9D0PTE +C+BR6VlXpVCTcAoiweuoDIxNTiJ5IbIJme3iMWxsAIJ4n10rSFFl9Cmmqbphp/6/ +XInB0X7Zyr1kBrwf+DH6DJhje5NXgGKVR9oe9jjW5v8V2tg1RrkzNU8iKBSxpvcI +x4mKhhRLYgoq/iNeYBVQrwJYktIbweVCQ5Spj7/20IrMkn3FAmMsZxGMZmLisJ9t +B0vvUkUgWxuJTsPJ2j+ytpGT0E2xIDYCpbG2EopDc8WvHcVNhagBvLC6xIjIKm7N +2zpBU2W3fPNXoToCAmaLDPYeRRpG6XaGFQAfvKUQRLBDGTfQ177qr34UBnmgvxDq +J2gA9rQm3XziLMuSlJexAQKCAQEA+yz49Ah7FFq0QffsoRb0qOJbfcmMGTRkaafb +ztto4EFSnjH2EwoSShu4DfqWw+ws1KxHlItNHHko5pVNpS4lj1OpnobW3QD7kEIV +mYKa3KowYUcCq1Gzq2RNDZqsC2BSXwx1MG0VVKYOahnu5bvzQq2Ft8W7CWBnbTbY +0Jxjs4KaOza+bH7Vfb5Yre0tlW7U5vI/YO8+YKxpxfOU9kVo8ZLQ/9r/YH8nnLa+ +Fd91+WjcUW8CTKU+Oz3lb/Vwcs6YOoAraq/wtOCqWURunBXkQtzOIn0bgBh3WEk1 +EQ+MVDHshlVVjv/rfnL571ZTT1amCJuEIwQRzLSvbso883srMQKCAQEAyLXaG3Pp +LYiRKu7Bqr5PPuqdT72UFabPpfgd5EtcFOL0xUpfRya6HyFdM25FWI8haXeg4e8N +0cIs3gMG+RRgm1xISJIZi92L0Cwj+kLFu2U5SkvAKMqZFh5q350FRi4Bp7ae4YrL +aguWLZCxhznh4D5xQGM6c8ObRfUUEMT+dnLPcj4zn9KHhoUudXjLKjPNw5v6nkbw +xtRdwANlHx/LX/d4+iwt2plDWmT+d2OLvqZcPyyghTMqV45L0p9XAXBsLnz4Zipx +7VJ8iH3jL5oaQ6YAFY+cXIrWBN0q3UYbXdkaA2ve6voioeF3KQNRmU10k7GKNRWl +pRNn62+rAR8isQKCAQAZnPVqFS9P3QwCqiCEMM4UJrkDs7jInTIcIBTnHDKuo5qk +LR4VxPImgnsbWdFj+0J7EXJfMHFVlPlZwiHf1TvZSMPEOaXRdZcxl7uSIuJd3DEA +ynf4NmWm9Zxx5bLjmhfsP1336TfCoQhZQ3m8DZV52C4Jlm1DQIRre6tSYpA8LvZB +UYzLjYeBwhZS7hu24E1vm4ZhASSQQSSsHfGzx1IzSDBt1swx7+V/MpdhrZ7fJxVI +bJSEcllNOzuZViL4Yh7d4FINGBHor/xPDA5ndkgHlXKjy7QxNM1+wEBcFATQVSL0 +c+E8qtY918Wq5VergH9/4zPvSivyfv5gwtjCT24RAoIBABP6HbJb0BqrHB/U0cvn +00Vk3rGAIgwhpUtUrcz6PzkI+enlJCSV0zKkBH3I/Pf6jw3LTWUPgSWemQ6j6H7E +K3VrMvqeKBLGw1K+Afq3yKyFP7WIYqDswV31Oxf0rgC1NY7220uBoAt3CcSRQUo/ +VZ8XN/h7p+a70mmdIhklMlqhxMoPLN48eybFfMFOe5JAw7szfDdiwjZYDti8vcTi +SkDMBeuImCvI025c3QMPEmqwbkAPdg6r8Av06tEU8PkAspPR9ntcwCgp7KE9Pm6P +fQu8qwd6WsrPOswTI2AQyUqHAFLU2sQyj13jbhPT87w5fF/y7NmpxOnwS4igfbnH +2pECggEBALO0FiJClb0GSqMXYJ+nbtRObD4AynYNVMEqYdZu5DBb6vb4T7uumTD5 +I1fKOa5SSELngUj23p2G6sVBsDyDHotGJYJwDGejHOFnEpY+J0Das0pGS40FsFC7 +qABIUaMoLKcIR9Ofcm9uu2n+koNULV2aaXj7A4OYhRCQi2PqiEx1wimdrLfGqTXn +O4rSf826ODch87vuPbfFPCaIFG28R3nByp/ZBH5QNiB3NBmc3A0tiHFnZW3cpOfW +Jm/Vu0PcNVVw32SroS2FCroR7qSWsvt61UzJtliLUiFHoUAxrXXiAxcZW1D2Hmpq +neUhT/t9hHdcMJgoxm2IITf6ip8nTnY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_tlsv1_3/configs/ssl_config.xml b/tests/integration/test_tlsv1_3/configs/ssl_config.xml new file mode 100644 index 00000000000..b18f04dd954 --- /dev/null +++ b/tests/integration/test_tlsv1_3/configs/ssl_config.xml @@ -0,0 +1,73 @@ + + + 8443 + + + + + + + + false + /etc/clickhouse-server/config.d/server-cert.pem + /etc/clickhouse-server/config.d/server-key.pem + /etc/clickhouse-server/config.d/dhparam4096.pem + /etc/clickhouse-server/config.d/ca-cert.pem + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 + + true + false + false + false + true + relaxed + + + + false + sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 + true + true + false + false + false + true + relaxed + + + + + + \ No newline at end of file diff --git a/tests/integration/test_tlsv1_3/configs/users_with_ssl_auth.xml b/tests/integration/test_tlsv1_3/configs/users_with_ssl_auth.xml new file mode 100644 index 00000000000..c41776f9e78 --- /dev/null +++ b/tests/integration/test_tlsv1_3/configs/users_with_ssl_auth.xml @@ -0,0 +1,22 @@ + + + + + + client1 + + + + + client2 + client3 + + + + + + + qwe123 + + + diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py new file mode 100644 index 00000000000..d48b84925f5 --- /dev/null +++ b/tests/integration/test_tlsv1_3/test.py @@ -0,0 +1,236 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import urllib.request, urllib.parse +import ssl +import os.path + +HTTPS_PORT = 8443 +NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "node", + ipv4_address=NODE_IP, + main_configs=[ + "configs/ssl_config.xml", + "certs/server-key.pem", + "certs/server-cert.pem", + "certs/ca-cert.pem", + "certs/dhparam4096.pem" + ], + user_configs=["configs/users_with_ssl_auth.xml"], +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_ssl_context(cert_name): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") + if cert_name: + context.load_cert_chain( + f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", + f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", + ) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + return context + + +def execute_query_https( + query, user, enable_ssl_auth=True, cert_name=None, password=None +): + url = f"https://{NODE_IP_WITH_HTTPS_PORT}/?query={urllib.parse.quote(query)}" + request = urllib.request.Request(url) + request.add_header("X-ClickHouse-User", user) + if enable_ssl_auth: + request.add_header("X-ClickHouse-SSL-Certificate-Auth", "on") + if password: + request.add_header("X-ClickHouse-Key", password) + response = urllib.request.urlopen( + request, context=get_ssl_context(cert_name) + ).read() + return response.decode("utf-8") + + +def test_https(): + assert ( + execute_query_https("SELECT currentUser()", user="john", cert_name="client1") + == "john\n" + ) + assert ( + execute_query_https("SELECT currentUser()", user="lucy", cert_name="client2") + == "lucy\n" + ) + assert ( + execute_query_https("SELECT currentUser()", user="lucy", cert_name="client3") + == "lucy\n" + ) + + +def test_https_wrong_cert(): + # Wrong certificate: different user's certificate + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client2") + assert "HTTP Error 403" in str(err.value) + + # Wrong certificate: self-signed certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="wrong") + assert "unknown ca" in str(err.value) + + # No certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john") + assert "HTTP Error 403" in str(err.value) + + # No header enabling SSL authentication. + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="john", + enable_ssl_auth=False, + cert_name="client1", + ) + + +def test_https_non_ssl_auth(): + # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). + # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + ) + == "jane\n" + ) + + # But we still can send a certificate if we want. + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client1", + ) + == "peter\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client2", + ) + == "peter\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client3", + ) + == "peter\n" + ) + + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client1", + ) + == "jane\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client2", + ) + == "jane\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client3", + ) + == "jane\n" + ) + + # However if we send a certificate it must not be wrong. + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="wrong", + ) + assert "unknown ca" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="wrong", + ) + assert "unknown ca" in str(err.value) + + +def test_create_user(): + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") + assert ( + execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") + == "emma\n" + ) + assert ( + instance.query("SHOW CREATE USER emma") + == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client3\\'\n" + ) + + instance.query("ALTER USER emma IDENTIFIED WITH ssl_certificate CN 'client2'") + assert ( + execute_query_https("SELECT currentUser()", user="emma", cert_name="client2") + == "emma\n" + ) + assert ( + instance.query("SHOW CREATE USER emma") + == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client2\\'\n" + ) + + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") + assert "HTTP Error 403" in str(err.value) + + assert ( + instance.query("SHOW CREATE USER lucy") + == "CREATE USER lucy IDENTIFIED WITH ssl_certificate CN \\'client2\\', \\'client3\\'\n" + ) + + assert ( + instance.query( + "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" + ) + == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' + 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + ) From e362e4fff1ade90eb36b762e3a51a0d197b64a15 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 23 Jun 2022 20:43:35 +0200 Subject: [PATCH 107/408] Fix --- src/Storages/RabbitMQ/RabbitMQSource.cpp | 3 -- .../ReadBufferFromRabbitMQConsumer.cpp | 6 +++ .../RabbitMQ/ReadBufferFromRabbitMQConsumer.h | 18 ++++---- .../integration/test_storage_rabbitmq/test.py | 41 +++++++++++++++++-- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index 71d80f0a632..bbcbaa2b32b 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -142,9 +142,6 @@ Chunk RabbitMQSource::generateImpl() while (true) { - if (buffer->eof()) - break; - auto new_rows = executor.execute(); if (new_rows) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index e2de5179990..3543085f5a0 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -47,6 +47,12 @@ ReadBufferFromRabbitMQConsumer::~ReadBufferFromRabbitMQConsumer() } +void ReadBufferFromRabbitMQConsumer::closeChannel() +{ + if (consumer_channel) + consumer_channel->close(); +} + void ReadBufferFromRabbitMQConsumer::subscribe() { for (const auto & queue_name : queues) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 8a527011a3c..bd55d169744 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -3,18 +3,24 @@ #include #include #include -#include -#include #include namespace Poco { - class Logger; +class Logger; +} + +namespace AMQP +{ +class TcpChannel; } namespace DB { +class RabbitMQHandler; +using ChannelPtr = std::unique_ptr; + class ReadBufferFromRabbitMQConsumer : public ReadBuffer { @@ -52,11 +58,7 @@ public: ChannelPtr & getChannel() { return consumer_channel; } void setupChannel(); bool needChannelUpdate(); - void closeChannel() - { - if (consumer_channel) - consumer_channel->close(); - } + void closeChannel(); void updateQueues(std::vector & queues_) { queues = queues_; } size_t queuesCount() { return queues.size(); } diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index c1bd136126f..59ac2f20886 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2745,7 +2745,40 @@ def test_rabbitmq_predefined_configuration(rabbitmq_cluster): break -if __name__ == "__main__": - cluster.start() - input("Cluster created, press any key to destroy...") - cluster.shutdown() +def test_rabbitmq_msgpack(rabbitmq_cluster): + + instance.query( + """ + drop table if exists rabbit_in; + drop table if exists rabbit_out; + create table + rabbit_in (val String) + engine=RabbitMQ + settings rabbitmq_host_port = 'localhost:5672', + rabbitmq_exchange_name = 'xhep', + rabbitmq_format = 'MsgPack', + rabbitmq_num_consumers = 1; + create table + rabbit_out (val String) + engine=RabbitMQ + settings rabbitmq_host_port = 'localhost:5672', + rabbitmq_exchange_name = 'xhep', + rabbitmq_format = 'MsgPack', + rabbitmq_num_consumers = 1; + set stream_like_engine_allow_direct_select=1; + insert into rabbit_out select 'kek'; + """ + ) + + result = "" + try_no = 0 + while True: + result = instance.query("select * from rabbit_in;") + if result.strip() == "kek": + break + else: + try_no = try_no + 1 + if try_no == 20: + break + sleep(1) + assert result.strip() == "kek" From 191936a806c6cdea328de07e0b58726b54ec4abb Mon Sep 17 00:00:00 2001 From: Larry Luo Date: Thu, 23 Jun 2022 11:50:06 -0700 Subject: [PATCH 108/408] clean up comments --- .../test_tlsv1_3/configs/ssl_config.xml | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/tests/integration/test_tlsv1_3/configs/ssl_config.xml b/tests/integration/test_tlsv1_3/configs/ssl_config.xml index b18f04dd954..e3d1831e08c 100644 --- a/tests/integration/test_tlsv1_3/configs/ssl_config.xml +++ b/tests/integration/test_tlsv1_3/configs/ssl_config.xml @@ -40,34 +40,6 @@ true relaxed - - \ No newline at end of file From 1239ee4b31311ef248d1a93f6829ed43ff8c0d9b Mon Sep 17 00:00:00 2001 From: Larry Luo Date: Thu, 23 Jun 2022 12:11:18 -0700 Subject: [PATCH 109/408] update ssl config --- tests/integration/test_tlsv1_3/configs/ssl_config.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_tlsv1_3/configs/ssl_config.xml b/tests/integration/test_tlsv1_3/configs/ssl_config.xml index e3d1831e08c..9e686b55567 100644 --- a/tests/integration/test_tlsv1_3/configs/ssl_config.xml +++ b/tests/integration/test_tlsv1_3/configs/ssl_config.xml @@ -28,7 +28,6 @@ true relaxed - false sslv2,sslv3,tlsv1,tlsv1_1,tlsv1_2 From 0d304f7b8cbfbbac48a8fe08a10b374c8987396c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 23 Jun 2022 21:19:07 +0200 Subject: [PATCH 110/408] fix tests --- tests/integration/test_atomic_drop_table/test.py | 2 +- .../test_attach_partition_with_large_destination/test.py | 2 +- tests/integration/test_backup_restore/test.py | 2 +- tests/integration/test_backup_with_other_granularity/test.py | 2 +- tests/integration/test_dictionaries_dependency/test.py | 2 +- tests/integration/test_distributed_ddl/test.py | 2 +- .../test_distributed_storage_configuration/test.py | 2 +- tests/integration/test_filesystem_layout/test.py | 2 +- tests/integration/test_force_drop_table/test.py | 2 +- .../integration/test_keeper_multinode_blocade_leader/test.py | 4 ++-- tests/integration/test_merge_tree_s3_restore/test.py | 2 +- tests/integration/test_partition/test.py | 2 +- tests/integration/test_polymorphic_parts/test.py | 2 +- .../test_replicated_merge_tree_compatibility/test.py | 2 +- .../integration/test_replicated_merge_tree_s3_restore/test.py | 2 +- tests/integration/test_system_merges/test.py | 4 ++-- tests/performance/merge_table_streams.xml | 1 + tests/queries/0_stateless/00090_union_race_conditions_1.sh | 1 + .../0_stateless/00215_primary_key_order_zookeeper_long.sql | 1 + tests/queries/0_stateless/00505_distributed_secure.data | 1 + tests/queries/0_stateless/00632_get_sample_block_cache.sql | 1 + tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh | 1 + 22 files changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index 1fe88dde099..5ef2f57b684 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -20,7 +20,7 @@ def start_cluster(): try: cluster.start() node1.query( - "CREATE DATABASE zktest ENGINE=Ordinary;" + "CREATE DATABASE zktest ENGINE=Ordinary;", settings={"allow_deprecated_database_ordinary": 1} ) # Different behaviour with Atomic node1.query( """ diff --git a/tests/integration/test_attach_partition_with_large_destination/test.py b/tests/integration/test_attach_partition_with_large_destination/test.py index 0a4ab9fada1..1f4e92ebc55 100644 --- a/tests/integration/test_attach_partition_with_large_destination/test.py +++ b/tests/integration/test_attach_partition_with_large_destination/test.py @@ -34,7 +34,7 @@ def create_force_drop_flag(node): @pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_attach_partition_with_large_destination(started_cluster, engine): # Initialize - node.query("CREATE DATABASE db ENGINE={}".format(engine)) + node.query("CREATE DATABASE db ENGINE={}".format(engine), settings={"allow_deprecated_database_ordinary": 1}) node.query( "CREATE TABLE db.destination (n UInt64) ENGINE=ReplicatedMergeTree('/test/destination', 'r1') ORDER BY n PARTITION BY n % 2" ) diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index 905abef05b0..a379c52579e 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -15,7 +15,7 @@ def started_cluster(): try: cluster.start() instance.query( - "CREATE DATABASE test ENGINE = Ordinary" + "CREATE DATABASE test ENGINE = Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different path in shadow/ with Atomic instance.query("DROP TABLE IF EXISTS test.tbl") instance.query( diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 9cb998fb505..fac5331cb7d 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -202,7 +202,7 @@ def test_backup_from_old_version_config(started_cluster): def test_backup_and_alter(started_cluster): node4.query( - "CREATE DATABASE test ENGINE=Ordinary" + "CREATE DATABASE test ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different path in shadow/ with Atomic node4.query( diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index f57d4e42813..be5767111a9 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -16,7 +16,7 @@ def start_cluster(): for node in nodes: node.query("CREATE DATABASE IF NOT EXISTS test") # Different internal dictionary name with Atomic - node.query("CREATE DATABASE IF NOT EXISTS test_ordinary ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS test_ordinary ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1}) node.query("CREATE DATABASE IF NOT EXISTS atest") node.query("CREATE DATABASE IF NOT EXISTS ztest") node.query("CREATE TABLE test.source(x UInt64, y UInt64) ENGINE=Log") diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 2789541b519..99ca6684e56 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -552,7 +552,7 @@ def test_replicated_without_arguments(test_cluster): ) test_cluster.ddl_check_query( - instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary" + instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) assert ( "are supported only for ON CLUSTER queries with Atomic database engine" diff --git a/tests/integration/test_distributed_storage_configuration/test.py b/tests/integration/test_distributed_storage_configuration/test.py index fa4e01bb7b3..1ce8846e815 100644 --- a/tests/integration/test_distributed_storage_configuration/test.py +++ b/tests/integration/test_distributed_storage_configuration/test.py @@ -20,7 +20,7 @@ def start_cluster(): try: cluster.start() node.query( - "CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary" + "CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different paths with Atomic yield cluster finally: diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 34e377e0ae4..78c7b74adcb 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -16,7 +16,7 @@ def started_cluster(): def test_file_path_escaping(started_cluster): - node.query("CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary", settings={"allow_deprecated_database_ordinary": 1}) node.query( """ CREATE TABLE test.`T.a_b,l-e!` (`~Id` UInt32) diff --git a/tests/integration/test_force_drop_table/test.py b/tests/integration/test_force_drop_table/test.py index c1eec1cd277..7731c5d7cf4 100644 --- a/tests/integration/test_force_drop_table/test.py +++ b/tests/integration/test_force_drop_table/test.py @@ -33,7 +33,7 @@ def create_force_drop_flag(node): @pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_drop_materialized_view(started_cluster, engine): - node.query("CREATE DATABASE d ENGINE={}".format(engine)) + node.query("CREATE DATABASE d ENGINE={}".format(engine), settings={"allow_deprecated_database_ordinary": 1}) node.query( "CREATE TABLE d.rmt (n UInt64) ENGINE=ReplicatedMergeTree('/test/rmt', 'r1') ORDER BY n PARTITION BY n % 2" ) diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index c2d4039e122..4b1b70ffeab 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -95,7 +95,7 @@ def test_blocade_leader(started_cluster): wait_nodes() try: for i, node in enumerate([node1, node2, node3]): - node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1}) node.query( "CREATE TABLE IF NOT EXISTS ordinary.t1 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t1', '{}') ORDER BY tuple()".format( i + 1 @@ -296,7 +296,7 @@ def test_blocade_leader_twice(started_cluster): wait_nodes() try: for i, node in enumerate([node1, node2, node3]): - node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") + node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1}) node.query( "CREATE TABLE IF NOT EXISTS ordinary.t2 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t2', '{}') ORDER BY tuple()".format( i + 1 diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index e6ca4a78c25..9eae7ce72ad 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -93,7 +93,7 @@ def create_table( node.query( "CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format( engine="Atomic" if db_atomic else "Ordinary" - ) + ), settings={"allow_deprecated_database_ordinary": 1} ) create_table_statement = """ diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index b396b58df10..43bbddfb2e1 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -14,7 +14,7 @@ def started_cluster(): try: cluster.start() q( - "CREATE DATABASE test ENGINE = Ordinary" + "CREATE DATABASE test ENGINE = Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different path in shadow/ with Atomic yield cluster diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index edd65ec002c..245b53d64bf 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -720,7 +720,7 @@ def test_in_memory_alters(start_cluster): def test_polymorphic_parts_index(start_cluster): node1.query( - "CREATE DATABASE test_index ENGINE=Ordinary" + "CREATE DATABASE test_index ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different paths with Atomic node1.query( """ diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index 00367daad33..476abd888ab 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -52,7 +52,7 @@ def test_replicated_merge_tree_defaults_compatibility(started_cluster): """ for node in (node1, node2): - node.query("CREATE DATABASE test ENGINE = Ordinary") + node.query("CREATE DATABASE test ENGINE = Ordinary", settings={"allow_deprecated_database_ordinary": 1}) node.query(create_query.format(replica=node.name)) node1.query("DETACH TABLE test.table") diff --git a/tests/integration/test_replicated_merge_tree_s3_restore/test.py b/tests/integration/test_replicated_merge_tree_s3_restore/test.py index 904bcfa4280..1454858f71e 100644 --- a/tests/integration/test_replicated_merge_tree_s3_restore/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_restore/test.py @@ -76,7 +76,7 @@ def create_table(node, table_name, schema, attach=False, db_atomic=False, uuid=" "CREATE DATABASE IF NOT EXISTS s3 {on_cluster} ENGINE = {engine}".format( engine="Atomic" if db_atomic else "Ordinary", on_cluster="ON CLUSTER '{cluster}'", - ) + ), settings={"allow_deprecated_database_ordinary": 1} ) create_table_statement = """ diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 775706f4df6..12f5cedbbb2 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -26,9 +26,9 @@ def started_cluster(): try: cluster.start() node1.query( - "CREATE DATABASE test ENGINE=Ordinary" + "CREATE DATABASE test ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1} ) # Different paths with Atomic - node2.query("CREATE DATABASE test ENGINE=Ordinary") + node2.query("CREATE DATABASE test ENGINE=Ordinary", settings={"allow_deprecated_database_ordinary": 1}) yield cluster finally: diff --git a/tests/performance/merge_table_streams.xml b/tests/performance/merge_table_streams.xml index efeb4547f37..1e053c98738 100644 --- a/tests/performance/merge_table_streams.xml +++ b/tests/performance/merge_table_streams.xml @@ -1,6 +1,7 @@ 5 + 1 + + + + json1 + + '{"k1":1, "k2": "some"}' + + + + json2 + + '{"col' || toString(number % 100) || '":' || toString(number) || '}' + + + + json3 + + '{"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}' + + + + CREATE TABLE t_json_1(data JSON) ENGINE = MergeTree ORDER BY tuple() CREATE TABLE t_json_2(data JSON) ENGINE = MergeTree ORDER BY tuple() CREATE TABLE t_json_3(data JSON) ENGINE = MergeTree ORDER BY tuple() - INSERT INTO t_json_1 SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers(200000) - INSERT INTO t_json_2 SELECT '{"col' || toString(number % 100) || '":' || toString(number) || '}' FROM numbers(100000) - INSERT INTO t_json_3 SELECT materialize('{"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}') FROM numbers_mt(100000) + INSERT INTO t_json_1 SELECT materialize({json1}) FROM numbers(200000) + INSERT INTO t_json_2 SELECT {json2} FROM numbers(100000) + INSERT INTO t_json_3 SELECT materialize({json3}) FROM numbers_mt(100000) DROP TABLE IF EXISTS t_json_1 DROP TABLE IF EXISTS t_json_2 From 4f54abf67fec04c3f3e34d6b8b4c5e57118a0a96 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 28 Jun 2022 16:25:29 +0200 Subject: [PATCH 213/408] better names for macros --- base/base/defines.h | 16 +++++++-------- src/Databases/DatabaseAtomic.cpp | 2 +- src/Databases/DatabaseOnDisk.cpp | 4 ++-- src/Databases/DatabaseOrdinary.cpp | 12 +++++------ src/Databases/DatabaseReplicated.cpp | 2 +- .../DatabaseMaterializedPostgreSQL.cpp | 10 +++++----- src/Interpreters/DatabaseCatalog.cpp | 2 +- src/Interpreters/DatabaseCatalog.h | 4 ++-- src/Interpreters/MergeTreeTransaction.cpp | 2 +- src/Interpreters/TransactionLog.cpp | 20 +++++++++---------- .../01174_select_insert_isolation.sh | 2 +- 11 files changed, 38 insertions(+), 38 deletions(-) diff --git a/base/base/defines.h b/base/base/defines.h index d0417885df6..a707e965675 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -136,14 +136,14 @@ /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function) /// Consider adding a comment before using these macros. -# define READ_NO_TSA(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> const auto & { return (x); }() -# define WRITE_NO_TSA(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> auto & { return (x); }() +# define TSA_SUPPRESS_WARNING_FOR_READ(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> const auto & { return (x); }() +# define TSA_SUPPRESS_WARNING_FOR_WRITE(x) [&]() TSA_NO_THREAD_SAFETY_ANALYSIS -> auto & { return (x); }() /// This macro is useful when only one thread writes to a member /// and you want to read this member from the same thread without locking a mutex. -/// It's safe (because no concurrent writes are possible), but TSA generates a waring. -/// (Seems like there's no way to verify it, but it makes sense to distinguish it from READ_NO_TSA for readability) -# define READ_ONE_THREAD(x) READ_NO_TSA(x) +/// It's safe (because no concurrent writes are possible), but TSA generates a warning. +/// (Seems like there's no way to verify it, but it makes sense to distinguish it from TSA_SUPPRESS_WARNING_FOR_READ for readability) +# define TSA_READ_ONE_THREAD(x) TSA_SUPPRESS_WARNING_FOR_READ(x) #else # define TSA_GUARDED_BY(...) @@ -152,9 +152,9 @@ # define TSA_REQUIRES_SHARED(...) # define TSA_NO_THREAD_SAFETY_ANALYSIS -# define READ_NO_TSA(x) -# define WRITE_NO_TSA(x) -# define TSA_READ_UNSAFE(x) +# define TSA_SUPPRESS_WARNING_FOR_READ(x) +# define TSA_SUPPRESS_WARNING_FOR_WRITE(x) +# define TSA_READ_ONE_THREAD(x) #endif /// A template function for suppressing warnings about unused variables or function results. diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index b0ac9e0e999..e92530662b2 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -73,7 +73,7 @@ String DatabaseAtomic::getTableDataPath(const ASTCreateQuery & query) const void DatabaseAtomic::drop(ContextPtr) { - assert(READ_NO_TSA(tables).empty()); + assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty()); try { fs::remove(path_to_metadata_symlink); diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index e9470a93af0..385fac37901 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -503,7 +503,7 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const void DatabaseOnDisk::drop(ContextPtr local_context) { - assert(READ_NO_TSA(tables).empty()); + assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty()); if (local_context->getSettingsRef().force_remove_data_recursively_on_drop) { fs::remove_all(local_context->getPath() + getDataPath()); @@ -757,7 +757,7 @@ void DatabaseOnDisk::modifySettingsMetadata(const SettingsChanges & settings_cha writeChar('\n', statement_buf); String statement = statement_buf.str(); - String database_name_escaped = escapeForFileName(READ_NO_TSA(database_name)); /// FIXME + String database_name_escaped = escapeForFileName(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); /// FIXME fs::path metadata_root_path = fs::canonical(query_context->getGlobalContext()->getPath()); fs::path metadata_file_tmp_path = fs::path(metadata_root_path) / "metadata" / (database_name_escaped + ".sql.tmp"); fs::path metadata_file_path = fs::path(metadata_root_path) / "metadata" / (database_name_escaped + ".sql"); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 9d6c86b1561..18b70222382 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -175,7 +175,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables { auto * create_query = ast->as(); /// NOTE No concurrent writes are possible during database loading - create_query->setDatabase(READ_NO_TSA(database_name)); + create_query->setDatabase(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); /// Even if we don't load the table we can still mark the uuid of it as taken. if (create_query->uuid != UUIDHelpers::Nil) @@ -202,7 +202,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables return; } - QualifiedTableName qualified_name{READ_NO_TSA(database_name), create_query->getTable()}; + QualifiedTableName qualified_name{TSA_SUPPRESS_WARNING_FOR_READ(database_name), create_query->getTable()}; TableNamesSet loading_dependencies = getDependenciesSetFromCreateQuery(getContext(), qualified_name, ast); std::lock_guard lock{metadata.mutex}; @@ -235,12 +235,12 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables size_t tables_in_database = objects_in_database - dictionaries_in_database; LOG_INFO(log, "Metadata processed, database {} has {} tables and {} dictionaries in total.", - READ_NO_TSA(database_name), tables_in_database, dictionaries_in_database); + TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database); } void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) { - assert(name.database == READ_NO_TSA(database_name)); + assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name)); const auto & create_query = ast->as(); tryAttachTable( @@ -257,7 +257,7 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_rest LOG_INFO(log, "Starting up tables."); /// NOTE No concurrent writes are possible during database loading - const size_t total_tables = READ_NO_TSA(tables).size(); + const size_t total_tables = TSA_SUPPRESS_WARNING_FOR_READ(tables).size(); if (!total_tables) return; @@ -273,7 +273,7 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_rest try { - for (const auto & table : READ_NO_TSA(tables)) + for (const auto & table : TSA_SUPPRESS_WARNING_FOR_READ(tables)) thread_pool.scheduleOrThrowOnError([&]() { startup_one_table(table.second); }); } catch (...) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index c9031fcaf27..7edcc0b693f 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -213,7 +213,7 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const treat_local_port_as_remote, cluster_auth_info.cluster_secure_connection, /*priority=*/1, - READ_NO_TSA(database_name), /// FIXME + TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME cluster_auth_info.cluster_secret); } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index 4d4b310ab61..f5985aa9df9 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -63,9 +63,9 @@ void DatabaseMaterializedPostgreSQL::startSynchronization() return; replication_handler = std::make_unique( - /* replication_identifier */ READ_NO_TSA(database_name), /// FIXME + /* replication_identifier */ TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME remote_database_name, - READ_NO_TSA(database_name), /// FIXME + TSA_SUPPRESS_WARNING_FOR_READ(database_name), /// FIXME connection_info, getContext(), is_attach, @@ -100,7 +100,7 @@ void DatabaseMaterializedPostgreSQL::startSynchronization() { /// Nested table does not exist and will be created by replication thread. /// FIXME TSA - storage = std::make_shared(StorageID(READ_NO_TSA(database_name), table_name), getContext(), remote_database_name, table_name); + storage = std::make_shared(StorageID(TSA_SUPPRESS_WARNING_FOR_READ(database_name), table_name), getContext(), remote_database_name, table_name); } /// Cache MaterializedPostgreSQL wrapper over nested table. @@ -212,7 +212,7 @@ ASTPtr DatabaseMaterializedPostgreSQL::getCreateTableQueryImpl(const String & ta std::lock_guard lock(handler_mutex); /// FIXME TSA - auto storage = std::make_shared(StorageID(READ_NO_TSA(database_name), table_name), getContext(), remote_database_name, table_name); + auto storage = std::make_shared(StorageID(TSA_SUPPRESS_WARNING_FOR_READ(database_name), table_name), getContext(), remote_database_name, table_name); auto ast_storage = replication_handler->getCreateNestedTableQuery(storage.get(), table_name); assert_cast(ast_storage.get())->uuid = UUIDHelpers::generateV4(); return ast_storage; @@ -236,7 +236,7 @@ ASTPtr DatabaseMaterializedPostgreSQL::createAlterSettingsQuery(const SettingCha auto * alter = query->as(); alter->alter_object = ASTAlterQuery::AlterObjectType::DATABASE; - alter->setDatabase(READ_NO_TSA(database_name)); /// FIXME + alter->setDatabase(TSA_SUPPRESS_WARNING_FOR_READ(database_name)); /// FIXME alter->set(alter->command_list, command_list); return query; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 4a0874f3050..a0579b813db 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -694,7 +694,7 @@ DDLGuardPtr DatabaseCatalog::getDDLGuard(const String & database, const String & { std::unique_lock lock(ddl_guards_mutex); /// TSA does not support unique_lock - auto db_guard_iter = WRITE_NO_TSA(ddl_guards).try_emplace(database).first; + auto db_guard_iter = TSA_SUPPRESS_WARNING_FOR_WRITE(ddl_guards).try_emplace(database).first; DatabaseGuard & db_guard = db_guard_iter->second; return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table, database); } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 7daa70fac13..4468cc3a5d8 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -233,8 +233,8 @@ private: static std::unique_ptr database_catalog; explicit DatabaseCatalog(ContextMutablePtr global_context_); - void assertDatabaseExistsUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex) TSA_REQUIRES(databases_mutex); - void assertDatabaseDoesntExistUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex) TSA_REQUIRES(databases_mutex); + void assertDatabaseExistsUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex); + void assertDatabaseDoesntExistUnlocked(const String & database_name) const TSA_REQUIRES(databases_mutex); void shutdownImpl(); diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index 5b657b88194..432116feaf5 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -225,7 +225,7 @@ void MergeTreeTransaction::afterCommit(CSN assigned_csn) noexcept DataPartsVector removed_parts; RunningMutationsList committed_mutations; { - /// We don't really need mutex here, because no concurrent modifications of transaction object may happen after comit. + /// We don't really need mutex here, because no concurrent modifications of transaction object may happen after commit. std::lock_guard lock{mutex}; created_parts = creating_parts; removed_parts = removing_parts; diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index a3a3e336693..a08f940a748 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -125,7 +125,7 @@ void TransactionLog::loadEntries(Strings::const_iterator beg, Strings::const_ite LOG_TRACE(log, "Loading {} entries from {}: {}..{}", entries_count, zookeeper_path_log, *beg, last_entry); futures.reserve(entries_count); for (auto it = beg; it != end; ++it) - futures.emplace_back(READ_ONE_THREAD(zookeeper)->asyncGet(fs::path(zookeeper_path_log) / *it)); + futures.emplace_back(TSA_READ_ONE_THREAD(zookeeper)->asyncGet(fs::path(zookeeper_path_log) / *it)); std::vector> loaded; loaded.reserve(entries_count); @@ -210,7 +210,7 @@ void TransactionLog::runUpdatingThread() try { /// Do not wait if we have some transactions to finalize - if (READ_ONE_THREAD(unknown_state_list_loaded).empty()) + if (TSA_READ_ONE_THREAD(unknown_state_list_loaded).empty()) log_updated_event->wait(); if (stop_flag.load()) @@ -227,7 +227,7 @@ void TransactionLog::runUpdatingThread() /// It's possible that we connected to different [Zoo]Keeper instance /// so we may read a bit stale state. - READ_ONE_THREAD(zookeeper)->sync(zookeeper_path_log); + TSA_READ_ONE_THREAD(zookeeper)->sync(zookeeper_path_log); } loadNewEntries(); @@ -252,13 +252,13 @@ void TransactionLog::runUpdatingThread() void TransactionLog::loadNewEntries() { - Strings entries_list = READ_ONE_THREAD(zookeeper)->getChildren(zookeeper_path_log, nullptr, log_updated_event); + Strings entries_list = TSA_READ_ONE_THREAD(zookeeper)->getChildren(zookeeper_path_log, nullptr, log_updated_event); chassert(!entries_list.empty()); ::sort(entries_list.begin(), entries_list.end()); - auto it = std::upper_bound(entries_list.begin(), entries_list.end(), READ_ONE_THREAD(last_loaded_entry)); + auto it = std::upper_bound(entries_list.begin(), entries_list.end(), TSA_READ_ONE_THREAD(last_loaded_entry)); loadEntries(it, entries_list.end()); - chassert(READ_ONE_THREAD(last_loaded_entry) == entries_list.back()); - chassert(latest_snapshot == deserializeCSN(READ_ONE_THREAD(last_loaded_entry))); + chassert(TSA_READ_ONE_THREAD(last_loaded_entry) == entries_list.back()); + chassert(latest_snapshot == deserializeCSN(TSA_READ_ONE_THREAD(last_loaded_entry))); latest_snapshot.notify_all(); } @@ -278,7 +278,7 @@ void TransactionLog::removeOldEntries() /// TODO we will need a bit more complex logic for multiple hosts Coordination::Stat stat; - CSN old_tail_ptr = deserializeCSN(READ_ONE_THREAD(zookeeper)->get(zookeeper_path + "/tail_ptr", &stat)); + CSN old_tail_ptr = deserializeCSN(TSA_READ_ONE_THREAD(zookeeper)->get(zookeeper_path + "/tail_ptr", &stat)); CSN new_tail_ptr = getOldestSnapshot(); if (new_tail_ptr < old_tail_ptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected tail_ptr {}, oldest snapshot is {}, it's a bug", old_tail_ptr, new_tail_ptr); @@ -287,7 +287,7 @@ void TransactionLog::removeOldEntries() /// (it's not supposed to fail with ZBADVERSION while there is only one host) LOG_TRACE(log, "Updating tail_ptr from {} to {}", old_tail_ptr, new_tail_ptr); - READ_ONE_THREAD(zookeeper)->set(zookeeper_path + "/tail_ptr", serializeCSN(new_tail_ptr), stat.version); + TSA_READ_ONE_THREAD(zookeeper)->set(zookeeper_path + "/tail_ptr", serializeCSN(new_tail_ptr), stat.version); tail_ptr.store(new_tail_ptr); /// Now we can find and remove old entries @@ -311,7 +311,7 @@ void TransactionLog::removeOldEntries() continue; LOG_TEST(log, "Removing entry {} -> {}", elem.second.tid, elem.second.csn); - auto code = READ_ONE_THREAD(zookeeper)->tryRemove(zookeeper_path_log + "/" + serializeCSN(elem.second.csn)); + auto code = TSA_READ_ONE_THREAD(zookeeper)->tryRemove(zookeeper_path_log + "/" + serializeCSN(elem.second.csn)); if (code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE) removed_entries.push_back(elem.first); } diff --git a/tests/queries/0_stateless/01174_select_insert_isolation.sh b/tests/queries/0_stateless/01174_select_insert_isolation.sh index d66741b08b7..cf1bb23f702 100755 --- a/tests/queries/0_stateless/01174_select_insert_isolation.sh +++ b/tests/queries/0_stateless/01174_select_insert_isolation.sh @@ -43,7 +43,7 @@ function thread_select() SELECT throwIf((SELECT sum(n) FROM mt) != 0) FORMAT Null; SELECT throwIf((SELECT count() FROM mt) % 2 != 0) FORMAT Null; SELECT arraySort(groupArray(n)), arraySort(groupArray(m)), arraySort(groupArray(_part)) FROM mt; - COMMIT;" + COMMIT;" | uniq | wc -l | grep -v "^1$" ||: done } From 9fc7c9f575f02112ea800462d70d9449e43c9d02 Mon Sep 17 00:00:00 2001 From: GruffGemini <43479425+GruffGemini@users.noreply.github.com> Date: Tue, 28 Jun 2022 17:55:16 +0300 Subject: [PATCH 214/408] Update group-by.md docs (group-by.md): fixed broken links --- docs/ru/sql-reference/statements/select/group-by.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/statements/select/group-by.md b/docs/ru/sql-reference/statements/select/group-by.md index 5a306e63955..01df1b969bf 100644 --- a/docs/ru/sql-reference/statements/select/group-by.md +++ b/docs/ru/sql-reference/statements/select/group-by.md @@ -264,10 +264,10 @@ GROUP BY вычисляет для каждого встретившегося ### Группировка во внешней памяти {#select-group-by-in-external-memory} Можно включить сброс временных данных на диск, чтобы ограничить потребление оперативной памяти при выполнении `GROUP BY`. -Настройка [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. +Настройка [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) определяет пороговое значение потребления RAM, по достижении которого временные данные `GROUP BY` сбрасываются в файловую систему. Если равно 0 (по умолчанию) - значит выключено. При использовании `max_bytes_before_external_group_by`, рекомендуем выставить `max_memory_usage` приблизительно в два раза больше. Это следует сделать, потому что агрегация выполняется в две стадии: чтение и формирование промежуточных данных (1) и слияние промежуточных данных (2). Сброс данных на файловую систему может производиться только на стадии 1. Если сброса временных данных не было, то на стадии 2 может потребляться до такого же объёма памяти, как на стадии 1. -Например, если [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а `max_memory_usage` в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. +Например, если [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) было выставлено в 10000000000, и вы хотите использовать внешнюю агрегацию, то имеет смысл выставить `max_bytes_before_external_group_by` в 10000000000, а `max_memory_usage` в 20000000000. При срабатывании внешней агрегации (если был хотя бы один сброс временных данных в файловую систему) максимальное потребление оперативки будет лишь чуть-чуть больше `max_bytes_before_external_group_by`. При распределённой обработке запроса внешняя агрегация производится на удалённых серверах. Для того чтобы на сервере-инициаторе запроса использовалось немного оперативки, нужно выставить настройку `distributed_aggregation_memory_efficient` в 1. From e1ac7477c9a27f6cb38aac3be7c45a11610beebf Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 28 Jun 2022 17:51:47 +0200 Subject: [PATCH 215/408] Avoid killing terminate-instances task --- tests/ci/worker/init_runner.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 13d873a0066..93c13b161de 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -50,7 +50,9 @@ ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}')) if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then echo "Going to terminate the runner, it has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /" INSTANCE_ID=$(ec2metadata --instance-id) - ( sleep 10 && aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" ) & + # We execute it with at to not have it as an orphan process + # GH Runners kill all remain processes + echo "sleep 10; aws ec2 terminate-instances --instance-ids $INSTANCE_ID" | at now exit 0 fi From 52d6b45a097d55889c884738a09457ec7b8996ec Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 28 Jun 2022 12:37:07 -0400 Subject: [PATCH 216/408] update poco --- contrib/poco | 2 +- contrib/poco-cmake/XML/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index de35b9fd72b..56a6a6c08d4 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit de35b9fd72b57127abdc3a5beaf0e320d767e356 +Subproject commit 56a6a6c08d4dab0262683264329c5db389ce2b82 diff --git a/contrib/poco-cmake/XML/CMakeLists.txt b/contrib/poco-cmake/XML/CMakeLists.txt index 45100f11eb7..97e655a0f04 100644 --- a/contrib/poco-cmake/XML/CMakeLists.txt +++ b/contrib/poco-cmake/XML/CMakeLists.txt @@ -11,6 +11,7 @@ add_library (_poco_xml_expat ${SRCS_EXPAT}) add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_include_directories (_poco_xml_expat PUBLIC "${LIBRARY_DIR}/XML/include") +target_include_directories (_poco_xml_expat PRIVATE "${LIBRARY_DIR}/Foundation/include") # Poco::XML From 30f136867fbfdee727decbe9fe4eb4a0ba19fde4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 28 Jun 2022 16:51:39 +0000 Subject: [PATCH 217/408] Fix build --- base/base/iostream_debug_helpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/base/iostream_debug_helpers.h b/base/base/iostream_debug_helpers.h index 6b12afa4bed..3a3f1a741ad 100644 --- a/base/base/iostream_debug_helpers.h +++ b/base/base/iostream_debug_helpers.h @@ -120,7 +120,7 @@ Out & dumpDispatchPriorities(Out & out, T && x, std::decay_t(out, x); } -struct LowPriority { explicit LowPriority(void *) {} }; +struct LowPriority { LowPriority(void *) {} }; template Out & dumpDispatchPriorities(Out & out, T && x, LowPriority) From e78db730782ab74c588578b3c31cf2d7879c1c1c Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 28 Jun 2022 12:56:43 -0400 Subject: [PATCH 218/408] update poco --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index 56a6a6c08d4..0e32cb42db7 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 56a6a6c08d4dab0262683264329c5db389ce2b82 +Subproject commit 0e32cb42db76ddaa76848470219056908053b676 From 2b11c0daa9c264ddc756ca6c6094347b3996ffe0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 28 Jun 2022 19:26:15 +0200 Subject: [PATCH 219/408] Updated tests --- .../02345_partial_sort_transform_optimization.reference | 5 +++++ .../02345_partial_sort_transform_optimization.sql | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.reference b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.reference index e6c99ff9291..7b50765be55 100644 --- a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.reference +++ b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.reference @@ -8,3 +8,8 @@ 0 999992 999992 0 999991 999991 0 999990 999990 +98974 +98973 +98972 +98971 +98970 diff --git a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql index e7855c47474..a53a352f57e 100644 --- a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql +++ b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql @@ -1,3 +1,3 @@ --- Regression for PartialSortingTransform optimization --- that requires at least 1500 rows. -select * from (select * from (select 0 a, toNullable(number) b, toString(number) c from numbers(1e6)) order by a desc, b desc, c limit 1500) limit 10; +-- Regression for PartialSortingTransform optimization that requires at least 1500 rows. +SELECT * FROM (SELECT * FROM (SELECT 0 a, toNullable(number) b, toString(number) c FROM numbers(1e6)) ORDER BY a DESC, b DESC, c LIMIT 1500) limit 10; +SELECT number FROM (SELECT number, 1 AS k FROM numbers(100000) ORDER BY k ASC, number DESC LIMIT 1025, 1023) LIMIT 5; From 0bb036dd211a488e5ff9a05458006fd1639debc7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 28 Jun 2022 20:35:20 +0300 Subject: [PATCH 220/408] Update Dockerfile --- docker/test/integration/runner/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 80a2158b17d..d6bd458a01b 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:20.04 # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" + RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list RUN apt-get update \ From 9b4da86e2e5cf74a8b5b1205bbabc909e357d654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 28 Jun 2022 20:12:07 +0200 Subject: [PATCH 221/408] Adapt table creation with replication to avoid issues with pre-22.4 replicas --- src/Storages/StorageReplicatedMergeTree.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6895142b2a4..c66fa16f323 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -598,6 +598,9 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() auto zookeeper = getZooKeeper(); std::vector futures; + /// We need to confirm /quorum exists here although it's called under createTableIfNotExists because in older CH releases (pre 22.4) + /// it was created here, so if metadata creation is done by an older replica the node might not exists when reaching this call + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum", String(), zkutil::CreateMode::Persistent)); futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); /// Nodes for remote fs zero-copy replication From f692ead6adf1e1e1afcb20e8adef2649df8d5376 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 28 Jun 2022 19:19:06 +0000 Subject: [PATCH 222/408] Don't use std::unique_lock unless we have to Replace where possible by std::lock_guard which is more light-weight. --- base/base/BorrowedObjectPool.h | 8 ++++---- src/Access/KerberosInit.cpp | 2 +- src/Common/OvercommitTracker.cpp | 2 +- src/Common/PoolBase.h | 4 ++-- src/Common/SystemLogBase.cpp | 2 +- src/Common/ThreadPool.cpp | 10 +++++----- src/Coordination/KeeperServer.cpp | 2 +- src/Daemon/BaseDaemon.cpp | 2 +- src/Databases/MySQL/DatabaseMaterializedMySQL.cpp | 4 ++-- src/Dictionaries/CacheDictionaryUpdateQueue.cpp | 4 ++-- src/Disks/DiskCacheWrapper.cpp | 4 ++-- src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp | 2 +- .../ObjectStorages/S3/ProxyResolverConfiguration.cpp | 4 ++-- src/IO/HTTPCommon.cpp | 2 +- src/IO/S3Common.cpp | 4 ++-- src/Interpreters/AsynchronousInsertQueue.cpp | 4 ++-- src/Interpreters/DatabaseCatalog.cpp | 2 +- src/Interpreters/EmbeddedDictionaries.cpp | 2 +- src/Interpreters/MergeJoin.cpp | 6 +++--- src/Interpreters/Set.cpp | 2 +- .../Formats/Impl/ParallelFormattingOutputFormat.cpp | 4 ++-- .../Formats/Impl/ParallelFormattingOutputFormat.h | 2 +- .../Formats/Impl/ParallelParsingInputFormat.cpp | 4 ++-- src/Server/HTTP/HTTPServerConnection.cpp | 2 +- .../ReplicatedMergeTreeRestartingThread.cpp | 2 +- src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 12 ++++++------ src/Storages/WindowView/StorageWindowView.cpp | 4 ++-- 28 files changed, 52 insertions(+), 52 deletions(-) diff --git a/base/base/BorrowedObjectPool.h b/base/base/BorrowedObjectPool.h index 10dcc41d862..bb4c9cd1c21 100644 --- a/base/base/BorrowedObjectPool.h +++ b/base/base/BorrowedObjectPool.h @@ -89,7 +89,7 @@ public: inline void returnObject(T && object_to_return) { { - std::lock_guard lock(objects_mutex); + std::lock_guard lock(objects_mutex); objects.emplace_back(std::move(object_to_return)); --borrowed_objects_size; @@ -107,14 +107,14 @@ public: /// Allocated objects size by the pool. If allocatedObjectsSize == maxSize then pool is full. inline size_t allocatedObjectsSize() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return allocated_objects_size; } /// Returns allocatedObjectsSize == maxSize inline bool isFull() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return allocated_objects_size == max_size; } @@ -122,7 +122,7 @@ public: /// Then client will wait during borrowObject function call. inline size_t borrowedObjectsSize() const { - std::unique_lock lock(objects_mutex); + std::lock_guard lock(objects_mutex); return borrowed_objects_size; } diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index ace03a5e0b5..1bba82bc46f 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -223,7 +223,7 @@ void kerberosInit(const String & keytab_file, const String & principal, const St { // Using mutex to prevent cache file corruptions static std::mutex kinit_mtx; - std::unique_lock lck(kinit_mtx); + std::lock_guard lck(kinit_mtx); KerberosInit k_init; k_init.init(keytab_file, principal, cache_name); } diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 4faed833428..0dea6589adc 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -137,7 +137,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) { DENY_ALLOCATIONS_IN_SCOPE; - std::unique_lock lk(overcommit_m); + std::lock_guard lk(overcommit_m); if (picked_tracker == tracker) { LOG_DEBUG_SAFE(getLogger(), "Picked query stopped"); diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h index 4b4e2c5cfa7..dac126fb5d7 100644 --- a/src/Common/PoolBase.h +++ b/src/Common/PoolBase.h @@ -55,7 +55,7 @@ private: explicit PoolEntryHelper(PooledObject & data_) : data(data_) { data.in_use = true; } ~PoolEntryHelper() { - std::unique_lock lock(data.pool.mutex); + std::lock_guard lock(data.pool.mutex); data.in_use = false; data.pool.available.notify_one(); } @@ -163,7 +163,7 @@ public: inline size_t size() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return items.size(); } diff --git a/src/Common/SystemLogBase.cpp b/src/Common/SystemLogBase.cpp index 7128d00714c..67aedbd5670 100644 --- a/src/Common/SystemLogBase.cpp +++ b/src/Common/SystemLogBase.cpp @@ -139,7 +139,7 @@ void SystemLogBase::flush(bool force) uint64_t this_thread_requested_offset; { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (is_shutdown) return; diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index a76037ae5cf..3f5091af0c9 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -209,7 +209,7 @@ template void ThreadPoolImpl::finalize() { { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); shutdown = true; } @@ -224,14 +224,14 @@ void ThreadPoolImpl::finalize() template size_t ThreadPoolImpl::active() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return scheduled_jobs; } template bool ThreadPoolImpl::finished() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); return shutdown; } @@ -290,7 +290,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ job = {}; { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!first_exception) first_exception = std::current_exception(); // NOLINT if (shutdown_on_exception) @@ -305,7 +305,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); --scheduled_jobs; if (threads.size() > scheduled_jobs + max_free_threads) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index d4c188fe8d9..8a46d8ee296 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -576,7 +576,7 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ auto set_initialized = [this]() { - std::unique_lock lock(initialized_mutex); + std::lock_guard lock(initialized_mutex); initialized_flag = true; initialized_cv.notify_all(); }; diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index e731787a5c1..445aa4463bd 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -925,7 +925,7 @@ void BaseDaemon::handleSignal(int signal_id) signal_id == SIGQUIT || signal_id == SIGTERM) { - std::unique_lock lock(signal_handler_mutex); + std::lock_guard lock(signal_handler_mutex); { ++terminate_signals_counter; sigint_signals_counter += signal_id == SIGINT; diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 13f55eab9e2..6fdedf6d38e 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -39,7 +39,7 @@ DatabaseMaterializedMySQL::DatabaseMaterializedMySQL( void DatabaseMaterializedMySQL::rethrowExceptionIfNeeded() const { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!settings->allows_query_when_mysql_lost && exception) { @@ -59,7 +59,7 @@ void DatabaseMaterializedMySQL::rethrowExceptionIfNeeded() const void DatabaseMaterializedMySQL::setException(const std::exception_ptr & exception_) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); exception = exception_; } diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index 2077f846f09..aee1f0de2f6 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -135,14 +135,14 @@ void CacheDictionaryUpdateQueue::updateThreadFunction() /// Notify thread about finished updating the bunch of ids /// where their own ids were included. - std::unique_lock lock(update_mutex); + std::lock_guard lock(update_mutex); unit_to_update->is_done = true; is_update_finished.notify_all(); } catch (...) { - std::unique_lock lock(update_mutex); + std::lock_guard lock(update_mutex); unit_to_update->current_exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing) is_update_finished.notify_all(); diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index 8e355f70432..45c98224966 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -90,7 +90,7 @@ DiskCacheWrapper::DiskCacheWrapper( std::shared_ptr DiskCacheWrapper::acquireDownloadMetadata(const String & path) const { - std::unique_lock lock{mutex}; + std::lock_guard lock{mutex}; auto it = file_downloads.find(path); if (it != file_downloads.end()) @@ -101,7 +101,7 @@ std::shared_ptr DiskCacheWrapper::acquireDownloadMetadata( new FileDownloadMetadata, [this, path] (FileDownloadMetadata * p) { - std::unique_lock erase_lock{mutex}; + std::lock_guard erase_lock{mutex}; file_downloads.erase(path); delete p; }); diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index 32e6fe5834d..22502a6e2d1 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -56,7 +56,7 @@ void MetadataStorageFromDiskTransaction::commit() toString(state), toString(MetadataFromDiskTransactionState::PREPARING)); { - std::unique_lock lock(metadata_storage.metadata_mutex); + std::lock_guard lock(metadata_storage.metadata_mutex); for (size_t i = 0; i < operations.size(); ++i) { try diff --git a/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp b/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp index 983e36489e6..109ccf0eba7 100644 --- a/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp +++ b/src/Disks/ObjectStorages/S3/ProxyResolverConfiguration.cpp @@ -28,7 +28,7 @@ ClientConfigurationPerRequest ProxyResolverConfiguration::getConfiguration(const { LOG_DEBUG(&Poco::Logger::get("AWSClient"), "Obtain proxy using resolver: {}", endpoint.toString()); - std::unique_lock lock(cache_mutex); + std::lock_guard lock(cache_mutex); std::chrono::time_point now = std::chrono::system_clock::now(); @@ -110,7 +110,7 @@ void ProxyResolverConfiguration::errorReport(const ClientConfigurationPerRequest if (config.proxy_host.empty()) return; - std::unique_lock lock(cache_mutex); + std::lock_guard lock(cache_mutex); if (!cache_ttl.count() || !cache_valid) return; diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 7ed2f343209..b31ee08eaf5 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -212,7 +212,7 @@ namespace size_t max_connections_per_endpoint, bool resolve_host = true) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); const std::string & host = uri.getHost(); UInt16 port = uri.getPort(); bool https = isHTTPS(uri); diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index a58732287e9..cea03277c91 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -148,7 +148,7 @@ public: { String credentials_string; { - std::unique_lock locker(token_mutex); + std::lock_guard locker(token_mutex); LOG_TRACE(logger, "Getting default credentials for EC2 instance."); auto result = GetResourceWithAWSWebServiceResult(endpoint.c_str(), EC2_SECURITY_CREDENTIALS_RESOURCE, nullptr); @@ -194,7 +194,7 @@ public: String new_token; { - std::unique_lock locker(token_mutex); + std::lock_guard locker(token_mutex); Aws::StringStream ss; ss << endpoint << EC2_IMDS_TOKEN_RESOURCE; diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index fd8f252d139..ae39c7035dd 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -215,7 +215,7 @@ void AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) } } - std::unique_lock write_lock(rwlock); + std::lock_guard write_lock(rwlock); auto it = queue.emplace(key, std::make_shared()).first; pushImpl(std::move(entry), it); } @@ -343,7 +343,7 @@ void AsynchronousInsertQueue::cleanup() if (!keys_to_remove.empty()) { - std::unique_lock write_lock(rwlock); + std::lock_guard write_lock(rwlock); size_t total_removed = 0; for (const auto & key : keys_to_remove) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 13340221f38..4cd61f7af32 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -698,7 +698,7 @@ std::unique_lock DatabaseCatalog::getExclusiveDDLGuardForData { DDLGuards::iterator db_guard_iter; { - std::unique_lock lock(ddl_guards_mutex); + std::lock_guard lock(ddl_guards_mutex); db_guard_iter = ddl_guards.try_emplace(database).first; assert(db_guard_iter->second.first.contains("")); } diff --git a/src/Interpreters/EmbeddedDictionaries.cpp b/src/Interpreters/EmbeddedDictionaries.cpp index b9bc5cd1e5c..0b2efaf3dbe 100644 --- a/src/Interpreters/EmbeddedDictionaries.cpp +++ b/src/Interpreters/EmbeddedDictionaries.cpp @@ -60,7 +60,7 @@ bool EmbeddedDictionaries::reloadDictionary( bool EmbeddedDictionaries::reloadImpl(const bool throw_on_error, const bool force_reload) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); /** If you can not update the directories, then despite this, do not throw an exception (use the old directories). * If there are no old correct directories, then when using functions that depend on them, diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 3dd6b7de574..9ddd4ac0be0 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -580,7 +580,7 @@ void MergeJoin::mergeRightBlocks() void MergeJoin::mergeInMemoryRightBlocks() { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (right_blocks.empty()) return; @@ -613,7 +613,7 @@ void MergeJoin::mergeInMemoryRightBlocks() void MergeJoin::mergeFlushedRightBlocks() { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); auto callback = [&](const Block & block) { @@ -638,7 +638,7 @@ bool MergeJoin::saveRightBlock(Block && block) { if (is_in_memory) { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (!is_in_memory) { diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 28bbea54110..ab443f58cf2 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -103,7 +103,7 @@ void NO_INLINE Set::insertFromBlockImplCase( void Set::setHeader(const ColumnsWithTypeAndName & header) { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (!data.empty()) return; diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp index 34b9305c0e1..32ab391cf8c 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp @@ -20,7 +20,7 @@ namespace DB } { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (background_exception) std::rethrow_exception(background_exception); @@ -30,7 +30,7 @@ namespace DB void ParallelFormattingOutputFormat::addChunk(Chunk chunk, ProcessingUnitType type, bool can_throw_exception) { { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (background_exception && can_throw_exception) std::rethrow_exception(background_exception); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index edde79be896..fb58f5765c1 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -236,7 +236,7 @@ private: void onBackgroundException() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!background_exception) { background_exception = std::current_exception(); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index d3e167d35c6..318bcaed466 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -121,7 +121,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa void ParallelParsingInputFormat::onBackgroundException(size_t offset) { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!background_exception) { background_exception = std::current_exception(); @@ -233,7 +233,7 @@ Chunk ParallelParsingInputFormat::generate() else { // Pass the unit back to the segmentator. - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); unit.status = READY_TO_INSERT; segmentator_condvar.notify_all(); } diff --git a/src/Server/HTTP/HTTPServerConnection.cpp b/src/Server/HTTP/HTTPServerConnection.cpp index 6b2ef32f6a4..92a994b3a4e 100644 --- a/src/Server/HTTP/HTTPServerConnection.cpp +++ b/src/Server/HTTP/HTTPServerConnection.cpp @@ -26,7 +26,7 @@ void HTTPServerConnection::run() { try { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (!stopped && tcp_server.isOpen() && session.connected()) { HTTPServerResponse response(session); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 4f500208215..dfb5eb0bd69 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -189,7 +189,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } catch (...) { - std::unique_lock lock(storage.last_queue_update_exception_lock); + std::lock_guard lock(storage.last_queue_update_exception_lock); storage.last_queue_update_exception = getCurrentExceptionMessage(false); throw; } diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 81cb58e4f5e..1bfaabe142b 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -336,7 +336,7 @@ StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_, void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , ContextPtr, TableExclusiveLockHolder &) { - std::unique_lock lock(rocksdb_ptr_mx); + std::lock_guard lock(rocksdb_ptr_mx); rocksdb_ptr->Close(); rocksdb_ptr = nullptr; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6895142b2a4..7cdfaa08e71 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2969,7 +2969,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke String StorageReplicatedMergeTree::getLastQueueUpdateException() const { - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); return last_queue_update_exception; } @@ -2991,7 +2991,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); last_queue_update_exception = getCurrentExceptionMessage(false); if (e.code == Coordination::Error::ZSESSIONEXPIRED) @@ -3006,7 +3006,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); - std::unique_lock lock(last_queue_update_exception_lock); + std::lock_guard lock(last_queue_update_exception_lock); last_queue_update_exception = getCurrentExceptionMessage(false); queue_updating_task->scheduleAfter(QUEUE_UPDATE_ERROR_SLEEP_MS); @@ -4340,7 +4340,7 @@ void StorageReplicatedMergeTree::shutdown() /// Ask all parts exchange handlers to finish asap. New ones will fail to start data_parts_exchange_ptr->blocker.cancelForever(); /// Wait for all of them - std::unique_lock lock(data_parts_exchange_ptr->rwlock); + std::lock_guard lock(data_parts_exchange_ptr->rwlock); } } @@ -7399,7 +7399,7 @@ void StorageReplicatedMergeTree::checkBrokenDisks() if (disk_ptr->isBroken()) { { - std::unique_lock lock(last_broken_disks_mutex); + std::lock_guard lock(last_broken_disks_mutex); if (!last_broken_disks.insert(disk_ptr->getName()).second) continue; } @@ -7419,7 +7419,7 @@ void StorageReplicatedMergeTree::checkBrokenDisks() else { { - std::unique_lock lock(last_broken_disks_mutex); + std::lock_guard lock(last_broken_disks_mutex); if (last_broken_disks.erase(disk_ptr->getName()) > 0) LOG_INFO( log, diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index cfb19869074..5f597737e4a 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -1017,7 +1017,7 @@ void StorageWindowView::threadFuncFireProc() if (shutdown_called) return; - std::unique_lock lock(fire_signal_mutex); + std::lock_guard lock(fire_signal_mutex); UInt32 timestamp_now = std::time(nullptr); while (next_fire_signal <= timestamp_now) @@ -1049,7 +1049,7 @@ void StorageWindowView::threadFuncFireProc() void StorageWindowView::threadFuncFireEvent() { - std::unique_lock lock(fire_signal_mutex); + std::lock_guard lock(fire_signal_mutex); LOG_TRACE(log, "Fire events: {}", fire_signal.size()); From d0d72e0b5e7bc02d57c2175c3e558f328dd6c021 Mon Sep 17 00:00:00 2001 From: Julian Gilyadov Date: Tue, 28 Jun 2022 15:27:53 -0400 Subject: [PATCH 223/408] Implement L2Squared Distance and Norm --- src/Functions/array/arrayDistance.cpp | 49 +++++-- src/Functions/array/arrayNorm.cpp | 20 +++ src/Functions/vectorFunctions.cpp | 129 ++++++++++++++++++ .../02011_tuple_vector_functions.reference | 5 + .../02011_tuple_vector_functions.sql | 5 + .../02282_array_distance.reference | 43 +++--- .../0_stateless/02282_array_distance.sql | 11 +- .../0_stateless/02283_array_norm.reference | 27 ++-- .../queries/0_stateless/02283_array_norm.sql | 8 +- 9 files changed, 251 insertions(+), 46 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 3f7900b6c62..fda93bceedd 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -70,6 +70,33 @@ struct L2Distance } }; +struct L2SquaredDistance +{ + static inline String name = "L2Squared"; + + struct ConstParams + { + }; + + template + struct State + { + FloatType sum = 0; + }; + + template + static void accumulate(State & state, ResultType x, ResultType y, const ConstParams &) + { + state.sum += (x - y) * (x - y); + } + + template + static ResultType finalize(const State & state, const ConstParams &) + { + return state.sum; + } +}; + struct LpDistance { static inline String name = "Lp"; @@ -195,12 +222,12 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - getName(), common_type->getName()); + getName(), + common_type->getName()); } } - ColumnPtr - executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { switch (result_type->getTypeId()) { @@ -248,7 +275,8 @@ private: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - getName(), type_x->getName()); + getName(), + type_x->getName()); } } @@ -273,7 +301,8 @@ private: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments of function {} has nested type {}. " "Support: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64.", - getName(), type_y->getName()); + getName(), + type_y->getName()); } } @@ -310,7 +339,9 @@ private: throw Exception( ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arguments of function {} have different array sizes: {} and {}", - getName(), offsets_x[row] - prev_offset, offsets_y[row] - prev_offset); + getName(), + offsets_x[row] - prev_offset, + offsets_y[row] - prev_offset); } } @@ -360,7 +391,9 @@ private: throw Exception( ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arguments of function {} have different array sizes: {} and {}", - getName(), offsets_x[0], offsets_y[row] - prev_offset); + getName(), + offsets_x[0], + offsets_y[row] - prev_offset); } prev_offset = offsets_y[row]; } @@ -430,8 +463,8 @@ LpDistance::ConstParams FunctionArrayDistance::initConstParams(const /// These functions are used by TupleOrArrayFunction FunctionPtr createFunctionArrayL1Distance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } FunctionPtr createFunctionArrayL2Distance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } +FunctionPtr createFunctionArrayL2SquaredDistance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } FunctionPtr createFunctionArrayLpDistance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } FunctionPtr createFunctionArrayLinfDistance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } FunctionPtr createFunctionArrayCosineDistance(ContextPtr context_) { return FunctionArrayDistance::create(context_); } - } diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 2142abc4c90..a9d1885c136 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -57,6 +57,25 @@ struct L2Norm } }; +struct L2SquaredNorm +{ + static inline String name = "L2Squared"; + + struct ConstParams {}; + + template + inline static ResultType accumulate(ResultType result, ResultType value, const ConstParams &) + { + return result + value * value; + } + + template + inline static ResultType finalize(ResultType result, const ConstParams &) + { + return result; + } +}; + struct LpNorm { @@ -265,6 +284,7 @@ LpNorm::ConstParams FunctionArrayNorm::initConstParams(const ColumnsWith /// These functions are used by TupleOrArrayFunction FunctionPtr createFunctionArrayL1Norm(ContextPtr context_) { return FunctionArrayNorm::create(context_); } FunctionPtr createFunctionArrayL2Norm(ContextPtr context_) { return FunctionArrayNorm::create(context_); } +FunctionPtr createFunctionArrayL2SquaredNorm(ContextPtr context_) { return FunctionArrayNorm::create(context_); } FunctionPtr createFunctionArrayLpNorm(ContextPtr context_) { return FunctionArrayNorm::create(context_); } FunctionPtr createFunctionArrayLinfNorm(ContextPtr context_) { return FunctionArrayNorm::create(context_); } diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp index 411b30040cc..d26eb40e1a8 100644 --- a/src/Functions/vectorFunctions.cpp +++ b/src/Functions/vectorFunctions.cpp @@ -25,6 +25,7 @@ struct DivideName { static constexpr auto name = "divide"; }; struct L1Label { static constexpr auto name = "1"; }; struct L2Label { static constexpr auto name = "2"; }; +struct L2SquaredLabel { static constexpr auto name = "2Squared"; }; struct LinfLabel { static constexpr auto name = "inf"; }; struct LpLabel { static constexpr auto name = "p"; }; @@ -625,6 +626,108 @@ public: }; using FunctionL2Norm = FunctionLNorm; +template <> +class FunctionLNorm : public ITupleFunction +{ +public: + static constexpr auto name = "L2SquaredNorm"; + + explicit FunctionLNorm(ContextPtr context_) : ITupleFunction(context_) {} + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + const auto * cur_tuple = checkAndGetDataType(arguments[0].type.get()); + + if (!cur_tuple) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument 0 of function {} should be tuple, got {}", + getName(), arguments[0].type->getName()); + + const auto & cur_types = cur_tuple->getElements(); + + Columns cur_elements; + if (arguments[0].column) + cur_elements = getTupleElements(*arguments[0].column); + + size_t tuple_size = cur_types.size(); + if (tuple_size == 0) + return std::make_shared(); + + auto multiply = FunctionFactory::instance().get("multiply", context); + auto plus = FunctionFactory::instance().get("plus", context); + DataTypePtr res_type; + for (size_t i = 0; i < tuple_size; ++i) + { + try + { + ColumnWithTypeAndName cur{cur_elements.empty() ? nullptr : cur_elements[i], cur_types[i], {}}; + auto elem_multiply = multiply->build(ColumnsWithTypeAndName{cur, cur}); + + if (i == 0) + { + res_type = elem_multiply->getResultType(); + continue; + } + + ColumnWithTypeAndName left_type{res_type, {}}; + ColumnWithTypeAndName right_type{elem_multiply->getResultType(), {}}; + auto plus_elem = plus->build({left_type, right_type}); + res_type = plus_elem->getResultType(); + } + catch (DB::Exception & e) + { + e.addMessage("While executing function {} for tuple element {}", getName(), i); + throw; + } + } + + return res_type; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * cur_tuple = checkAndGetDataType(arguments[0].type.get()); + const auto & cur_types = cur_tuple->getElements(); + auto cur_elements = getTupleElements(*arguments[0].column); + + size_t tuple_size = cur_elements.size(); + if (tuple_size == 0) + return DataTypeUInt8().createColumnConstWithDefaultValue(input_rows_count); + + auto multiply = FunctionFactory::instance().get("multiply", context); + auto plus = FunctionFactory::instance().get("plus", context); + ColumnWithTypeAndName res; + for (size_t i = 0; i < tuple_size; ++i) + { + ColumnWithTypeAndName cur{cur_elements[i], cur_types[i], {}}; + auto elem_multiply = multiply->build(ColumnsWithTypeAndName{cur, cur}); + + ColumnWithTypeAndName column; + column.type = elem_multiply->getResultType(); + column.column = elem_multiply->execute({cur, cur}, column.type, input_rows_count); + + if (i == 0) + { + res = std::move(column); + } + else + { + auto plus_elem = plus->build({res, column}); + auto res_type = plus_elem->getResultType(); + res.column = plus_elem->execute({res, column}, res_type, input_rows_count); + res.type = res_type; + } + } + + return res.column; + } +}; +using FunctionL2SquaredNorm = FunctionLNorm; + template <> class FunctionLNorm : public ITupleFunction { @@ -930,6 +1033,8 @@ using FunctionL1Distance = FunctionLDistance; using FunctionL2Distance = FunctionLDistance; +using FunctionL2SquaredDistance = FunctionLDistance; + using FunctionLinfDistance = FunctionLDistance; using FunctionLpDistance = FunctionLDistance; @@ -1111,11 +1216,13 @@ private: extern FunctionPtr createFunctionArrayL1Norm(ContextPtr context_); extern FunctionPtr createFunctionArrayL2Norm(ContextPtr context_); +extern FunctionPtr createFunctionArrayL2SquaredNorm(ContextPtr context_); extern FunctionPtr createFunctionArrayLpNorm(ContextPtr context_); extern FunctionPtr createFunctionArrayLinfNorm(ContextPtr context_); extern FunctionPtr createFunctionArrayL1Distance(ContextPtr context_); extern FunctionPtr createFunctionArrayL2Distance(ContextPtr context_); +extern FunctionPtr createFunctionArrayL2SquaredDistance(ContextPtr context_); extern FunctionPtr createFunctionArrayLpDistance(ContextPtr context_); extern FunctionPtr createFunctionArrayLinfDistance(ContextPtr context_); extern FunctionPtr createFunctionArrayCosineDistance(ContextPtr context_); @@ -1136,6 +1243,14 @@ struct L2NormTraits static constexpr auto CreateArrayFunction = createFunctionArrayL2Norm; }; +struct L2SquaredNormTraits +{ + static inline String name = "L2SquaredNorm"; + + static constexpr auto CreateTupleFunction = FunctionL2SquaredNorm::create; + static constexpr auto CreateArrayFunction = createFunctionArrayL2SquaredNorm; +}; + struct LpNormTraits { static inline String name = "LpNorm"; @@ -1168,6 +1283,14 @@ struct L2DistanceTraits static constexpr auto CreateArrayFunction = createFunctionArrayL2Distance; }; +struct L2SquaredDistanceTraits +{ + static inline String name = "L2SquaredDistance"; + + static constexpr auto CreateTupleFunction = FunctionL2SquaredDistance::create; + static constexpr auto CreateArrayFunction = createFunctionArrayL2SquaredDistance; +}; + struct LpDistanceTraits { static inline String name = "LpDistance"; @@ -1194,11 +1317,13 @@ struct CosineDistanceTraits using TupleOrArrayFunctionL1Norm = TupleOrArrayFunction; using TupleOrArrayFunctionL2Norm = TupleOrArrayFunction; +using TupleOrArrayFunctionL2SquaredNorm = TupleOrArrayFunction; using TupleOrArrayFunctionLpNorm = TupleOrArrayFunction; using TupleOrArrayFunctionLinfNorm = TupleOrArrayFunction; using TupleOrArrayFunctionL1Distance = TupleOrArrayFunction; using TupleOrArrayFunctionL2Distance = TupleOrArrayFunction; +using TupleOrArrayFunctionL2SquaredDistance = TupleOrArrayFunction; using TupleOrArrayFunctionLpDistance = TupleOrArrayFunction; using TupleOrArrayFunctionLinfDistance = TupleOrArrayFunction; using TupleOrArrayFunctionCosineDistance = TupleOrArrayFunction; @@ -1221,21 +1346,25 @@ void registerVectorFunctions(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerAlias("normL1", TupleOrArrayFunctionL1Norm::name, FunctionFactory::CaseInsensitive); factory.registerAlias("normL2", TupleOrArrayFunctionL2Norm::name, FunctionFactory::CaseInsensitive); + factory.registerAlias("normL2Squared", TupleOrArrayFunctionL2SquaredNorm::name, FunctionFactory::CaseInsensitive); factory.registerAlias("normLinf", TupleOrArrayFunctionLinfNorm::name, FunctionFactory::CaseInsensitive); factory.registerAlias("normLp", FunctionLpNorm::name, FunctionFactory::CaseInsensitive); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerAlias("distanceL1", FunctionL1Distance::name, FunctionFactory::CaseInsensitive); factory.registerAlias("distanceL2", FunctionL2Distance::name, FunctionFactory::CaseInsensitive); + factory.registerAlias("distanceL2Squared", FunctionL2SquaredDistance::name, FunctionFactory::CaseInsensitive); factory.registerAlias("distanceLinf", FunctionLinfDistance::name, FunctionFactory::CaseInsensitive); factory.registerAlias("distanceLp", FunctionLpDistance::name, FunctionFactory::CaseInsensitive); diff --git a/tests/queries/0_stateless/02011_tuple_vector_functions.reference b/tests/queries/0_stateless/02011_tuple_vector_functions.reference index 4c5067c7da2..1b54179cc87 100644 --- a/tests/queries/0_stateless/02011_tuple_vector_functions.reference +++ b/tests/queries/0_stateless/02011_tuple_vector_functions.reference @@ -21,7 +21,9 @@ 6 7.1 1.4142135623730951 +2 13 +169 1.5 -3 2.3 @@ -34,8 +36,10 @@ 2.0000887587111964 4 2.8284271247461903 +8 1 0 +0 -4.413254828250501e-8 (0.2,-0.8) (0.6,0.8) @@ -48,6 +52,7 @@ (NULL,NULL) \N \N +\N (2,4,NULL) \N \N diff --git a/tests/queries/0_stateless/02011_tuple_vector_functions.sql b/tests/queries/0_stateless/02011_tuple_vector_functions.sql index f34fb91586c..14f013937bb 100644 --- a/tests/queries/0_stateless/02011_tuple_vector_functions.sql +++ b/tests/queries/0_stateless/02011_tuple_vector_functions.sql @@ -28,7 +28,9 @@ SELECT scalarProduct(tuple(1), tuple(0)); SELECT L1Norm((-1, 2, -3)); SELECT L1Norm((-1, 2.5, -3.6)); SELECT L2Norm((1, 1.0)); +SELECT L2SquaredNorm((1, 1.0)); SELECT L2Norm(materialize((-12, 5))); +SELECT L2SquaredNorm(materialize((-12, 5))); SELECT max2(materialize(1), 1.5); SELECT min2(-1, -3); @@ -44,8 +46,10 @@ SELECT LpNorm((-1, -2), 11.); SELECT L1Distance((1, 2, 3), (2, 3, 1)); SELECT L2Distance(materialize((1, 1)), (3, -1)); +SELECT L2SquaredDistance(materialize((1, 1)), (3, -1)); SELECT LinfDistance((1, 1), (1, 2)); SELECT L2Distance((5, 5), (5, 5)); +SELECT L2SquaredDistance((5, 5), (5, 5)); SELECT LpDistance((1800, 1900), (18, 59), 12) - LpDistance(tuple(-22), tuple(1900), 12.); SELECT L1Normalize(materialize((1, -4))); @@ -61,6 +65,7 @@ SELECT cosineDistance((1, 0), (0.5, sqrt(3) / 2)); SELECT (NULL, 1) + (1, NULL); SELECT (NULL, 1) * materialize((1, NULL)); SELECT L2Norm((NULL, 3, 4)); +SELECT L2SquaredNorm((NULL, 3, 4)); SELECT 2 * (1, 2, NULL); SELECT (1, 1.0, NULL) / NULL; SELECT (1, 1.0, NULL) / materialize(NULL); diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index ebce2788fe9..e5bc46df295 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -1,17 +1,22 @@ 6 3.7416573867739413 +14 3.2071843327373397 3 0.00258509695694209 \N +\N nan -0 0 0 0 +0 0 0 0 0 12 14 21 7.0710678118654755 9.16515138991168 12.12435565298214 +50 +84 +147 5.917593844525055 8.308858759453505 9.932246380845738 @@ -27,21 +32,21 @@ nan 0.020204102886728692 0.11808289631180313 0 -1 1 218.74642854227358 -1 2 1348.2117786164013 -2 1 219.28064210048274 -2 2 1347.4008312302617 -3 1 214.35251339790725 -3 2 1342.8856987845243 -1 1 218.74642854227358 -1 2 1348.2117786164013 -2 1 219.28064210048274 -2 2 1347.4008312302617 -3 1 214.35251339790725 -3 2 1342.8856987845243 -1 1 218.74642854227358 -1 2 1348.2117786164013 -2 1 219.28064210048274 -2 2 1347.4008312302617 -3 1 214.35251339790725 -3 2 1342.8856987845243 +1 1 218.74642854227358 47850 +1 2 1348.2117786164013 1817675 +2 1 219.28064210048274 48084 +2 2 1347.4008312302617 1815489 +3 1 214.35251339790725 45947 +3 2 1342.8856987845243 1803342 +1 1 218.74642854227358 47850 +1 2 1348.2117786164013 1817675 +2 1 219.28064210048274 48084 +2 2 1347.4008312302617 1815489 +3 1 214.35251339790725 45947 +3 2 1342.8856987845243 1803342 +1 1 218.74642854227358 47850 +1 2 1348.2117786164013 1817675 +2 1 219.28064210048274 48084 +2 2 1347.4008312302617 1815489 +3 1 214.35251339790725 45947 +3 2 1342.8856987845243 1803342 diff --git a/tests/queries/0_stateless/02282_array_distance.sql b/tests/queries/0_stateless/02282_array_distance.sql index 75e4b0d653e..431704abd40 100644 --- a/tests/queries/0_stateless/02282_array_distance.sql +++ b/tests/queries/0_stateless/02282_array_distance.sql @@ -1,10 +1,12 @@ SELECT L1Distance([0, 0, 0], [1, 2, 3]); SELECT L2Distance([1, 2, 3], [0, 0, 0]); +SELECT L2SquaredDistance([1, 2, 3], [0, 0, 0]); SELECT LpDistance([1, 2, 3], [0, 0, 0], 3.5); SELECT LinfDistance([1, 2, 3], [0, 0, 0]); SELECT cosineDistance([1, 2, 3], [3, 5, 7]); SELECT L2Distance([1, 2, 3], NULL); +SELECT L2SquaredDistance([1, 2, 3], NULL); SELECT cosineDistance([1, 2, 3], [0, 0, 0]); -- Overflows @@ -12,6 +14,7 @@ WITH CAST([-547274980, 1790553898, 1981517754, 1908431500, 1352428565, -57341255 SELECT L1Distance(a,a), L2Distance(a,a), + L2SquaredDistance(a,a), LinfDistance(a,a), cosineDistance(a, a); @@ -27,6 +30,7 @@ CREATE TABLE vec2d (id UInt64, v Array(Float64)) ENGINE = Memory; INSERT INTO vec1 VALUES (1, [3, 4, 5]), (2, [2, 4, 8]), (3, [7, 7, 7]); SELECT L1Distance(v, [0, 0, 0]) FROM vec1; SELECT L2Distance(v, [0, 0, 0]) FROM vec1; +SELECT L2SquaredDistance(v, [0, 0, 0]) FROM vec1; SELECT LpDistance(v, [0, 0, 0], 3.14) FROM vec1; SELECT LinfDistance([5, 4, 3], v) FROM vec1; SELECT cosineDistance([3, 2, 1], v) FROM vec1; @@ -34,16 +38,17 @@ SELECT LinfDistance(v, materialize([0, -2, 0])) FROM vec1; SELECT cosineDistance(v, materialize([1., 1., 1.])) FROM vec1; INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; +SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; INSERT INTO vec2f VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2f v2; +SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2f v2; INSERT INTO vec2d VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v) as dist FROM vec1 v1, vec2d v2; +SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2d v2; SELECT L1Distance([0, 0], [1]); -- { serverError 190 } SELECT L2Distance([1, 2], (3,4)); -- { serverError 43 } +SELECT L2SquaredDistance([1, 2], (3,4)); -- { serverError 43 } SELECT LpDistance([1, 2], [3,4]); -- { serverError 42 } SELECT LpDistance([1, 2], [3,4], -1.); -- { serverError 69 } SELECT LpDistance([1, 2], [3,4], 'aaa'); -- { serverError 43 } diff --git a/tests/queries/0_stateless/02283_array_norm.reference b/tests/queries/0_stateless/02283_array_norm.reference index ebaadee321f..86a77e27172 100644 --- a/tests/queries/0_stateless/02283_array_norm.reference +++ b/tests/queries/0_stateless/02283_array_norm.reference @@ -1,28 +1,29 @@ 6 7.0710678118654755 +50 10.882246697870885 2 -10803059573 4234902446.7343364 10803059573 4234902446.7343364 3122003357.3280888 2096941042 -1 7 5 4.601724723020627 4 -2 2 2 2 2 -3 9 5.196152422706632 4.506432087111623 3 -4 0 0 0 0 +10803059573 4234902446.7343364 17934398733356468000 10803059573 4234902446.7343364 3122003357.3280888 2096941042 +1 7 5 25 4.601724723020627 4 +2 2 2 4 2 2 +3 9 5.196152422706632 27 4.506432087111623 3 +4 0 0 0 0 0 1 11 2 11 3 11 4 11 -1 7 5 4.601724723020627 4 -2 2 2 2 2 -3 9 5.196152422706632 4.506432087111623 3 -4 0 0 0 0 +1 7 5 25 4.601724723020627 4 +2 2 2 4 2 2 +3 9 5.196152422706632 27 4.506432087111623 3 +4 0 0 0 0 0 1 11 2 11 3 11 4 11 -1 7 5 4.601724723020627 4 -2 2 2 2 2 -3 9 5.196152422706632 4.506432087111623 3 -4 0 0 0 0 +1 7 5 25 4.601724723020627 4 +2 2 2 4 2 2 +3 9 5.196152422706632 27 4.506432087111623 3 +4 0 0 0 0 0 1 11 2 11 3 11 diff --git a/tests/queries/0_stateless/02283_array_norm.sql b/tests/queries/0_stateless/02283_array_norm.sql index 6938618d633..2f5753f9943 100644 --- a/tests/queries/0_stateless/02283_array_norm.sql +++ b/tests/queries/0_stateless/02283_array_norm.sql @@ -1,5 +1,6 @@ SELECT L1Norm([1, 2, 3]); SELECT L2Norm([3., 4., 5.]); +SELECT L2SquaredNorm([3., 4., 5.]); SELECT LpNorm([3., 4., 5.], 1.1); SELECT LinfNorm([0, 0, 2]); @@ -8,6 +9,7 @@ WITH CAST([-547274980, 1790553898, 1981517754, 1908431500, 1352428565, -57341255 SELECT L1Norm(a), L2Norm(a), + L2SquaredNorm(a), LpNorm(a,1), LpNorm(a,2), LpNorm(a,3.14), @@ -23,13 +25,13 @@ INSERT INTO vec1 VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); INSERT INTO vec1f VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); INSERT INTO vec1d VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); -SELECT id, L1Norm(v), L2Norm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1; +SELECT id, L1Norm(v), L2Norm(v), L2SquaredNorm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1; SELECT id, L1Norm(materialize([5., 6.])) FROM vec1; -SELECT id, L1Norm(v), L2Norm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1f; +SELECT id, L1Norm(v), L2Norm(v), L2SquaredNorm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1f; SELECT id, L1Norm(materialize([5., 6.])) FROM vec1f; -SELECT id, L1Norm(v), L2Norm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1d; +SELECT id, L1Norm(v), L2Norm(v), L2SquaredNorm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1d; SELECT id, L1Norm(materialize([5., 6.])) FROM vec1d; SELECT L1Norm(1, 2); -- { serverError 42 } From 1648cd0eb0e5814eb0c0ccab7b1442e84b03d134 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 27 Jun 2022 23:52:49 +0300 Subject: [PATCH 224/408] tests: cleanup tmp data in 02335_column_ttl_expired_column_optimization v2: remove trap usage as requested by @tavplubix Signed-off-by: Azat Khuzhin --- .../02335_column_ttl_expired_column_optimization.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02335_column_ttl_expired_column_optimization.sh b/tests/queries/0_stateless/02335_column_ttl_expired_column_optimization.sh index e2f7209d036..96f80d65878 100755 --- a/tests/queries/0_stateless/02335_column_ttl_expired_column_optimization.sh +++ b/tests/queries/0_stateless/02335_column_ttl_expired_column_optimization.sh @@ -4,7 +4,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_LOCAL --path "$CLICKHOUSE_TEST_UNIQUE_NAME" -nm -q " +data_path="$CLICKHOUSE_TMP/local" + +$CLICKHOUSE_LOCAL --path "$data_path" -nm -q " create table ttl_02335 ( date Date, key Int, @@ -25,5 +27,5 @@ $CLICKHOUSE_LOCAL --path "$CLICKHOUSE_TEST_UNIQUE_NAME" -nm -q " optimize table ttl_02335 final; " -test -f "$CLICKHOUSE_TEST_UNIQUE_NAME"/data/_local/ttl_02335/all_1_1_3/value.bin && echo "[FAIL] value column should not exist" -exit 0 +test -f "$data_path"/data/_local/ttl_02335/all_1_1_3/value.bin && echo "[FAIL] value column should not exist" +rm -fr "${data_path:?}" From 75f7662ebb5ad64bf2b4300cf2e7075beaf239cd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 28 Jun 2022 19:47:00 +0000 Subject: [PATCH 225/408] Rename slightly weirdly named "BuilderBinTidy" to "BuilderClangTidy" --- .github/workflows/master.yml | 4 ++-- .github/workflows/pull_request.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index cd516ba9674..5dd1ec73fde 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -643,7 +643,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinTidy: + BuilderClangTidy: needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: @@ -1011,7 +1011,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderBinTidy + - BuilderClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] steps: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 79d54d77f06..50f87549811 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -707,7 +707,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinTidy: + BuilderClangTidy: needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] steps: @@ -1065,7 +1065,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderBinTidy + - BuilderClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] if: ${{ success() || failure() }} From ee0f2651ee243d8ca4bec4ce97ea67bf2c9e4257 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 4 Jun 2022 16:13:01 +0300 Subject: [PATCH 226/408] Revert "Fix converting types for UNION queries (may produce LOGICAL_ERROR)" This fix is incorrect, and it introduce new issues, in particular it may breaks UNION queries w/o column aliases, i.e.: SELECT a, b, c FROM (SELECT 3 AS a, 2147483647 AS b, 1048575 AS c UNION ALL SELECT -2, NULL, -2) AS js1 ORDER BY a CI: https://s3.amazonaws.com/clickhouse-test-reports/37796/e637489f81768df582fe7389e57f7ed12893087c/fuzzer_astfuzzerdebug,actions//report.html Reverts: #37593/#34775 (2613149f6bf4f242bbbf2c3c8539b5176fd77286) Signed-off-by: Azat Khuzhin --- .../InterpreterSelectWithUnionQuery.cpp | 4 --- src/Interpreters/SelectQueryOptions.h | 8 ------ src/Interpreters/TreeRewriter.cpp | 28 ++----------------- 3 files changed, 2 insertions(+), 38 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index f0150fe663f..28b1f7667a8 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -52,10 +52,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_children) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - /// This is required for UNION to match headers correctly. - if (num_children > 1) - options.reorderColumns(); - /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index 032befe1993..6b5a6a7f8eb 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -31,8 +31,6 @@ struct SelectQueryOptions bool only_analyze = false; bool modify_inplace = false; bool remove_duplicates = false; - /// This is required for UNION to match headers correctly. - bool reorder_columns_as_required_header = false; bool ignore_quota = false; bool ignore_limits = false; /// This flag is needed to analyze query ignoring table projections. @@ -99,12 +97,6 @@ struct SelectQueryOptions return *this; } - SelectQueryOptions & reorderColumns(bool value = true) - { - reorder_columns_as_required_header = value; - return *this; - } - SelectQueryOptions & noSubquery() { subquery_depth = 0; diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 7fa1fe2f012..869b6b7a51f 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -429,7 +429,7 @@ void renameDuplicatedColumns(const ASTSelectQuery * select_query) /// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result. /// Also we have to remove duplicates in case of GLOBAL subqueries. Their results are placed into tables so duplicates are impossible. /// Also remove all INTERPOLATE columns which are not in SELECT anymore. -void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups, bool reorder_columns_as_required_header) +void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups) { ASTs & elements = select_query->select()->children; @@ -460,29 +460,6 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const NameSet remove_columns; - /// Resort columns according to required_result_columns. - if (reorder_columns_as_required_header && !required_result_columns.empty()) - { - std::unordered_map name_pos; - { - size_t pos = 0; - for (const auto & name : required_result_columns) - name_pos[name] = pos++; - } - ::sort(elements.begin(), elements.end(), [&](const auto & lhs, const auto & rhs) - { - String lhs_name = lhs->getAliasOrColumnName(); - String rhs_name = rhs->getAliasOrColumnName(); - size_t lhs_pos = name_pos.size(); - size_t rhs_pos = name_pos.size(); - if (auto it = name_pos.find(lhs_name); it != name_pos.end()) - lhs_pos = it->second; - if (auto it = name_pos.find(rhs_name); it != name_pos.end()) - rhs_pos = it->second; - return lhs_pos < rhs_pos; - }); - } - for (const auto & elem : elements) { String name = elem->getAliasOrColumnName(); @@ -1193,7 +1170,6 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( size_t subquery_depth = select_options.subquery_depth; bool remove_duplicates = select_options.remove_duplicates; - bool reorder_columns_as_required_header = select_options.reorder_columns_as_required_header; const auto & settings = getContext()->getSettingsRef(); @@ -1245,7 +1221,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) /// and before 'executeScalarSubqueries', 'analyzeAggregation', etc. to avoid excessive calculations. - removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates, reorder_columns_as_required_header); + removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates); /// Executing scalar subqueries - replacing them with constant values. executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze); From 563fac53337477f6c54665c488fba8be5b0be3ba Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 6 Jun 2022 16:17:17 +0300 Subject: [PATCH 227/408] tests: add a test for incorrect header in UNION queries Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02340_union_header.reference | 7 +++++++ tests/queries/0_stateless/02340_union_header.sql | 3 +++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/02340_union_header.reference create mode 100644 tests/queries/0_stateless/02340_union_header.sql diff --git a/tests/queries/0_stateless/02340_union_header.reference b/tests/queries/0_stateless/02340_union_header.reference new file mode 100644 index 00000000000..b7d6846023d --- /dev/null +++ b/tests/queries/0_stateless/02340_union_header.reference @@ -0,0 +1,7 @@ +-- { echo } +SELECT a, b, c FROM (SELECT 3 AS a, 2147483647 AS b, 1048575 AS c UNION ALL SELECT -2, NULL, -2) AS js1 ORDER BY a; +-2 \N -2 +3 2147483647 1048575 +SELECT a, b, c, d FROM (SELECT 3 AS a, 2147483647 AS b, 1048575 AS c UNION ALL SELECT -2, NULL, -2) AS js1 ALL LEFT JOIN (SELECT 100 AS a, -9223372036854775808 AS b, NULL AS d UNION ALL SELECT 256, 256, NULL) AS js2 USING (a, b) ORDER BY a DESC NULLS FIRST, '-0.02' ASC, b ASC NULLS FIRST, c DESC NULLS FIRST, 1048575 ASC NULLS LAST, d DESC SETTINGS enable_positional_arguments=0; +3 2147483647 1048575 \N +-2 \N -2 \N diff --git a/tests/queries/0_stateless/02340_union_header.sql b/tests/queries/0_stateless/02340_union_header.sql new file mode 100644 index 00000000000..3481d51d669 --- /dev/null +++ b/tests/queries/0_stateless/02340_union_header.sql @@ -0,0 +1,3 @@ +-- { echo } +SELECT a, b, c FROM (SELECT 3 AS a, 2147483647 AS b, 1048575 AS c UNION ALL SELECT -2, NULL, -2) AS js1 ORDER BY a; +SELECT a, b, c, d FROM (SELECT 3 AS a, 2147483647 AS b, 1048575 AS c UNION ALL SELECT -2, NULL, -2) AS js1 ALL LEFT JOIN (SELECT 100 AS a, -9223372036854775808 AS b, NULL AS d UNION ALL SELECT 256, 256, NULL) AS js2 USING (a, b) ORDER BY a DESC NULLS FIRST, '-0.02' ASC, b ASC NULLS FIRST, c DESC NULLS FIRST, 1048575 ASC NULLS LAST, d DESC SETTINGS enable_positional_arguments=0; From d98336ad8318e7033556dad322ea94c6c67510de Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 6 Jun 2022 16:11:13 +0300 Subject: [PATCH 228/408] Fix incorrect columns order in subqueries of UNION MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consider the following query: SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y) Here is UNION from two SELECT queries - `SELECT NULL, 255 AS x, 1 AS y` - `SELECT y, NULL AS x, 1 AS y` UNION queries matches columns by positions, not names, so the following columns should be used by `avgWeighted()`: - `255 AS x, 1 AS y` - `NULL AS x, 1 AS y` Result types of arguments should be: - `x Nullable(UInt8)` - `y UInt8` And in case of UNION query is a subselect itself, it will return only required columns, for the example above it needs only `x` and `y`. For this it will get positions of these arguments from the first query, and then use those positions to get required column names from the second query (since there is no ability to get columns by positions instead of names internally), and due to duplicated columns the second query will return (`y`, `x`) not (`x`, `y`), and this will make the result incorrect: EXPLAIN header = 1, optimize = 0, actions=1 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y) Aggregates: avgWeighted(x, y) Function: avgWeighted(Nullable(UInt8), UInt8) → Nullable(Float64) Arguments: x, y Argument positions: 0, 1 Expression (Before GROUP BY) Header: x UInt8 y Nullable(UInt8) ... Union Header: x UInt8 y Nullable(UInt8) Expression (Conversion before UNION) Header: x UInt8 y Nullable(UInt8) Expression (Conversion before UNION) Header: x UInt8 y Nullable(UInt8) And the query itself fails with an error: Logical error: 'Bad cast from type DB::ColumnVector to DB::ColumnNullable'. _NOTE: `avgWeighted()` here is required to trigger `LOGICAL_ERROR`_ CI: https://s3.amazonaws.com/clickhouse-test-reports/37796/e637489f81768df582fe7389e57f7ed12893087c/fuzzer_astfuzzerdebug,actions//report.html Fixes: 02227_union_match_by_name v2: fix untuple() (reserve space for output_columns_positions too) Signed-off-by: Azat Khuzhin --- src/Interpreters/TreeRewriter.cpp | 50 +++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 869b6b7a51f..9a4a956f3dc 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -433,17 +433,24 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const { ASTs & elements = select_query->select()->children; - std::map required_columns_with_duplicate_count; + std::unordered_map required_columns_with_duplicate_count; + /// Order of output columns should match order in required_result_columns, + /// otherwise UNION queries may have incorrect header when subselect has duplicated columns. + /// + /// NOTE: multimap is required since there can be duplicated column names. + std::unordered_multimap output_columns_positions; if (!required_result_columns.empty()) { /// Some columns may be queried multiple times, like SELECT x, y, y FROM table. - for (const auto & name : required_result_columns) + for (size_t i = 0; i < required_result_columns.size(); ++i) { + const auto & name = required_result_columns[i]; if (remove_dups) required_columns_with_duplicate_count[name] = 1; else ++required_columns_with_duplicate_count[name]; + output_columns_positions.emplace(name, i); } } else if (remove_dups) @@ -455,8 +462,8 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const else return; - ASTs new_elements; - new_elements.reserve(elements.size()); + ASTs new_elements(elements.size() + output_columns_positions.size()); + size_t new_elements_size = 0; NameSet remove_columns; @@ -464,17 +471,35 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const { String name = elem->getAliasOrColumnName(); + /// Columns that are presented in output_columns_positions should + /// appears in the same order in the new_elements, hence default + /// result_index goes after all elements of output_columns_positions + /// (it is for columns that are not located in + /// output_columns_positions, i.e. untuple()) + size_t result_index = output_columns_positions.size() + new_elements_size; + + /// Note, order of duplicated columns is not important here (since they + /// are the same), only order for unique columns is important, so it is + /// fine to use multimap here. + if (auto it = output_columns_positions.find(name); it != output_columns_positions.end()) + { + result_index = it->second; + output_columns_positions.erase(it); + } + auto it = required_columns_with_duplicate_count.find(name); if (required_columns_with_duplicate_count.end() != it && it->second) { - new_elements.push_back(elem); + new_elements[result_index] = elem; --it->second; + ++new_elements_size; } else if (select_query->distinct || hasArrayJoin(elem)) { /// ARRAY JOIN cannot be optimized out since it may change number of rows, /// so as DISTINCT. - new_elements.push_back(elem); + new_elements[result_index] = elem; + ++new_elements_size; } else { @@ -485,13 +510,20 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const /// Never remove untuple. It's result column may be in required columns. /// It is not easy to analyze untuple here, because types were not calculated yet. if (func && func->name == "untuple") - new_elements.push_back(elem); - + { + new_elements[result_index] = elem; + ++new_elements_size; + } /// removing aggregation can change number of rows, so `count()` result in outer sub-query would be wrong if (func && AggregateUtils::isAggregateFunction(*func) && !select_query->groupBy()) - new_elements.push_back(elem); + { + new_elements[result_index] = elem; + ++new_elements_size; + } } } + /// Remove empty nodes. + std::erase(new_elements, ASTPtr{}); if (select_query->interpolate()) { From f86ddebadaa12dc57b0bf4d60f96aa0d2eac6191 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 6 Jun 2022 19:32:41 +0300 Subject: [PATCH 229/408] Add sanity check for UNION header Signed-off-by: Azat Khuzhin --- .../InterpreterSelectWithUnionQuery.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 28b1f7667a8..e886027683f 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -159,7 +159,22 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( { Blocks headers(num_children); for (size_t query_num = 0; query_num < num_children; ++query_num) + { headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); + const auto & current_required_result_column_names = required_result_column_names_for_other_selects[query_num]; + if (!current_required_result_column_names.empty()) + { + const auto & header_columns = headers[query_num].getNames(); + if (current_required_result_column_names != header_columns) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Different order of columns in UNION subquery: {} and {}", + fmt::join(current_required_result_column_names, ", "), + fmt::join(header_columns, ", ")); + } + + } + } result_header = getCommonHeaderForUnion(headers); } From 4a52d3f6b9510ce4ba250fcfe8b633de66e68900 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Jun 2022 07:55:28 +0300 Subject: [PATCH 230/408] tests: update reference for 00597_push_down_predicate_long CI: https://s3.amazonaws.com/clickhouse-test-reports/37887/6b4613c88056f3901f6d8a832c9e2bc15e22206d/stateless_tests__aarch64__actions_.html Signed-off-by: Azat Khuzhin --- .../00597_push_down_predicate_long.reference | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.reference b/tests/queries/0_stateless/00597_push_down_predicate_long.reference index f6f1320c2f8..af4e9b5496e 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate_long.reference +++ b/tests/queries/0_stateless/00597_push_down_predicate_long.reference @@ -402,8 +402,8 @@ FROM ANY LEFT JOIN ( SELECT - date, id, + date, name, value FROM test_00597 @@ -472,8 +472,8 @@ FROM ANY LEFT JOIN ( SELECT - date, id, + date, name, value FROM test_00597 @@ -537,10 +537,10 @@ FROM ANY LEFT JOIN ( SELECT - date, - id, name, - value + value, + date, + id FROM test_00597 ) AS b ON id = b.id WHERE id = 1 @@ -567,8 +567,8 @@ FROM SEMI LEFT JOIN ( SELECT - date, id, + date, name, value FROM From d80a21a44552ba1484a99bef5d0d95e6d5a09b10 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 26 Jun 2022 08:23:52 +0000 Subject: [PATCH 231/408] Distinct sorted: calculate column positions once in constructor - instead of calculating them on every chunk --- .../Transforms/DistinctSortedTransform.cpp | 29 +++++++++---------- .../Transforms/DistinctSortedTransform.h | 4 ++- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 13d039ebcae..a4e5983de82 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -13,10 +13,20 @@ DistinctSortedTransform::DistinctSortedTransform( : ISimpleTransform(header_, header_, true) , header(std::move(header_)) , description(std::move(sort_description)) - , columns_names(columns) + , column_names(columns) , limit_hint(limit_hint_) , set_size_limits(set_size_limits_) { + /// pre-calculate column positions to process during chunk transformation + const size_t num_columns = column_names.empty() ? header_.columns() : column_names.size(); + column_positions.reserve(num_columns); + for (size_t i = 0; i < num_columns; ++i) + { + auto pos = column_names.empty() ? i : header_.getPositionByName(column_names[i]); + const auto & col = header_.getByPosition(pos).column; + if (col && !isColumnConst(*col)) + column_positions.emplace_back(pos); + } } void DistinctSortedTransform::transform(Chunk & chunk) @@ -119,24 +129,13 @@ bool DistinctSortedTransform::buildFilter( ColumnRawPtrs DistinctSortedTransform::getKeyColumns(const Chunk & chunk) const { - size_t columns = columns_names.empty() ? chunk.getNumColumns() : columns_names.size(); - ColumnRawPtrs column_ptrs; - column_ptrs.reserve(columns); - - for (size_t i = 0; i < columns; ++i) + column_ptrs.reserve(column_positions.size()); + for (const auto pos : column_positions) { - auto pos = i; - if (!columns_names.empty()) - pos = input.getHeader().getPositionByName(columns_names[i]); - const auto & column = chunk.getColumns()[pos]; - - /// Ignore all constant columns. - if (!isColumnConst(*column)) - column_ptrs.emplace_back(column.get()); + column_ptrs.emplace_back(column.get()); } - return column_ptrs; } diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 0530a6689e9..72acde0716b 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -57,7 +58,8 @@ private: }; PreviousChunk prev_chunk; - Names columns_names; + Names column_names; + ColumnNumbers column_positions; ClearableSetVariants data; Sizes key_sizes; UInt64 limit_hint; From c1840e798ceeec72e315503d635a2e53e6d890c7 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 Jun 2022 20:04:27 +0000 Subject: [PATCH 232/408] Fix: wrong header variable was used --- src/Processors/Transforms/DistinctSortedTransform.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index a4e5983de82..73e89c774ae 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -17,13 +17,13 @@ DistinctSortedTransform::DistinctSortedTransform( , limit_hint(limit_hint_) , set_size_limits(set_size_limits_) { - /// pre-calculate column positions to process during chunk transformation - const size_t num_columns = column_names.empty() ? header_.columns() : column_names.size(); + /// pre-calculate column positions to use during chunk transformation + const size_t num_columns = column_names.empty() ? header.columns() : column_names.size(); column_positions.reserve(num_columns); for (size_t i = 0; i < num_columns; ++i) { - auto pos = column_names.empty() ? i : header_.getPositionByName(column_names[i]); - const auto & col = header_.getByPosition(pos).column; + auto pos = column_names.empty() ? i : header.getPositionByName(column_names[i]); + const auto & col = header.getByPosition(pos).column; if (col && !isColumnConst(*col)) column_positions.emplace_back(pos); } From 5048d384b233c817a255c938fb6c3c7d485b48bb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 28 Jun 2022 22:28:19 +0200 Subject: [PATCH 233/408] BuilderBinClangTidy --- .github/workflows/master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5dd1ec73fde..66ba8547894 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -643,7 +643,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderClangTidy: + BuilderBinClangTidy: needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: @@ -1011,7 +1011,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderClangTidy + - BuilderBinClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] steps: From f5bfc2c27b4ad0adcaa49e24bda64a5d627225ea Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 28 Jun 2022 22:29:17 +0200 Subject: [PATCH 234/408] BuilderBinClangTidy --- .github/workflows/pull_request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 50f87549811..9cd8fd6f49d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -707,7 +707,7 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderClangTidy: + BuilderBinClangTidy: needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] steps: @@ -1065,7 +1065,7 @@ jobs: - BuilderBinFreeBSD # - BuilderBinGCC - BuilderBinPPC64 - - BuilderClangTidy + - BuilderBinClangTidy - BuilderDebSplitted runs-on: [self-hosted, style-checker] if: ${{ success() || failure() }} From 4a00e33e6b5159c8773cf2fa6453f9a315d20bf2 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 Jun 2022 21:42:46 +0000 Subject: [PATCH 235/408] Fixes for some review comments --- src/Core/Settings.h | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 3 ++- src/Processors/Transforms/DistinctSortedChunkTransform.cpp | 2 +- src/Processors/Transforms/DistinctSortedChunkTransform.h | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 617b22206de..f33b087e571 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -605,7 +605,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ - M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if columns in DISTINCT are sorted", 1) \ + M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 1) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4f0e929d587..4a1772759bc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -140,7 +140,7 @@ ReadFromMergeTree::ReadFromMergeTree( { /// build sort description for output stream SortDescription sort_description; const Names & sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); - Block const & header = output_stream->header; + const Block & header = output_stream->header; const int sort_direction = getSortDirection(query_info); for (const auto & column_name : sorting_key_columns) { @@ -150,6 +150,7 @@ ReadFromMergeTree::ReadFromMergeTree( sort_description.emplace_back(column_name, sort_direction); } output_stream->sort_description = std::move(sort_description); + output_stream->sort_mode = DataStream::SortMode::Chunk; } } diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp index 09b894c261e..b76c895c00e 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp @@ -140,7 +140,7 @@ size_t DistinctSortedChunkTransform::getRangeEnd(size_t range_begin, size_t rang return range_end; } -std::tuple DistinctSortedChunkTransform::continueWithPrevRange(const size_t chunk_rows, IColumn::Filter & filter) +std::pair DistinctSortedChunkTransform::continueWithPrevRange(const size_t chunk_rows, IColumn::Filter & filter) { /// current_key is empty on very first transform() call /// or first row doesn't match a key from previous transform() diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.h b/src/Processors/Transforms/DistinctSortedChunkTransform.h index 8f437d6fb9c..1d9162afd30 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.h +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.h @@ -32,7 +32,7 @@ protected: private: void initChunkProcessing(const Columns & input_columns); - std::tuple continueWithPrevRange(size_t chunk_rows, IColumn::Filter & filter); + std::pair continueWithPrevRange(size_t chunk_rows, IColumn::Filter & filter); size_t ordinaryDistinctOnRange(IColumn::Filter & filter, size_t range_begin, size_t range_end, bool clear_data); inline void setCurrentKey(size_t row_pos); inline bool isCurrentKey(size_t row_pos) const; From f5d26572df57a538486069557c214f43d51e442f Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 29 Jun 2022 01:16:30 +0200 Subject: [PATCH 236/408] Quick fix for aggregation pipeline (#38295) --- src/Interpreters/InterpreterSelectQuery.cpp | 24 +++- src/Processors/QueryPlan/AggregatingStep.cpp | 13 +- src/Processors/QueryPlan/AggregatingStep.h | 7 +- .../QueryPlan/MergingAggregatedStep.cpp | 13 +- .../QueryPlan/MergingAggregatedStep.h | 7 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 7 +- .../performance/queries_over_aggregation.xml | 7 + .../01915_for_each_crakjie.reference | 2 +- .../0_stateless/01915_for_each_crakjie.sql | 3 +- .../02343_aggregation_pipeline.reference | 127 ++++++++++++++++++ .../02343_aggregation_pipeline.sql | 58 ++++++++ 11 files changed, 249 insertions(+), 19 deletions(-) create mode 100644 tests/performance/queries_over_aggregation.xml create mode 100644 tests/queries/0_stateless/02343_aggregation_pipeline.reference create mode 100644 tests/queries/0_stateless/02343_aggregation_pipeline.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 047e50272fe..5863dd20bb9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1590,7 +1590,8 @@ static void executeMergeAggregatedImpl( bool has_grouping_sets, const Settings & settings, const NamesAndTypesList & aggregation_keys, - const AggregateDescriptions & aggregates) + const AggregateDescriptions & aggregates, + bool should_produce_results_in_order_of_bucket_number) { auto keys = aggregation_keys.getNames(); if (has_grouping_sets) @@ -1619,7 +1620,8 @@ static void executeMergeAggregatedImpl( final, settings.distributed_aggregation_memory_efficient && is_remote_storage, settings.max_threads, - settings.aggregation_memory_efficient_merge_threads); + settings.aggregation_memory_efficient_merge_threads, + should_produce_results_in_order_of_bucket_number); query_plan.addStep(std::move(merging_aggregated)); } @@ -1680,6 +1682,9 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( query_plan.addStep(std::move(expression_before_aggregation)); } + // Let's just choose the safe option since we don't know the value of `to_stage` here. + const bool should_produce_results_in_order_of_bucket_number = true; + executeMergeAggregatedImpl( query_plan, query_info.projection->aggregate_overflow_row, @@ -1688,7 +1693,8 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan( false, context_->getSettingsRef(), query_info.projection->aggregation_keys, - query_info.projection->aggregate_descriptions); + query_info.projection->aggregate_descriptions, + should_produce_results_in_order_of_bucket_number); } } } @@ -2268,6 +2274,9 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); + const bool should_produce_results_in_order_of_bucket_number + = options.to_stage == QueryProcessingStage::WithMergeableState && settings.distributed_aggregation_memory_efficient; + auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), std::move(aggregator_params), @@ -2279,7 +2288,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac temporary_data_merge_threads, storage_has_evenly_distributed_read, std::move(group_by_info), - std::move(group_by_sort_description)); + std::move(group_by_sort_description), + should_produce_results_in_order_of_bucket_number); query_plan.addStep(std::move(aggregating_step)); } @@ -2292,6 +2302,9 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool if (query_info.projection && query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) return; + const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState + && context->getSettingsRef().distributed_aggregation_memory_efficient; + executeMergeAggregatedImpl( query_plan, overflow_row, @@ -2300,7 +2313,8 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool has_grouping_sets, context->getSettingsRef(), query_analyzer->aggregationKeys(), - query_analyzer->aggregates()); + query_analyzer->aggregates(), + should_produce_results_in_order_of_bucket_number); } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 1e62673bc26..0a4b12084eb 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -19,13 +19,13 @@ namespace DB { -static ITransformingStep::Traits getTraits() +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) { return ITransformingStep::Traits { { .preserves_distinct_columns = false, /// Actually, we may check that distinct names are in aggregation keys - .returns_single_stream = true, + .returns_single_stream = should_produce_results_in_order_of_bucket_number, /// Actually, may also return single stream if should_produce_results_in_order_of_bucket_number = false .preserves_number_of_streams = false, .preserves_sorting = false, }, @@ -75,9 +75,10 @@ AggregatingStep::AggregatingStep( size_t temporary_data_merge_threads_, bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, - SortDescription group_by_sort_description_) + SortDescription group_by_sort_description_, + bool should_produce_results_in_order_of_bucket_number_) : ITransformingStep( - input_stream_, appendGroupingColumn(params_.getHeader(input_stream_.header, final_), grouping_sets_params_), getTraits(), false) + input_stream_, appendGroupingColumn(params_.getHeader(input_stream_.header, final_), grouping_sets_params_), getTraits(should_produce_results_in_order_of_bucket_number_), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(final_) @@ -88,6 +89,7 @@ AggregatingStep::AggregatingStep( , storage_has_evenly_distributed_read(storage_has_evenly_distributed_read_) , group_by_info(std::move(group_by_info_)) , group_by_sort_description(std::move(group_by_sort_description_)) + , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) { } @@ -351,7 +353,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B return std::make_shared(header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads); }); - pipeline.resize(1); + /// We add the explicit resize here, but not in case of aggregating in order, since AIO don't use two-level hash tables and thus returns only buckets with bucket_number = -1. + pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : pipeline.getNumStreams(), true /* force */); aggregating = collector.detachProcessors(0); } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 2879cd1e0e9..0e982d76940 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -36,7 +36,8 @@ public: size_t temporary_data_merge_threads_, bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, - SortDescription group_by_sort_description_); + SortDescription group_by_sort_description_, + bool should_produce_results_in_order_of_bucket_number_); String getName() const override { return "Aggregating"; } @@ -65,6 +66,10 @@ private: InputOrderInfoPtr group_by_info; SortDescription group_by_sort_description; + /// It determines if we should resize pipeline to 1 at the end. + /// Needed in case of distributed memory efficient aggregation. + const bool should_produce_results_in_order_of_bucket_number; + Processors aggregating_in_order; Processors aggregating_sorted; Processors finalizing; diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index c898b901a6a..d74a6174f00 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -7,13 +7,13 @@ namespace DB { -static ITransformingStep::Traits getTraits() +static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_of_bucket_number) { return ITransformingStep::Traits { { .preserves_distinct_columns = false, - .returns_single_stream = true, + .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, }, @@ -29,13 +29,16 @@ MergingAggregatedStep::MergingAggregatedStep( bool final_, bool memory_efficient_aggregation_, size_t max_threads_, - size_t memory_efficient_merge_threads_) - : ITransformingStep(input_stream_, params_.getHeader(input_stream_.header, final_), getTraits()) + size_t memory_efficient_merge_threads_, + bool should_produce_results_in_order_of_bucket_number_) + : ITransformingStep( + input_stream_, params_.getHeader(input_stream_.header, final_), getTraits(should_produce_results_in_order_of_bucket_number_)) , params(std::move(params_)) , final(final_) , memory_efficient_aggregation(memory_efficient_aggregation_) , max_threads(max_threads_) , memory_efficient_merge_threads(memory_efficient_merge_threads_) + , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) { /// Aggregation keys are distinct for (const auto & key : params.keys) @@ -62,6 +65,8 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c pipeline.addMergingAggregatedMemoryEfficientTransform(transform_params, num_merge_threads); } + + pipeline.resize(should_produce_results_in_order_of_bucket_number ? 1 : max_threads); } void MergingAggregatedStep::describeActions(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 136422c8c27..419b43615bd 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -19,7 +19,8 @@ public: bool final_, bool memory_efficient_aggregation_, size_t max_threads_, - size_t memory_efficient_merge_threads_); + size_t memory_efficient_merge_threads_, + bool should_produce_results_in_order_of_bucket_number_); String getName() const override { return "MergingAggregated"; } @@ -36,6 +37,10 @@ private: bool memory_efficient_aggregation; size_t max_threads; size_t memory_efficient_merge_threads; + + /// It determines if we should resize pipeline to 1 at the end. + /// Needed in case of distributed memory efficient aggregation over distributed table. + const bool should_produce_results_in_order_of_bucket_number; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 6590445572d..85231aca253 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -369,6 +369,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( else group_by_info = nullptr; + // We don't have information regarding the `to_stage` of the query processing, only about `from_stage` (which is passed through `processed_stage` argument). + // Thus we cannot assign false here since it may be a query over distributed table. + const bool should_produce_results_in_order_of_bucket_number = true; + auto aggregating_step = std::make_unique( query_plan->getCurrentDataStream(), std::move(params), @@ -380,7 +384,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( temporary_data_merge_threads, /* storage_has_evenly_distributed_read_= */ false, std::move(group_by_info), - std::move(group_by_sort_description)); + std::move(group_by_sort_description), + should_produce_results_in_order_of_bucket_number); query_plan->addStep(std::move(aggregating_step)); }; diff --git a/tests/performance/queries_over_aggregation.xml b/tests/performance/queries_over_aggregation.xml new file mode 100644 index 00000000000..2a92ea26819 --- /dev/null +++ b/tests/performance/queries_over_aggregation.xml @@ -0,0 +1,7 @@ + + select * from (select * from numbers_mt(1e7) group by number) group by number format Null + select * from (select * from numbers_mt(1e7) group by number) order by number format Null + select * from (select * from numbers_mt(1e7) group by number) group by number format Null settings max_bytes_before_external_group_by = 1 + select * from (select * from remote('127.0.0.{{1,2}}', numbers_mt(1e7)) group by number) group by number format Null settings distributed_aggregation_memory_efficient = 1 + select * from (select * from remote('127.0.0.{{1,2}}', numbers_mt(1e7)) group by number) group by number format Null settings distributed_aggregation_memory_efficient = 0 + diff --git a/tests/queries/0_stateless/01915_for_each_crakjie.reference b/tests/queries/0_stateless/01915_for_each_crakjie.reference index 905658eb3f7..7e57fec849e 100644 --- a/tests/queries/0_stateless/01915_for_each_crakjie.reference +++ b/tests/queries/0_stateless/01915_for_each_crakjie.reference @@ -1,2 +1,2 @@ -b [2,2.2,2.260035] a [2,2.2,2.260035] +b [2,2.2,2.260035] diff --git a/tests/queries/0_stateless/01915_for_each_crakjie.sql b/tests/queries/0_stateless/01915_for_each_crakjie.sql index f65e08af61b..7d6ce0746b9 100644 --- a/tests/queries/0_stateless/01915_for_each_crakjie.sql +++ b/tests/queries/0_stateless/01915_for_each_crakjie.sql @@ -7,4 +7,5 @@ FROM SELECT sumStateForEach([1., 1.1, 1.1300175]) AS x FROM remote('127.0.0.{1,2}', system.one) ) -GROUP BY z; +GROUP BY z +ORDER BY z; diff --git a/tests/queries/0_stateless/02343_aggregation_pipeline.reference b/tests/queries/0_stateless/02343_aggregation_pipeline.reference new file mode 100644 index 00000000000..67bd9c414ba --- /dev/null +++ b/tests/queries/0_stateless/02343_aggregation_pipeline.reference @@ -0,0 +1,127 @@ +-- { echoOn } + +explain pipeline select * from (select * from numbers_mt(1e8) group by number) group by number; +(Expression) +ExpressionTransform × 16 + (Aggregating) + Resize 16 → 16 + AggregatingTransform × 16 + StrictResize 16 → 16 + (Expression) + ExpressionTransform × 16 + (Aggregating) + Resize 16 → 16 + AggregatingTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromStorage) + NumbersMt × 16 0 → 1 +explain pipeline select * from (select * from numbers_mt(1e8) group by number) order by number; +(Expression) +ExpressionTransform + (Sorting) + MergingSortedTransform 16 → 1 + MergeSortingTransform × 16 + LimitsCheckingTransform × 16 + PartialSortingTransform × 16 + (Expression) + ExpressionTransform × 16 + (Aggregating) + Resize 16 → 16 + AggregatingTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromStorage) + NumbersMt × 16 0 → 1 +explain pipeline select number from remote('127.0.0.{1,2,3}', system, numbers_mt) group by number settings distributed_aggregation_memory_efficient = 1; +(Expression) +ExpressionTransform × 16 + (MergingAggregated) + Resize 1 → 16 + SortingAggregatedTransform 16 → 1 + MergingAggregatedBucketTransform × 16 + Resize 1 → 16 + GroupingAggregatedTransform 3 → 1 + (Union) + (Aggregating) + Resize 16 → 1 + AggregatingTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromStorage) + Numbers × 16 0 → 1 + (ReadFromRemote) +explain pipeline select number from remote('127.0.0.{1,2,3}', system, numbers_mt) group by number settings distributed_aggregation_memory_efficient = 0; +(Expression) +ExpressionTransform × 16 + (MergingAggregated) + Resize 1 → 16 + MergingAggregatedTransform + Resize 18 → 1 + (Union) + (Aggregating) + Resize 16 → 16 + AggregatingTransform × 16 + (Expression) + ExpressionTransform × 16 + (ReadFromStorage) + Numbers × 16 0 → 1 + (ReadFromRemote) +-- { echoOn } + +explain pipeline SELECT k1, k3, sum(value) v FROM remote('127.0.0.{1,2}', currentDatabase(), proj_agg_02343) GROUP BY k1, k3 SETTINGS distributed_aggregation_memory_efficient = 0; +(Expression) +ExpressionTransform × 16 + (MergingAggregated) + Resize 1 → 16 + MergingAggregatedTransform + Resize 2 → 1 + (Union) + (ReadFromStorage) + AggregatingTransform + ExpressionTransform + MergeTreeInOrder 0 → 1 + (ReadFromRemote) +explain pipeline SELECT k1, k3, sum(value) v FROM remote('127.0.0.{1,2}', currentDatabase(), proj_agg_02343) GROUP BY k1, k3 SETTINGS distributed_aggregation_memory_efficient = 1; +(Expression) +ExpressionTransform × 16 + (MergingAggregated) + Resize 1 → 16 + SortingAggregatedTransform 16 → 1 + MergingAggregatedBucketTransform × 16 + Resize 1 → 16 + GroupingAggregatedTransform 2 → 1 + (Union) + (ReadFromStorage) + AggregatingTransform + ExpressionTransform + MergeTreeInOrder 0 → 1 + (ReadFromRemote) +-- { echoOn } + +explain pipeline select a from remote('127.0.0.{1,2}', currentDatabase(), dist_t) group by a settings max_threads = 2, distributed_aggregation_memory_efficient = 1; +(Expression) +ExpressionTransform × 2 + (MergingAggregated) + Resize 1 → 2 + SortingAggregatedTransform 2 → 1 + MergingAggregatedBucketTransform × 2 + Resize 1 → 2 + GroupingAggregatedTransform 2 → 1 + (Union) + (MergingAggregated) + SortingAggregatedTransform 2 → 1 + MergingAggregatedBucketTransform × 2 + Resize 1 → 2 + GroupingAggregatedTransform 2 → 1 + (Union) + (Aggregating) + Resize 2 → 1 + AggregatingTransform × 2 + StrictResize 2 → 2 + (Expression) + ExpressionTransform × 2 + (ReadFromMergeTree) + MergeTreeThread × 2 0 → 1 + (ReadFromRemote) + (ReadFromRemote) diff --git a/tests/queries/0_stateless/02343_aggregation_pipeline.sql b/tests/queries/0_stateless/02343_aggregation_pipeline.sql new file mode 100644 index 00000000000..d259889b042 --- /dev/null +++ b/tests/queries/0_stateless/02343_aggregation_pipeline.sql @@ -0,0 +1,58 @@ +set max_threads = 16; +set prefer_localhost_replica = 1; +set optimize_aggregation_in_order = 0; + +-- { echoOn } + +explain pipeline select * from (select * from numbers_mt(1e8) group by number) group by number; + +explain pipeline select * from (select * from numbers_mt(1e8) group by number) order by number; + +explain pipeline select number from remote('127.0.0.{1,2,3}', system, numbers_mt) group by number settings distributed_aggregation_memory_efficient = 1; + +explain pipeline select number from remote('127.0.0.{1,2,3}', system, numbers_mt) group by number settings distributed_aggregation_memory_efficient = 0; + +-- { echoOff } + +DROP TABLE IF EXISTS proj_agg_02343; + +CREATE TABLE proj_agg_02343 +( + k1 UInt32, + k2 UInt32, + k3 UInt32, + value UInt32, + PROJECTION aaaa + ( + SELECT + k1, + k2, + k3, + sum(value) + GROUP BY k1, k2, k3 + ) +) +ENGINE = MergeTree +ORDER BY tuple(); + +INSERT INTO proj_agg_02343 SELECT 1, number % 2, number % 4, number FROM numbers(100000); +OPTIMIZE TABLE proj_agg_02343 FINAL; + +-- { echoOn } + +explain pipeline SELECT k1, k3, sum(value) v FROM remote('127.0.0.{1,2}', currentDatabase(), proj_agg_02343) GROUP BY k1, k3 SETTINGS distributed_aggregation_memory_efficient = 0; + +explain pipeline SELECT k1, k3, sum(value) v FROM remote('127.0.0.{1,2}', currentDatabase(), proj_agg_02343) GROUP BY k1, k3 SETTINGS distributed_aggregation_memory_efficient = 1; + +-- { echoOff } + +create table t(a UInt64) engine = MergeTree order by (a); +system stop merges t; +create table dist_t as t engine = Distributed(test_cluster_two_shards, currentDatabase(), t, a % 2); +system stop merges dist_t; +insert into dist_t select number from numbers_mt(10); +insert into dist_t select number from numbers_mt(10); + +-- { echoOn } + +explain pipeline select a from remote('127.0.0.{1,2}', currentDatabase(), dist_t) group by a settings max_threads = 2, distributed_aggregation_memory_efficient = 1; From 7274169c33f10837ec5c0fb8b8edc62a01cac442 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Wed, 29 Jun 2022 10:03:10 +0800 Subject: [PATCH 237/408] update test scripts --- .../integration/test_hive_query/data/prepare_hive_data.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_hive_query/data/prepare_hive_data.sh b/tests/integration/test_hive_query/data/prepare_hive_data.sh index 39d435eb05a..d65ec9cc153 100755 --- a/tests/integration/test_hive_query/data/prepare_hive_data.sh +++ b/tests/integration/test_hive_query/data/prepare_hive_data.sh @@ -1,15 +1,15 @@ #!/bin/bash hive -e "create database test" -hive -e "create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; " -hive -e "create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" -hive -e "create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text " +hive -e "drop table if exists test.demo; create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; " +hive -e "drop table if exists test.parquet_demo; create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" +hive -e "drop table if exists test.demo_text; create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text " hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo partition(day) select * from test.demo_text; insert into test.demo_orc partition(day) select * from test.demo_text" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" -hive -e "CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" +hive -e "drop table if test.test_hive_types; exists CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 10, 'y', 'xyz')); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 11, 'y', 'abc'));" From ee30c4a3f7039ca6a4e029d3f973f8a943125e21 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Wed, 29 Jun 2022 10:23:00 +0800 Subject: [PATCH 238/408] update test scripts --- tests/integration/test_hive_query/data/prepare_hive_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_hive_query/data/prepare_hive_data.sh b/tests/integration/test_hive_query/data/prepare_hive_data.sh index d65ec9cc153..495ea201870 100755 --- a/tests/integration/test_hive_query/data/prepare_hive_data.sh +++ b/tests/integration/test_hive_query/data/prepare_hive_data.sh @@ -9,7 +9,7 @@ hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo pa hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" -hive -e "drop table if test.test_hive_types; exists CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" +hive -e "drop table if exists test.test_hive_types; CREATE TABLE test.test_hive_types( f_tinyint tinyint, f_smallint smallint, f_int int, f_integer int, f_bigint bigint, f_float float, f_double double, f_decimal decimal(10,0), f_timestamp timestamp, f_date date, f_string string, f_varchar varchar(100), f_char char(100), f_bool boolean, f_array_int array, f_array_string array, f_array_float array, f_map_int map, f_map_string map, f_map_float map, f_struct struct>) PARTITIONED BY( day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';" hive -e "insert into test.test_hive_types partition(day='2022-02-20') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-20 14:47:04', '2022-02-20', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 10, 'y', 'xyz')); insert into test.test_hive_types partition(day='2022-02-19') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, '2022-02-19 14:47:04', '2022-02-19', 'hello world', 'hello world', 'hello world', true, array(1,2,3), array('hello world', 'hello world'), array(float(1.1),float(1.2)), map('a', 100, 'b', 200, 'c', 300), map('a', 'aa', 'b', 'bb', 'c', 'cc'), map('a', float(111.1), 'b', float(222.2), 'c', float(333.3)), named_struct('a', 'aaa', 'b', 200, 'c', float(333.3), 'd', named_struct('x', 11, 'y', 'abc'));" From d5886e4861d5a16e05f01cffab8f0dece84f7e6a Mon Sep 17 00:00:00 2001 From: HeenaBansal2009 Date: Tue, 28 Jun 2022 19:24:06 -0700 Subject: [PATCH 239/408] Updated Index file for GeoFunctions --- docs/en/sql-reference/functions/geo/index.md | 70 +++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/geo/index.md b/docs/en/sql-reference/functions/geo/index.md index f76c3a3f731..ea43b3ef96c 100644 --- a/docs/en/sql-reference/functions/geo/index.md +++ b/docs/en/sql-reference/functions/geo/index.md @@ -3,6 +3,74 @@ sidebar_label: Geo sidebar_position: 62 --- -# Geo Functions +# Geo Functions + +## Geographical Coordinates Functions + +- [greatCircleDistance](./coordinates.md#greatCircleDistance) +- [geoDistance](./coordinates.md#geoDistance) +- [greatCircleAngle](./coordinates.md#greatCircleAngle) +- [pointInEllipses](./coordinates.md#pointInEllipses) +- [pointInPolygon](./coordinates.md#pointInPolygon) + +## Geohash Functions +- [geohashEncode](./geohash.md#geohashEncode) +- [geohashDecode](./geohash.md#geohashDecode) +- [geohashesInBox](./geohash.md#geohashesInBox) + +## H3 Indexes Functions + +- [h3IsValid](./h3#h3IsValid) +- [h3GetResolution](./h3#h3GetResolution) +- [h3EdgeAngle](./h3#h3EdgeAngle) +- [h3EdgeLengthM​](./h3#h3EdgeLengthM​) +- [h3EdgeLengthKm] (./h3#h3EdgeLengthKm) +- [geoToH3](./h3#geoToH3) +- [h3ToGeo](./h3#h3ToGeo) +- [h3ToGeoBoundary](./h3#h3ToGeoBoundary) +- [h3kRing](./h3#h3kRing) +- [h3GetBaseCell](./h3#h3GetBaseCell) +- [h3HexAreaM2](./h3#h3HexAreaM2) +- [h3HexAreaKm2](./h3#h3HexAreaKm2) +- [h3IndexesAreNeighbors](./h3#h3IndexesAreNeighbors) +- [h3ToChildren](./h3#h3ToChildren) +- [h3ToParent](./h3#h3ToParent) +- [h3ToString](./h3#h3ToString) +- [stringToH3](./h3#stringToH3) +- [h3GetResolution](./h3#h3GetResolution) +- [h3IsResClassIII](./h3#h3IsResClassIII) +- [h3IsPentagon](./h3#h3IsPentagon) +- [h3GetFaces](./h3#h3GetFaces) +- [h3CellAreaM2](./h3#h3CellAreaM2) +- [h3CellAreaRads2](./h3#h3CellAreaRads2) +- [h3ToCenterChild](./h3#h3ToCenterChild) +- [h3ExactEdgeLengthM](./h3#h3ExactEdgeLengthM) +- [h3ExactEdgeLengthKm](./h3#h3ExactEdgeLengthKm) +- [h3ExactEdgeLengthRads](./h3#h3ExactEdgeLengthRads) +- [h3NumHexagons](./h3#h3NumHexagons) +- [h3Line](./h3#h3Line) +- [h3Distance](./h3#h3Distance) +- [h3HexRing](./h3#h3HexRing) +- [h3GetUnidirectionalEdge](./h3#h3GetUnidirectionalEdge) +- [h3UnidirectionalEdgeIsValid](./h3#h3UnidirectionalEdgeIsValid) +- [h3GetOriginIndexFromUnidirectionalEdge](./h3#h3GetOriginIndexFromUnidirectionalEdge) +- [h3GetDestinationIndexFromUnidirectionalEdge](./h3#h3GetDestinationIndexFromUnidirectionalEdge) +- [h3GetIndexesFromUnidirectionalEdge](./h3#h3GetIndexesFromUnidirectionalEdge) +- [h3GetUnidirectionalEdgesFromHexagon](./h3#h3GetUnidirectionalEdgesFromHexagon) +- [h3GetUnidirectionalEdgeBoundary](./h3#h3GetUnidirectionalEdgeBoundary) + +## S2 Index Functions + +- [geoToS2](./s2#geoToS2) +- [s2ToGeo](./s2#s2ToGeo) +- [s2GetNeighbors](./s2#s2GetNeighbors) +- [s2CellsIntersect](./s2#s2CellsIntersect) +- [s2CapContains](./s2#s2CapContains) +- [s2CapUnion](./s2#s2CapUnion) +- [s2RectAdd](./s2#s2RectAdd) +- [s2RectContains](./s2#s2RectContains) +- [s2RectUinion](./s2#s2RectUinion) +- [s2RectIntersection](./s2#s2RectIntersection) + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/) From 4d0635371fcc3afe8c6fccbe0d5df048343f8e2f Mon Sep 17 00:00:00 2001 From: Jianmei Zhang Date: Thu, 10 Mar 2022 15:31:27 +0800 Subject: [PATCH 240/408] Support sql stanard create and drop index --- src/Interpreters/InterpreterAlterQuery.cpp | 2 + src/Parsers/ASTAlterQuery.cpp | 43 +++++++- src/Parsers/ASTAlterQuery.h | 8 ++ src/Parsers/ASTIndexDeclaration.cpp | 16 ++- src/Parsers/IAST.h | 1 + src/Parsers/ParserAlterQuery.cpp | 110 ++++++++++++++++++++- src/Parsers/ParserAlterQuery.h | 24 ++++- src/Parsers/formatAST.cpp | 3 +- src/Parsers/formatAST.h | 2 +- src/Storages/AlterCommands.cpp | 4 +- src/Storages/MutationCommands.cpp | 4 +- 11 files changed, 200 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 056a3d9f7b4..23b002350cf 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -295,11 +295,13 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS break; } case ASTAlterCommand::ADD_INDEX: + case ASTAlterCommand::STD_CREATE_INDEX: { required_access.emplace_back(AccessType::ALTER_ADD_INDEX, database, table); break; } case ASTAlterCommand::DROP_INDEX: + case ASTAlterCommand::STD_DROP_INDEX: { if (command.clear_index) required_access.emplace_back(AccessType::ALTER_CLEAR_INDEX, database, table); diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index f53c39b192f..36193049a0a 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -1,9 +1,9 @@ +#include #include #include #include #include - namespace DB { namespace ErrorCodes @@ -100,7 +100,9 @@ const char * ASTAlterCommand::typeToString(ASTAlterCommand::Type type) case REMOVE_TTL: return "REMOVE_TTL"; case REMOVE_SAMPLE_BY: return "REMOVE_SAMPLE_BY"; case ADD_INDEX: return "ADD_INDEX"; + case STD_CREATE_INDEX: return "ADD_INDEX"; case DROP_INDEX: return "DROP_INDEX"; + case STD_DROP_INDEX: return "DROP_INDEX"; case MATERIALIZE_INDEX: return "MATERIALIZE_INDEX"; case ADD_CONSTRAINT: return "ADD_CONSTRAINT"; case DROP_CONSTRAINT: return "DROP_CONSTRAINT"; @@ -227,6 +229,15 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & index->formatImpl(settings, state, frame); } } + else if (type == ASTAlterCommand::STD_CREATE_INDEX) + { + auto name = index_decl->as()->name; + index_decl->as()->name = ""; + + index_decl->formatImpl(settings, state, frame); + + index_decl->as()->name = name; + } else if (type == ASTAlterCommand::DROP_INDEX) { settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_index ? "CLEAR " : "DROP ") << "INDEX " @@ -238,6 +249,14 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & partition->formatImpl(settings, state, frame); } } + else if (type == ASTAlterCommand::STD_DROP_INDEX) + { + if (settings.is_translate) + { + settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); + index->formatImpl(settings, state, frame); + } + } else if (type == ASTAlterCommand::MATERIALIZE_INDEX) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE INDEX " << (settings.hilite ? hilite_none : ""); @@ -556,12 +575,26 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; switch (alter_object) { case AlterObjectType::TABLE: - settings.ostr << "ALTER TABLE "; + if (command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) + { + settings.ostr << "CREATE INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + } + else if (command_type == ASTAlterCommand::Type::STD_DROP_INDEX) + { + settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + } + else + settings.ostr << "ALTER TABLE "; break; case AlterObjectType::DATABASE: settings.ostr << "ALTER DATABASE "; @@ -594,6 +627,12 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; frame_nested.expression_list_always_start_on_new_line = true; + + if ((command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) || (command_type == ASTAlterCommand::Type::STD_DROP_INDEX)){ + frame_nested.expression_list_always_start_on_new_line = false; + settings.ostr << " "; + } + static_cast(command_list)->formatImplMultiline(settings, state, frame_nested); } diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 956f07811ae..0bcb7473e41 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -46,6 +46,8 @@ public: ADD_INDEX, DROP_INDEX, MATERIALIZE_INDEX, + STD_CREATE_INDEX, + STD_DROP_INDEX, ADD_CONSTRAINT, DROP_CONSTRAINT, @@ -226,6 +228,12 @@ public: }; AlterObjectType alter_object = AlterObjectType::UNKNOWN; + ASTAlterCommand::Type command_type = ASTAlterCommand::NO_TYPE; + + /// Used for SQL-standard CREATE INDEX and DROP INDEX + ASTPtr index_name; + bool if_not_exists = false; /// option for CREATE INDEX + bool if_exists = false; /// option for DROP INDEX ASTExpressionList * command_list = nullptr; diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index d8ebf825674..a9b30a0f433 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -25,9 +25,19 @@ ASTPtr ASTIndexDeclaration::clone() const void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - s.ostr << backQuoteIfNeed(name); - s.ostr << " "; - expr->formatImpl(s, state, frame); + /// '' is from CREATE INDEX + if (name != "") + { + s.ostr << backQuoteIfNeed(name); + s.ostr << " "; + expr->formatImpl(s, state, frame); + } + else + { + s.ostr << "("; + expr->formatImpl(s, state, frame); + s.ostr << ")"; + } s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); type->formatImpl(s, state, frame); s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index b73919f4f36..07b6bed4f5b 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -184,6 +184,7 @@ public: bool hilite = false; bool one_line; bool always_quote_identifiers = false; + bool is_translate = false; //convert current standard SQL to clickhouse dialect. IdentifierQuotingStyle identifier_quoting_style = IdentifierQuotingStyle::Backticks; // Newline or whitespace. diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 9a3c1dd8bed..6a5ea0c2286 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -94,6 +95,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_delete("DELETE"); ParserKeyword s_update("UPDATE"); ParserKeyword s_where("WHERE"); + ParserKeyword s_create_index("CREATE INDEX"); ParserKeyword s_to("TO"); ParserKeyword s_remove("REMOVE"); @@ -112,6 +114,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserIdentifier parser_remove_property; ParserCompoundColumnDeclaration parser_col_decl; ParserIndexDeclaration parser_idx_decl; + ParserCreateIndexDeclaration parser_create_idx_decl; ParserConstraintDeclaration parser_constraint_decl; ParserProjectionDeclaration parser_projection_decl; ParserCompoundColumnDeclaration parser_modify_col_decl(false, false, true); @@ -156,7 +159,18 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected break; case ASTAlterQuery::AlterObjectType::TABLE: { - if (s_add_column.ignore(pos, expected)) + if (command_type == ASTAlterCommand::STD_CREATE_INDEX) + { + if (!parser_create_idx_decl.parse(pos, command->index_decl, expected)) + return false; + + command->type = ASTAlterCommand::STD_CREATE_INDEX; + } + else if (command_type == ASTAlterCommand::STD_DROP_INDEX) + { + command->type = ASTAlterCommand::STD_DROP_INDEX; + } + else if (s_add_column.ignore(pos, expected)) { if (s_if_not_exists.ignore(pos, expected)) command->if_not_exists = true; @@ -840,6 +854,44 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } +bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_type("TYPE"); + ParserKeyword s_granularity("GRANULARITY"); + + ParserDataType data_type_p; + ParserExpression expression_p; + ParserUnsignedInteger granularity_p; + + ASTPtr expr; + ASTPtr type; + ASTPtr granularity; + + /// Skip name parser for SQL-standard CREATE INDEX + + if (!expression_p.parse(pos, expr, expected)) + return false; + + if (!s_type.ignore(pos, expected)) + return false; + + if (!data_type_p.parse(pos, type, expected)) + return false; + + if (!s_granularity.ignore(pos, expected)) + return false; + + if (!granularity_p.parse(pos, granularity, expected)) + return false; + + auto index = std::make_shared(); + index->granularity = granularity->as().value.safeGet(); + index->set(index->expr, expr); + index->set(index->type, type); + node = index; + + return true; +} bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -849,10 +901,45 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_alter_table("ALTER TABLE"); ParserKeyword s_alter_live_view("ALTER LIVE VIEW"); ParserKeyword s_alter_database("ALTER DATABASE"); + ParserKeyword s_create_index("CREATE INDEX"); + ParserKeyword s_drop_index("DROP INDEX"); + ParserKeyword s_on("ON"); + ParserKeyword s_if_not_exists("IF NOT EXISTS"); + ParserKeyword s_if_exists("IF EXISTS"); + + ParserCompoundIdentifier parser_name; ASTAlterQuery::AlterObjectType alter_object_type; - if (s_alter_table.ignore(pos, expected)) + if (s_create_index.ignore(pos, expected)) + { + alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; + if (s_if_not_exists.ignore(pos, expected)) + query->if_not_exists = true; + + if (!parser_name.parse(pos, query->index_name, expected)) + return false; + + if (!s_on.ignore(pos, expected)) + return false; + + query->command_type = ASTAlterCommand::STD_CREATE_INDEX; + } + else if (s_drop_index.ignore(pos, expected)) + { + alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; + if (s_if_exists.ignore(pos, expected)) + query->if_exists = true; + + if (!parser_name.parse(pos, query->index_name, expected)) + return false; + + if (!s_on.ignore(pos, expected)) + return false; + + query->command_type = ASTAlterCommand::STD_DROP_INDEX; + } + else if (s_alter_table.ignore(pos, expected)) { alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; } @@ -891,6 +978,25 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!p_command_list.parse(pos, command_list, expected)) return false; + /// Set the index_name and exists flags for CREATE and DROP INDEX + if (query->command_type == ASTAlterCommand::STD_CREATE_INDEX) + { + ASTAlterCommand * command_ast = command_list->as()->children[0]->as(); + + command_ast->if_not_exists = query->if_not_exists; + + auto & ast_index_decl = command_ast->index_decl->as(); + ast_index_decl.name = query->index_name->as().name(); + + } + else if (query->command_type == ASTAlterCommand::STD_DROP_INDEX) + { + ASTAlterCommand * command_ast = command_list->as()->children[0]->as(); + + command_ast->if_exists = query->if_exists; + command_ast->as()->index = query->index_name; + } + query->set(query->command_list, command_list); query->alter_object = alter_object_type; diff --git a/src/Parsers/ParserAlterQuery.h b/src/Parsers/ParserAlterQuery.h index b0029ff88fd..25505b1c7f9 100644 --- a/src/Parsers/ParserAlterQuery.h +++ b/src/Parsers/ParserAlterQuery.h @@ -30,6 +30,9 @@ namespace DB * [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]] * ALTER LIVE VIEW [db.name] * [REFRESH] + * + * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value + * DROP INDEX [IF EXISTS] name on [db].name */ class ParserAlterQuery : public IParserBase @@ -48,9 +51,10 @@ protected: public: ASTAlterQuery::AlterObjectType alter_object; + ASTAlterCommand::Type command_type; - ParserAlterCommandList(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE) - : alter_object(alter_object_) {} + ParserAlterCommandList(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE, ASTAlterCommand::Type command_type_= ASTAlterCommand::NO_TYPE) + : alter_object(alter_object_), command_type(command_type_) {} }; @@ -62,10 +66,22 @@ protected: public: ASTAlterQuery::AlterObjectType alter_object; + ASTAlterCommand::Type command_type; - ParserAlterCommand(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE) - : alter_object(alter_object_) {} + ParserAlterCommand(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE, ASTAlterCommand::Type command_type_= ASTAlterCommand::NO_TYPE) + : alter_object(alter_object_), command_type(command_type_) {} }; +/** Part of CREATE INDEX expr TYPE typename(arg1, arg2, ...) GRANULARITY value */ +class ParserCreateIndexDeclaration : public IParserBase +{ +public: + ParserCreateIndexDeclaration() {} + +protected: + const char * getName() const override { return "index declaration in create index"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + } diff --git a/src/Parsers/formatAST.cpp b/src/Parsers/formatAST.cpp index fca8ea0aa35..6f5eb3a60b0 100644 --- a/src/Parsers/formatAST.cpp +++ b/src/Parsers/formatAST.cpp @@ -4,10 +4,11 @@ namespace DB { -void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line) +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line, bool is_translate) { IAST::FormatSettings settings(buf, one_line); settings.hilite = hilite; + settings.is_translate = is_translate; ast.format(settings); } diff --git a/src/Parsers/formatAST.h b/src/Parsers/formatAST.h index 28af2400a4c..0bf845cd04d 100644 --- a/src/Parsers/formatAST.h +++ b/src/Parsers/formatAST.h @@ -11,7 +11,7 @@ class WriteBuffer; /** Takes a syntax tree and turns it back into text. * In case of INSERT query, the data will be missing. */ -void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false); +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false, bool is_translate = false); String serializeAST(const IAST & ast, bool one_line = true); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 845aae52582..1b04fe3ede4 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -211,7 +211,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.type = AlterCommand::REMOVE_SAMPLE_BY; return command; } - else if (command_ast->type == ASTAlterCommand::ADD_INDEX) + else if (command_ast->type == ASTAlterCommand::ADD_INDEX || command_ast->type == ASTAlterCommand::STD_CREATE_INDEX) { AlterCommand command; command.ast = command_ast->clone(); @@ -274,7 +274,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_INDEX) + else if (command_ast->type == ASTAlterCommand::DROP_INDEX || command_ast->type == ASTAlterCommand::STD_DROP_INDEX) { AlterCommand command; command.ast = command_ast->clone(); diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 28dfe488869..0f9d17c5d1d 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -103,7 +103,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, return res; } - else if (parse_alter_commands && command->type == ASTAlterCommand::DROP_INDEX) + else if (parse_alter_commands && (command->type == ASTAlterCommand::DROP_INDEX || command->type == ASTAlterCommand::STD_DROP_INDEX)) { MutationCommand res; res.ast = command->ptr(); @@ -160,7 +160,7 @@ std::shared_ptr MutationCommands::ast() const void MutationCommands::writeText(WriteBuffer & out) const { WriteBufferFromOwnString commands_buf; - formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true); + formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true, /* is_translate = */ true); writeEscapedString(commands_buf.str(), out); } From 13231d6a85485a7b80481313ce46080a028c60b2 Mon Sep 17 00:00:00 2001 From: Jianmei Zhang Date: Thu, 10 Mar 2022 16:08:50 +0800 Subject: [PATCH 241/408] Pass command type for create and drop index --- src/Parsers/ASTAlterQuery.cpp | 3 ++- src/Parsers/ParserAlterQuery.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 36193049a0a..465cd24c9d6 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -628,7 +628,8 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState frame_nested.need_parens = false; frame_nested.expression_list_always_start_on_new_line = true; - if ((command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) || (command_type == ASTAlterCommand::Type::STD_DROP_INDEX)){ + if ((command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) || (command_type == ASTAlterCommand::Type::STD_DROP_INDEX)) + { frame_nested.expression_list_always_start_on_new_line = false; settings.ostr << " "; } diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 6a5ea0c2286..99105d27bca 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -839,7 +839,7 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe node = command_list; ParserToken s_comma(TokenType::Comma); - ParserAlterCommand p_command(alter_object); + ParserAlterCommand p_command(alter_object, command_type); do { @@ -973,7 +973,7 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->cluster = cluster_str; } - ParserAlterCommandList p_command_list(alter_object_type); + ParserAlterCommandList p_command_list(alter_object_type, query->command_type); ASTPtr command_list; if (!p_command_list.parse(pos, command_list, expected)) return false; From d6b0bc2942992dac782c5a49a049b211b3814650 Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Wed, 8 Jun 2022 16:27:00 +0800 Subject: [PATCH 242/408] Add test case --- ...319_sql_standard_create_drop_index.reference | 4 ++++ .../02319_sql_standard_create_drop_index.sql | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/02319_sql_standard_create_drop_index.reference create mode 100644 tests/queries/0_stateless/02319_sql_standard_create_drop_index.sql diff --git a/tests/queries/0_stateless/02319_sql_standard_create_drop_index.reference b/tests/queries/0_stateless/02319_sql_standard_create_drop_index.reference new file mode 100644 index 00000000000..6565857f89d --- /dev/null +++ b/tests/queries/0_stateless/02319_sql_standard_create_drop_index.reference @@ -0,0 +1,4 @@ +CREATE TABLE default.t_index\n(\n `a` Int32,\n `b` String,\n INDEX i_a a TYPE minmax GRANULARITY 4,\n INDEX i_b b TYPE bloom_filter GRANULARITY 2\n)\nENGINE = MergeTree\nORDER BY a\nSETTINGS index_granularity = 8192 +t_index i_a minmax a 4 +t_index i_b bloom_filter b 2 +t_index i_b bloom_filter b 2 diff --git a/tests/queries/0_stateless/02319_sql_standard_create_drop_index.sql b/tests/queries/0_stateless/02319_sql_standard_create_drop_index.sql new file mode 100644 index 00000000000..a33505ced3a --- /dev/null +++ b/tests/queries/0_stateless/02319_sql_standard_create_drop_index.sql @@ -0,0 +1,17 @@ +drop table if exists t_index; +create table t_index(a int, b String) engine=MergeTree() order by a; + +create index i_a on t_index(a) TYPE minmax GRANULARITY 4; +create index if not exists i_a on t_index(a) TYPE minmax GRANULARITY 2; + +create index i_b on t_index(b) TYPE bloom_filter GRANULARITY 2; + +show create table t_index; +select table, name, type, expr, granularity from system.data_skipping_indices where database = currentDatabase() and table = 't_index'; + +drop index i_a on t_index; +drop index if exists i_a on t_index; + +select table, name, type, expr, granularity from system.data_skipping_indices where database = currentDatabase() and table = 't_index'; + +drop table t_index; From 9154aec2010c3efa24a3f4bd9a9f648a14ec38a1 Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Thu, 9 Jun 2022 11:27:46 +0800 Subject: [PATCH 243/408] Fix compile error --- src/Parsers/ASTIndexDeclaration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index a9b30a0f433..87500a565e4 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -26,7 +26,7 @@ ASTPtr ASTIndexDeclaration::clone() const void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { /// '' is from CREATE INDEX - if (name != "") + if (!name.empty()) { s.ostr << backQuoteIfNeed(name); s.ostr << " "; From 589cba8045ba06efc5e50c12124ed6c557733f57 Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Thu, 16 Jun 2022 17:58:14 +0800 Subject: [PATCH 244/408] Support sql standard create index in seprate parser files --- .../InterpreterCreateIndexQuery.cpp | 82 ++++++++++++ .../InterpreterCreateIndexQuery.h | 25 ++++ src/Interpreters/InterpreterFactory.cpp | 6 + src/Parsers/ASTCreateIndexQuery.cpp | 65 ++++++++++ src/Parsers/ASTCreateIndexQuery.h | 40 ++++++ src/Parsers/ASTIndexDeclaration.cpp | 16 +-- src/Parsers/ASTIndexDeclaration.h | 1 + src/Parsers/ParserAlterQuery.cpp | 68 +--------- src/Parsers/ParserAlterQuery.h | 12 -- src/Parsers/ParserCreateIndexQuery.cpp | 120 ++++++++++++++++++ src/Parsers/ParserCreateIndexQuery.h | 31 +++++ src/Parsers/ParserQuery.cpp | 3 + 12 files changed, 383 insertions(+), 86 deletions(-) create mode 100644 src/Interpreters/InterpreterCreateIndexQuery.cpp create mode 100644 src/Interpreters/InterpreterCreateIndexQuery.h create mode 100644 src/Parsers/ASTCreateIndexQuery.cpp create mode 100644 src/Parsers/ASTCreateIndexQuery.h create mode 100644 src/Parsers/ParserCreateIndexQuery.cpp create mode 100644 src/Parsers/ParserCreateIndexQuery.h diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp new file mode 100644 index 00000000000..8a237162b54 --- /dev/null +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int INCORRECT_QUERY; + extern const int NOT_IMPLEMENTED; + extern const int TABLE_IS_READ_ONLY; +} + + +BlockIO InterpreterCreateIndexQuery::execute() +{ + FunctionNameNormalizer().visit(query_ptr.get()); + const auto & create_index = query_ptr->as(); + + AccessRightsElements required_access; + required_access.emplace_back(AccessType::ALTER_ADD_INDEX, create_index.getDatabase(), create_index.getTable()); + + if (!create_index.cluster.empty()) + { + DDLQueryOnClusterParams params; + params.access_to_check = std::move(required_access); + return executeDDLQueryOnCluster(query_ptr, getContext(), params); + } + + getContext()->checkAccess(required_access); + auto table_id = getContext()->resolveStorageID(create_index, Context::ResolveOrdinary); + query_ptr->as().setDatabase(table_id.database_name); + + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + if (typeid_cast(database.get()) + && !getContext()->getClientInfo().is_replicated_database_internal) + { + auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); + guard->releaseTableLock(); + return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + } + + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + if (table->isStaticStorage()) + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); + + /// Convert ASTCreateIndexQuery to AlterCommand. + AlterCommands alter_commands; + + AlterCommand command; + command.index_decl = create_index.index_decl; + command.type = AlterCommand::ADD_INDEX; + command.index_name = create_index.index_name->as().name(); + command.if_not_exists = create_index.if_not_exists; + + /// Fill name in ASTIndexDeclaration + auto & ast_index_decl = command.index_decl->as(); + ast_index_decl.name = command.index_name; + + alter_commands.emplace_back(std::move(command)); + + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, getContext()); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, getContext()); + table->alter(alter_commands, getContext(), alter_lock); + + return {}; +} + +} diff --git a/src/Interpreters/InterpreterCreateIndexQuery.h b/src/Interpreters/InterpreterCreateIndexQuery.h new file mode 100644 index 00000000000..c6cb7285590 --- /dev/null +++ b/src/Interpreters/InterpreterCreateIndexQuery.h @@ -0,0 +1,25 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +class InterpreterCreateIndexQuery : public IInterpreter, WithContext +{ +public: + InterpreterCreateIndexQuery(const ASTPtr & query_ptr_, ContextPtr context_) + : WithContext(context_) + , query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index c212eb50b97..a2edecbf1e4 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -298,6 +300,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Parsers/ASTCreateIndexQuery.cpp b/src/Parsers/ASTCreateIndexQuery.cpp new file mode 100644 index 00000000000..e83e95f1dc7 --- /dev/null +++ b/src/Parsers/ASTCreateIndexQuery.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNEXPECTED_AST_STRUCTURE; +} + +/** Get the text that identifies this element. */ +String ASTCreateIndexQuery::getID(char delim) const +{ + return "CreateIndexQuery" + (delim + getDatabase()) + delim + getTable(); +} + +ASTPtr ASTCreateIndexQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + res->index_name = index_name->clone(); + res->children.push_back(res->index_name); + + res->index_decl = index_decl->clone(); + res->children.push_back(res->index_decl); + return res; +} + +void ASTCreateIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + + std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + + settings.ostr << "CREATE INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + + settings.ostr << (settings.hilite ? hilite_none : ""); + + if (table) + { + if (database) + { + settings.ostr << indent_str << backQuoteIfNeed(getDatabase()); + settings.ostr << "."; + } + settings.ostr << indent_str << backQuoteIfNeed(getTable()); + } + + formatOnCluster(settings); + + if (!cluster.empty()) + settings.ostr << " "; + + index_decl->formatImpl(settings, state, frame); +} + +} diff --git a/src/Parsers/ASTCreateIndexQuery.h b/src/Parsers/ASTCreateIndexQuery.h new file mode 100644 index 00000000000..aa2bbfe8573 --- /dev/null +++ b/src/Parsers/ASTCreateIndexQuery.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/** + * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value + */ + +class ASTCreateIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster +{ +public: + bool if_not_exists{false}; + + ASTPtr index_name; + + /// Stores the IndexDeclaration here. + ASTPtr index_decl; + + String getID(char delim) const override; + + ASTPtr clone() const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override + { + return removeOnCluster(clone(), params.default_database); + } + + virtual QueryKind getQueryKind() const override { return QueryKind::Create; } + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index 87500a565e4..cc988d1d307 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -25,19 +25,19 @@ ASTPtr ASTIndexDeclaration::clone() const void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - /// '' is from CREATE INDEX - if (!name.empty()) - { - s.ostr << backQuoteIfNeed(name); - s.ostr << " "; - expr->formatImpl(s, state, frame); - } - else + if (from_create_index) { s.ostr << "("; expr->formatImpl(s, state, frame); s.ostr << ")"; } + else + { + s.ostr << backQuoteIfNeed(name); + s.ostr << " "; + expr->formatImpl(s, state, frame); + } + s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : ""); type->formatImpl(s, state, frame); s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index 8416ec6b0a6..31d5ef0e7f8 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -16,6 +16,7 @@ public: IAST * expr; ASTFunction * type; UInt64 granularity; + bool from_create_index = false; /** Get the text that identifies this element. */ String getID(char) const override { return "Index"; } diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index 99105d27bca..f40a63ef584 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -95,7 +95,6 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_delete("DELETE"); ParserKeyword s_update("UPDATE"); ParserKeyword s_where("WHERE"); - ParserKeyword s_create_index("CREATE INDEX"); ParserKeyword s_to("TO"); ParserKeyword s_remove("REMOVE"); @@ -114,7 +113,6 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserIdentifier parser_remove_property; ParserCompoundColumnDeclaration parser_col_decl; ParserIndexDeclaration parser_idx_decl; - ParserCreateIndexDeclaration parser_create_idx_decl; ParserConstraintDeclaration parser_constraint_decl; ParserProjectionDeclaration parser_projection_decl; ParserCompoundColumnDeclaration parser_modify_col_decl(false, false, true); @@ -159,14 +157,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected break; case ASTAlterQuery::AlterObjectType::TABLE: { - if (command_type == ASTAlterCommand::STD_CREATE_INDEX) - { - if (!parser_create_idx_decl.parse(pos, command->index_decl, expected)) - return false; - - command->type = ASTAlterCommand::STD_CREATE_INDEX; - } - else if (command_type == ASTAlterCommand::STD_DROP_INDEX) + if (command_type == ASTAlterCommand::STD_DROP_INDEX) { command->type = ASTAlterCommand::STD_DROP_INDEX; } @@ -854,45 +845,6 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } -bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) -{ - ParserKeyword s_type("TYPE"); - ParserKeyword s_granularity("GRANULARITY"); - - ParserDataType data_type_p; - ParserExpression expression_p; - ParserUnsignedInteger granularity_p; - - ASTPtr expr; - ASTPtr type; - ASTPtr granularity; - - /// Skip name parser for SQL-standard CREATE INDEX - - if (!expression_p.parse(pos, expr, expected)) - return false; - - if (!s_type.ignore(pos, expected)) - return false; - - if (!data_type_p.parse(pos, type, expected)) - return false; - - if (!s_granularity.ignore(pos, expected)) - return false; - - if (!granularity_p.parse(pos, granularity, expected)) - return false; - - auto index = std::make_shared(); - index->granularity = granularity->as().value.safeGet(); - index->set(index->expr, expr); - index->set(index->type, type); - node = index; - - return true; -} - bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { auto query = std::make_shared(); @@ -901,31 +853,15 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_alter_table("ALTER TABLE"); ParserKeyword s_alter_live_view("ALTER LIVE VIEW"); ParserKeyword s_alter_database("ALTER DATABASE"); - ParserKeyword s_create_index("CREATE INDEX"); ParserKeyword s_drop_index("DROP INDEX"); ParserKeyword s_on("ON"); - ParserKeyword s_if_not_exists("IF NOT EXISTS"); ParserKeyword s_if_exists("IF EXISTS"); ParserCompoundIdentifier parser_name; ASTAlterQuery::AlterObjectType alter_object_type; - if (s_create_index.ignore(pos, expected)) - { - alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; - if (s_if_not_exists.ignore(pos, expected)) - query->if_not_exists = true; - - if (!parser_name.parse(pos, query->index_name, expected)) - return false; - - if (!s_on.ignore(pos, expected)) - return false; - - query->command_type = ASTAlterCommand::STD_CREATE_INDEX; - } - else if (s_drop_index.ignore(pos, expected)) + if (s_drop_index.ignore(pos, expected)) { alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; if (s_if_exists.ignore(pos, expected)) diff --git a/src/Parsers/ParserAlterQuery.h b/src/Parsers/ParserAlterQuery.h index 25505b1c7f9..22d3f2a13c2 100644 --- a/src/Parsers/ParserAlterQuery.h +++ b/src/Parsers/ParserAlterQuery.h @@ -31,7 +31,6 @@ namespace DB * ALTER LIVE VIEW [db.name] * [REFRESH] * - * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value * DROP INDEX [IF EXISTS] name on [db].name */ @@ -73,15 +72,4 @@ public: }; -/** Part of CREATE INDEX expr TYPE typename(arg1, arg2, ...) GRANULARITY value */ -class ParserCreateIndexDeclaration : public IParserBase -{ -public: - ParserCreateIndexDeclaration() {} - -protected: - const char * getName() const override { return "index declaration in create index"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; -}; - } diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp new file mode 100644 index 00000000000..fe765281e21 --- /dev/null +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -0,0 +1,120 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_type("TYPE"); + ParserKeyword s_granularity("GRANULARITY"); + + ParserDataType data_type_p; + ParserExpression expression_p; + ParserUnsignedInteger granularity_p; + + ASTPtr expr; + ASTPtr type; + ASTPtr granularity; + + /// Skip name parser for SQL-standard CREATE INDEX + if (!expression_p.parse(pos, expr, expected)) + return false; + + if (!s_type.ignore(pos, expected)) + return false; + + if (!data_type_p.parse(pos, type, expected)) + return false; + + if (!s_granularity.ignore(pos, expected)) + return false; + + if (!granularity_p.parse(pos, granularity, expected)) + return false; + + auto index = std::make_shared(); + index->from_create_index = true; + index->granularity = granularity->as().value.safeGet(); + index->set(index->expr, expr); + index->set(index->type, type); + node = index; + + return true; +} + +bool ParserCreateIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + node = query; + + ParserKeyword s_create("CREATE"); + ParserKeyword s_index("INDEX"); + ParserKeyword s_if_not_exists("IF NOT EXISTS"); + ParserKeyword s_on("ON"); + ParserIdentifier index_name_p; + ParserCreateIndexDeclaration parser_create_idx_decl; + + ASTPtr index_name; + ASTPtr index_decl; + + String cluster_str; + bool if_not_exists = false; + + if (!s_create.ignore(pos, expected)) + return false; + + if (!s_index.ignore(pos, expected)) + return false; + + if (s_if_not_exists.ignore(pos, expected)) + if_not_exists = true; + + if (!index_name_p.parse(pos, index_name, expected)) + return false; + + /// ON [db.] table_name + if (!s_on.ignore(pos, expected)) + return false; + + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + return false; + + /// [ON cluster_name] + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + + if (!parser_create_idx_decl.parse(pos, index_decl, expected)) + return false; + + query->index_name = index_name; + query->children.push_back(index_name); + + query->index_decl = index_decl; + query->children.push_back(index_decl); + + query->if_not_exists = if_not_exists; + query->cluster = cluster_str; + + if (query->database) + query->children.push_back(query->database); + + if (query->table) + query->children.push_back(query->table); + + return true; +} + +} diff --git a/src/Parsers/ParserCreateIndexQuery.h b/src/Parsers/ParserCreateIndexQuery.h new file mode 100644 index 00000000000..3dfdccc301f --- /dev/null +++ b/src/Parsers/ParserCreateIndexQuery.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace DB +{ + +/** Query like this: + * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value + */ + +class ParserCreateIndexQuery : public IParserBase +{ +protected: + const char * getName() const override{ return "CREATE INDEX query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + + +/** Parser for index declaration in create index, where name is ignored. */ +class ParserCreateIndexDeclaration : public IParserBase +{ +public: + ParserCreateIndexDeclaration() {} + +protected: + const char * getName() const override { return "index declaration in create index"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserQuery.cpp b/src/Parsers/ParserQuery.cpp index eaea5dd0f5f..3edaa384c36 100644 --- a/src/Parsers/ParserQuery.cpp +++ b/src/Parsers/ParserQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserCreateSettingsProfileQuery create_settings_profile_p; ParserCreateFunctionQuery create_function_p; ParserDropFunctionQuery drop_function_p; + ParserCreateIndexQuery create_index_p; ParserDropAccessEntityQuery drop_access_entity_p; ParserGrantQuery grant_p; ParserSetRoleQuery set_role_p; @@ -63,6 +65,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) || create_settings_profile_p.parse(pos, node, expected) || create_function_p.parse(pos, node, expected) || drop_function_p.parse(pos, node, expected) + || create_index_p.parse(pos, node, expected) || drop_access_entity_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected) || external_ddl_p.parse(pos, node, expected) From 8d49aa4697e17a7cb588f7393a2f85832a43dd8c Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Fri, 17 Jun 2022 15:46:44 +0800 Subject: [PATCH 245/408] Support sql standard drop index in seprate parser files --- src/Interpreters/InterpreterAlterQuery.cpp | 2 - .../InterpreterCreateIndexQuery.cpp | 5 -- .../InterpreterCreateIndexQuery.h | 1 - .../InterpreterDropIndexQuery.cpp | 71 +++++++++++++++++++ src/Interpreters/InterpreterDropIndexQuery.h | 24 +++++++ src/Interpreters/InterpreterFactory.cpp | 6 ++ src/Parsers/ASTAlterQuery.cpp | 42 +---------- src/Parsers/ASTAlterQuery.h | 8 --- src/Parsers/ASTCreateIndexQuery.cpp | 4 -- src/Parsers/ASTDropIndexQuery.cpp | 63 ++++++++++++++++ src/Parsers/ASTDropIndexQuery.h | 44 ++++++++++++ src/Parsers/IAST.h | 1 - src/Parsers/ParserAlterQuery.cpp | 51 ++----------- src/Parsers/ParserAlterQuery.h | 12 ++-- src/Parsers/ParserCreateIndexQuery.cpp | 2 +- src/Parsers/ParserDropIndexQuery.cpp | 67 +++++++++++++++++ src/Parsers/ParserDropIndexQuery.h | 19 +++++ src/Parsers/ParserQuery.cpp | 3 + src/Parsers/formatAST.cpp | 3 +- src/Parsers/formatAST.h | 2 +- src/Storages/AlterCommands.cpp | 4 +- src/Storages/MutationCommands.cpp | 4 +- 22 files changed, 314 insertions(+), 124 deletions(-) create mode 100644 src/Interpreters/InterpreterDropIndexQuery.cpp create mode 100644 src/Interpreters/InterpreterDropIndexQuery.h create mode 100644 src/Parsers/ASTDropIndexQuery.cpp create mode 100644 src/Parsers/ASTDropIndexQuery.h create mode 100644 src/Parsers/ParserDropIndexQuery.cpp create mode 100644 src/Parsers/ParserDropIndexQuery.h diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 23b002350cf..056a3d9f7b4 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -295,13 +295,11 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS break; } case ASTAlterCommand::ADD_INDEX: - case ASTAlterCommand::STD_CREATE_INDEX: { required_access.emplace_back(AccessType::ALTER_ADD_INDEX, database, table); break; } case ASTAlterCommand::DROP_INDEX: - case ASTAlterCommand::STD_DROP_INDEX: { if (command.clear_index) required_access.emplace_back(AccessType::ALTER_CLEAR_INDEX, database, table); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 8a237162b54..29c151d1e4d 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -15,16 +14,12 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; - extern const int INCORRECT_QUERY; - extern const int NOT_IMPLEMENTED; extern const int TABLE_IS_READ_ONLY; } BlockIO InterpreterCreateIndexQuery::execute() { - FunctionNameNormalizer().visit(query_ptr.get()); const auto & create_index = query_ptr->as(); AccessRightsElements required_access; diff --git a/src/Interpreters/InterpreterCreateIndexQuery.h b/src/Interpreters/InterpreterCreateIndexQuery.h index c6cb7285590..63eaaf5f1c2 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.h +++ b/src/Interpreters/InterpreterCreateIndexQuery.h @@ -18,7 +18,6 @@ public: BlockIO execute() override; private: - ASTPtr query_ptr; }; diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp new file mode 100644 index 00000000000..6cc9334fad2 --- /dev/null +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TABLE_IS_READ_ONLY; +} + + +BlockIO InterpreterDropIndexQuery::execute() +{ + const auto & drop_index = query_ptr->as(); + + AccessRightsElements required_access; + required_access.emplace_back(AccessType::ALTER_DROP_INDEX, drop_index.getDatabase(), drop_index.getTable()); + + if (!drop_index.cluster.empty()) + { + DDLQueryOnClusterParams params; + params.access_to_check = std::move(required_access); + return executeDDLQueryOnCluster(query_ptr, getContext(), params); + } + + getContext()->checkAccess(required_access); + auto table_id = getContext()->resolveStorageID(drop_index, Context::ResolveOrdinary); + query_ptr->as().setDatabase(table_id.database_name); + + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + if (typeid_cast(database.get()) + && !getContext()->getClientInfo().is_replicated_database_internal) + { + auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); + guard->releaseTableLock(); + return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + } + + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + if (table->isStaticStorage()) + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); + + /// Convert ASTDropIndexQuery to AlterCommand. + AlterCommands alter_commands; + + AlterCommand command; + command.ast = drop_index.convertToASTAlterCommand(); + command.type = AlterCommand::DROP_INDEX; + command.index_name = drop_index.index_name->as().name(); + command.if_exists = drop_index.if_exists; + + alter_commands.emplace_back(std::move(command)); + + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, getContext()); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, getContext()); + table->alter(alter_commands, getContext(), alter_lock); + + return {}; +} + +} diff --git a/src/Interpreters/InterpreterDropIndexQuery.h b/src/Interpreters/InterpreterDropIndexQuery.h new file mode 100644 index 00000000000..c6fb3add72f --- /dev/null +++ b/src/Interpreters/InterpreterDropIndexQuery.h @@ -0,0 +1,24 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +class InterpreterDropIndexQuery : public IInterpreter, WithContext +{ +public: + InterpreterDropIndexQuery(const ASTPtr & query_ptr_, ContextPtr context_) + : WithContext(context_) + , query_ptr(query_ptr_) {} + + BlockIO execute() override; + +private: + ASTPtr query_ptr; +}; + +} diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index a2edecbf1e4..6b081467ae7 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -304,6 +306,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 465cd24c9d6..cfcc5decdf5 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -4,6 +4,7 @@ #include #include + namespace DB { namespace ErrorCodes @@ -100,9 +101,7 @@ const char * ASTAlterCommand::typeToString(ASTAlterCommand::Type type) case REMOVE_TTL: return "REMOVE_TTL"; case REMOVE_SAMPLE_BY: return "REMOVE_SAMPLE_BY"; case ADD_INDEX: return "ADD_INDEX"; - case STD_CREATE_INDEX: return "ADD_INDEX"; case DROP_INDEX: return "DROP_INDEX"; - case STD_DROP_INDEX: return "DROP_INDEX"; case MATERIALIZE_INDEX: return "MATERIALIZE_INDEX"; case ADD_CONSTRAINT: return "ADD_CONSTRAINT"; case DROP_CONSTRAINT: return "DROP_CONSTRAINT"; @@ -229,15 +228,6 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & index->formatImpl(settings, state, frame); } } - else if (type == ASTAlterCommand::STD_CREATE_INDEX) - { - auto name = index_decl->as()->name; - index_decl->as()->name = ""; - - index_decl->formatImpl(settings, state, frame); - - index_decl->as()->name = name; - } else if (type == ASTAlterCommand::DROP_INDEX) { settings.ostr << (settings.hilite ? hilite_keyword : "") << (clear_index ? "CLEAR " : "DROP ") << "INDEX " @@ -249,14 +239,6 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState & partition->formatImpl(settings, state, frame); } } - else if (type == ASTAlterCommand::STD_DROP_INDEX) - { - if (settings.is_translate) - { - settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : "") << (settings.hilite ? hilite_none : ""); - index->formatImpl(settings, state, frame); - } - } else if (type == ASTAlterCommand::MATERIALIZE_INDEX) { settings.ostr << (settings.hilite ? hilite_keyword : "") << "MATERIALIZE INDEX " << (settings.hilite ? hilite_none : ""); @@ -581,20 +563,7 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState switch (alter_object) { case AlterObjectType::TABLE: - if (command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) - { - settings.ostr << "CREATE INDEX " << (if_not_exists ? "IF NOT EXISTS " : ""); - index_name->formatImpl(settings, state, frame); - settings.ostr << " ON "; - } - else if (command_type == ASTAlterCommand::Type::STD_DROP_INDEX) - { - settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); - index_name->formatImpl(settings, state, frame); - settings.ostr << " ON "; - } - else - settings.ostr << "ALTER TABLE "; + settings.ostr << "ALTER TABLE "; break; case AlterObjectType::DATABASE: settings.ostr << "ALTER DATABASE "; @@ -627,13 +596,6 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; frame_nested.expression_list_always_start_on_new_line = true; - - if ((command_type == ASTAlterCommand::Type::STD_CREATE_INDEX) || (command_type == ASTAlterCommand::Type::STD_DROP_INDEX)) - { - frame_nested.expression_list_always_start_on_new_line = false; - settings.ostr << " "; - } - static_cast(command_list)->formatImplMultiline(settings, state, frame_nested); } diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 0bcb7473e41..956f07811ae 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -46,8 +46,6 @@ public: ADD_INDEX, DROP_INDEX, MATERIALIZE_INDEX, - STD_CREATE_INDEX, - STD_DROP_INDEX, ADD_CONSTRAINT, DROP_CONSTRAINT, @@ -228,12 +226,6 @@ public: }; AlterObjectType alter_object = AlterObjectType::UNKNOWN; - ASTAlterCommand::Type command_type = ASTAlterCommand::NO_TYPE; - - /// Used for SQL-standard CREATE INDEX and DROP INDEX - ASTPtr index_name; - bool if_not_exists = false; /// option for CREATE INDEX - bool if_exists = false; /// option for DROP INDEX ASTExpressionList * command_list = nullptr; diff --git a/src/Parsers/ASTCreateIndexQuery.cpp b/src/Parsers/ASTCreateIndexQuery.cpp index e83e95f1dc7..7a5c80551d6 100644 --- a/src/Parsers/ASTCreateIndexQuery.cpp +++ b/src/Parsers/ASTCreateIndexQuery.cpp @@ -6,10 +6,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int UNEXPECTED_AST_STRUCTURE; -} /** Get the text that identifies this element. */ String ASTCreateIndexQuery::getID(char delim) const diff --git a/src/Parsers/ASTDropIndexQuery.cpp b/src/Parsers/ASTDropIndexQuery.cpp new file mode 100644 index 00000000000..eca8c1676d5 --- /dev/null +++ b/src/Parsers/ASTDropIndexQuery.cpp @@ -0,0 +1,63 @@ +#include +#include +#include + + +namespace DB +{ + +/** Get the text that identifies this element. */ +String ASTDropIndexQuery::getID(char delim) const +{ + return "CreateIndexQuery" + (delim + getDatabase()) + delim + getTable(); +} + +ASTPtr ASTDropIndexQuery::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + + res->index_name = index_name->clone(); + res->children.push_back(res->index_name); + + return res; +} + +void ASTDropIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const +{ + frame.need_parens = false; + + std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); + + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; + + settings.ostr << "DROP INDEX " << (if_exists ? "IF EXISTS " : ""); + index_name->formatImpl(settings, state, frame); + settings.ostr << " ON "; + + settings.ostr << (settings.hilite ? hilite_none : ""); + + if (table) + { + if (database) + { + settings.ostr << indent_str << backQuoteIfNeed(getDatabase()); + settings.ostr << "."; + } + settings.ostr << indent_str << backQuoteIfNeed(getTable()); + } + + formatOnCluster(settings); +} + +ASTPtr ASTDropIndexQuery::convertToASTAlterCommand() const +{ + auto command = std::make_shared(); + command->index = index_name->clone(); + command->if_exists = if_exists; + command->type = ASTAlterCommand::DROP_INDEX; + + return command; +} + +} diff --git a/src/Parsers/ASTDropIndexQuery.h b/src/Parsers/ASTDropIndexQuery.h new file mode 100644 index 00000000000..2a771c643ed --- /dev/null +++ b/src/Parsers/ASTDropIndexQuery.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + + +namespace DB +{ + +/** + * DROP INDEX [IF EXISTS] name on [db].name + */ + +class ASTDropIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster +{ +public: + bool if_exists{false}; + + ASTPtr index_name; + + String getID(char delim) const override; + + ASTPtr clone() const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override + { + return removeOnCluster(clone(), params.default_database); + } + + virtual QueryKind getQueryKind() const override { return QueryKind::Drop; } + + /// Convert ASTDropIndexQuery to ASTAlterCommand. + ASTPtr convertToASTAlterCommand() const; + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + +} diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 07b6bed4f5b..b73919f4f36 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -184,7 +184,6 @@ public: bool hilite = false; bool one_line; bool always_quote_identifiers = false; - bool is_translate = false; //convert current standard SQL to clickhouse dialect. IdentifierQuotingStyle identifier_quoting_style = IdentifierQuotingStyle::Backticks; // Newline or whitespace. diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp index f40a63ef584..bc3af03a3c4 100644 --- a/src/Parsers/ParserAlterQuery.cpp +++ b/src/Parsers/ParserAlterQuery.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -157,11 +156,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected break; case ASTAlterQuery::AlterObjectType::TABLE: { - if (command_type == ASTAlterCommand::STD_DROP_INDEX) - { - command->type = ASTAlterCommand::STD_DROP_INDEX; - } - else if (s_add_column.ignore(pos, expected)) + if (s_add_column.ignore(pos, expected)) { if (s_if_not_exists.ignore(pos, expected)) command->if_not_exists = true; @@ -830,7 +825,7 @@ bool ParserAlterCommandList::parseImpl(Pos & pos, ASTPtr & node, Expected & expe node = command_list; ParserToken s_comma(TokenType::Comma); - ParserAlterCommand p_command(alter_object, command_type); + ParserAlterCommand p_command(alter_object); do { @@ -853,29 +848,10 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_alter_table("ALTER TABLE"); ParserKeyword s_alter_live_view("ALTER LIVE VIEW"); ParserKeyword s_alter_database("ALTER DATABASE"); - ParserKeyword s_drop_index("DROP INDEX"); - ParserKeyword s_on("ON"); - ParserKeyword s_if_exists("IF EXISTS"); - - ParserCompoundIdentifier parser_name; ASTAlterQuery::AlterObjectType alter_object_type; - if (s_drop_index.ignore(pos, expected)) - { - alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; - if (s_if_exists.ignore(pos, expected)) - query->if_exists = true; - - if (!parser_name.parse(pos, query->index_name, expected)) - return false; - - if (!s_on.ignore(pos, expected)) - return false; - - query->command_type = ASTAlterCommand::STD_DROP_INDEX; - } - else if (s_alter_table.ignore(pos, expected)) + if (s_alter_table.ignore(pos, expected)) { alter_object_type = ASTAlterQuery::AlterObjectType::TABLE; } @@ -909,30 +885,11 @@ bool ParserAlterQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->cluster = cluster_str; } - ParserAlterCommandList p_command_list(alter_object_type, query->command_type); + ParserAlterCommandList p_command_list(alter_object_type); ASTPtr command_list; if (!p_command_list.parse(pos, command_list, expected)) return false; - /// Set the index_name and exists flags for CREATE and DROP INDEX - if (query->command_type == ASTAlterCommand::STD_CREATE_INDEX) - { - ASTAlterCommand * command_ast = command_list->as()->children[0]->as(); - - command_ast->if_not_exists = query->if_not_exists; - - auto & ast_index_decl = command_ast->index_decl->as(); - ast_index_decl.name = query->index_name->as().name(); - - } - else if (query->command_type == ASTAlterCommand::STD_DROP_INDEX) - { - ASTAlterCommand * command_ast = command_list->as()->children[0]->as(); - - command_ast->if_exists = query->if_exists; - command_ast->as()->index = query->index_name; - } - query->set(query->command_list, command_list); query->alter_object = alter_object_type; diff --git a/src/Parsers/ParserAlterQuery.h b/src/Parsers/ParserAlterQuery.h index 22d3f2a13c2..b0029ff88fd 100644 --- a/src/Parsers/ParserAlterQuery.h +++ b/src/Parsers/ParserAlterQuery.h @@ -30,8 +30,6 @@ namespace DB * [MATERIALIZE INDEX [IF EXISTS] index_name [IN PARTITION partition]] * ALTER LIVE VIEW [db.name] * [REFRESH] - * - * DROP INDEX [IF EXISTS] name on [db].name */ class ParserAlterQuery : public IParserBase @@ -50,10 +48,9 @@ protected: public: ASTAlterQuery::AlterObjectType alter_object; - ASTAlterCommand::Type command_type; - ParserAlterCommandList(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE, ASTAlterCommand::Type command_type_= ASTAlterCommand::NO_TYPE) - : alter_object(alter_object_), command_type(command_type_) {} + ParserAlterCommandList(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE) + : alter_object(alter_object_) {} }; @@ -65,10 +62,9 @@ protected: public: ASTAlterQuery::AlterObjectType alter_object; - ASTAlterCommand::Type command_type; - ParserAlterCommand(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE, ASTAlterCommand::Type command_type_= ASTAlterCommand::NO_TYPE) - : alter_object(alter_object_), command_type(command_type_) {} + ParserAlterCommand(ASTAlterQuery::AlterObjectType alter_object_ = ASTAlterQuery::AlterObjectType::TABLE) + : alter_object(alter_object_) {} }; diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp index fe765281e21..22411c71ee5 100644 --- a/src/Parsers/ParserCreateIndexQuery.cpp +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -70,7 +70,7 @@ bool ParserCreateIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect String cluster_str; bool if_not_exists = false; - if (!s_create.ignore(pos, expected)) + if (!s_create.ignore(pos, expected)) return false; if (!s_index.ignore(pos, expected)) diff --git a/src/Parsers/ParserDropIndexQuery.cpp b/src/Parsers/ParserDropIndexQuery.cpp new file mode 100644 index 00000000000..19f31d6128d --- /dev/null +++ b/src/Parsers/ParserDropIndexQuery.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ParserDropIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + node = query; + + ParserKeyword s_drop("DROP"); + ParserKeyword s_index("INDEX"); + ParserKeyword s_on("ON"); + ParserKeyword s_if_exists("IF EXISTS"); + ParserIdentifier index_name_p; + + String cluster_str; + bool if_exists = false; + + if (!s_drop.ignore(pos, expected)) + return false; + + if (!s_index.ignore(pos, expected)) + return false; + + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + + if (!index_name_p.parse(pos, query->index_name, expected)) + return false; + + /// ON [db.] table_name + if (!s_on.ignore(pos, expected)) + return false; + + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + return false; + + /// [ON cluster_name] + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + + query->cluster = std::move(cluster_str); + } + + if(query->index_name) + query->children.push_back(query->index_name); + + query->if_exists = if_exists; + + if (query->database) + query->children.push_back(query->database); + + if (query->table) + query->children.push_back(query->table); + + return true; +} + +} diff --git a/src/Parsers/ParserDropIndexQuery.h b/src/Parsers/ParserDropIndexQuery.h new file mode 100644 index 00000000000..1b6535c7efb --- /dev/null +++ b/src/Parsers/ParserDropIndexQuery.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace DB +{ + +/** Query like this: + * DROP INDEX [IF EXISTS] name ON [db].name + */ + +class ParserDropIndexQuery : public IParserBase +{ +protected: + const char * getName() const override{ return "DROP INDEX query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserQuery.cpp b/src/Parsers/ParserQuery.cpp index 3edaa384c36..a3cafee65d7 100644 --- a/src/Parsers/ParserQuery.cpp +++ b/src/Parsers/ParserQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserCreateFunctionQuery create_function_p; ParserDropFunctionQuery drop_function_p; ParserCreateIndexQuery create_index_p; + ParserDropIndexQuery drop_index_p; ParserDropAccessEntityQuery drop_access_entity_p; ParserGrantQuery grant_p; ParserSetRoleQuery set_role_p; @@ -66,6 +68,7 @@ bool ParserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) || create_function_p.parse(pos, node, expected) || drop_function_p.parse(pos, node, expected) || create_index_p.parse(pos, node, expected) + || drop_index_p.parse(pos, node, expected) || drop_access_entity_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected) || external_ddl_p.parse(pos, node, expected) diff --git a/src/Parsers/formatAST.cpp b/src/Parsers/formatAST.cpp index 6f5eb3a60b0..fca8ea0aa35 100644 --- a/src/Parsers/formatAST.cpp +++ b/src/Parsers/formatAST.cpp @@ -4,11 +4,10 @@ namespace DB { -void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line, bool is_translate) +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line) { IAST::FormatSettings settings(buf, one_line); settings.hilite = hilite; - settings.is_translate = is_translate; ast.format(settings); } diff --git a/src/Parsers/formatAST.h b/src/Parsers/formatAST.h index 0bf845cd04d..28af2400a4c 100644 --- a/src/Parsers/formatAST.h +++ b/src/Parsers/formatAST.h @@ -11,7 +11,7 @@ class WriteBuffer; /** Takes a syntax tree and turns it back into text. * In case of INSERT query, the data will be missing. */ -void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false, bool is_translate = false); +void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false); String serializeAST(const IAST & ast, bool one_line = true); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 1b04fe3ede4..845aae52582 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -211,7 +211,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.type = AlterCommand::REMOVE_SAMPLE_BY; return command; } - else if (command_ast->type == ASTAlterCommand::ADD_INDEX || command_ast->type == ASTAlterCommand::STD_CREATE_INDEX) + else if (command_ast->type == ASTAlterCommand::ADD_INDEX) { AlterCommand command; command.ast = command_ast->clone(); @@ -274,7 +274,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ return command; } - else if (command_ast->type == ASTAlterCommand::DROP_INDEX || command_ast->type == ASTAlterCommand::STD_DROP_INDEX) + else if (command_ast->type == ASTAlterCommand::DROP_INDEX) { AlterCommand command; command.ast = command_ast->clone(); diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 0f9d17c5d1d..28dfe488869 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -103,7 +103,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, return res; } - else if (parse_alter_commands && (command->type == ASTAlterCommand::DROP_INDEX || command->type == ASTAlterCommand::STD_DROP_INDEX)) + else if (parse_alter_commands && command->type == ASTAlterCommand::DROP_INDEX) { MutationCommand res; res.ast = command->ptr(); @@ -160,7 +160,7 @@ std::shared_ptr MutationCommands::ast() const void MutationCommands::writeText(WriteBuffer & out) const { WriteBufferFromOwnString commands_buf; - formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true, /* is_translate = */ true); + formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true); writeEscapedString(commands_buf.str(), out); } From d75af09f7435c8229ff3482bfa0fd80b6bd866f2 Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Sat, 18 Jun 2022 18:18:31 +0800 Subject: [PATCH 246/408] Fix style error --- src/Parsers/ASTCreateIndexQuery.h | 3 +-- src/Parsers/ASTDropIndexQuery.cpp | 2 +- src/Parsers/ASTDropIndexQuery.h | 4 +--- src/Parsers/ParserDropIndexQuery.cpp | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Parsers/ASTCreateIndexQuery.h b/src/Parsers/ASTCreateIndexQuery.h index aa2bbfe8573..f3c6a7830a4 100644 --- a/src/Parsers/ASTCreateIndexQuery.h +++ b/src/Parsers/ASTCreateIndexQuery.h @@ -8,8 +8,7 @@ namespace DB { -/** - * CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value +/** CREATE INDEX [IF NOT EXISTS] name ON [db].name (expression) TYPE type GRANULARITY value */ class ASTCreateIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster diff --git a/src/Parsers/ASTDropIndexQuery.cpp b/src/Parsers/ASTDropIndexQuery.cpp index eca8c1676d5..78152d213b8 100644 --- a/src/Parsers/ASTDropIndexQuery.cpp +++ b/src/Parsers/ASTDropIndexQuery.cpp @@ -24,7 +24,7 @@ ASTPtr ASTDropIndexQuery::clone() const } void ASTDropIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const -{ +{ frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); diff --git a/src/Parsers/ASTDropIndexQuery.h b/src/Parsers/ASTDropIndexQuery.h index 2a771c643ed..d7e39f797b5 100644 --- a/src/Parsers/ASTDropIndexQuery.h +++ b/src/Parsers/ASTDropIndexQuery.h @@ -8,12 +8,10 @@ #include - namespace DB { -/** - * DROP INDEX [IF EXISTS] name on [db].name +/** DROP INDEX [IF EXISTS] name on [db].name */ class ASTDropIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster diff --git a/src/Parsers/ParserDropIndexQuery.cpp b/src/Parsers/ParserDropIndexQuery.cpp index 19f31d6128d..0844ea16ae0 100644 --- a/src/Parsers/ParserDropIndexQuery.cpp +++ b/src/Parsers/ParserDropIndexQuery.cpp @@ -50,7 +50,7 @@ bool ParserDropIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected query->cluster = std::move(cluster_str); } - if(query->index_name) + if (query->index_name) query->children.push_back(query->index_name); query->if_exists = if_exists; From f7ef571842f788993718b658dbf83b0b530f26cf Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Wed, 29 Jun 2022 03:10:10 +0300 Subject: [PATCH 247/408] Don't spoil return code of integration tests runner with redundant tee. --- tests/ci/integration_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index cf0dfe51e9b..9fda2d09ae6 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -225,7 +225,7 @@ if __name__ == "__main__": output_path_log = os.path.join(result_path, "main_script_log.txt") runner_path = os.path.join(repo_path, "tests/integration", "ci-runner.py") - run_command = f"sudo -E {runner_path} | tee {output_path_log}" + run_command = f"sudo -E {runner_path}" logging.info("Going to run command: `%s`", run_command) logging.info( "ENV parameters for runner:\n%s", From dea3b5bfcecfa8ce778bd61fab86e448ad648fec Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 29 Jun 2022 08:56:15 +0300 Subject: [PATCH 248/408] Revert "Non Negative Derivative window function" --- src/Common/IntervalKind.cpp | 27 +---- src/Common/IntervalKind.h | 5 +- src/Processors/Transforms/WindowTransform.cpp | 112 ------------------ .../02232_non_negative_derivative.reference | 64 ---------- .../02232_non_negative_derivative.sql | 63 ---------- 5 files changed, 5 insertions(+), 266 deletions(-) delete mode 100644 tests/queries/0_stateless/02232_non_negative_derivative.reference delete mode 100644 tests/queries/0_stateless/02232_non_negative_derivative.sql diff --git a/src/Common/IntervalKind.cpp b/src/Common/IntervalKind.cpp index 75c2a83e9fb..d3cd4eeff8f 100644 --- a/src/Common/IntervalKind.cpp +++ b/src/Common/IntervalKind.cpp @@ -9,13 +9,13 @@ namespace ErrorCodes extern const int SYNTAX_ERROR; } -Float64 IntervalKind::toAvgSeconds() const +Int32 IntervalKind::toAvgSeconds() const { switch (kind) { - case IntervalKind::Nanosecond: return 0.000000001; - case IntervalKind::Microsecond: return 0.000001; - case IntervalKind::Millisecond: return 0.001; + case IntervalKind::Nanosecond: + case IntervalKind::Microsecond: + case IntervalKind::Millisecond: return 0; /// fractional parts of seconds have 0 seconds case IntervalKind::Second: return 1; case IntervalKind::Minute: return 60; case IntervalKind::Hour: return 3600; @@ -28,25 +28,6 @@ Float64 IntervalKind::toAvgSeconds() const __builtin_unreachable(); } -bool IntervalKind::isFixedLength() const -{ - switch (kind) - { - case IntervalKind::Nanosecond: - case IntervalKind::Microsecond: - case IntervalKind::Millisecond: - case IntervalKind::Second: - case IntervalKind::Minute: - case IntervalKind::Hour: - case IntervalKind::Day: - case IntervalKind::Week: return true; - case IntervalKind::Month: - case IntervalKind::Quarter: - case IntervalKind::Year: return false; - } - __builtin_unreachable(); -} - IntervalKind IntervalKind::fromAvgSeconds(Int64 num_seconds) { if (num_seconds) diff --git a/src/Common/IntervalKind.h b/src/Common/IntervalKind.h index 65c14515e34..d5f2b5672cd 100644 --- a/src/Common/IntervalKind.h +++ b/src/Common/IntervalKind.h @@ -31,15 +31,12 @@ struct IntervalKind /// Returns number of seconds in one interval. /// For `Month`, `Quarter` and `Year` the function returns an average number of seconds. - Float64 toAvgSeconds() const; + Int32 toAvgSeconds() const; /// Chooses an interval kind based on number of seconds. /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Hour`. static IntervalKind fromAvgSeconds(Int64 num_seconds); - /// Returns whether IntervalKind has a fixed number of seconds (e.g. Day) or non-fixed(e.g. Month) - bool isFixedLength() const; - /// Returns an uppercased version of what `toString()` returns. const char * toKeyword() const; diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 09805696472..3eb0f62cb01 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -28,7 +27,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; } // Interface for true window functions. It's not much of an interface, they just @@ -2202,109 +2200,6 @@ struct WindowFunctionNthValue final : public WindowFunction } }; -struct NonNegativeDerivativeState -{ - Float64 previous_metric = 0; - Float64 previous_timestamp = 0; -}; - -// nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL 1 SECOND]) -struct WindowFunctionNonNegativeDerivative final : public StatefulWindowFunction -{ - static constexpr size_t ARGUMENT_METRIC = 0; - static constexpr size_t ARGUMENT_TIMESTAMP = 1; - static constexpr size_t ARGUMENT_INTERVAL = 2; - - WindowFunctionNonNegativeDerivative(const std::string & name_, - const DataTypes & argument_types_, const Array & parameters_) - : StatefulWindowFunction(name_, argument_types_, parameters_) - { - if (!parameters.empty()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Function {} cannot be parameterized", name_); - } - - if (argument_types.size() != 2 && argument_types.size() != 3) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Function {} takes 2 or 3 arguments", name_); - } - - if (!isNumber(argument_types[ARGUMENT_METRIC])) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be a number, '{}' given", - ARGUMENT_METRIC, - argument_types[ARGUMENT_METRIC]->getName()); - } - - if (!isDateTime(argument_types[ARGUMENT_TIMESTAMP]) && !isDateTime64(argument_types[ARGUMENT_TIMESTAMP])) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be DateTime or DateTime64, '{}' given", - ARGUMENT_TIMESTAMP, - argument_types[ARGUMENT_TIMESTAMP]->getName()); - } - - if (argument_types.size() == 3) - { - const DataTypeInterval * interval_datatype = checkAndGetDataType(argument_types[ARGUMENT_INTERVAL].get()); - if (!interval_datatype) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Argument {} must be an INTERVAL, '{}' given", - ARGUMENT_INTERVAL, - argument_types[ARGUMENT_INTERVAL]->getName()); - } - if (!interval_datatype->getKind().isFixedLength()) - { - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "The INTERVAL must be a week or shorter, '{}' given", - argument_types[ARGUMENT_INTERVAL]->getName()); - } - interval_length = interval_datatype->getKind().toAvgSeconds(); - interval_specified = true; - } - } - - - DataTypePtr getReturnType() const override { return argument_types[0]; } - - bool allocatesMemoryInArena() const override { return false; } - - void windowInsertResultInto(const WindowTransform * transform, - size_t function_index) override - { - const auto & current_block = transform->blockAt(transform->current_row); - const auto & workspace = transform->workspaces[function_index]; - auto & state = getState(workspace); - - auto interval_duration = interval_specified ? interval_length * - (*current_block.input_columns[workspace.argument_column_indices[ARGUMENT_INTERVAL]]).getFloat64(0) : 1; - - Float64 last_metric = state.previous_metric; - Float64 last_timestamp = state.previous_timestamp; - - Float64 curr_metric = WindowFunctionHelpers::getValue(transform, function_index, ARGUMENT_METRIC, transform->current_row); - Float64 curr_timestamp = WindowFunctionHelpers::getValue(transform, function_index, ARGUMENT_TIMESTAMP, transform->current_row); - - Float64 time_elapsed = curr_timestamp - last_timestamp; - Float64 metric_diff = curr_metric - last_metric; - Float64 result = (time_elapsed != 0) ? (metric_diff / time_elapsed * interval_duration) : 0; - - state.previous_metric = curr_metric; - state.previous_timestamp = curr_timestamp; - - WindowFunctionHelpers::setValueToOutputColumn(transform, function_index, result >= 0 ? result : 0); - } -private: - Float64 interval_length = 1; - bool interval_specified = false; -}; - void registerWindowFunctions(AggregateFunctionFactory & factory) { @@ -2404,13 +2299,6 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) return std::make_shared( name, argument_types, parameters); }, properties}); - - factory.registerFunction("nonNegativeDerivative", {[](const std::string & name, - const DataTypes & argument_types, const Array & parameters, const Settings *) - { - return std::make_shared( - name, argument_types, parameters); - }, properties}); } } diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.reference b/tests/queries/0_stateless/02232_non_negative_derivative.reference deleted file mode 100644 index 7559f527c7a..00000000000 --- a/tests/queries/0_stateless/02232_non_negative_derivative.reference +++ /dev/null @@ -1,64 +0,0 @@ -1 -1979-12-12 21:21:21.123 1.1 3.5045052519931732e-9 -1979-12-12 21:21:22.000 1.3345 0.26738883339230357 -1979-12-12 21:21:23.000 1.54 0.20550000000000002 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 453.33916989529325 -1979-12-12 21:21:21.123 1.1 1.0513515755979521e-17 -1979-12-12 21:21:22.000 1.3345 8.021665001769108e-10 -1979-12-12 21:21:23.000 1.54 6.165000000000001e-10 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 0.0000013600175096858798 -1979-12-12 21:21:21.123 1.1 1.4018021007972692e-14 -1979-12-12 21:21:22.000 1.3345 0.0000010695553335692141 -1979-12-12 21:21:23.000 1.54 8.22e-7 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 0.001813356679581173 -1979-12-12 21:21:21.123 1.1 1.7522526259965866e-11 -1979-12-12 21:21:22.000 1.3345 0.0013369441669615178 -1979-12-12 21:21:23.000 1.54 0.0010275000000000002 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 2.2666958494764664 -1979-12-12 21:21:21.123 1.1 2.102703151195904e-8 -1979-12-12 21:21:22.000 1.3345 1.6043330003538214 -1979-12-12 21:21:23.000 1.54 1.233 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 2720.0350193717595 -1979-12-12 21:21:21.123 1.1 0.0000014718922058371327 -1979-12-12 21:21:22.000 1.3345 112.3033100247675 -1979-12-12 21:21:23.000 1.54 86.31 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:21.124 2.34 0 -1979-12-12 21:21:21.127 3.7 190402.45135602317 -1979-12-12 21:21:21.123 1.1 0.0001009297512574034 -1979-12-12 21:21:21.124 2.34 35712459.78375156 -1979-12-12 21:21:21.127 3.7 13056168.092984445 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:22.000 1.3345 0 -1979-12-12 21:21:23.000 1.54 5918.400000000001 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.123 1.1 0.0027251032839498914 -1979-12-12 21:21:21.124 2.34 964236414.1612921 -1979-12-12 21:21:21.127 3.7 352516538.51058006 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:22.000 1.3345 0 -1979-12-12 21:21:23.000 1.54 159796.80000000002 -1979-12-12 21:21:23.000 1.54 0 -1979-12-12 21:21:21.123 1.1 0.021195247764054712 -1979-12-12 21:21:21.124 2.34 7499616554.587828 -1979-12-12 21:21:21.127 3.7 2741795299.5267334 -1979-12-12 21:21:21.129 2.1 0 -1979-12-12 21:21:22.000 1.3345 0 -1979-12-12 21:21:23.000 1.54 1242864 -1979-12-12 21:21:23.000 1.54 0 diff --git a/tests/queries/0_stateless/02232_non_negative_derivative.sql b/tests/queries/0_stateless/02232_non_negative_derivative.sql deleted file mode 100644 index c4cbadb68a8..00000000000 --- a/tests/queries/0_stateless/02232_non_negative_derivative.sql +++ /dev/null @@ -1,63 +0,0 @@ -DROP TABLE IF EXISTS nnd; - -CREATE TABLE nnd -( - id Int8, ts DateTime64(3, 'UTC'), metric Float64 -) -ENGINE=MergeTree() -ORDER BY id; - -INSERT INTO nnd VALUES (1, toDateTime64('1979-12-12 21:21:21.123', 3, 'UTC'), 1.1), (2, toDateTime64('1979-12-12 21:21:21.124', 3, 'UTC'), 2.34), (3, toDateTime64('1979-12-12 21:21:21.127', 3, 'UTC'), 3.7); -INSERT INTO nnd VALUES (4, toDateTime64('1979-12-12 21:21:21.129', 3, 'UTC'), 2.1), (5, toDateTime('1979-12-12 21:21:22', 'UTC'), 1.3345), (6, toDateTime('1979-12-12 21:21:23', 'UTC'), 1.54), (7, toDateTime('1979-12-12 21:21:23', 'UTC'), 1.54); - --- shall work for precise intervals --- INTERVAL 1 SECOND shall be default -SELECT ( - SELECT - ts, - metric, - nonNegativeDerivative(metric, ts) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv - FROM nnd - LIMIT 5, 1 - ) = ( - SELECT - ts, - metric, - nonNegativeDerivative(metric, ts, toIntervalSecond(1)) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv - FROM nnd - LIMIT 5, 1 - ); -SELECT ts, metric, nonNegativeDerivative(metric, ts) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Nanosecond -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Microsecond -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 4 MICROSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Millisecond -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 5 MILLISECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Second -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 6 SECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Minute -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 7 MINUTE) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Hour -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 8 HOUR) OVER (ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Day -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 9 DAY) OVER (ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; --- Week -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 10 WEEK) OVER (ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; - --- shall not work for month, quarter, year (intervals with floating number of seconds) --- Month -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 11 MONTH) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } --- Quarter -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 12 QUARTER) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } --- Year -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 13 YEAR) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } - --- test against wrong arguments/types -SELECT ts, metric, nonNegativeDerivative(metric, 1, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS } -SELECT ts, metric, nonNegativeDerivative('string not datetime', ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS } -SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND, id) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS } -SELECT ts, metric, nonNegativeDerivative(metric) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS } - --- cleanup -DROP TABLE IF EXISTS nnd; From e7dbe526f8b5d108d8e42086ee17d0b8090fe3f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 29 Jun 2022 09:46:45 +0300 Subject: [PATCH 249/408] Update ParserCreateQuery.cpp --- src/Parsers/ParserCreateQuery.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 08227aa0c2f..4b6ab67e22f 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -575,6 +575,8 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (!table_properties_p.parse(pos, columns_list, expected)) return false; + /// We allow a trailing comma in the columns list for user convenience. + /// Although it diverges from the SQL standard slightly. s_comma.ignore(pos, expected); if (!s_rparen.ignore(pos, expected)) From 00372e4646f06008867da06ebcb938b0f118abf6 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 11:05:36 +0200 Subject: [PATCH 250/408] Fixed tests --- .../0_stateless/02345_partial_sort_transform_optimization.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql index a53a352f57e..fe2ab096ab7 100644 --- a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql +++ b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check:22.6.1 + -- Regression for PartialSortingTransform optimization that requires at least 1500 rows. SELECT * FROM (SELECT * FROM (SELECT 0 a, toNullable(number) b, toString(number) c FROM numbers(1e6)) ORDER BY a DESC, b DESC, c LIMIT 1500) limit 10; SELECT number FROM (SELECT number, 1 AS k FROM numbers(100000) ORDER BY k ASC, number DESC LIMIT 1025, 1023) LIMIT 5; From 78ea290789eba0f60ac42e2bf45c3439fd61abd6 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 29 Jun 2022 06:47:21 -0400 Subject: [PATCH 251/408] add remove command --- programs/self-extracting/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/self-extracting/CMakeLists.txt b/programs/self-extracting/CMakeLists.txt index 2aec7938114..89f6b56b826 100644 --- a/programs/self-extracting/CMakeLists.txt +++ b/programs/self-extracting/CMakeLists.txt @@ -1,5 +1,6 @@ add_custom_target (self-extracting ALL - ${CMAKE_BINARY_DIR}/utils/self-extracting-executable/compressor clickhouse ../clickhouse + ${CMAKE_COMMAND} -E remove clickhouse + COMMAND ${CMAKE_BINARY_DIR}/utils/self-extracting-executable/compressor clickhouse ../clickhouse DEPENDS clickhouse compressor ) From 498a9cc8d409b90072328078359590531600eb8e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 29 Jun 2022 12:51:36 +0200 Subject: [PATCH 252/408] Terminate instance if docker does not start on post-hook --- tests/ci/worker/init_runner.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh index 93c13b161de..74ad4be2547 100644 --- a/tests/ci/worker/init_runner.sh +++ b/tests/ci/worker/init_runner.sh @@ -45,15 +45,20 @@ cat > /tmp/actions-hooks/post-run.sh << 'EOF' #!/bin/bash set -xuo pipefail -# Free KiB, free percents -ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}')) -if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then - echo "Going to terminate the runner, it has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /" +terminate-and-exit() { + echo "Going to terminate the runner" INSTANCE_ID=$(ec2metadata --instance-id) # We execute it with at to not have it as an orphan process # GH Runners kill all remain processes echo "sleep 10; aws ec2 terminate-instances --instance-ids $INSTANCE_ID" | at now exit 0 +} + +# Free KiB, free percents +ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}')) +if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then + echo "The runner has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /" + terminate-and-exit fi # shellcheck disable=SC2046 @@ -64,7 +69,9 @@ docker rm -f $(docker ps -a -q) ||: # If we have hanged containers after the previous commands, than we have a hanged one # and should restart the daemon if [ "$(docker ps -a -q)" ]; then - for i in {1..5}; + # Systemd service of docker has StartLimitBurst=3 and StartLimitInterval=60s, + # that's why we try restarting it for long + for i in {1..25}; do sudo systemctl restart docker && break || sleep 5 done @@ -73,6 +80,8 @@ if [ "$(docker ps -a -q)" ]; then do docker info && break || sleep 2 done + # Last chance, otherwise we have to terminate poor instance + docker info 1>/dev/null || { echo Docker unable to start; terminate-and-exit; } fi EOF From 30fd086b5e78978fa79432c3225a2a3fb6db92c0 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 29 Jun 2022 12:59:18 +0200 Subject: [PATCH 253/408] Add logging in Epoll and TimerDescriptor in case of EINTR --- src/Common/Epoll.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index d2c16c186ce..e5c0b668150 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -70,6 +70,9 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki if (ready_size == -1 && errno != EINTR) throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR); + + if (errno == EINTR) + LOG_DEBUG(&Poco::Logger::get("Epoll"), "EINTR"); } while (ready_size <= 0 && (ready_size != 0 || blocking)); From 77873e75bf91bb89d738ff5c1d09a12265773f0f Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 29 Jun 2022 13:01:14 +0200 Subject: [PATCH 254/408] Update TimerDescriptor.cpp --- src/Common/TimerDescriptor.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp index a7c74dab8be..53e7cefc358 100644 --- a/src/Common/TimerDescriptor.cpp +++ b/src/Common/TimerDescriptor.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -70,6 +72,8 @@ void TimerDescriptor::drain() const if (errno != EINTR) throwFromErrno("Cannot drain timer_fd", ErrorCodes::CANNOT_READ_FROM_SOCKET); + else + LOG_DEBUG(&Poco::Logger::get("TimerDescriptor"), "EINTR"); } } } From eeae73e0cf9470490a7dd7957306b3b18efe5f44 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 13:10:30 +0200 Subject: [PATCH 255/408] SQL create drop index update implementation --- .../InterpreterCreateIndexQuery.cpp | 26 +++++++++---------- .../InterpreterDropIndexQuery.cpp | 21 ++++++++------- src/Parsers/ASTAlterQuery.cpp | 2 -- src/Parsers/ASTCreateIndexQuery.cpp | 15 +++++++++++ src/Parsers/ASTCreateIndexQuery.h | 9 ++++--- src/Parsers/ASTDropIndexQuery.cpp | 4 ++- src/Parsers/ASTDropIndexQuery.h | 7 ++--- src/Parsers/ASTIndexDeclaration.cpp | 2 +- src/Parsers/ASTIndexDeclaration.h | 2 +- src/Parsers/ParserCreateIndexQuery.cpp | 11 +++++--- src/Parsers/ParserCreateIndexQuery.h | 4 +-- src/Parsers/ParserDropIndexQuery.cpp | 6 ++--- src/Parsers/ParserDropIndexQuery.h | 2 +- 13 files changed, 66 insertions(+), 45 deletions(-) diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 29c151d1e4d..ef19eaa1c42 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes BlockIO InterpreterCreateIndexQuery::execute() { + auto current_context = getContext(); const auto & create_index = query_ptr->as(); AccessRightsElements required_access; @@ -29,23 +30,23 @@ BlockIO InterpreterCreateIndexQuery::execute() { DDLQueryOnClusterParams params; params.access_to_check = std::move(required_access); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + return executeDDLQueryOnCluster(query_ptr, current_context, params); } - getContext()->checkAccess(required_access); - auto table_id = getContext()->resolveStorageID(create_index, Context::ResolveOrdinary); + current_context->checkAccess(required_access); + auto table_id = current_context->resolveStorageID(create_index, Context::ResolveOrdinary); query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal) + && !current_context->getClientInfo().is_replicated_database_internal) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); } - StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); @@ -53,23 +54,20 @@ BlockIO InterpreterCreateIndexQuery::execute() AlterCommands alter_commands; AlterCommand command; + command.ast = create_index.convertToASTAlterCommand(); command.index_decl = create_index.index_decl; command.type = AlterCommand::ADD_INDEX; command.index_name = create_index.index_name->as().name(); command.if_not_exists = create_index.if_not_exists; - /// Fill name in ASTIndexDeclaration - auto & ast_index_decl = command.index_decl->as(); - ast_index_decl.name = command.index_name; - alter_commands.emplace_back(std::move(command)); - auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); - alter_commands.validate(table, getContext()); + alter_commands.validate(table, current_context); alter_commands.prepare(metadata); - table->checkAlterIsPossible(alter_commands, getContext()); - table->alter(alter_commands, getContext(), alter_lock); + table->checkAlterIsPossible(alter_commands, current_context); + table->alter(alter_commands, current_context, alter_lock); return {}; } diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 6cc9334fad2..2339e0dc68e 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes BlockIO InterpreterDropIndexQuery::execute() { + auto current_context = getContext(); const auto & drop_index = query_ptr->as(); AccessRightsElements required_access; @@ -27,23 +28,23 @@ BlockIO InterpreterDropIndexQuery::execute() { DDLQueryOnClusterParams params; params.access_to_check = std::move(required_access); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + return executeDDLQueryOnCluster(query_ptr, current_context, params); } - getContext()->checkAccess(required_access); - auto table_id = getContext()->resolveStorageID(drop_index, Context::ResolveOrdinary); + current_context->checkAccess(required_access); + auto table_id = current_context->resolveStorageID(drop_index, Context::ResolveOrdinary); query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal) + && !current_context->getClientInfo().is_replicated_database_internal) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); - return typeid_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext()); + return assert_cast(database.get())->tryEnqueueReplicatedDDL(query_ptr, current_context); } - StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + StoragePtr table = DatabaseCatalog::instance().getTable(table_id, current_context); if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); @@ -58,12 +59,12 @@ BlockIO InterpreterDropIndexQuery::execute() alter_commands.emplace_back(std::move(command)); - auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); - alter_commands.validate(table, getContext()); + alter_commands.validate(table, current_context); alter_commands.prepare(metadata); - table->checkAlterIsPossible(alter_commands, getContext()); - table->alter(alter_commands, getContext(), alter_lock); + table->checkAlterIsPossible(alter_commands, current_context); + table->alter(alter_commands, current_context, alter_lock); return {}; } diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index cfcc5decdf5..f53c39b192f 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -557,7 +556,6 @@ void ASTAlterQuery::formatQueryImpl(const FormatSettings & settings, FormatState frame.need_parens = false; std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); - settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str; switch (alter_object) diff --git a/src/Parsers/ASTCreateIndexQuery.cpp b/src/Parsers/ASTCreateIndexQuery.cpp index 7a5c80551d6..50470fbc1e4 100644 --- a/src/Parsers/ASTCreateIndexQuery.cpp +++ b/src/Parsers/ASTCreateIndexQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB @@ -23,6 +24,9 @@ ASTPtr ASTCreateIndexQuery::clone() const res->index_decl = index_decl->clone(); res->children.push_back(res->index_decl); + + cloneTableOptions(*res); + return res; } @@ -58,4 +62,15 @@ void ASTCreateIndexQuery::formatQueryImpl(const FormatSettings & settings, Forma index_decl->formatImpl(settings, state, frame); } +ASTPtr ASTCreateIndexQuery::convertToASTAlterCommand() const +{ + auto command = std::make_shared(); + command->type = ASTAlterCommand::ADD_INDEX; + command->index = index_name->clone(); + command->index_decl = index_decl->clone(); + command->if_not_exists = if_not_exists; + + return command; +} + } diff --git a/src/Parsers/ASTCreateIndexQuery.h b/src/Parsers/ASTCreateIndexQuery.h index f3c6a7830a4..424a0e493d9 100644 --- a/src/Parsers/ASTCreateIndexQuery.h +++ b/src/Parsers/ASTCreateIndexQuery.h @@ -14,13 +14,13 @@ namespace DB class ASTCreateIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster { public: - bool if_not_exists{false}; - ASTPtr index_name; /// Stores the IndexDeclaration here. ASTPtr index_decl; + bool if_not_exists{false}; + String getID(char delim) const override; ASTPtr clone() const override; @@ -30,7 +30,10 @@ public: return removeOnCluster(clone(), params.default_database); } - virtual QueryKind getQueryKind() const override { return QueryKind::Create; } + QueryKind getQueryKind() const override { return QueryKind::Create; } + + /// Convert ASTCreateIndexQuery to ASTAlterCommand + ASTPtr convertToASTAlterCommand() const; protected: void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; diff --git a/src/Parsers/ASTDropIndexQuery.cpp b/src/Parsers/ASTDropIndexQuery.cpp index 78152d213b8..a07336a2d26 100644 --- a/src/Parsers/ASTDropIndexQuery.cpp +++ b/src/Parsers/ASTDropIndexQuery.cpp @@ -20,6 +20,8 @@ ASTPtr ASTDropIndexQuery::clone() const res->index_name = index_name->clone(); res->children.push_back(res->index_name); + cloneTableOptions(*res); + return res; } @@ -53,9 +55,9 @@ void ASTDropIndexQuery::formatQueryImpl(const FormatSettings & settings, FormatS ASTPtr ASTDropIndexQuery::convertToASTAlterCommand() const { auto command = std::make_shared(); + command->type = ASTAlterCommand::DROP_INDEX; command->index = index_name->clone(); command->if_exists = if_exists; - command->type = ASTAlterCommand::DROP_INDEX; return command; } diff --git a/src/Parsers/ASTDropIndexQuery.h b/src/Parsers/ASTDropIndexQuery.h index d7e39f797b5..6c2aaeb5936 100644 --- a/src/Parsers/ASTDropIndexQuery.h +++ b/src/Parsers/ASTDropIndexQuery.h @@ -17,10 +17,11 @@ namespace DB class ASTDropIndexQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster { public: - bool if_exists{false}; ASTPtr index_name; + bool if_exists{false}; + String getID(char delim) const override; ASTPtr clone() const override; @@ -30,9 +31,9 @@ public: return removeOnCluster(clone(), params.default_database); } - virtual QueryKind getQueryKind() const override { return QueryKind::Drop; } + QueryKind getQueryKind() const override { return QueryKind::Drop; } - /// Convert ASTDropIndexQuery to ASTAlterCommand. + /// Convert ASTDropIndexQuery to ASTAlterCommand ASTPtr convertToASTAlterCommand() const; protected: diff --git a/src/Parsers/ASTIndexDeclaration.cpp b/src/Parsers/ASTIndexDeclaration.cpp index cc988d1d307..d223661451e 100644 --- a/src/Parsers/ASTIndexDeclaration.cpp +++ b/src/Parsers/ASTIndexDeclaration.cpp @@ -25,7 +25,7 @@ ASTPtr ASTIndexDeclaration::clone() const void ASTIndexDeclaration::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const { - if (from_create_index) + if (part_of_create_index_query) { s.ostr << "("; expr->formatImpl(s, state, frame); diff --git a/src/Parsers/ASTIndexDeclaration.h b/src/Parsers/ASTIndexDeclaration.h index 31d5ef0e7f8..e22c1da4489 100644 --- a/src/Parsers/ASTIndexDeclaration.h +++ b/src/Parsers/ASTIndexDeclaration.h @@ -16,7 +16,7 @@ public: IAST * expr; ASTFunction * type; UInt64 granularity; - bool from_create_index = false; + bool part_of_create_index_query = false; /** Get the text that identifies this element. */ String getID(char) const override { return "Index"; } diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp index 22411c71ee5..af0d9064626 100644 --- a/src/Parsers/ParserCreateIndexQuery.cpp +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -43,7 +43,7 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected return false; auto index = std::make_shared(); - index->from_create_index = true; + index->part_of_create_index_query = true; index->granularity = granularity->as().value.safeGet(); index->set(index->expr, expr); index->set(index->type, type); @@ -87,18 +87,21 @@ bool ParserCreateIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect return false; if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) - return false; + return false; /// [ON cluster_name] if (s_on.ignore(pos, expected)) { - if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) - return false; + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; } if (!parser_create_idx_decl.parse(pos, index_decl, expected)) return false; + auto & ast_index_decl = index_decl->as(); + ast_index_decl.name = index_name->as().name(); + query->index_name = index_name; query->children.push_back(index_name); diff --git a/src/Parsers/ParserCreateIndexQuery.h b/src/Parsers/ParserCreateIndexQuery.h index 3dfdccc301f..3cb91cd03c6 100644 --- a/src/Parsers/ParserCreateIndexQuery.h +++ b/src/Parsers/ParserCreateIndexQuery.h @@ -12,7 +12,7 @@ namespace DB class ParserCreateIndexQuery : public IParserBase { protected: - const char * getName() const override{ return "CREATE INDEX query"; } + const char * getName() const override { return "CREATE INDEX query"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -21,7 +21,7 @@ protected: class ParserCreateIndexDeclaration : public IParserBase { public: - ParserCreateIndexDeclaration() {} + ParserCreateIndexDeclaration() = default; protected: const char * getName() const override { return "index declaration in create index"; } diff --git a/src/Parsers/ParserDropIndexQuery.cpp b/src/Parsers/ParserDropIndexQuery.cpp index 0844ea16ae0..89ed4f01838 100644 --- a/src/Parsers/ParserDropIndexQuery.cpp +++ b/src/Parsers/ParserDropIndexQuery.cpp @@ -39,13 +39,13 @@ bool ParserDropIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected return false; if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) - return false; + return false; /// [ON cluster_name] if (s_on.ignore(pos, expected)) { - if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) - return false; + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; query->cluster = std::move(cluster_str); } diff --git a/src/Parsers/ParserDropIndexQuery.h b/src/Parsers/ParserDropIndexQuery.h index 1b6535c7efb..fc25ace469b 100644 --- a/src/Parsers/ParserDropIndexQuery.h +++ b/src/Parsers/ParserDropIndexQuery.h @@ -12,7 +12,7 @@ namespace DB class ParserDropIndexQuery : public IParserBase { protected: - const char * getName() const override{ return "DROP INDEX query"; } + const char * getName() const override { return "DROP INDEX query"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; From 656d0c0f04b66b8423a7e5b8c1d628326597d8c2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 29 Jun 2022 11:24:30 +0000 Subject: [PATCH 256/408] Update version_date.tsv and changelogs after v22.6.2.12-stable --- docs/changelogs/v22.6.2.12-stable.md | 22 ++++++++++++++++++++++ utils/list-versions/version_date.tsv | 1 + 2 files changed, 23 insertions(+) create mode 100644 docs/changelogs/v22.6.2.12-stable.md diff --git a/docs/changelogs/v22.6.2.12-stable.md b/docs/changelogs/v22.6.2.12-stable.md new file mode 100644 index 00000000000..224367b994a --- /dev/null +++ b/docs/changelogs/v22.6.2.12-stable.md @@ -0,0 +1,22 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.6.2.12-stable FIXME as compared to v22.6.1.1985-stable + +#### Improvement +* Backported in [#38484](https://github.com/ClickHouse/ClickHouse/issues/38484): Improve the stability for hive storage integration test. Move the data prepare step into test.py. [#38260](https://github.com/ClickHouse/ClickHouse/pull/38260) ([lgbo](https://github.com/lgbo-ustc)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#38404](https://github.com/ClickHouse/ClickHouse/issues/38404): Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove processor description from span attributes - it is not working [#38157](https://github.com/ClickHouse/ClickHouse/pull/38157) ([Ilya Yatsishin](https://github.com/qoega)). +* Checkout full repositories for performance tests [#38327](https://github.com/ClickHouse/ClickHouse/pull/38327) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to fix 02305_schema_inference_with_globs [#38337](https://github.com/ClickHouse/ClickHouse/pull/38337) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index b33cbcebdb7..64e2050f683 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v22.6.2.12-stable 2022-06-29 v22.6.1.1985-stable 2022-06-16 v22.5.1.2079-stable 2022-05-19 v22.4.5.9-stable 2022-05-06 From ac963a766370388f1f64efc31eb963e76d6fc2e3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 13:55:20 +0200 Subject: [PATCH 257/408] Remove redundant check --- src/Storages/MergeTree/DataPartStorageOnDisk.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 780e443266d..1297a5ab58d 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -709,12 +709,6 @@ void DataPartStorageBuilderOnDisk::rename( bool remove_new_dir_if_exists, bool fsync_part_dir) { - if (!exists()) - throw Exception( - ErrorCodes::FILE_DOESNT_EXIST, - "Part directory {} doesn't exist. Most likely it is a logical error.", - std::string(fs::path(volume->getDisk()->getPath()) / root_path / part_dir)); - String to = fs::path(new_root_path) / new_part_dir / ""; if (volume->getDisk()->exists(to)) From 0b400518830a83f4d8f6a264b9541cdf119b5c1c Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Wed, 29 Jun 2022 06:04:02 -0600 Subject: [PATCH 258/408] Update zh/tutorial.md Fix `include` from old mkdocs version --- docs/zh/getting-started/tutorial.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index 3d66f79cb46..2a0095bd3a9 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -16,7 +16,17 @@ sidebar_label: 使用教程 例如，您选择`deb`安装包，执行: ``` bash -{% include 'install/deb.sh' %} +sudo apt-get install -y apt-transport-https ca-certificates dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + +echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ + /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client + +sudo service clickhouse-server start +clickhouse-client # or "clickhouse-client --password" if you've set up a password. ``` 在我们安装的软件中包含这些包: From c5e0869c46663dc49c87a5d8e45386f16132ff86 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 29 Jun 2022 14:20:26 +0200 Subject: [PATCH 259/408] fix flaky test --- tests/queries/0_stateless/01592_long_window_functions1.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01592_long_window_functions1.sql b/tests/queries/0_stateless/01592_long_window_functions1.sql index 4911b7aa792..022d8071ffa 100644 --- a/tests/queries/0_stateless/01592_long_window_functions1.sql +++ b/tests/queries/0_stateless/01592_long_window_functions1.sql @@ -4,6 +4,10 @@ drop table if exists stack; set max_insert_threads = 4; +-- Temporary disable aggregation in order, +-- because it may fail with UBSan. +set optimize_aggregation_in_order = 0; + create table stack(item_id Int64, brand_id Int64, rack_id Int64, dt DateTime, expiration_dt DateTime, quantity UInt64) Engine = MergeTree partition by toYYYYMM(dt) From 3ff26939fc4aa4da32253acae7a2a56d32f4e0d7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 29 Jun 2022 15:46:40 +0300 Subject: [PATCH 260/408] Update run.sh --- docker/test/stress/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 119ea04080f..0b517fb4af8 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -42,6 +42,7 @@ function install_packages() function configure() { # install test configs + export USE_DATABASE_ORDINARY=1 /usr/share/clickhouse-test/config/install.sh # we mount tests folder from repo to /usr/share From c86c6cc2d95db02a6434fe99532223c220e6272f Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 29 Jun 2022 12:54:45 +0000 Subject: [PATCH 261/408] Improve postgres tests. Fix flaky test_postgresql_database_engine --- .../configs/dictionaries/postgres_dict.xml | 4 +- .../test_dictionaries_postgresql/test.py | 154 +++++++----------- .../test_dictionaries_redis/test_long.py | 2 +- .../test_postgresql_database_engine/test.py | 154 ++++++++---------- 4 files changed, 138 insertions(+), 176 deletions(-) diff --git a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml index 8b69d74b67c..dc4e474f125 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml @@ -4,7 +4,7 @@ dict0 - clickhouse + postgres_database postgres1 5432 postgres @@ -38,7 +38,7 @@ dict1 - clickhouse + postgres_database postgres mysecretpassword test1
diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 49a75a09e4e..46f7f7254eb 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -1,9 +1,11 @@ import pytest import time +import logging import psycopg2 from multiprocessing.dummy import Pool from helpers.cluster import ClickHouseCluster +from helpers.postgres_utility import get_postgres_conn from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) @@ -18,63 +20,39 @@ node1 = cluster.add_instance( with_postgres_cluster=True, ) -postgres_dict_table_template = """ - CREATE TABLE IF NOT EXISTS {} ( - id Integer NOT NULL, key Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) - """ -click_dict_table_template = """ - CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` ( - `key` UInt32, `value` UInt32 - ) ENGINE = Dictionary({}) - """ - - -def get_postgres_conn(ip, port, database=False): - if database == True: - conn_string = "host={} port={} dbname='clickhouse' user='postgres' password='mysecretpassword'".format( - ip, port - ) - else: - conn_string = ( - "host={} port={} user='postgres' password='mysecretpassword'".format( - ip, port - ) - ) - - conn = psycopg2.connect(conn_string) - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - conn.autocommit = True - return conn - - def create_postgres_db(conn, name): cursor = conn.cursor() - cursor.execute("CREATE DATABASE {}".format(name)) + cursor.execute(f"CREATE DATABASE {name}") def create_postgres_table(cursor, table_name): - cursor.execute(postgres_dict_table_template.format(table_name)) + cursor.execute( + f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id Integer NOT NULL, key Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) + """ + ) def create_and_fill_postgres_table(cursor, table_name, port, host): create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check - table_func = """postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')""".format( - host, port, table_name - ) + table_func = f"""postgresql('{host}:{port}', 'postgres_database', '{table_name}', 'postgres', 'mysecretpassword')""" node1.query( - """INSERT INTO TABLE FUNCTION {} SELECT number, number, number from numbers(10000) - """.format( - table_func, table_name - ) + f"""INSERT INTO TABLE FUNCTION {table_func} SELECT number, number, number from numbers(10000)""" ) - result = node1.query("SELECT count() FROM {}".format(table_func)) + result = node1.query(f"SELECT count() FROM {table_func}") assert result.rstrip() == "10000" def create_dict(table_name, index=0): - node1.query(click_dict_table_template.format(table_name, "dict" + str(index))) - + node1.query( + f""" + CREATE TABLE IF NOT EXISTS `test`.`dict_table_{table_name}` ( + `key` UInt32, `value` UInt32 + ) ENGINE = Dictionary(dict{str(index)}) + """ + ) @pytest.fixture(scope="module") def started_cluster(): @@ -85,14 +63,14 @@ def started_cluster(): postgres_conn = get_postgres_conn( ip=cluster.postgres_ip, port=cluster.postgres_port ) - print("postgres1 connected") - create_postgres_db(postgres_conn, "clickhouse") + logging.debug("postgres1 connected") + create_postgres_db(postgres_conn, "postgres_database") - postgres_conn = get_postgres_conn( + postgres2_conn = get_postgres_conn( ip=cluster.postgres2_ip, port=cluster.postgres_port ) - print("postgres2 connected") - create_postgres_db(postgres_conn, "clickhouse") + logging.debug("postgres2 connected") + create_postgres_db(postgres2_conn, "postgres_database") yield cluster @@ -117,27 +95,27 @@ def test_load_dictionaries(started_cluster): create_dict(table_name) dict_name = "dict0" - node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) + node1.query(f"SYSTEM RELOAD DICTIONARY {dict_name}") assert ( node1.query( - "SELECT count() FROM `test`.`dict_table_{}`".format(table_name) + f"SELECT count() FROM `test`.`dict_table_{table_name}`" ).rstrip() == "10000" ) assert ( - node1.query("SELECT dictGetUInt32('{}', 'key', toUInt64(0))".format(dict_name)) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'key', toUInt64(0))") == "0\n" ) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(9999))" ) == "9999\n" ) - cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) - node1.query("DROP TABLE IF EXISTS {}".format(table_name)) - node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) + cursor.execute(f"DROP TABLE IF EXISTS {table_name}") + node1.query(f"DROP TABLE IF EXISTS {table_name}") + node1.query(f"DROP DICTIONARY IF EXISTS {dict_name}") def test_postgres_dictionaries_custom_query_full_load(started_cluster): @@ -159,7 +137,7 @@ def test_postgres_dictionaries_custom_query_full_load(started_cluster): query = node1.query query( - """ + f""" CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -169,16 +147,14 @@ def test_postgres_dictionaries_custom_query_full_load(started_cluster): PRIMARY KEY id LAYOUT(FLAT()) SOURCE(PostgreSQL( - DB 'clickhouse' - HOST '{}' - PORT {} + DB 'postgres_database' + HOST '{started_cluster.postgres_ip}' + PORT {started_cluster.postgres_port} USER 'postgres' PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id);$doc$)) LIFETIME(0) - """.format( - started_cluster.postgres_ip, started_cluster.postgres_port - ) + """ ) result = query("SELECT id, value_1, value_2 FROM test_dictionary_custom_query") @@ -210,7 +186,7 @@ def test_postgres_dictionaries_custom_query_partial_load_simple_key(started_clus query = node1.query query( - """ + f""" CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -220,15 +196,13 @@ def test_postgres_dictionaries_custom_query_partial_load_simple_key(started_clus PRIMARY KEY id LAYOUT(DIRECT()) SOURCE(PostgreSQL( - DB 'clickhouse' - HOST '{}' - PORT {} + DB 'postgres_database' + HOST '{started_cluster.postgres_ip}' + PORT {started_cluster.postgres_port} USER 'postgres' PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id) WHERE {{condition}};$doc$)) - """.format( - started_cluster.postgres_ip, started_cluster.postgres_port - ) + """ ) result = query( @@ -262,7 +236,7 @@ def test_postgres_dictionaries_custom_query_partial_load_complex_key(started_clu query = node1.query query( - """ + f""" CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -273,15 +247,13 @@ def test_postgres_dictionaries_custom_query_partial_load_complex_key(started_clu PRIMARY KEY id, key LAYOUT(COMPLEX_KEY_DIRECT()) SOURCE(PostgreSQL( - DB 'clickhouse' - HOST '{}' - PORT {} + DB 'postgres_database' + HOST '{started_cluster.postgres_ip}' + PORT {started_cluster.postgres_port} USER 'postgres' PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, key, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id, key) WHERE {{condition}};$doc$)) - """.format( - started_cluster.postgres_ip, started_cluster.postgres_port - ) + """ ) result = query( @@ -314,70 +286,70 @@ def test_invalidate_query(started_cluster): # invalidate query: SELECT value FROM test0 WHERE id = 0 dict_name = "dict0" create_dict(table_name) - node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) + node1.query(f"SYSTEM RELOAD DICTIONARY {dict_name}") assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" ) == "0\n" ) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" ) == "1\n" ) # update should happen - cursor.execute("UPDATE {} SET value=value+1 WHERE id = 0".format(table_name)) + cursor.execute(f"UPDATE {table_name} SET value=value+1 WHERE id = 0") while True: result = node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" ) if result != "0\n": break assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" ) == "1\n" ) # no update should happen - cursor.execute("UPDATE {} SET value=value*2 WHERE id != 0".format(table_name)) + cursor.execute(f"UPDATE {table_name} SET value=value*2 WHERE id != 0") time.sleep(5) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" ) == "1\n" ) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" ) == "1\n" ) # update should happen - cursor.execute("UPDATE {} SET value=value+1 WHERE id = 0".format(table_name)) + cursor.execute(f"UPDATE {table_name} SET value=value+1 WHERE id = 0") time.sleep(5) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" ) == "2\n" ) assert ( node1.query( - "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" ) == "2\n" ) - node1.query("DROP TABLE IF EXISTS {}".format(table_name)) - node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) - cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query(f"DROP TABLE IF EXISTS {table_name}") + node1.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + cursor.execute(f"DROP TABLE IF EXISTS {table_name}") def test_dictionary_with_replicas(started_cluster): @@ -446,7 +418,7 @@ def test_postgres_schema(started_cluster): host 'postgres1' user 'postgres' password 'mysecretpassword' - db 'clickhouse' + db 'postgres_database' table 'test_schema.test_table')) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); @@ -566,7 +538,7 @@ def test_bad_configuration(started_cluster): host 'postgres1' user 'postgres' password 'mysecretpassword' - dbbb 'clickhouse' + dbbb 'postgres_database' table 'test_schema.test_table')) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); diff --git a/tests/integration/test_dictionaries_redis/test_long.py b/tests/integration/test_dictionaries_redis/test_long.py index 094df789704..19b03322b4d 100644 --- a/tests/integration/test_dictionaries_redis/test_long.py +++ b/tests/integration/test_dictionaries_redis/test_long.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster import redis -cluster = ClickHouseCluster(__file__) +cluster = ClickHouseCluster(__file__, name="long") node = cluster.add_instance("node", with_redis=True) diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index aabf3507d8f..723c4f7cd09 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -3,6 +3,7 @@ import psycopg2 from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry +from helpers.postgres_utility import get_postgres_conn from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) @@ -20,17 +21,6 @@ postgres_drop_table_template = """ """ -def get_postgres_conn(cluster, database=False): - if database == True: - conn_string = f"host={cluster.postgres_ip} port={cluster.postgres_port} dbname='test_database' user='postgres' password='mysecretpassword'" - else: - conn_string = f"host={cluster.postgres_ip} port={cluster.postgres_port} user='postgres' password='mysecretpassword'" - conn = psycopg2.connect(conn_string) - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - conn.autocommit = True - return conn - - def create_postgres_db(cursor, name): cursor.execute("CREATE DATABASE {}".format(name)) @@ -49,9 +39,9 @@ def drop_postgres_table(cursor, table_name): def started_cluster(): try: cluster.start() - conn = get_postgres_conn(cluster) + conn = get_postgres_conn(cluster.postgres_ip, cluster.postgres_port) cursor = conn.cursor() - create_postgres_db(cursor, "test_database") + create_postgres_db(cursor, "postgres_database") yield cluster finally: @@ -60,93 +50,93 @@ def started_cluster(): def test_postgres_database_engine_with_postgres_ddl(started_cluster): # connect to database as well - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')" ) - assert "test_database" in node1.query("SHOW DATABASES") + assert "postgres_database" in node1.query("SHOW DATABASES") create_postgres_table(cursor, "test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") cursor.execute("ALTER TABLE test_table ADD COLUMN data Text") assert "data" in node1.query( - "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'postgres_database'" ) cursor.execute("ALTER TABLE test_table DROP COLUMN data") assert "data" not in node1.query( - "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'postgres_database'" ) - node1.query("DROP DATABASE test_database") - assert "test_database" not in node1.query("SHOW DATABASES") + node1.query("DROP DATABASE postgres_database") + assert "postgres_database" not in node1.query("SHOW DATABASES") drop_postgres_table(cursor, "test_table") def test_postgresql_database_engine_with_clickhouse_ddl(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')" ) create_postgres_table(cursor, "test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") - node1.query("DROP TABLE test_database.test_table") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + node1.query("DROP TABLE postgres_database.test_table") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("ATTACH TABLE test_database.test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + node1.query("ATTACH TABLE postgres_database.test_table") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") - node1.query("DETACH TABLE test_database.test_table") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + node1.query("DETACH TABLE postgres_database.test_table") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("ATTACH TABLE test_database.test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + node1.query("ATTACH TABLE postgres_database.test_table") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") - node1.query("DROP DATABASE test_database") - assert "test_database" not in node1.query("SHOW DATABASES") + node1.query("DROP DATABASE postgres_database") + assert "postgres_database" not in node1.query("SHOW DATABASES") drop_postgres_table(cursor, "test_table") def test_postgresql_database_engine_queries(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')" ) create_postgres_table(cursor, "test_table") - assert node1.query("SELECT count() FROM test_database.test_table").rstrip() == "0" + assert node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "0" node1.query( - "INSERT INTO test_database.test_table SELECT number, number from numbers(10000)" + "INSERT INTO postgres_database.test_table SELECT number, number from numbers(10000)" ) assert ( - node1.query("SELECT count() FROM test_database.test_table").rstrip() == "10000" + node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "10000" ) drop_postgres_table(cursor, "test_table") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("DROP DATABASE test_database") - assert "test_database" not in node1.query("SHOW DATABASES") + node1.query("DROP DATABASE postgres_database") + assert "postgres_database" not in node1.query("SHOW DATABASES") def test_get_create_table_query_with_multidim_arrays(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword')" ) cursor.execute( @@ -157,11 +147,11 @@ def test_get_create_table_query_with_multidim_arrays(started_cluster): )""" ) - node1.query("DETACH TABLE test_database.array_columns") - node1.query("ATTACH TABLE test_database.array_columns") + node1.query("DETACH TABLE postgres_database.array_columns") + node1.query("ATTACH TABLE postgres_database.array_columns") node1.query( - "INSERT INTO test_database.array_columns " + "INSERT INTO postgres_database.array_columns " "VALUES (" "[[[1, 1], [1, 1]], [[3, 3], [3, 3]], [[4, 4], [5, 5]]], " "[[[1, NULL], [NULL, 1]], [[NULL, NULL], [NULL, NULL]], [[4, 4], [5, 5]]] " @@ -169,7 +159,7 @@ def test_get_create_table_query_with_multidim_arrays(started_cluster): ) result = node1.query( """ - SELECT * FROM test_database.array_columns""" + SELECT * FROM postgres_database.array_columns""" ) expected = ( "[[[1,1],[1,1]],[[3,3],[3,3]],[[4,4],[5,5]]]\t" @@ -177,64 +167,64 @@ def test_get_create_table_query_with_multidim_arrays(started_cluster): ) assert result == expected - node1.query("DROP DATABASE test_database") - assert "test_database" not in node1.query("SHOW DATABASES") + node1.query("DROP DATABASE postgres_database") + assert "postgres_database" not in node1.query("SHOW DATABASES") drop_postgres_table(cursor, "array_columns") def test_postgresql_database_engine_table_cache(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', '', 1)" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword', '', 1)" ) create_postgres_table(cursor, "test_table") assert ( - node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + node1.query("DESCRIBE TABLE postgres_database.test_table").rstrip() == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)" ) cursor.execute("ALTER TABLE test_table ADD COLUMN data Text") assert ( - node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + node1.query("DESCRIBE TABLE postgres_database.test_table").rstrip() == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)" ) - node1.query("DETACH TABLE test_database.test_table") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + node1.query("DETACH TABLE postgres_database.test_table") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("ATTACH TABLE test_database.test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + node1.query("ATTACH TABLE postgres_database.test_table") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") assert ( - node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + node1.query("DESCRIBE TABLE postgres_database.test_table").rstrip() == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)\t\t\t\t\t\ndata\tNullable(String)" ) - node1.query("DROP TABLE test_database.test_table") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + node1.query("DROP TABLE postgres_database.test_table") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("ATTACH TABLE test_database.test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") + node1.query("ATTACH TABLE postgres_database.test_table") + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") node1.query( - "INSERT INTO test_database.test_table SELECT number, number, toString(number) from numbers(10000)" + "INSERT INTO postgres_database.test_table SELECT number, number, toString(number) from numbers(10000)" ) assert ( - node1.query("SELECT count() FROM test_database.test_table").rstrip() == "10000" + node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "10000" ) cursor.execute("DROP TABLE test_table;") - assert "test_table" not in node1.query("SHOW TABLES FROM test_database") + assert "test_table" not in node1.query("SHOW TABLES FROM postgres_database") - node1.query("DROP DATABASE test_database") - assert "test_database" not in node1.query("SHOW DATABASES") + node1.query("DROP DATABASE postgres_database") + assert "postgres_database" not in node1.query("SHOW DATABASES") def test_postgresql_database_with_schema(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() cursor.execute("CREATE SCHEMA test_schema") @@ -243,17 +233,17 @@ def test_postgresql_database_with_schema(started_cluster): cursor.execute("CREATE TABLE table3 (a integer)") node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 'test_schema')" + "CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword', 'test_schema')" ) - assert node1.query("SHOW TABLES FROM test_database") == "table1\ntable2\n" + assert node1.query("SHOW TABLES FROM postgres_database") == "table1\ntable2\n" - node1.query("INSERT INTO test_database.table1 SELECT number from numbers(10000)") - assert node1.query("SELECT count() FROM test_database.table1").rstrip() == "10000" - node1.query("DETACH TABLE test_database.table1") - node1.query("ATTACH TABLE test_database.table1") - assert node1.query("SELECT count() FROM test_database.table1").rstrip() == "10000" - node1.query("DROP DATABASE test_database") + node1.query("INSERT INTO postgres_database.table1 SELECT number from numbers(10000)") + assert node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + node1.query("DETACH TABLE postgres_database.table1") + node1.query("ATTACH TABLE postgres_database.table1") + assert node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + node1.query("DROP DATABASE postgres_database") cursor.execute("DROP SCHEMA test_schema CASCADE") cursor.execute("DROP TABLE table3") @@ -321,18 +311,18 @@ def test_predefined_connection_configuration(started_cluster): def test_postgres_database_old_syntax(started_cluster): - conn = get_postgres_conn(started_cluster, True) + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) cursor = conn.cursor() node1.query( """ - DROP DATABASE IF EXISTS test_database; - CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 1); + CREATE DATABASE postgres_database ENGINE = PostgreSQL('postgres1:5432', 'postgres_database', 'postgres', 'mysecretpassword', 1); """ ) create_postgres_table(cursor, "test_table") - assert "test_table" in node1.query("SHOW TABLES FROM test_database") - + assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") + cursor.execute(f"DROP TABLE test_table ") + node1.query("DROP DATABASE IF EXISTS postgres_database;") if __name__ == "__main__": cluster.start() From 2f03e821bd5ad44f82f0d68ecf43859c3af8117c Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 15:08:16 +0200 Subject: [PATCH 262/408] Fix stupid bug --- src/Storages/MergeTree/DataPartsExchange.cpp | 11 +++++++---- src/Storages/MergeTree/MergeTreePartsMover.h | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 313bde658cb..68e9f0a78d1 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -869,12 +869,12 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( part_relative_path, part_dir); - if (data_part_storage_builder->exists()) - throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", data_part_storage_builder->getFullPath()); + if (data_part_storage->exists()) + throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", data_part_storage->getFullPath()); CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; - data_part_storage_builder->createDirectories(); + volume->getDisk()->createDirectories(data_part_storage->getFullPath()); size_t files; readBinary(files, in); @@ -887,7 +887,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( readStringBinary(file_name, in); readBinary(file_size, in); - String metadata_file = fs::path(data_part_storage_builder->getFullPath()) / file_name; + String metadata_file = fs::path(data_part_storage->getFullPath()) / file_name; { auto file_out = std::make_unique(metadata_file, DBMS_DEFAULT_BUFFER_SIZE, -1, 0666, nullptr, 0); @@ -902,6 +902,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( /// performing a poll with a not very large timeout. /// And now we check it only between read chunks (in the `copyData` function). data_part_storage_builder->removeSharedRecursive(true); + data_part_storage_builder->commit(); throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); } @@ -919,6 +920,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( assertEOF(in); + data_part_storage_builder->commit(); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, data_part_storage); new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr); new_data_part->is_temp = true; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.h b/src/Storages/MergeTree/MergeTreePartsMover.h index 45bbce01d00..6ad658c2cb3 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.h +++ b/src/Storages/MergeTree/MergeTreePartsMover.h @@ -50,14 +50,14 @@ public: const std::lock_guard & moving_parts_lock); /// Copies part to selected reservation in detached folder. Throws exception if part already exists. - std::shared_ptr clonePart(const MergeTreeMoveEntry & moving_part) const; + MergeTreeDataPartPtr clonePart(const MergeTreeMoveEntry & moving_part) const; /// Replaces cloned part from detached directory into active data parts set. /// Replacing part changes state to DeleteOnDestroy and will be removed from disk after destructor of ///IMergeTreeDataPart called. If replacing part doesn't exists or not active (committed) than /// cloned part will be removed and log message will be reported. It may happen in case of concurrent /// merge or mutation. - void swapClonedPart(const std::shared_ptr & cloned_parts) const; + void swapClonedPart(const MergeTreeDataPartPtr & cloned_parts) const; /// Can stop background moves and moves from queries ActionBlocker moves_blocker; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2fee9782e77..a711ca803d4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4242,6 +4242,7 @@ DataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( auto replaced_path = fs::path(replaced_part_path); auto builder = part->data_part_storage->getBuilder(); builder->rename(replaced_path.parent_path(), replaced_path.filename(), nullptr, true, false); + part->data_part_storage->onRename(replaced_path.parent_path(), replaced_path.filename()); builder->commit(); } catch (const Exception & e) From 3627c6ff3684cfa307b96b0ab536ddc05caba8c8 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 29 Jun 2022 13:13:39 +0000 Subject: [PATCH 263/408] Perf tests with high cardinality --- .../Transforms/DistinctSortedChunkTransform.cpp | 2 +- .../Transforms/DistinctSortedChunkTransform.h | 15 ++++++++++++--- tests/performance/distinct_in_order.xml | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 tests/performance/distinct_in_order.xml diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp index b76c895c00e..70548aa4396 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp @@ -27,7 +27,7 @@ DistinctSortedChunkTransform::DistinctSortedChunkTransform( sorted_columns_pos.emplace_back(pos); } - /// calculate not-sorted columns positions + /// calculate non-sorted columns positions other_columns_pos.reserve(source_columns.size()); for (const auto & source_column : source_columns) { diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.h b/src/Processors/Transforms/DistinctSortedChunkTransform.h index 1d9162afd30..2e21c36f7dc 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.h +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.h @@ -8,12 +8,21 @@ namespace DB { - /// /// DISTINCT optimization for sorted chunks -/// It also checks if current chunks is continuation of previous one, +/// +/// (1) distinct columns are split into two groups - sorted i.e. belong to sorting prefix, +/// and non-sorted (other columns w/o sorting guarantees). +/// +/// (2) Rows are split into ranges. Range is a set of rows where the sorting prefix value is the same. +/// If there are no non-sorted columns, then we just skip all rows in range except one. +/// If there are non-sorted columns, then for each range, we use a hash table to find unique rows in a range. +/// +/// (3) The implementation also checks if current chunks is continuation of previous one, /// i.e. sorting prefix value of last row in previous chunk is the same as of first row in current one, -/// so it can correctly process sorted stream as well +/// so it can correctly process sorted stream as well. +/// For this, we don't clear sorting prefix value and hash table after a range is processed, +/// only right before a new range processing /// class DistinctSortedChunkTransform : public ISimpleTransform { diff --git a/tests/performance/distinct_in_order.xml b/tests/performance/distinct_in_order.xml new file mode 100644 index 00000000000..742fcefc0df --- /dev/null +++ b/tests/performance/distinct_in_order.xml @@ -0,0 +1,15 @@ + + CREATE TABLE distinct_cardinality (high UInt64, middle UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, middle, low) + INSERT INTO distinct_cardinality SELECT number, number % 1000, number % 10 from numbers(1000000) + + select distinct high from distinct_cardinality + select distinct high, low from distinct_cardinality + select distinct high, middle from distinct_cardinality + select distinct high, middle, low from distinct_cardinality + + select distinct high, middle from distinct_cardinality order by middle + select distinct high, low from distinct_cardinality order by low + select distinct high, middle, low from distinct_cardinality order by high + + DROP TABLE IF EXISTS distinct_cardinality + From 67f2fa0ef014f6a55c6bef7a5a13c0180aa26171 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 15:18:48 +0200 Subject: [PATCH 264/408] Fix style --- src/Storages/MergeTree/DataPartStorageOnDisk.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 1297a5ab58d..347f43a03e4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -18,7 +18,6 @@ namespace DB namespace ErrorCodes { - extern const int FILE_DOESNT_EXIST; extern const int DIRECTORY_ALREADY_EXISTS; extern const int NOT_ENOUGH_SPACE; extern const int LOGICAL_ERROR; From baeb1811e194df4784339b737d135a9c30f58c50 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 29 Jun 2022 16:59:16 +0300 Subject: [PATCH 265/408] try to fix performance tests --- docker/test/performance-comparison/download.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index 463c08c5304..8d6c48935b7 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -73,6 +73,7 @@ function download ) & wait + cd db0 && echo "ATTACH DATABASE default ENGINE=Ordinary" > metadata/default.sql } download From 6f6e814542cec112c2e7b4b0c156ceaeaccfedff Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:18:12 +0000 Subject: [PATCH 266/408] Fix test_dictionaries_postgresql --- .../configs/named_collections.xml | 4 ++-- tests/integration/test_dictionaries_postgresql/test.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_dictionaries_postgresql/configs/named_collections.xml b/tests/integration/test_dictionaries_postgresql/configs/named_collections.xml index 647840848fd..d08bc5b32c2 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/named_collections.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/named_collections.xml @@ -5,7 +5,7 @@ mysecretpassword postgres1 5432 - clickhouse + postgres_database test_table
@@ -13,7 +13,7 @@ mysecretpassword postgres1 5432 - clickhouse + postgres_database test_table
test_schema diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 46f7f7254eb..5d6bbf70a40 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -430,6 +430,9 @@ def test_postgres_schema(started_cluster): result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") assert int(result.strip()) == 99 node1.query("DROP DICTIONARY IF EXISTS postgres_dict") + cursor.execute("DROP TABLE test_schema.test_table") + cursor.execute("DROP SCHEMA test_schema") + def test_predefined_connection_configuration(started_cluster): From 628ac3e1b8dd78f25cca79a128526dc93cd27243 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Wed, 29 Jun 2022 10:34:38 -0400 Subject: [PATCH 267/408] Apply suggestions from code review --- docs/zh/getting-started/tutorial.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index 2a0095bd3a9..5fa77bffa78 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -24,9 +24,6 @@ echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ sudo apt-get update sudo apt-get install -y clickhouse-server clickhouse-client - -sudo service clickhouse-server start -clickhouse-client # or "clickhouse-client --password" if you've set up a password. ``` 在我们安装的软件中包含这些包: From f4691b0b0020df7ffd5b8c32662141931f3e4050 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 29 Jun 2022 15:08:24 +0000 Subject: [PATCH 268/408] fix black --- .../test_dictionaries_postgresql/test.py | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 5d6bbf70a40..516ac27ea26 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -20,6 +20,7 @@ node1 = cluster.add_instance( with_postgres_cluster=True, ) + def create_postgres_db(conn, name): cursor = conn.cursor() cursor.execute(f"CREATE DATABASE {name}") @@ -27,7 +28,7 @@ def create_postgres_db(conn, name): def create_postgres_table(cursor, table_name): cursor.execute( - f""" + f""" CREATE TABLE IF NOT EXISTS {table_name} ( id Integer NOT NULL, key Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) """ @@ -47,13 +48,14 @@ def create_and_fill_postgres_table(cursor, table_name, port, host): def create_dict(table_name, index=0): node1.query( - f""" + f""" CREATE TABLE IF NOT EXISTS `test`.`dict_table_{table_name}` ( `key` UInt32, `value` UInt32 ) ENGINE = Dictionary(dict{str(index)}) """ ) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -97,19 +99,14 @@ def test_load_dictionaries(started_cluster): node1.query(f"SYSTEM RELOAD DICTIONARY {dict_name}") assert ( - node1.query( - f"SELECT count() FROM `test`.`dict_table_{table_name}`" - ).rstrip() + node1.query(f"SELECT count() FROM `test`.`dict_table_{table_name}`").rstrip() == "10000" ) assert ( - node1.query(f"SELECT dictGetUInt32('{dict_name}', 'key', toUInt64(0))") - == "0\n" + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'key', toUInt64(0))") == "0\n" ) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(9999))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(9999))") == "9999\n" ) @@ -288,15 +285,11 @@ def test_invalidate_query(started_cluster): create_dict(table_name) node1.query(f"SYSTEM RELOAD DICTIONARY {dict_name}") assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))") == "0\n" ) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))") == "1\n" ) @@ -309,9 +302,7 @@ def test_invalidate_query(started_cluster): if result != "0\n": break assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))") == "1\n" ) @@ -319,15 +310,11 @@ def test_invalidate_query(started_cluster): cursor.execute(f"UPDATE {table_name} SET value=value*2 WHERE id != 0") time.sleep(5) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))") == "1\n" ) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))") == "1\n" ) @@ -335,15 +322,11 @@ def test_invalidate_query(started_cluster): cursor.execute(f"UPDATE {table_name} SET value=value+1 WHERE id = 0") time.sleep(5) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(0))") == "2\n" ) assert ( - node1.query( - f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))" - ) + node1.query(f"SELECT dictGetUInt32('{dict_name}', 'value', toUInt64(1))") == "2\n" ) @@ -434,7 +417,6 @@ def test_postgres_schema(started_cluster): cursor.execute("DROP SCHEMA test_schema") - def test_predefined_connection_configuration(started_cluster): conn = get_postgres_conn( ip=started_cluster.postgres_ip, From 41460dcaca29b84d36fa52155b4dd4c9ef7675bd Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 29 Jun 2022 17:37:29 +0200 Subject: [PATCH 269/408] quick fix for 02112_with_fill_interval Seems like the problem is that now data more often come to FillingTransform in multiple chunks. Don't know why it affects the results, will continue investigation. --- tests/queries/0_stateless/02112_with_fill_interval.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02112_with_fill_interval.sql b/tests/queries/0_stateless/02112_with_fill_interval.sql index d2416f9a84b..16773780515 100644 --- a/tests/queries/0_stateless/02112_with_fill_interval.sql +++ b/tests/queries/0_stateless/02112_with_fill_interval.sql @@ -1,3 +1,5 @@ +SET max_threads = 1; + DROP TABLE IF EXISTS with_fill_date; CREATE TABLE with_fill_date (d Date, d32 Date32) ENGINE = Memory; From 65110fdffc7be3a39b2fcc4c0a4cd019c1d1c190 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 17:50:25 +0200 Subject: [PATCH 270/408] ColumnVector refactor replicate SSE42 optimization --- src/Columns/ColumnVector.cpp | 220 ++++++++++++++++++----------------- src/Columns/ColumnVector.h | 4 +- 2 files changed, 114 insertions(+), 110 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 8a986c1ca86..7cfb90d4371 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -577,6 +577,113 @@ ColumnPtr ColumnVector::index(const IColumn & indexes, size_t limit) const return selectIndexImpl(*this, indexes, limit); } +#ifdef __SSE2__ + +namespace +{ + /** Optimization for ColumnVector replicate using SIMD instructions. + * For such optimization it is important that data is right padded with 15 bytes. + * + * Replicate span size is offsets[i] - offsets[i - 1]. + * + * Split spans into 3 categories. + * 1. Span with 0 size. Continue iteration. + * + * 2. Span with 1 size. Update pointer from which data must be copied into result. + * Then if we see span with size 1 or greater than 1 copy data directly into result data and reset pointer. + * Example: + * Data: 1 2 3 4 + * Offsets: 1 2 3 4 + * Result data: 1 2 3 4 + * + * 3. Span with size greater than 1. Save single data element into register and copy it into result data. + * Example: + * Data: 1 2 3 4 + * Offsets: 4 4 4 4 + * Result data: 1 1 1 1 + * + * Additional handling for tail is needed if pointer from which data must be copied from span with size 1 is not null. + */ + template + requires (std::is_same_v || std::is_same_v) + void replicateSSE42Int32(const IntType * __restrict data, IntType * __restrict result_data, const IColumn::Offsets & offsets) + { + const IntType * data_copy_begin_ptr = nullptr; + size_t offsets_size = offsets.size(); + + for (size_t offset_index = 0; offset_index < offsets_size; ++offset_index) + { + size_t span = offsets[offset_index] - offsets[offset_index - 1]; + if (span == 1) + { + if (!data_copy_begin_ptr) + data_copy_begin_ptr = data + offset_index; + + continue; + } + + /// Copy data + + if (data_copy_begin_ptr) + { + size_t copy_size = (data + offset_index) - data_copy_begin_ptr; + bool remainder = copy_size % 4; + size_t sse_copy_counter = (copy_size / 4) + remainder; + auto * result_data_copy = result_data; + + while (sse_copy_counter) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_copy), *(reinterpret_cast(data_copy_begin_ptr))); + result_data_copy += 4; + data_copy_begin_ptr += 4; + --sse_copy_counter; + } + + result_data += copy_size; + data_copy_begin_ptr = nullptr; + } + + if (span == 0) + continue; + + /// Copy single data element into result data + + bool span_remainder = span % 4; + size_t copy_counter = (span / 4) + span_remainder; + auto * result_data_tmp = result_data; + __m128i copy_element_data = _mm_set1_epi32(data[offset_index]); + + while (copy_counter) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_tmp), copy_element_data); + result_data_tmp += 4; + --copy_counter; + } + + result_data += span; + } + + /// Copy tail if needed + + if (data_copy_begin_ptr) + { + size_t copy_size = (data + offsets_size) - data_copy_begin_ptr; + bool remainder = copy_size % 4; + size_t sse_copy_counter = (copy_size / 4) + remainder; + + while (sse_copy_counter) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data), *(reinterpret_cast(data_copy_begin_ptr))); + result_data += 4; + data_copy_begin_ptr += 4; + --sse_copy_counter; + } + } + } +} + +#endif + template ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const { @@ -587,13 +694,16 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const if (0 == size) return this->create(); + auto res = this->create(offsets.back()); + #ifdef __SSE2__ if constexpr (std::is_same_v) - return replicateSSE2(offsets); + { + replicateSSE42Int32(getData().data(), res->getData().data(), offsets); + return res; + } #endif - auto res = this->create(offsets.back()); - auto it = res->getData().begin(); // NOLINT for (size_t i = 0; i < size; ++i) { @@ -605,110 +715,6 @@ ColumnPtr ColumnVector::replicate(const IColumn::Offsets & offsets) const return res; } - -#ifdef __SSE2__ - -template -ColumnPtr ColumnVector::replicateSSE2(const IColumn::Offsets & offsets) const -{ - auto res = this->create(offsets.back()); - - auto it = res->getData().begin(); // NOLINT - - /// Column is using PaddedPODArray, so we don't have to worry about the 4 out of range elements. - - IColumn::Offset prev_offset = 0; - std::optional copy_begin; - size_t size = offsets.size(); - for (size_t i = 0; i < size; ++i) - { - size_t span = offsets[i] - prev_offset; - prev_offset = offsets[i]; - if (span == 1) - { - if (!copy_begin) - copy_begin = i; - continue; - } - - /// data : 11 22 33 44 55 - /// offsets: 0 1 2 3 3 - /// res: 22 33 44 - if (copy_begin) - { - size_t copy_size = i - (*copy_begin); - bool remain = (copy_size & 3); - size_t sse_copy_counter = (copy_size >> 2); - sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); - auto it_tmp = it; // NOLINT - size_t data_start = *copy_begin; - copy_begin.reset(); - constexpr const int copy_mask = _MM_SHUFFLE(3, 2, 1, 0); - while (sse_copy_counter) - { - __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); - it_tmp += 4; - data_start += 4; - --sse_copy_counter; - } - - it += copy_size; - } - - if (span == 0) - continue; - - /// data : 11 22 33 - /// offsets: 0 0 4 - /// res: 33 33 33 33 - size_t shuffle_size = span; - bool shuffle_remain = (shuffle_size & 3); - size_t sse_shuffle_counter = (shuffle_size >> 2); - sse_shuffle_counter = shuffle_remain * (sse_shuffle_counter + 1) + (!shuffle_remain) * (sse_shuffle_counter); - auto it_tmp = it; // NOLINT - constexpr const int shuffle_mask = (_MM_SHUFFLE(0, 0, 0, 0)); - __m128i data_to_shuffle = _mm_loadu_si128(reinterpret_cast(&data[i])); - auto shuffle_result = _mm_shuffle_epi32(data_to_shuffle, shuffle_mask); - while (sse_shuffle_counter) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), shuffle_result); - it_tmp += 4; - --sse_shuffle_counter; - } - it += shuffle_size; - } - - /// data : 11 22 33 44 55 - /// offsets: 1 2 3 4 5 - /// res: 11 22 33 44 55 - if (copy_begin) - { - size_t copy_size = (size - (*copy_begin)); - bool remain = (copy_size & 3); - size_t sse_copy_counter = (copy_size >> 2); - sse_copy_counter = remain * (sse_copy_counter + 1) + (!remain) * (sse_copy_counter); - auto it_tmp = it; // NOLINT - size_t data_start = *copy_begin; - constexpr const int copy_mask = (_MM_SHUFFLE(3, 2, 1, 0)); - while (sse_copy_counter) - { - __m128i data_to_copy = _mm_loadu_si128(reinterpret_cast(&data[data_start])); - auto copy_result = _mm_shuffle_epi32(data_to_copy, copy_mask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(it_tmp), copy_result); - it_tmp += 4; - data_start += 4; - --sse_copy_counter; - } - it += copy_size; - } - - return res; -} -#endif - - template void ColumnVector::gather(ColumnGathererStream & gatherer) { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 6aae3a3e3fb..6ba9abaca32 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -136,9 +136,7 @@ private: /// Sugar constructor. ColumnVector(std::initializer_list il) : data{il} {} - #ifdef __SSE2__ - ColumnPtr replicateSSE2(const IColumn::Offsets & offsets) const; - #endif + public: bool isNumeric() const override { return is_arithmetic_v; } From 45c18145ecc7b87d94aec801f07b68ba68ff510d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 17:51:27 +0200 Subject: [PATCH 271/408] Fixed tests --- .../0_stateless/02345_partial_sort_transform_optimization.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql index fe2ab096ab7..eb395e5ec41 100644 --- a/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql +++ b/tests/queries/0_stateless/02345_partial_sort_transform_optimization.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check:22.6.1 +-- Tags: no-backward-compatibility-check -- Regression for PartialSortingTransform optimization that requires at least 1500 rows. SELECT * FROM (SELECT * FROM (SELECT 0 a, toNullable(number) b, toString(number) c FROM numbers(1e6)) ORDER BY a DESC, b DESC, c LIMIT 1500) limit 10; From 62e7a89f262909aa6d2c56ef9e88960d7d401500 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 17:53:08 +0200 Subject: [PATCH 272/408] Proper fix for ipv4/ipv6 conversion error --- src/Databases/TablesLoader.cpp | 13 +++++--- src/Databases/TablesLoader.h | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 5 +++ .../__init__.py | 1 + .../test.py | 31 +++++++++++++++++++ ...2316_cast_to_ip_address_default_column.sql | 17 +++++++++- 6 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_server_start_and_ip_conversions/__init__.py create mode 100644 tests/integration/test_server_start_and_ip_conversions/test.py diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index e973c9211be..7e9b83d423a 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -171,6 +171,11 @@ void TablesLoader::removeUnresolvableDependencies(bool remove_loaded) void TablesLoader::loadTablesInTopologicalOrder(ThreadPool & pool) { + /// Compatibility setting which should be enabled by default on attach + /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types of columns + ContextMutablePtr load_context = Context::createCopy(global_context); + load_context->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); + /// Load independent tables in parallel. /// Then remove loaded tables from dependency graph, find tables/dictionaries that do not have unresolved dependencies anymore, /// move them to the list of independent tables and load. @@ -183,7 +188,7 @@ void TablesLoader::loadTablesInTopologicalOrder(ThreadPool & pool) assert(metadata.parsed_tables.size() == tables_processed + metadata.independent_database_objects.size() + getNumberOfTablesWithDependencies()); logDependencyGraph(); - startLoadingIndependentTables(pool, level); + startLoadingIndependentTables(pool, level, load_context); TableNames new_independent_database_objects; for (const auto & table_name : metadata.independent_database_objects) @@ -237,7 +242,7 @@ DependenciesInfosIter TablesLoader::removeResolvedDependency(const DependenciesI return metadata.dependencies_info.erase(info_it); } -void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level) +void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level, ContextMutablePtr load_context) { size_t total_tables = metadata.parsed_tables.size(); @@ -245,10 +250,10 @@ void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level for (const auto & table_name : metadata.independent_database_objects) { - pool.scheduleOrThrowOnError([this, total_tables, &table_name]() + pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]() { const auto & path_and_query = metadata.parsed_tables[table_name]; - databases[table_name.database]->loadTableFromMetadata(global_context, path_and_query.path, table_name, path_and_query.ast, force_restore); + databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, force_restore); logAboutProgress(log, ++tables_processed, total_tables, stopwatch); }); } diff --git a/src/Databases/TablesLoader.h b/src/Databases/TablesLoader.h index 189906df6ff..43e8bfdb92c 100644 --- a/src/Databases/TablesLoader.h +++ b/src/Databases/TablesLoader.h @@ -104,7 +104,7 @@ private: DependenciesInfosIter removeResolvedDependency(const DependenciesInfosIter & info_it, TableNames & independent_database_objects); - void startLoadingIndependentTables(ThreadPool & pool, size_t level); + void startLoadingIndependentTables(ThreadPool & pool, size_t level, ContextMutablePtr load_context); void checkCyclicDependencies() const; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 20d88c91709..0fb048332b7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -996,6 +996,11 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) throw Exception("Temporary tables cannot be inside a database. You should not specify a database for a temporary table.", ErrorCodes::BAD_DATABASE_FOR_TEMPORARY_TABLE); + /// Compatibility setting which should be enabled by default on attach + /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types + if (create.attach) + getContext()->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); + String current_database = getContext()->getCurrentDatabase(); auto database_name = create.database ? create.getDatabase() : current_database; diff --git a/tests/integration/test_server_start_and_ip_conversions/__init__.py b/tests/integration/test_server_start_and_ip_conversions/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_server_start_and_ip_conversions/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_server_start_and_ip_conversions/test.py b/tests/integration/test_server_start_and_ip_conversions/test.py new file mode 100644 index 00000000000..f91617f60b8 --- /dev/null +++ b/tests/integration/test_server_start_and_ip_conversions/test.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import logging +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", stay_alive=True) + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_restart_success(): + node.query(""" + CREATE TABLE ipv4_test + ( + id UInt64, + value String + ) ENGINE=MergeTree ORDER BY id""", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + + node.query("ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''", settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + + node.restart_clickhouse() + + assert node.query("SELECT 1") == "1\n" diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql index 200cec8fed9..128acd7d132 100644 --- a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql +++ b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql @@ -11,6 +11,13 @@ CREATE TABLE ipv4_test ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''; +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +DETACH TABLE ipv4_test; +ATTACH TABLE ipv4_test; + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + DROP TABLE ipv4_test; DROP TABLE IF EXISTS ipv6_test; @@ -20,7 +27,15 @@ CREATE TABLE ipv6_test value String ) ENGINE=MergeTree ORDER BY id; -ALTER TABLE ipv6_test MODIFY COLUMN value IPv4 DEFAULT ''; +ALTER TABLE ipv6_test MODIFY COLUMN value IPv6 DEFAULT ''; + +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +DETACH TABLE ipv6_test; +ATTACH TABLE ipv6_test; + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + SELECT * FROM ipv6_test; DROP TABLE ipv6_test; From 1ef92b4c2796ac0a9b2826277e606c194de3689b Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 29 Jun 2022 15:56:29 +0000 Subject: [PATCH 273/408] fix black again --- .../test_postgresql_database_engine/test.py | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index 723c4f7cd09..f6cfa1b230f 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -50,7 +50,9 @@ def started_cluster(): def test_postgres_database_engine_with_postgres_ddl(started_cluster): # connect to database as well - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -78,7 +80,9 @@ def test_postgres_database_engine_with_postgres_ddl(started_cluster): def test_postgresql_database_engine_with_clickhouse_ddl(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -107,7 +111,9 @@ def test_postgresql_database_engine_with_clickhouse_ddl(started_cluster): def test_postgresql_database_engine_queries(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -115,13 +121,16 @@ def test_postgresql_database_engine_queries(started_cluster): ) create_postgres_table(cursor, "test_table") - assert node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "0" + assert ( + node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "0" + ) node1.query( "INSERT INTO postgres_database.test_table SELECT number, number from numbers(10000)" ) assert ( - node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "10000" + node1.query("SELECT count() FROM postgres_database.test_table").rstrip() + == "10000" ) drop_postgres_table(cursor, "test_table") @@ -132,7 +141,9 @@ def test_postgresql_database_engine_queries(started_cluster): def test_get_create_table_query_with_multidim_arrays(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -173,7 +184,9 @@ def test_get_create_table_query_with_multidim_arrays(started_cluster): def test_postgresql_database_engine_table_cache(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -213,7 +226,8 @@ def test_postgresql_database_engine_table_cache(started_cluster): "INSERT INTO postgres_database.test_table SELECT number, number, toString(number) from numbers(10000)" ) assert ( - node1.query("SELECT count() FROM postgres_database.test_table").rstrip() == "10000" + node1.query("SELECT count() FROM postgres_database.test_table").rstrip() + == "10000" ) cursor.execute("DROP TABLE test_table;") @@ -224,7 +238,9 @@ def test_postgresql_database_engine_table_cache(started_cluster): def test_postgresql_database_with_schema(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() cursor.execute("CREATE SCHEMA test_schema") @@ -238,11 +254,17 @@ def test_postgresql_database_with_schema(started_cluster): assert node1.query("SHOW TABLES FROM postgres_database") == "table1\ntable2\n" - node1.query("INSERT INTO postgres_database.table1 SELECT number from numbers(10000)") - assert node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + node1.query( + "INSERT INTO postgres_database.table1 SELECT number from numbers(10000)" + ) + assert ( + node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + ) node1.query("DETACH TABLE postgres_database.table1") node1.query("ATTACH TABLE postgres_database.table1") - assert node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + assert ( + node1.query("SELECT count() FROM postgres_database.table1").rstrip() == "10000" + ) node1.query("DROP DATABASE postgres_database") cursor.execute("DROP SCHEMA test_schema CASCADE") @@ -311,7 +333,9 @@ def test_predefined_connection_configuration(started_cluster): def test_postgres_database_old_syntax(started_cluster): - conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=True + ) cursor = conn.cursor() node1.query( @@ -324,6 +348,7 @@ def test_postgres_database_old_syntax(started_cluster): cursor.execute(f"DROP TABLE test_table ") node1.query("DROP DATABASE IF EXISTS postgres_database;") + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") From 9b387a57ed79ff37a1530ac46592412799e8634b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 17:56:59 +0200 Subject: [PATCH 274/408] Test ipv6 as well --- .../test.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_server_start_and_ip_conversions/test.py b/tests/integration/test_server_start_and_ip_conversions/test.py index f91617f60b8..73604394d9d 100644 --- a/tests/integration/test_server_start_and_ip_conversions/test.py +++ b/tests/integration/test_server_start_and_ip_conversions/test.py @@ -15,7 +15,7 @@ def start_cluster(): cluster.shutdown() -def test_restart_success(): +def test_restart_success_ipv4(): node.query(""" CREATE TABLE ipv4_test ( @@ -29,3 +29,19 @@ def test_restart_success(): node.restart_clickhouse() assert node.query("SELECT 1") == "1\n" + + +def test_restart_success_ipv6(): + node.query(""" + CREATE TABLE ipv6_test + ( + id UInt64, + value String + ) ENGINE=MergeTree ORDER BY id""", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + + node.query("ALTER TABLE ipv6_test MODIFY COLUMN value IPv6 DEFAULT ''", settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + + node.restart_clickhouse() + + assert node.query("SELECT 1") == "1\n" From eb3f49426c3064df729f168e2b42b718d5bd4f1c Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Wed, 29 Jun 2022 09:58:55 -0700 Subject: [PATCH 275/408] Remove zlib in mariadb-connector-c --- contrib/mariadb-connector-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 5f4034a3a63..e39608998f5 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 5f4034a3a6376416504f17186c55fe401c6d8e5e +Subproject commit e39608998f5f6944ece9ec61f48e9172ec1de660 From 09be594c8135b4b40f4ddaefc1357cbdc75e50b4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 19:15:47 +0200 Subject: [PATCH 276/408] Dictionaries added TSA annotations --- src/Dictionaries/CacheDictionary.h | 2 +- .../CassandraDictionarySource.cpp | 6 +---- src/Dictionaries/CassandraDictionarySource.h | 5 +++-- src/Dictionaries/IDictionary.h | 22 +++++++++---------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 0dbaf2716ba..d4716999b47 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -190,7 +190,7 @@ private: /// Dictionary source should be used with mutex mutable std::mutex source_mutex; - mutable DictionarySourcePtr source_ptr; + mutable DictionarySourcePtr source_ptr TSA_GUARDED_BY(source_mutex); CacheDictionaryStoragePtr cache_storage_ptr; mutable CacheDictionaryUpdateQueue update_queue; diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index e51caf7112a..e0cf2483b3d 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -194,12 +194,8 @@ QueryPipeline CassandraDictionarySource::loadUpdatedAll() CassSessionShared CassandraDictionarySource::getSession() { /// Reuse connection if exists, create new one if not - auto session = maybe_session.lock(); - if (session) - return session; - std::lock_guard lock(connect_mutex); - session = maybe_session.lock(); + auto session = maybe_session.lock(); if (session) return session; diff --git a/src/Dictionaries/CassandraDictionarySource.h b/src/Dictionaries/CassandraDictionarySource.h index c2038a966ea..e73383aa75c 100644 --- a/src/Dictionaries/CassandraDictionarySource.h +++ b/src/Dictionaries/CassandraDictionarySource.h @@ -82,9 +82,10 @@ private: Block sample_block; ExternalQueryBuilder query_builder; - std::mutex connect_mutex; CassClusterPtr cluster; - CassSessionWeak maybe_session; + + std::mutex connect_mutex; + CassSessionWeak maybe_session TSA_GUARDED_BY(connect_mutex); }; } diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index 480befdcfb2..3f3c60206d6 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -62,26 +62,26 @@ public: std::string getFullName() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id.getNameForLogs(); } StorageID getDictionaryID() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id; } void updateDictionaryName(const StorageID & new_name) const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; assert(new_name.uuid == dictionary_id.uuid && dictionary_id.uuid != UUIDHelpers::Nil); dictionary_id = new_name; } std::string getLoadableName() const final { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_id.getInternalDictionaryName(); } @@ -92,6 +92,8 @@ public: std::string getDatabaseOrNoDatabaseTag() const { + std::lock_guard lock{mutex}; + if (!dictionary_id.database_name.empty()) return dictionary_id.database_name; @@ -278,22 +280,20 @@ public: void setDictionaryComment(String new_comment) { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; dictionary_comment = std::move(new_comment); } String getDictionaryComment() const { - std::lock_guard lock{name_mutex}; + std::lock_guard lock{mutex}; return dictionary_comment; } private: - mutable std::mutex name_mutex; - mutable StorageID dictionary_id; - -protected: - String dictionary_comment; + mutable std::mutex mutex; + mutable StorageID dictionary_id TSA_GUARDED_BY(mutex); + String dictionary_comment TSA_GUARDED_BY(mutex); }; } From cd2cb10f605bcd706d9be731e0488d6c2bd3c369 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 29 Jun 2022 20:55:46 +0300 Subject: [PATCH 277/408] another try to fix performance tests --- docker/test/performance-comparison/download.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index 8d6c48935b7..a125b1816e5 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -73,7 +73,16 @@ function download ) & wait + pwd + ls + ls db0 + ls db0/metadata cd db0 && echo "ATTACH DATABASE default ENGINE=Ordinary" > metadata/default.sql + cd db0 && echo "ATTACH DATABASE datasets ENGINE=Ordinary" > metadata/datasets.sql + pwd + ls + ls db0 + ls db0/metadata } download From 8f5582f95efd18eac8c926a89574b20e617b939c Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 20:29:50 +0200 Subject: [PATCH 278/408] Review and style fixes --- src/Interpreters/InterpreterCreateQuery.cpp | 9 ++++---- .../test.py | 23 ++++++++++++++----- ...2316_cast_to_ip_address_default_column.sql | 1 + 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 0fb048332b7..7a00bbf524c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -996,11 +996,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) throw Exception("Temporary tables cannot be inside a database. You should not specify a database for a temporary table.", ErrorCodes::BAD_DATABASE_FOR_TEMPORARY_TABLE); - /// Compatibility setting which should be enabled by default on attach - /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types - if (create.attach) - getContext()->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); - String current_database = getContext()->getCurrentDatabase(); auto database_name = create.database ? create.getDatabase() : current_database; @@ -1043,6 +1038,10 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) create.attach = true; create.attach_short_syntax = true; create.if_not_exists = if_not_exists; + + /// Compatibility setting which should be enabled by default on attach + /// Otherwise server will be unable to start for some old-format of IPv6/IPv4 types + getContext()->setSetting("cast_ipv4_ipv6_default_on_conversion_error", 1); } /// TODO throw exception if !create.attach_short_syntax && !create.attach_from_path && !internal diff --git a/tests/integration/test_server_start_and_ip_conversions/test.py b/tests/integration/test_server_start_and_ip_conversions/test.py index 73604394d9d..abb6a546f64 100644 --- a/tests/integration/test_server_start_and_ip_conversions/test.py +++ b/tests/integration/test_server_start_and_ip_conversions/test.py @@ -6,6 +6,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", stay_alive=True) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): try: @@ -16,15 +17,20 @@ def start_cluster(): def test_restart_success_ipv4(): - node.query(""" + node.query( + """ CREATE TABLE ipv4_test ( id UInt64, value String ) ENGINE=MergeTree ORDER BY id""", - settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) - node.query("ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''", settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + node.query( + "ALTER TABLE ipv4_test MODIFY COLUMN value IPv4 DEFAULT ''", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) node.restart_clickhouse() @@ -32,15 +38,20 @@ def test_restart_success_ipv4(): def test_restart_success_ipv6(): - node.query(""" + node.query( + """ CREATE TABLE ipv6_test ( id UInt64, value String ) ENGINE=MergeTree ORDER BY id""", - settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) - node.query("ALTER TABLE ipv6_test MODIFY COLUMN value IPv6 DEFAULT ''", settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}) + node.query( + "ALTER TABLE ipv6_test MODIFY COLUMN value IPv6 DEFAULT ''", + settings={"cast_ipv4_ipv6_default_on_conversion_error": 1}, + ) node.restart_clickhouse() diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql index 128acd7d132..eabe6ed1d65 100644 --- a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql +++ b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql @@ -1,4 +1,5 @@ -- Tags: no-backward-compatibility-check +-- TODO: remove after new 22.6 release SET cast_ipv4_ipv6_default_on_conversion_error = 1; From 615070425e8c759e282af1c3917c4cab6482b89b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jun 2022 20:30:23 +0200 Subject: [PATCH 279/408] Fix comment --- .../0_stateless/02316_cast_to_ip_address_default_column.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql index eabe6ed1d65..35f210be43d 100644 --- a/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql +++ b/tests/queries/0_stateless/02316_cast_to_ip_address_default_column.sql @@ -1,5 +1,5 @@ -- Tags: no-backward-compatibility-check --- TODO: remove after new 22.6 release +-- TODO: remove no-backward-compatibility-check after new 22.6 release SET cast_ipv4_ipv6_default_on_conversion_error = 1; From 6bf9c7cd4d5de11b94b777c5d37158f9eeb9f084 Mon Sep 17 00:00:00 2001 From: HeenaBansal2009 Date: Wed, 29 Jun 2022 12:21:05 -0700 Subject: [PATCH 280/408] Updated index file for Geo functions --- docs/en/sql-reference/functions/geo/index.md | 96 ++++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/index.md b/docs/en/sql-reference/functions/geo/index.md index ea43b3ef96c..c0162cb5b63 100644 --- a/docs/en/sql-reference/functions/geo/index.md +++ b/docs/en/sql-reference/functions/geo/index.md @@ -20,57 +20,57 @@ sidebar_position: 62 ## H3 Indexes Functions -- [h3IsValid](./h3#h3IsValid) -- [h3GetResolution](./h3#h3GetResolution) -- [h3EdgeAngle](./h3#h3EdgeAngle) -- [h3EdgeLengthM​](./h3#h3EdgeLengthM​) -- [h3EdgeLengthKm] (./h3#h3EdgeLengthKm) -- [geoToH3](./h3#geoToH3) -- [h3ToGeo](./h3#h3ToGeo) -- [h3ToGeoBoundary](./h3#h3ToGeoBoundary) -- [h3kRing](./h3#h3kRing) -- [h3GetBaseCell](./h3#h3GetBaseCell) -- [h3HexAreaM2](./h3#h3HexAreaM2) -- [h3HexAreaKm2](./h3#h3HexAreaKm2) -- [h3IndexesAreNeighbors](./h3#h3IndexesAreNeighbors) -- [h3ToChildren](./h3#h3ToChildren) -- [h3ToParent](./h3#h3ToParent) -- [h3ToString](./h3#h3ToString) -- [stringToH3](./h3#stringToH3) -- [h3GetResolution](./h3#h3GetResolution) -- [h3IsResClassIII](./h3#h3IsResClassIII) -- [h3IsPentagon](./h3#h3IsPentagon) -- [h3GetFaces](./h3#h3GetFaces) -- [h3CellAreaM2](./h3#h3CellAreaM2) -- [h3CellAreaRads2](./h3#h3CellAreaRads2) -- [h3ToCenterChild](./h3#h3ToCenterChild) -- [h3ExactEdgeLengthM](./h3#h3ExactEdgeLengthM) -- [h3ExactEdgeLengthKm](./h3#h3ExactEdgeLengthKm) -- [h3ExactEdgeLengthRads](./h3#h3ExactEdgeLengthRads) -- [h3NumHexagons](./h3#h3NumHexagons) -- [h3Line](./h3#h3Line) -- [h3Distance](./h3#h3Distance) -- [h3HexRing](./h3#h3HexRing) -- [h3GetUnidirectionalEdge](./h3#h3GetUnidirectionalEdge) -- [h3UnidirectionalEdgeIsValid](./h3#h3UnidirectionalEdgeIsValid) -- [h3GetOriginIndexFromUnidirectionalEdge](./h3#h3GetOriginIndexFromUnidirectionalEdge) -- [h3GetDestinationIndexFromUnidirectionalEdge](./h3#h3GetDestinationIndexFromUnidirectionalEdge) -- [h3GetIndexesFromUnidirectionalEdge](./h3#h3GetIndexesFromUnidirectionalEdge) -- [h3GetUnidirectionalEdgesFromHexagon](./h3#h3GetUnidirectionalEdgesFromHexagon) -- [h3GetUnidirectionalEdgeBoundary](./h3#h3GetUnidirectionalEdgeBoundary) +- [h3IsValid](./h3.md#h3IsValid) +- [h3GetResolution](./h3.md#h3GetResolution) +- [h3EdgeAngle](./h3.md#h3EdgeAngle) +- [h3EdgeLengthM​](./h3.md#h3EdgeLengthM​) +- [h3EdgeLengthKm](./h3.md#h3EdgeLengthKm) +- [geoToH3](./h3.md#geoToH3) +- [h3ToGeo](./h3.md#h3ToGeo) +- [h3ToGeoBoundary](./h3.md#h3ToGeoBoundary) +- [h3kRing](./h3.md#h3kRing) +- [h3GetBaseCell](./h3.md#h3GetBaseCell) +- [h3HexAreaM2](./h3.md#h3HexAreaM2) +- [h3HexAreaKm2](./h3.md#h3HexAreaKm2) +- [h3IndexesAreNeighbors](./h3.md#h3IndexesAreNeighbors) +- [h3ToChildren](./h3.md#h3ToChildren) +- [h3ToParent](./h3.md#h3ToParent) +- [h3ToString](./h3.md#h3ToString) +- [stringToH3](./h3.md#stringToH3) +- [h3GetResolution](./h3.md#h3GetResolution) +- [h3IsResClassIII](./h3.md#h3IsResClassIII) +- [h3IsPentagon](./h3.md#h3IsPentagon) +- [h3GetFaces](./h3.md#h3GetFaces) +- [h3CellAreaM2](./h3.md#h3CellAreaM2) +- [h3CellAreaRads2](./h3.md#h3CellAreaRads2) +- [h3ToCenterChild](./h3.md#h3ToCenterChild) +- [h3ExactEdgeLengthM](./h3.md#h3ExactEdgeLengthM) +- [h3ExactEdgeLengthKm](./h3.md#h3ExactEdgeLengthKm) +- [h3ExactEdgeLengthRads](./h3.md#h3ExactEdgeLengthRads) +- [h3NumHexagons](./h3.md#h3NumHexagons) +- [h3Line](./h3.md#h3Line) +- [h3Distance](./h3.md#h3Distance) +- [h3HexRing](./h3.md#h3HexRing) +- [h3GetUnidirectionalEdge](./h3.md#h3GetUnidirectionalEdge) +- [h3UnidirectionalEdgeIsValid](./h3.md#h3UnidirectionalEdgeIsValid) +- [h3GetOriginIndexFromUnidirectionalEdge](./h3.md#h3GetOriginIndexFromUnidirectionalEdge) +- [h3GetDestinationIndexFromUnidirectionalEdge](./h3.md#h3GetDestinationIndexFromUnidirectionalEdge) +- [h3GetIndexesFromUnidirectionalEdge](./h3.md#h3GetIndexesFromUnidirectionalEdge) +- [h3GetUnidirectionalEdgesFromHexagon](./h3.md#h3GetUnidirectionalEdgesFromHexagon) +- [h3GetUnidirectionalEdgeBoundary](./h3.md#h3GetUnidirectionalEdgeBoundary) ## S2 Index Functions -- [geoToS2](./s2#geoToS2) -- [s2ToGeo](./s2#s2ToGeo) -- [s2GetNeighbors](./s2#s2GetNeighbors) -- [s2CellsIntersect](./s2#s2CellsIntersect) -- [s2CapContains](./s2#s2CapContains) -- [s2CapUnion](./s2#s2CapUnion) -- [s2RectAdd](./s2#s2RectAdd) -- [s2RectContains](./s2#s2RectContains) -- [s2RectUinion](./s2#s2RectUinion) -- [s2RectIntersection](./s2#s2RectIntersection) +- [geoToS2](./s2.md#geoToS2) +- [s2ToGeo](./s2.md#s2ToGeo) +- [s2GetNeighbors](./s2.md#s2GetNeighbors) +- [s2CellsIntersect](./s2.md#s2CellsIntersect) +- [s2CapContains](./s2.md#s2CapContains) +- [s2CapUnion](./s2.md#s2CapUnion) +- [s2RectAdd](./s2.md#s2RectAdd) +- [s2RectContains](./s2.md#s2RectContains) +- [s2RectUinion](./s2.md#s2RectUinion) +- [s2RectIntersection](./s2.md#s2RectIntersection) [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/) From 7d2e8b08830a9b7e1bea94e4988a5fb8f38b7f99 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 29 Jun 2022 22:15:12 +0200 Subject: [PATCH 281/408] clean up --- src/Common/HashTable/HashTable.h | 50 ++++++++++--------- .../cache_values_inside_hash_table_grower.xml | 8 --- 2 files changed, 26 insertions(+), 32 deletions(-) delete mode 100644 tests/performance/cache_values_inside_hash_table_grower.xml diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index b802eb0a860..e8a204c4043 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -240,40 +240,42 @@ struct HashTableGrower static constexpr auto performs_linear_probing_with_single_step = true; /// The size of the hash table in the cells. - size_t bufSize() const { return 1ULL << size_degree; } + size_t bufSize() const { return 1ULL << size_degree; } - size_t maxFill() const { return 1ULL << (size_degree - 1); } - size_t mask() const { return bufSize() - 1; } + size_t maxFill() const { return 1ULL << (size_degree - 1); } + size_t mask() const { return bufSize() - 1; } /// From the hash value, get the cell number in the hash table. - size_t place(size_t x) const { return x & mask(); } + size_t place(size_t x) const { return x & mask(); } /// The next cell in the collision resolution chain. - size_t next(size_t pos) const - { - ++pos; - return pos & mask(); - } + size_t next(size_t pos) const { ++pos; return pos & mask(); } /// Whether the hash table is sufficiently full. You need to increase the size of the hash table, or remove something unnecessary from it. - bool overflow(size_t elems) const { return elems > maxFill(); } + bool overflow(size_t elems) const { return elems > maxFill(); } /// Increase the size of the hash table. - void increaseSize() { size_degree += size_degree >= 23 ? 1 : 2; } + void increaseSize() + { + size_degree += size_degree >= 23 ? 1 : 2; + } /// Set the buffer size by the number of elements in the hash table. Used when deserializing a hash table. void set(size_t num_elems) { size_degree = num_elems <= 1 - ? initial_size_degree - : ((initial_size_degree > static_cast(log2(num_elems - 1)) + 2) ? initial_size_degree - : (static_cast(log2(num_elems - 1)) + 2)); + ? initial_size_degree + : ((initial_size_degree > static_cast(log2(num_elems - 1)) + 2) + ? initial_size_degree + : (static_cast(log2(num_elems - 1)) + 2)); } - void setBufSize(size_t buf_size_) { size_degree = static_cast(log2(buf_size_ - 1) + 1); } + void setBufSize(size_t buf_size_) + { + size_degree = static_cast(log2(buf_size_ - 1) + 1); + } }; - /** Determines the size of the hash table, and when and how much it should be resized. * This structure is aligned to cache line boundary and also occupies it all. * Precalculates some values to speed up lookups and insertion into the HashTable (and thus has bigger memory footprint than HashTableGrower). @@ -284,8 +286,8 @@ class alignas(64) HashTableGrowerWithPrecalculation /// The state of this structure is enough to get the buffer size of the hash table. UInt8 size_degree = initial_size_degree; - size_t cached_mask = (1ULL << initial_size_degree) - 1; - size_t cached_max_fill = 1ULL << (initial_size_degree - 1); + size_t precalculated_mask = (1ULL << initial_size_degree) - 1; + size_t precalculated_max_fill = 1ULL << (initial_size_degree - 1); public: UInt8 sizeDegree() const { return size_degree; } @@ -293,8 +295,8 @@ public: void increaseSizeDegree(UInt8 delta) { size_degree += delta; - cached_mask = (1ULL << size_degree) - 1; - cached_max_fill = 1ULL << (size_degree - 1); + precalculated_mask = (1ULL << size_degree) - 1; + precalculated_max_fill = 1ULL << (size_degree - 1); } static constexpr auto initial_count = 1ULL << initial_size_degree; @@ -303,16 +305,16 @@ public: static constexpr auto performs_linear_probing_with_single_step = true; /// The size of the hash table in the cells. - size_t bufSize() const { return 1ULL << size_degree; } + size_t bufSize() const { return 1ULL << size_degree; } /// From the hash value, get the cell number in the hash table. - size_t place(size_t x) const { return x & cached_mask; } + size_t place(size_t x) const { return x & precalculated_mask; } /// The next cell in the collision resolution chain. - size_t next(size_t pos) const { return (pos + 1) & cached_mask; } + size_t next(size_t pos) const { return (pos + 1) & precalculated_mask; } /// Whether the hash table is sufficiently full. You need to increase the size of the hash table, or remove something unnecessary from it. - bool overflow(size_t elems) const { return elems > cached_max_fill; } + bool overflow(size_t elems) const { return elems > precalculated_max_fill; } /// Increase the size of the hash table. void increaseSize() { increaseSizeDegree(size_degree >= 23 ? 1 : 2); } diff --git a/tests/performance/cache_values_inside_hash_table_grower.xml b/tests/performance/cache_values_inside_hash_table_grower.xml deleted file mode 100644 index f915c1268bf..00000000000 --- a/tests/performance/cache_values_inside_hash_table_grower.xml +++ /dev/null @@ -1,8 +0,0 @@ - - select * from numbers_mt(200) group by number format Null - select * from numbers_mt(1e4) group by number format Null - select * from numbers_mt(1e5) group by number format Null - select * from numbers_mt(1e6) group by number format Null - select * from numbers_mt(1e7) group by number format Null - select * from numbers_mt(1e8) group by number format Null - From 81bb2242fdd2c98af7ad0dcd66d3b2275f0aadba Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 29 Jun 2022 15:08:16 +0000 Subject: [PATCH 282/408] Fix countSubstrings() & position() on patterns with 0-bytes SQL functions countSubstrings(), countSubstringsCaseInsensitive(), countSubstringsUTF8(), position(), positionCaseInsensitive(), positionUTF8() with non-const pattern argument use fallback sorters LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher which call ::strstr(), resp. ::strcasestr(). These functions assume that the haystack is 0-terminated and they even document that. However, the callers did not check if the haystack contains 0-byte (perhaps because its sort of expensive). As a consequence, if the haystack contained a zero byte in it's payload, matches behind this zero byte were ignored. create table t (id UInt32, pattern String) engine = MergeTree() order by id; insert into t values (1, 'x'); select countSubstrings('aaaxxxaa\0xxx', pattern) from t; We returned 3 before this commit, now we return 6 --- src/Common/StringSearcher.h | 69 +++++++------------ src/Functions/PositionImpl.h | 6 +- ...sition_countsubstrings_zero_byte.reference | 12 ++++ ...346_position_countsubstrings_zero_byte.sql | 24 +++++++ 4 files changed, 62 insertions(+), 49 deletions(-) create mode 100644 tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.reference create mode 100644 tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.sql diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index a82115a9923..7d669ddd369 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -826,66 +826,43 @@ using UTF8CaseInsensitiveStringSearcher = StringSearcher; using ASCIICaseSensitiveTokenSearcher = TokenSearcher; using ASCIICaseInsensitiveTokenSearcher = TokenSearcher; - -/** Uses functions from libc. - * It makes sense to use only with short haystacks when cheap initialization is required. - * There is no option for case-insensitive search for UTF-8 strings. - * It is required that strings are zero-terminated. - */ - -struct LibCASCIICaseSensitiveStringSearcher : public StringSearcherBase +/// Use only with short haystacks where cheap initialization is required. +template +struct StdLibASCIIStringSearcher : public StringSearcherBase { - const char * const needle; + const char * const needle_start; + const char * const needle_end; template requires (sizeof(CharT) == 1) - LibCASCIICaseSensitiveStringSearcher(const CharT * const needle_, const size_t /* needle_size */) - : needle(reinterpret_cast(needle_)) {} + StdLibASCIIStringSearcher(const CharT * const needle_start_, const size_t needle_size_) + : needle_start{reinterpret_cast(needle_start_)} + , needle_end{reinterpret_cast(needle_start) + needle_size_} + {} template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const + const CharT * search(const CharT * haystack_start, const CharT * const haystack_end) const { - const auto * res = strstr(reinterpret_cast(haystack), reinterpret_cast(needle)); - if (!res) - return haystack_end; - return reinterpret_cast(res); + if constexpr (CaseInsensitive) + { + return std::search( + haystack_start, haystack_end, needle_start, needle_end, + [](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);}); + } + else + { + return std::search( + haystack_start, haystack_end, needle_start, needle_end); + } } template requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const + const CharT * search(const CharT * haystack_start, const size_t haystack_length) const { - return search(haystack, haystack + haystack_size); + return search(haystack_start, haystack_start + haystack_length); } }; -struct LibCASCIICaseInsensitiveStringSearcher : public StringSearcherBase -{ - const char * const needle; - - template - requires (sizeof(CharT) == 1) - LibCASCIICaseInsensitiveStringSearcher(const CharT * const needle_, const size_t /* needle_size */) - : needle(reinterpret_cast(needle_)) {} - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const CharT * const haystack_end) const - { - const auto * res = strcasestr(reinterpret_cast(haystack), reinterpret_cast(needle)); - if (!res) - return haystack_end; - return reinterpret_cast(res); - } - - template - requires (sizeof(CharT) == 1) - const CharT * search(const CharT * haystack, const size_t haystack_size) const - { - return search(haystack, haystack + haystack_size); - } -}; - - } diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 5380fcc36d9..76f10373a58 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -26,7 +26,7 @@ struct PositionCaseSensitiveASCII using MultiSearcherInBigHaystack = MultiVolnitsky; /// For searching single substring, that is different each time. This object is created for each row of data. It must have cheap initialization. - using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint) { @@ -62,7 +62,7 @@ struct PositionCaseInsensitiveASCII /// `Volnitsky` is not used here, because one person has measured that this is better. It will be good if you question it. using SearcherInBigHaystack = ASCIICaseInsensitiveStringSearcher; using MultiSearcherInBigHaystack = MultiVolnitskyCaseInsensitive; - using SearcherInSmallHaystack = LibCASCIICaseInsensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t /*haystack_size_hint*/) { @@ -94,7 +94,7 @@ struct PositionCaseSensitiveUTF8 { using SearcherInBigHaystack = VolnitskyUTF8; using MultiSearcherInBigHaystack = MultiVolnitskyUTF8; - using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher; + using SearcherInSmallHaystack = StdLibASCIIStringSearcher; static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint) { diff --git a/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.reference b/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.reference new file mode 100644 index 00000000000..2b70bdc272e --- /dev/null +++ b/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.reference @@ -0,0 +1,12 @@ +6 +6 +6 +6 +6 +6 +7 +7 +7 +7 +7 +7 diff --git a/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.sql b/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.sql new file mode 100644 index 00000000000..6208baf41c4 --- /dev/null +++ b/tests/queries/0_stateless/02346_position_countsubstrings_zero_byte.sql @@ -0,0 +1,24 @@ +drop table if exists tab; + +create table tab (id UInt32, haystack String, pattern String) engine = MergeTree() order by id; +insert into tab values (1, 'aaaxxxaa\0xxx', 'x'); + +select countSubstrings('aaaxxxaa\0xxx', pattern) from tab where id = 1; +select countSubstringsCaseInsensitive('aaaxxxaa\0xxx', pattern) from tab where id = 1; +select countSubstringsCaseInsensitiveUTF8('aaaxxxaa\0xxx', pattern) from tab where id = 1; + +select countSubstrings(haystack, pattern) from tab where id = 1; +select countSubstringsCaseInsensitive(haystack, pattern) from tab where id = 1; +select countSubstringsCaseInsensitiveUTF8(haystack, pattern) from tab where id = 1; + +insert into tab values (2, 'aaaaa\0x', 'x'); + +select position('aaaaa\0x', pattern) from tab where id = 2; +select positionCaseInsensitive('aaaaa\0x', pattern) from tab where id = 2; +select positionCaseInsensitiveUTF8('aaaaa\0x', pattern) from tab where id = 2; + +select position(haystack, pattern) from tab where id = 2; +select positionCaseInsensitive(haystack, pattern) from tab where id = 2; +select positionCaseInsensitiveUTF8(haystack, pattern) from tab where id = 2; + +drop table if exists tab; From 4bf1443daecea075ca31de92285b65d0a59b66f5 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jun 2022 01:00:08 +0300 Subject: [PATCH 283/408] Update download.sh --- docker/test/performance-comparison/download.sh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index a125b1816e5..da866aabd9a 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -73,15 +73,8 @@ function download ) & wait - pwd - ls - ls db0 - ls db0/metadata - cd db0 && echo "ATTACH DATABASE default ENGINE=Ordinary" > metadata/default.sql - cd db0 && echo "ATTACH DATABASE datasets ENGINE=Ordinary" > metadata/datasets.sql - pwd - ls - ls db0 + echo "ATTACH DATABASE default ENGINE=Ordinary" > db0/metadata/default.sql + echo "ATTACH DATABASE datasets ENGINE=Ordinary" > db0/metadata/datasets.sql ls db0/metadata } From 8726c7df262975184f06aee43332a7b80fd95265 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 01:21:49 +0200 Subject: [PATCH 284/408] Strangest fix for HDFS test --- .../ObjectStorages/HDFS/registerDiskHDFS.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp index 106ecee4702..2c3a7454d4b 100644 --- a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp +++ b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace DB @@ -37,11 +38,12 @@ void registerDiskHDFS(DiskFactory & factory) /// FIXME Cache currently unsupported :( ObjectStoragePtr hdfs_storage = std::make_unique(nullptr, uri, std::move(settings), config); - auto metadata_disk = prepareForLocalMetadata(name, config, config_prefix, context_).second; + auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context_); + auto metadata_storage = std::make_shared(metadata_disk, uri); uint64_t copy_thread_pool_size = config.getUInt(config_prefix + ".thread_pool_size", 16); - return std::make_shared( + std::shared_ptr disk_result = std::make_shared( name, uri, "DiskHDFS", @@ -50,6 +52,22 @@ void registerDiskHDFS(DiskFactory & factory) DiskType::HDFS, /* send_metadata = */ false, copy_thread_pool_size); + +#ifdef NDEBUG + bool use_cache = true; +#else + /// Current S3 cache implementation lead to allocations in destructor of + /// read buffer. + bool use_cache = false; +#endif + + if (config.getBool(config_prefix + ".cache_enabled", use_cache)) + { + String cache_path = config.getString(config_prefix + ".cache_path", context_->getPath() + "disks/" + name + "/cache/"); + disk_result = wrapWithCache(disk_result, "hdfs-cache", cache_path, metadata_path); + } + + return std::make_shared(disk_result); }; factory.registerDiskType("hdfs", creator); From d435532c68afad97e1527864a678a12b15b85a23 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 29 Jun 2022 23:24:20 +0000 Subject: [PATCH 285/408] Adapt range search algorithm to high cardinality case + range search done in steps of some number of rows. Controled by new setting `distinct_in_order_range_search_step`. By default 0, i.e. whole chunk + before start binary search, linear probing is done on each step (32 rows currently) --- src/Core/Settings.h | 3 +- src/Interpreters/InterpreterSelectQuery.cpp | 8 ++- .../InterpreterSelectWithUnionQuery.cpp | 8 ++- src/Processors/QueryPlan/DistinctStep.cpp | 6 +- src/Processors/QueryPlan/DistinctStep.h | 4 +- .../DistinctSortedChunkTransform.cpp | 65 +++++++++++++++---- .../Transforms/DistinctSortedChunkTransform.h | 4 +- tests/performance/distinct_in_order.xml | 38 ++++++++--- 8 files changed, 105 insertions(+), 31 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a4a19c280d3..d8080fc427d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -606,7 +606,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ - M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 1) \ + M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ + M(UInt64, distinct_in_order_range_search_step, 0, "Setting for DISTINCT in order optimization. TBD", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a2902b235c8..c7149cd30dc 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2573,7 +2573,13 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); auto distinct_step = std::make_unique( - query_plan.getCurrentDataStream(), limits, limit_for_distinct, columns, pre_distinct, settings.optimize_distinct_in_order); + query_plan.getCurrentDataStream(), + limits, + limit_for_distinct, + columns, + pre_distinct, + settings.optimize_distinct_in_order, + settings.distinct_in_order_range_search_step); if (pre_distinct) distinct_step->setStepDescription("Preliminary DISTINCT"); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index c9ba5bcd653..2929d3cc893 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -319,7 +319,13 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); auto distinct_step = std::make_unique( - query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, settings.optimize_distinct_in_order); + query_plan.getCurrentDataStream(), + limits, + 0, + result_header.getNames(), + false, + settings.optimize_distinct_in_order, + settings.distinct_in_order_range_search_step); query_plan.addStep(std::move(distinct_step)); } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index f4baa175ab6..c182d336b35 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -59,7 +59,8 @@ DistinctStep::DistinctStep( UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, - bool optimize_distinct_in_order_) + bool optimize_distinct_in_order_, + UInt64 distinct_in_order_range_search_step_) : ITransformingStep( input_stream_, input_stream_.header, @@ -67,6 +68,7 @@ DistinctStep::DistinctStep( , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) + , distinct_in_order_range_search_step(distinct_in_order_range_search_step_) , pre_distinct(pre_distinct_) , optimize_distinct_in_order(optimize_distinct_in_order_) { @@ -105,7 +107,7 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil return nullptr; return std::make_shared( - header, set_size_limits, limit_hint, distinct_sort_desc, columns); + header, set_size_limits, limit_hint, distinct_sort_desc, columns, distinct_in_order_range_search_step); }); return; } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index dc734a58704..1d678ac3144 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -15,7 +15,8 @@ public: UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, /// If is enabled, execute distinct for separate streams. Otherwise, merge streams. - bool optimize_distinct_in_order_); + bool optimize_distinct_in_order_, + UInt64 distinct_in_order_range_search_step); String getName() const override { return "Distinct"; } @@ -30,6 +31,7 @@ private: SizeLimits set_size_limits; UInt64 limit_hint; Names columns; + UInt64 distinct_in_order_range_search_step = 0; bool pre_distinct; bool optimize_distinct_in_order; }; diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp index 70548aa4396..89fa675dbc7 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp @@ -13,11 +13,13 @@ DistinctSortedChunkTransform::DistinctSortedChunkTransform( const SizeLimits & output_size_limits_, UInt64 limit_hint_, const SortDescription & sorted_columns_descr_, - const Names & source_columns) + const Names & source_columns, + size_t range_search_step_) : ISimpleTransform(header_, header_, true) , limit_hint(limit_hint_) , output_size_limits(output_size_limits_) , sorted_columns_descr(sorted_columns_descr_) + , range_search_step(range_search_step_) { /// calculate sorted columns positions sorted_columns_pos.reserve(sorted_columns_descr.size()); @@ -124,20 +126,55 @@ bool DistinctSortedChunkTransform::isCurrentKey(const size_t row_pos) const size_t DistinctSortedChunkTransform::getRangeEnd(size_t range_begin, size_t range_end) const { - size_t low = range_begin; - size_t high = range_end - 1; - while (low <= high) - { - size_t mid = low + (high - low) / 2; - if (isCurrentKey(mid)) - low = mid + 1; - else - { - high = mid - 1; - range_end = mid; - } + assert(range_begin < range_end); + + // probe latest row + if (isCurrentKey(range_end-1)) { + return range_end; } - return range_end; + + auto find_range_end = [this](size_t begin, size_t end) -> size_t + { + const size_t linear_probe_threadhold = 32; + size_t linear_probe_end = begin + linear_probe_threadhold; + if (linear_probe_end > end) + linear_probe_end = end; + + for(size_t pos=begin; pos < linear_probe_end; ++pos) + { + if (!isCurrentKey(pos)) + return pos; + } + + size_t low = linear_probe_end; + size_t high = end - 1; + while (low <= high) + { + size_t mid = low + (high - low) / 2; + if (isCurrentKey(mid)) + low = mid + 1; + else + { + high = mid - 1; + end = mid; + } + } + return end; + }; + + const size_t step = range_search_step; + if (!step) + return find_range_end(range_begin, range_end); + + size_t begin = range_begin; + while (begin + step <= range_end) + { + const size_t pos = find_range_end(begin, begin + step); + if (pos < begin + step) + return pos; + begin += step; + } + return find_range_end(begin, range_end); } std::pair DistinctSortedChunkTransform::continueWithPrevRange(const size_t chunk_rows, IColumn::Filter & filter) diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.h b/src/Processors/Transforms/DistinctSortedChunkTransform.h index 2e21c36f7dc..983b976c49f 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.h +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.h @@ -32,7 +32,8 @@ public: const SizeLimits & output_size_limits_, UInt64 limit_hint_, const SortDescription & sorted_columns_descr_, - const Names & source_columns_); + const Names & source_columns_, + size_t range_search_step); String getName() const override { return "DistinctSortedChunkTransform"; } @@ -67,6 +68,7 @@ private: ColumnRawPtrs other_columns; // used during processing MutableColumns current_key; + const size_t range_search_step = 0; }; } diff --git a/tests/performance/distinct_in_order.xml b/tests/performance/distinct_in_order.xml index 742fcefc0df..ea76fedcc34 100644 --- a/tests/performance/distinct_in_order.xml +++ b/tests/performance/distinct_in_order.xml @@ -1,15 +1,33 @@ - CREATE TABLE distinct_cardinality (high UInt64, middle UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, middle, low) - INSERT INTO distinct_cardinality SELECT number, number % 1000, number % 10 from numbers(1000000) + + DROP TABLE IF EXISTS distinct_cardinality_high + CREATE TABLE distinct_cardinality_high (high UInt64, medium UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, medium, low) + INSERT INTO distinct_cardinality_high SELECT number % 10000, number % 1000, number % 100 from numbers(1000000) - select distinct high from distinct_cardinality - select distinct high, low from distinct_cardinality - select distinct high, middle from distinct_cardinality - select distinct high, middle, low from distinct_cardinality + select distinct high from distinct_cardinality_high + select distinct high, low from distinct_cardinality_high + select distinct high, medium from distinct_cardinality_high + select distinct high, medium, low from distinct_cardinality_high - select distinct high, middle from distinct_cardinality order by middle - select distinct high, low from distinct_cardinality order by low - select distinct high, middle, low from distinct_cardinality order by high + select distinct high, medium from distinct_cardinality_high order by medium + select distinct high, low from distinct_cardinality_high order by low + select distinct high, medium, low from distinct_cardinality_high order by high - DROP TABLE IF EXISTS distinct_cardinality + DROP TABLE IF EXISTS distinct_cardinality_high + + + DROP TABLE IF EXISTS distinct_cardinality_low + CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium, high) + INSERT INTO distinct_cardinality_low SELECT number % 100, number % 1000, number % 10000 from numbers(1000000) + + select distinct low from distinct_cardinality_low + select distinct low, medium from distinct_cardinality_low + select distinct low, high from distinct_cardinality_low + select distinct low, medium, high from distinct_cardinality_low + + select distinct low, medium from distinct_cardinality_low order by medium + select distinct low, high from distinct_cardinality_low order by high + select distinct low, medium, high from distinct_cardinality_low order by low + + DROP TABLE IF EXISTS distinct_cardinality_low From 572e5402173cee7f45afe9a0b5c38e0ac825acee Mon Sep 17 00:00:00 2001 From: GruffGemini <43479425+GruffGemini@users.noreply.github.com> Date: Thu, 30 Jun 2022 09:29:18 +0300 Subject: [PATCH 286/408] docs (en, group-by.md): fixed broken links --- docs/en/sql-reference/statements/select/group-by.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index e02db6d4f6b..1d6edc5fa3d 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -312,11 +312,11 @@ The aggregation can be performed more effectively, if a table is sorted by some ### GROUP BY in External Memory You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`. -The [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. +The [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled. When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the data and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn’t dumped, then stage 2 might require up to the same amount of memory as in stage 1. -For example, if [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and `max_memory_usage` to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. +For example, if [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and `max_memory_usage` to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`. With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1. From c64cfbcf4fec90c3acbb3e1f0ec27ee3df04130e Mon Sep 17 00:00:00 2001 From: GruffGemini <43479425+GruffGemini@users.noreply.github.com> Date: Thu, 30 Jun 2022 09:32:42 +0300 Subject: [PATCH 287/408] Update group-by.md --- docs/zh/sql-reference/statements/select/group-by.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/sql-reference/statements/select/group-by.md b/docs/zh/sql-reference/statements/select/group-by.md index bebe4b164bf..33fa27b8dd9 100644 --- a/docs/zh/sql-reference/statements/select/group-by.md +++ b/docs/zh/sql-reference/statements/select/group-by.md @@ -116,11 +116,11 @@ GROUP BY domain ### 在外部存储器中分组 {#select-group-by-in-external-memory} 您可以启用将临时数据转储到磁盘以限制内存使用期间 `GROUP BY`. -该 [max_bytes_before_external_group_by](../../../operations/settings/settings.md#settings-max_bytes_before_external_group_by) 设置确定倾销的阈值RAM消耗 `GROUP BY` 临时数据到文件系统。 如果设置为0（默认值），它将被禁用。 +该 [max_bytes_before_external_group_by](../../../operations/settings/query-complexity.md#settings-max_bytes_before_external_group_by) 设置确定倾销的阈值RAM消耗 `GROUP BY` 临时数据到文件系统。 如果设置为0（默认值），它将被禁用。 使用时 `max_bytes_before_external_group_by`，我们建议您设置 `max_memory_usage` 大约两倍高。 这是必要的，因为聚合有两个阶段：读取数据和形成中间数据（1）和合并中间数据（2）。 将数据转储到文件系统只能在阶段1中发生。 如果未转储临时数据，则阶段2可能需要与阶段1相同的内存量。 -例如，如果 [max_memory_usage](../../../operations/settings/settings.md#settings_max_memory_usage) 设置为10000000000，你想使用外部聚合，这是有意义的设置 `max_bytes_before_external_group_by` 到10000000000，和 `max_memory_usage` 到20000000000。 当触发外部聚合（如果至少有一个临时数据转储）时，RAM的最大消耗仅略高于 `max_bytes_before_external_group_by`. +例如，如果 [max_memory_usage](../../../operations/settings/query-complexity.md#settings_max_memory_usage) 设置为10000000000，你想使用外部聚合，这是有意义的设置 `max_bytes_before_external_group_by` 到10000000000，和 `max_memory_usage` 到20000000000。 当触发外部聚合（如果至少有一个临时数据转储）时，RAM的最大消耗仅略高于 `max_bytes_before_external_group_by`. 通过分布式查询处理，在远程服务器上执行外部聚合。 为了使请求者服务器只使用少量的RAM，设置 `distributed_aggregation_memory_efficient` 到1。 From 47ac47350bbd59766fa1cb2c67ebae13dfb2140c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 22 Jun 2022 22:30:50 +0200 Subject: [PATCH 288/408] Store projections in backups. --- .../MergeTree/DataPartStorageOnDisk.cpp | 36 +++++++++---- .../MergeTree/DataPartStorageOnDisk.h | 1 + src/Storages/MergeTree/IDataPartStorage.h | 1 + src/Storages/MergeTree/MergeTreeData.cpp | 47 ++++++++++++----- .../test_backup_restore_new/test.py | 52 +++++++++++++++++++ .../test_backup_restore_on_cluster/test.py | 43 +++++++++++++++ 6 files changed, 156 insertions(+), 24 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 083cbc90cb1..68a2a0cbd15 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -615,36 +615,50 @@ void DataPartStorageOnDisk::backup( TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const { - auto disk = volume->getDisk(); + fs::path part_path_on_disk = fs::path{root_path} / part_dir; + fs::path part_path_in_backup = fs::path{path_in_backup} / part_dir; + auto disk = volume->getDisk(); auto temp_dir_it = temp_dirs.find(disk); if (temp_dir_it == temp_dirs.end()) - temp_dir_it = temp_dirs.emplace(disk, std::make_shared(disk, "tmp/backup_")).first; + temp_dir_it = temp_dirs.emplace(disk, std::make_shared(disk, "tmp/backup/")).first; auto temp_dir_owner = temp_dir_it->second; fs::path temp_dir = temp_dir_owner->getPath(); - - fs::path temp_part_dir = temp_dir / part_dir; + fs::path temp_part_dir = temp_dir / part_path_in_backup.relative_path(); disk->createDirectories(temp_part_dir); + /// For example, + /// part_path_in_backup = /data/test/table/0_1_1_0 + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0 + /// tmp_part_dir = tmp/backup/1aaaaaa/data/test/table/0_1_1_0 + /// Or, for projections: + /// part_path_in_backup = /data/test/table/0_1_1_0/prjmax.proj + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0/prjmax.proj + /// tmp_part_dir = tmp/backup/1aaaaaa/data/test/table/0_1_1_0/prjmax.proj + for (const auto & [filepath, checksum] : checksums.files) { - String relative_filepath = fs::path(part_dir) / filepath; - String full_filepath = fs::path(root_path) / part_dir / filepath; + if (filepath.ends_with(".proj")) + continue; /// Skip *.proj files - they're actually directories and will be handled. + String filepath_on_disk = part_path_on_disk / filepath; + String filepath_in_backup = part_path_in_backup / filepath; String hardlink_filepath = temp_part_dir / filepath; - disk->createHardLink(full_filepath, hardlink_filepath); + + disk->createHardLink(filepath_on_disk, hardlink_filepath); UInt128 file_hash{checksum.file_hash.first, checksum.file_hash.second}; backup_entries.emplace_back( - relative_filepath, + filepath_in_backup, std::make_unique(disk, hardlink_filepath, checksum.file_size, file_hash, temp_dir_owner)); } for (const auto & filepath : files_without_checksums) { - String relative_filepath = fs::path(part_dir) / filepath; - String full_filepath = fs::path(root_path) / part_dir / filepath; - backup_entries.emplace_back(relative_filepath, std::make_unique(disk, full_filepath)); + String filepath_on_disk = part_path_on_disk / filepath; + String filepath_in_backup = part_path_in_backup / filepath; + backup_entries.emplace_back(filepath_in_backup, std::make_unique(disk, filepath_on_disk)); } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index d6fcb2f1442..bb1a8879d63 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -88,6 +88,7 @@ public: TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const override; DataPartStoragePtr freeze( diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 0e165e74ed0..72810680812 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -173,6 +173,7 @@ public: TemporaryFilesOnDisks & temp_dirs, const MergeTreeDataPartChecksums & checksums, const NameSet & files_without_checksums, + const String & path_in_backup, BackupEntries & backup_entries) const = 0; /// Creates hardlinks into 'to/dir_path' for every file in data part. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2c4dcfa05ee..02d4d9ebe8b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3993,14 +3993,23 @@ BackupEntries MergeTreeData::backupParts(const ContextPtr & local_context, const BackupEntries backup_entries; std::map> temp_dirs; - fs::path data_path_in_backup_fs = data_path_in_backup; for (const auto & part : data_parts) - part->data_part_storage->backup(temp_dirs, part->checksums, part->getFileNamesWithoutChecksums(), backup_entries); + { + part->data_part_storage->backup( + temp_dirs, part->checksums, part->getFileNamesWithoutChecksums(), data_path_in_backup, backup_entries); - /// TODO: try to write better code later. - for (auto & entry : backup_entries) - entry.first = data_path_in_backup_fs / entry.first; + auto projection_parts = part->getProjectionParts(); + for (const auto & [projection_name, projection_part] : projection_parts) + { + projection_part->data_part_storage->backup( + temp_dirs, + projection_part->checksums, + projection_part->getFileNamesWithoutChecksums(), + fs::path{data_path_in_backup} / part->name, + backup_entries); + } + } return backup_entries; } @@ -4116,27 +4125,39 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r auto disk = reservation->getDisk(); String part_name = part_info.getPartName(); - auto temp_part_dir_owner = std::make_shared(disk, relative_data_path + "restoring_" + part_name + "_"); - String temp_part_dir = temp_part_dir_owner->getPath(); + auto temp_part_dir_owner = std::make_shared(disk, fs::path{relative_data_path} / ("restoring_" + part_name + "_")); + fs::path temp_part_dir = temp_part_dir_owner->getPath(); disk->createDirectories(temp_part_dir); + std::unordered_set subdirs; - assert(temp_part_dir.starts_with(relative_data_path)); - String relative_temp_part_dir = temp_part_dir.substr(relative_data_path.size()); + /// temp_part_name = "restoring__", for example "restoring_0_1_1_0_1baaaaa" + String temp_part_name = temp_part_dir.filename(); for (const String & filename : filenames) { + /// Needs to create subdirectories before copying the files. Subdirectories are used to represent projections. + auto separator_pos = filename.rfind('/'); + if (separator_pos != String::npos) + { + String subdir = filename.substr(0, separator_pos); + if (subdirs.emplace(subdir).second) + disk->createDirectories(temp_part_dir / subdir); + } + + /// TODO Transactions: Decide what to do with version metadata (if any). Let's just skip it for now. + if (filename.ends_with(IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME)) + continue; + auto backup_entry = backup->readFile(part_path_in_backup_fs / filename); auto read_buffer = backup_entry->getReadBuffer(); - auto write_buffer = disk->writeFile(fs::path(temp_part_dir) / filename); + auto write_buffer = disk->writeFile(temp_part_dir / filename); copyData(*read_buffer, *write_buffer); reservation->update(reservation->getSize() - backup_entry->getSize()); } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, relative_temp_part_dir); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, temp_part_name); auto part = createPart(part_name, part_info, data_part_storage); - /// TODO Transactions: Decide what to do with version metadata (if any). Let's just remove it for now. - disk->removeFileIfExists(fs::path(temp_part_dir) / IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME); part->version.setCreationTID(Tx::PrehistoricTID, nullptr); part->loadColumnsChecksumsIndexes(false, true); diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 3996a31e7c9..35545e95537 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -711,3 +711,55 @@ def test_system_users_async(): instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH sha256_password SETTINGS custom_c = 3\n" ) + + +def test_projection(): + create_and_fill_table(n=3) + + instance.query("ALTER TABLE test.table ADD PROJECTION prjmax (SELECT MAX(x))") + instance.query(f"INSERT INTO test.table VALUES (100, 'a'), (101, 'b')") + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "2\n" + ) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + + assert os.path.exists( + os.path.join( + get_path_to_backup(backup_name), "data/test/table/1_5_5_0/data.bin" + ) + ) + + assert os.path.exists( + os.path.join( + get_path_to_backup(backup_name), + "data/test/table/1_5_5_0/prjmax.proj/data.bin", + ) + ) + + instance.query("DROP TABLE test.table") + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "0\n" + ) + + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[0, "0"], [1, "1"], [2, "2"], [100, "a"], [101, "b"]] + ) + + assert ( + instance.query( + "SELECT count() FROM system.projection_parts WHERE database='test' AND table='table' AND name='prjmax'" + ) + == "2\n" + ) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 6264959fbce..63198f40af9 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -514,3 +514,46 @@ def test_system_users(): node1.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS custom_a = 123\n" ) assert node1.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON default.tbl TO u1\n" + + +def test_projection(): + node1.query( + "CREATE TABLE tbl ON CLUSTER 'cluster' (x UInt32, y String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}') " + "ORDER BY y PARTITION BY x%10" + ) + node1.query(f"INSERT INTO tbl SELECT number, toString(number) FROM numbers(3)") + + node1.query("ALTER TABLE tbl ADD PROJECTION prjmax (SELECT MAX(x))") + node1.query(f"INSERT INTO tbl VALUES (100, 'a'), (101, 'b')") + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "2\n" + ) + + backup_name = new_backup_name() + node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") + + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "0\n" + ) + + node1.query(f"RESTORE TABLE tbl FROM {backup_name}") + + assert node1.query("SELECT * FROM tbl ORDER BY x") == TSV( + [[0, "0"], [1, "1"], [2, "2"], [100, "a"], [101, "b"]] + ) + + assert ( + node1.query( + "SELECT count() FROM system.projection_parts WHERE database='default' AND table='tbl' AND name='prjmax'" + ) + == "2\n" + ) From 64b51a3772f3c03043f9530dd7d434075b045fb8 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 23 Jun 2022 00:56:41 +0200 Subject: [PATCH 289/408] Improve gathering metadata for backup. --- src/Backups/BackupEntriesCollector.cpp | 537 ++++++++++-------- src/Backups/BackupEntriesCollector.h | 56 +- src/Backups/RestorerFromBackup.cpp | 7 +- src/Backups/RestorerFromBackup.h | 1 + src/Databases/DatabaseMemory.cpp | 48 ++ src/Databases/DatabaseMemory.h | 2 + src/Databases/DatabaseReplicated.cpp | 28 +- src/Databases/DatabaseReplicated.h | 2 +- src/Databases/DatabasesCommon.cpp | 45 +- src/Databases/DatabasesCommon.h | 5 +- src/Databases/IDatabase.cpp | 18 +- src/Databases/IDatabase.h | 14 +- src/Storages/IStorage.cpp | 29 +- src/Storages/IStorage.h | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 76 ++- src/Storages/StorageReplicatedMergeTree.h | 4 +- .../test_backup_restore_new/test.py | 28 +- 17 files changed, 528 insertions(+), 378 deletions(-) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 322bc00ee3c..9ee57cb4fd5 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -27,15 +27,16 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } - -bool BackupEntriesCollector::TableKey::operator ==(const TableKey & right) const +namespace { - return (name == right.name) && (is_temporary == right.is_temporary); -} + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_char_uppercase) + { + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + return fmt::format("{}emporary table {}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(table_name)); + else + return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + } -bool BackupEntriesCollector::TableKey::operator <(const TableKey & right) const -{ - return (name < right.name) || ((name == right.name) && (is_temporary < right.is_temporary)); } std::string_view BackupEntriesCollector::toString(Stage stage) @@ -86,7 +87,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() /// Find databases and tables which we're going to put to the backup. setStage(Stage::kFindingTables); - collectDatabasesAndTablesInfo(); + gatherMetadataAndCheckConsistency(); /// Make backup entries for the definitions of the found databases. makeBackupEntriesForDatabasesDefs(); @@ -100,7 +101,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() /// Run all the tasks added with addPostCollectingTask(). setStage(Stage::kRunningPostTasks); - runPostCollectingTasks(); + runPostTasks(); /// No more backup entries or tasks are allowed after this point. setStage(Stage::kWritingBackup); @@ -156,56 +157,30 @@ void BackupEntriesCollector::calculateRootPathInBackup() } /// Finds databases and tables which we will put to the backup. -void BackupEntriesCollector::collectDatabasesAndTablesInfo() +void BackupEntriesCollector::gatherMetadataAndCheckConsistency() { bool use_timeout = (timeout.count() >= 0); auto start_time = std::chrono::steady_clock::now(); - int pass = 0; - do + int pass = 1; + for (;;) { - database_infos.clear(); - table_infos.clear(); - consistent = true; + consistency = true; /// Collect information about databases and tables specified in the BACKUP query. - for (const auto & element : backup_query_elements) - { - switch (element.type) - { - case ASTBackupQuery::ElementType::TABLE: - { - collectTableInfo({element.database_name, element.table_name}, false, element.partitions, true); - break; - } - - case ASTBackupQuery::ElementType::TEMPORARY_TABLE: - { - collectTableInfo({"", element.table_name}, true, element.partitions, true); - break; - } - - case ASTBackupQuery::ElementType::DATABASE: - { - collectDatabaseInfo(element.database_name, element.except_tables, true); - break; - } - - case ASTBackupQuery::ElementType::ALL: - { - collectAllDatabasesInfo(element.except_databases, element.except_tables); - break; - } - } - } + gatherDatabasesMetadata(); + gatherTablesMetadata(); /// We have to check consistency of collected information to protect from the case when some table or database is /// renamed during this collecting making the collected information invalid. checkConsistency(); + if (consistency) + break; + /// Two passes is absolute minimum (see `previous_table_names` & `previous_database_names`). auto elapsed = std::chrono::steady_clock::now() - start_time; - if (!consistent && (pass >= 2) && use_timeout) + if ((pass >= 2) && use_timeout) { if (elapsed > timeout) throw Exception( @@ -218,224 +193,298 @@ void BackupEntriesCollector::collectDatabasesAndTablesInfo() if (pass >= 2) LOG_WARNING(log, "Couldn't collect tables and databases to make a backup (pass #{}, elapsed {})", pass, to_string(elapsed)); ++pass; - } while (!consistent); + } LOG_INFO(log, "Will backup {} databases and {} tables", database_infos.size(), table_infos.size()); } -void BackupEntriesCollector::collectTableInfo( - const QualifiedTableName & table_name, bool is_temporary_table, const std::optional & partitions, bool throw_if_not_found) +void BackupEntriesCollector::gatherDatabasesMetadata() { - /// Gather information about the table. - DatabasePtr database; - StoragePtr storage; - TableLockHolder table_lock; - ASTPtr create_table_query; + database_infos.clear(); - TableKey table_key{table_name, is_temporary_table}; - - if (throw_if_not_found) + /// Collect information about databases and tables specified in the BACKUP query. + for (const auto & element : backup_query_elements) { - auto resolved_id = is_temporary_table - ? context->resolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) - : context->resolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); - std::tie(database, storage) = DatabaseCatalog::instance().getDatabaseAndTable(resolved_id, context); - table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - create_table_query = storage->getCreateQueryForBackup(*this); + switch (element.type) + { + case ASTBackupQuery::ElementType::TABLE: + { + gatherDatabaseMetadata( + element.database_name, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ false, + element.table_name, + /* throw_if_table_not_found= */ true, + element.partitions, + /* all_tables= */ false, + /* except_table_names= */ {}); + break; + } + + case ASTBackupQuery::ElementType::TEMPORARY_TABLE: + { + gatherDatabaseMetadata( + DatabaseCatalog::TEMPORARY_DATABASE, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ false, + element.table_name, + /* throw_if_table_not_found= */ true, + element.partitions, + /* all_tables= */ false, + /* except_table_names= */ {}); + break; + } + + case ASTBackupQuery::ElementType::DATABASE: + { + gatherDatabaseMetadata( + element.database_name, + /* throw_if_database_not_found= */ true, + /* backup_create_database_query= */ true, + /* table_name= */ {}, + /* throw_if_table_not_found= */ false, + /* partitions= */ {}, + /* all_tables= */ true, + /* except_table_names= */ element.except_tables); + break; + } + + case ASTBackupQuery::ElementType::ALL: + { + for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) + { + if (!element.except_databases.contains(database_name)) + { + gatherDatabaseMetadata( + database_name, + /* throw_if_database_not_found= */ false, + /* backup_create_database_query= */ true, + /* table_name= */ {}, + /* throw_if_table_not_found= */ false, + /* partitions= */ {}, + /* all_tables= */ true, + /* except_table_names= */ element.except_tables); + if (!consistency) + return; + } + } + break; + } + } + + if (!consistency) + return; } - else - { - auto resolved_id = is_temporary_table - ? context->tryResolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) - : context->tryResolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); - if (!resolved_id.empty()) - std::tie(database, storage) = DatabaseCatalog::instance().tryGetDatabaseAndTable(resolved_id, context); +} +void BackupEntriesCollector::gatherDatabaseMetadata( + const String & database_name, + bool throw_if_database_not_found, + bool backup_create_database_query, + const std::optional & table_name, + bool throw_if_table_not_found, + const std::optional & partitions, + bool all_tables, + const std::set & except_table_names) +{ + auto it = database_infos.find(database_name); + if (it == database_infos.end()) + { + DatabasePtr database; + if (throw_if_database_not_found) + { + database = DatabaseCatalog::instance().getDatabase(database_name); + } + else + { + database = DatabaseCatalog::instance().tryGetDatabase(database_name); + if (!database) + return; + } + + DatabaseInfo new_database_info; + new_database_info.database = database; + it = database_infos.emplace(database_name, new_database_info).first; + } + + DatabaseInfo & database_info = it->second; + + if (backup_create_database_query && !database_info.create_database_query && !DatabaseCatalog::isPredefinedDatabaseName(database_name)) + { + ASTPtr create_database_query; + try + { + create_database_query = database_info.database->getCreateDatabaseQueryForBackup(); + } + catch (...) + { + /// The database has been dropped recently. + consistency = false; + return; + } + + database_info.create_database_query = create_database_query; + const auto & create = create_database_query->as(); + + if (create.getDatabase() != database_name) + { + /// The database has been renamed recently. + consistency = false; + return; + } + } + + if (table_name) + { + auto & table_params = database_info.tables[*table_name]; + if (throw_if_table_not_found) + table_params.throw_if_table_not_found = true; + if (partitions) + { + table_params.partitions.emplace(); + insertAtEnd(*table_params.partitions, *partitions); + } + database_info.except_table_names.emplace(*table_name); + } + + if (all_tables) + { + database_info.all_tables = all_tables; + for (const auto & except_table_name : except_table_names) + if (except_table_name.first == database_name) + database_info.except_table_names.emplace(except_table_name.second); + } +} + +void BackupEntriesCollector::gatherTablesMetadata() +{ + if (!consistency) + return; + + table_infos.clear(); + for (const auto & [database_name, database_info] : database_infos) + { + const auto & database = database_info.database; + bool is_temporary_database = (database_name == DatabaseCatalog::TEMPORARY_DATABASE); + + auto filter_by_table_name = [database_info = &database_info](const String & table_name) + { + /// We skip inner tables of materialized views. + if (table_name.starts_with(".inner_id.")) + return false; + + if (database_info->tables.contains(table_name)) + return true; + + if (database_info->all_tables) + return !database_info->except_table_names.contains(table_name); + + return false; + }; + + auto db_tables = database->getTablesForBackup(filter_by_table_name, context, consistency); + + if (!consistency) + return; + + /// Check that all tables were found. + std::unordered_set found_table_names; + for (const auto & db_table : db_tables) + { + const auto & create_table_query = db_table.first; + const auto & create = create_table_query->as(); + found_table_names.emplace(create.getTable()); + + if ((is_temporary_database && !create.temporary) || (!is_temporary_database && (create.getDatabase() != database_name))) + { + consistency = false; + return; + } + } + + for (const auto & [table_name, table_info] : database_info.tables) + { + if (table_info.throw_if_table_not_found && !found_table_names.contains(table_name)) + throw Exception(ErrorCodes::UNKNOWN_TABLE, "{} not found", tableNameWithTypeToString(database_name, table_name, true)); + } + + for (const auto & db_table : db_tables) + { + const auto & create_table_query = db_table.first; + const auto & create = create_table_query->as(); + String table_name = create.getTable(); + + fs::path data_path_in_backup; + if (is_temporary_database) + { + auto table_name_in_backup = renaming_map.getNewTemporaryTableName(table_name); + data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup); + } + else + { + auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name}); + data_path_in_backup + = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); + } + + /// Add information to `table_infos`. + auto & res_table_info = table_infos[QualifiedTableName{database_name, table_name}]; + res_table_info.database = database; + res_table_info.storage = db_table.second; + res_table_info.create_table_query = create_table_query; + res_table_info.data_path_in_backup = data_path_in_backup; + + auto partitions_it = database_info.tables.find(table_name); + if (partitions_it != database_info.tables.end()) + res_table_info.partitions = partitions_it->second.partitions; + } + } +} + +void BackupEntriesCollector::lockTablesForReading() +{ + if (!consistency) + return; + + for (auto & table_info : table_infos | boost::adaptors::map_values) + { + auto storage = table_info.storage; + TableLockHolder table_lock; if (storage) { try { table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - create_table_query = storage->getCreateQueryForBackup(*this); } catch (Exception & e) { if (e.code() != ErrorCodes::TABLE_IS_DROPPED) throw; + consistency = false; + return; } } - - if (!create_table_query) - { - consistent &= !table_infos.contains(table_key); - return; - } - } - - fs::path data_path_in_backup; - if (is_temporary_table) - { - auto table_name_in_backup = renaming_map.getNewTemporaryTableName(table_name.table); - data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup); - } - else - { - auto table_name_in_backup = renaming_map.getNewTableName(table_name); - data_path_in_backup - = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); - } - - /// Check that information is consistent. - const auto & create = create_table_query->as(); - if ((create.getTable() != table_name.table) || (is_temporary_table != create.temporary) || (create.getDatabase() != table_name.database)) - { - /// Table was renamed recently. - consistent = false; - return; - } - - if (auto it = table_infos.find(table_key); it != table_infos.end()) - { - const auto & table_info = it->second; - if ((table_info.database != database) || (table_info.storage != storage)) - { - /// Table was renamed recently. - consistent = false; - return; - } - } - - /// Add information to `table_infos`. - auto & res_table_info = table_infos[table_key]; - res_table_info.database = database; - res_table_info.storage = storage; - res_table_info.table_lock = table_lock; - res_table_info.create_table_query = create_table_query; - res_table_info.data_path_in_backup = data_path_in_backup; - - if (partitions) - { - if (!res_table_info.partitions) - res_table_info.partitions.emplace(); - insertAtEnd(*res_table_info.partitions, *partitions); - } -} - -void BackupEntriesCollector::collectDatabaseInfo(const String & database_name, const std::set & except_table_names, bool throw_if_not_found) -{ - /// Gather information about the database. - DatabasePtr database; - ASTPtr create_database_query; - - if (throw_if_not_found) - { - database = DatabaseCatalog::instance().getDatabase(database_name); - create_database_query = database->getCreateDatabaseQueryForBackup(); - } - else - { - database = DatabaseCatalog::instance().tryGetDatabase(database_name); - if (!database) - { - consistent &= !database_infos.contains(database_name); - return; - } - - try - { - create_database_query = database->getCreateDatabaseQueryForBackup(); - } - catch (...) - { - /// The database has been dropped recently. - consistent &= !database_infos.contains(database_name); - return; - } - } - - /// Check that information is consistent. - const auto & create = create_database_query->as(); - if (create.getDatabase() != database_name) - { - /// Database was renamed recently. - consistent = false; - return; - } - - if (auto it = database_infos.find(database_name); it != database_infos.end()) - { - const auto & database_info = it->second; - if (database_info.database != database) - { - /// Database was renamed recently. - consistent = false; - return; - } - } - - /// Add information to `database_infos`. - auto & res_database_info = database_infos[database_name]; - res_database_info.database = database; - res_database_info.create_database_query = create_database_query; - - /// Add information about tables too. - for (auto it = database->getTablesIteratorForBackup(*this); it->isValid(); it->next()) - { - if (except_table_names.contains({database_name, it->name()})) - continue; - - collectTableInfo({database_name, it->name()}, /* is_temporary_table= */ false, {}, /* throw_if_not_found= */ false); - if (!consistent) - return; - } -} - -void BackupEntriesCollector::collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names) -{ - for (const auto & [database_name, database] : DatabaseCatalog::instance().getDatabases()) - { - if (except_database_names.contains(database_name)) - continue; - collectDatabaseInfo(database_name, except_table_names, false); - if (!consistent) - return; } } /// Check for consistency of collected information about databases and tables. void BackupEntriesCollector::checkConsistency() { - if (!consistent) + if (!consistency) return; /// Already inconsistent, no more checks necessary - /// Databases found while we were scanning tables and while we were scanning databases - must be the same. - for (const auto & [key, table_info] : table_infos) - { - auto it = database_infos.find(key.name.database); - if (it != database_infos.end()) - { - const auto & database_info = it->second; - if (database_info.database != table_info.database) - { - consistent = false; - return; - } - } - } - /// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed /// while we were scanning. std::set database_names; - std::set table_names; + std::set table_names; boost::range::copy(database_infos | boost::adaptors::map_keys, std::inserter(database_names, database_names.end())); boost::range::copy(table_infos | boost::adaptors::map_keys, std::inserter(table_names, table_names.end())); - if (!previous_database_names || !previous_table_names || (*previous_database_names != database_names) - || (*previous_table_names != table_names)) + if ((previous_database_names != database_names) || (previous_table_names != table_names)) { previous_database_names = std::move(database_names); previous_table_names = std::move(table_names); - consistent = false; + consistency = false; } } @@ -444,6 +493,9 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() { for (const auto & [database_name, database_info] : database_infos) { + if (!database_info.create_database_query) + continue; /// We don't store CREATE queries for predefined databases (see DatabaseCatalog::isPredefinedDatabaseName()). + LOG_TRACE(log, "Adding definition of database {}", backQuoteIfNeed(database_name)); ASTPtr new_create_query = database_info.create_database_query; @@ -459,22 +511,23 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() /// Calls IDatabase::backupTable() for all the tables found to make backup entries for tables. void BackupEntriesCollector::makeBackupEntriesForTablesDefs() { - for (const auto & [key, table_info] : table_infos) + for (const auto & [table_name, table_info] : table_infos) { - LOG_TRACE(log, "Adding definition of {}table {}", (key.is_temporary ? "temporary " : ""), key.name.getFullName()); + LOG_TRACE(log, "Adding definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); + bool is_temporary_database = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE); ASTPtr new_create_query = table_info.create_table_query; renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); fs::path metadata_path_in_backup; - if (key.is_temporary) + if (is_temporary_database) { - auto new_name = renaming_map.getNewTemporaryTableName(key.name.table); + auto new_name = renaming_map.getNewTemporaryTableName(table_name.table); metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(new_name) + ".sql"); } else { - auto new_name = renaming_map.getNewTableName(key.name); + auto new_name = renaming_map.getNewTableName({table_name.database, table_name.table}); metadata_path_in_backup = root_path_in_backup / "metadata" / escapeForFileName(new_name.database) / (escapeForFileName(new_name.table) + ".sql"); } @@ -488,10 +541,18 @@ void BackupEntriesCollector::makeBackupEntriesForTablesData() if (backup_settings.structure_only) return; - for (const auto & [key, table_info] : table_infos) + for (const auto & [table_name, table_info] : table_infos) { - LOG_TRACE(log, "Adding data of {}table {}", (key.is_temporary ? "temporary " : ""), key.name.getFullName()); const auto & storage = table_info.storage; + if (!storage) + { + /// This storage exists on other replica and has not been created on this replica yet. + /// We store metadata only for such tables. + /// TODO: Need special processing if it's a ReplicatedMergeTree. + continue; + } + + LOG_TRACE(log, "Adding data of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); const auto & data_path_in_backup = table_info.data_path_in_backup; const auto & partitions = table_info.partitions; storage->backupData(*this, data_path_in_backup, partitions); @@ -519,21 +580,21 @@ void BackupEntriesCollector::addBackupEntries(BackupEntries && backup_entries_) insertAtEnd(backup_entries, std::move(backup_entries_)); } -void BackupEntriesCollector::addPostCollectingTask(std::function task) +void BackupEntriesCollector::addPostTask(std::function task) { if (current_stage == Stage::kWritingBackup) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding post tasks is not allowed"); - post_collecting_tasks.push(std::move(task)); + post_tasks.push(std::move(task)); } /// Runs all the tasks added with addPostCollectingTask(). -void BackupEntriesCollector::runPostCollectingTasks() +void BackupEntriesCollector::runPostTasks() { /// Post collecting tasks can add other post collecting tasks, our code is fine with that. - while (!post_collecting_tasks.empty()) + while (!post_tasks.empty()) { - auto task = std::move(post_collecting_tasks.front()); - post_collecting_tasks.pop(); + auto task = std::move(post_tasks.front()); + post_tasks.pop(); std::move(task)(); } } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 1466815f3a7..c34c6204abb 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -49,7 +49,7 @@ public: /// Adds a function which must be called after all IStorage::backup() have finished their work on all hosts. /// This function is designed to help making a consistent in some complex cases like /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. - void addPostCollectingTask(std::function task); + void addPostTask(std::function task); /// Writing a backup includes a few stages: enum class Stage @@ -79,16 +79,31 @@ public: private: void setStage(Stage new_stage, const String & error_message = {}); + void calculateRootPathInBackup(); - void collectDatabasesAndTablesInfo(); - void collectTableInfo(const QualifiedTableName & table_name, bool is_temporary_table, const std::optional & partitions, bool throw_if_not_found); - void collectDatabaseInfo(const String & database_name, const std::set & except_table_names, bool throw_if_not_found); - void collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names); + + void gatherMetadataAndCheckConsistency(); + + void gatherDatabasesMetadata(); + + void gatherDatabaseMetadata( + const String & database_name, + bool throw_if_database_not_found, + bool backup_create_database_query, + const std::optional & table_name, + bool throw_if_table_not_found, + const std::optional & partitions, + bool all_tables, + const std::set & except_table_names); + + void gatherTablesMetadata(); + void lockTablesForReading(); void checkConsistency(); + void makeBackupEntriesForDatabasesDefs(); void makeBackupEntriesForTablesDefs(); void makeBackupEntriesForTablesData(); - void runPostCollectingTasks(); + void runPostTasks(); const ASTBackupQuery::Elements backup_query_elements; const BackupSettings backup_settings; @@ -105,6 +120,17 @@ private: { DatabasePtr database; ASTPtr create_database_query; + + struct TableParams + { + bool throw_if_table_not_found = false; + std::optional partitions; + }; + + std::unordered_map tables; + + bool all_tables = false; + std::unordered_set except_table_names; }; struct TableInfo @@ -117,22 +143,14 @@ private: std::optional partitions; }; - struct TableKey - { - QualifiedTableName name; - bool is_temporary = false; - bool operator ==(const TableKey & right) const; - bool operator <(const TableKey & right) const; - }; - std::unordered_map database_infos; - std::map table_infos; - std::optional> previous_database_names; - std::optional> previous_table_names; - bool consistent = false; + std::map table_infos; + std::set previous_database_names; + std::set previous_table_names; + bool consistency = false; BackupEntries backup_entries; - std::queue> post_collecting_tasks; + std::queue> post_tasks; }; } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 14f5b7f48f0..16ffead3976 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -102,6 +102,7 @@ RestorerFromBackup::RestorerFromBackup( , backup(backup_) , context(context_) , timeout(timeout_) + , create_table_timeout_ms(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(&Poco::Logger::get("RestorerFromBackup")) { } @@ -674,7 +675,7 @@ void RestorerFromBackup::createTables() table_key.name.getFullName(), serializeAST(*create_table_query)); - database->createTableRestoredFromBackup(create_table_query, *this); + database->createTableRestoredFromBackup(create_table_query, context, restore_coordination, create_table_timeout_ms); } table_info.created = true; @@ -689,7 +690,9 @@ void RestorerFromBackup::createTables() if (!restore_settings.allow_different_table_def) { - ASTPtr create_table_query = storage->getCreateQueryForBackup(context, nullptr); + ASTPtr create_table_query = database->getCreateTableQuery(resolved_id.table_name, context); + bool consistency = true; + storage->adjustCreateQueryForBackup(create_table_query, consistency); ASTPtr expected_create_query = table_info.create_table_query; if (serializeAST(*create_table_query) != serializeAST(*expected_create_query)) { diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 65139e0b946..86edf08b484 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -94,6 +94,7 @@ private: BackupPtr backup; ContextMutablePtr context; std::chrono::seconds timeout; + UInt64 create_table_timeout_ms; Poco::Logger * log; Stage current_stage = Stage::kPreparing; diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 5268252731f..62cee31bbad 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -145,4 +145,52 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl DatabaseCatalog::instance().updateLoadingDependencies(table_id, std::move(new_dependencies)); } +std::vector> DatabaseMemory::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const +{ + /// We need a special processing for the temporary database. + if (getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE) + return DatabaseWithOwnTablesBase::getTablesForBackup(filter, local_context, consistency); + + std::vector> res; + + /// `this->tables` for the temporary database doesn't contain real names of tables. + /// That's why we need to call Context::getExternalTables() and then resolve those names using tryResolveStorageID() below. + auto external_tables = local_context->getExternalTables(); + + for (const auto & [table_name, storage] : external_tables) + { + if (!filter(table_name)) + continue; + + bool ok = false; + + if (auto storage_id = local_context->tryResolveStorageID(StorageID{"", table_name}, Context::ResolveExternal)) + { + /// Here `storage_id.table_name` looks like looks like "_tmp_ab9b15a3-fb43-4670-abec-14a0e9eb70f1" + /// it's not the real name of the table. + if (auto create_table_query = tryGetCreateTableQuery(storage_id.table_name, local_context)) + { + const auto & create = create_table_query->as(); + if (create.getTable() == table_name) + { + storage->adjustCreateQueryForBackup(create_table_query, consistency); + if (consistency) + { + res.emplace_back(create_table_query, storage); + ok = true; + } + } + } + } + + if (!ok) + { + consistency = false; + return {}; + } + } + + return res; +} + } diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index eef9f306343..8ec216b165d 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -50,6 +50,8 @@ public: void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const override; + private: const String data_path; using NameToASTCreate = std::unordered_map; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 5a22eeaf570..6286723aaa3 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -923,7 +923,11 @@ String DatabaseReplicated::readMetadataFile(const String & table_name) const } -void DatabaseReplicated::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) +void DatabaseReplicated::createTableRestoredFromBackup( + const ASTPtr & create_table_query, + ContextMutablePtr local_context, + std::shared_ptr restore_coordination, + UInt64 timeout_ms) { /// Because of the replication multiple nodes can try to restore the same tables again and failed with "Table already exists" /// because of some table could be restored already on other node and then replicated to this node. @@ -931,29 +935,25 @@ void DatabaseReplicated::createTableRestoredFromBackup(const ASTPtr & create_tab /// IRestoreCoordination::acquireCreatingTableInReplicatedDatabase() and then for other nodes this function returns false which means /// this table is already being created by some other node. String table_name = create_table_query->as().getTable(); - if (restorer.getRestoreCoordination()->acquireCreatingTableInReplicatedDatabase(getZooKeeperPath(), table_name)) + if (restore_coordination->acquireCreatingTableInReplicatedDatabase(getZooKeeperPath(), table_name)) { - restorer.executeCreateQuery(create_table_query); + DatabaseAtomic::createTableRestoredFromBackup(create_table_query, local_context, restore_coordination, timeout_ms); } /// Wait until the table is actually created no matter if it's created by the current or another node and replicated to the /// current node afterwards. We have to wait because `RestorerFromBackup` is going to restore data of the table then. /// TODO: The following code doesn't look very reliable, probably we need to rewrite it somehow. - auto timeout = restorer.getTimeout(); - bool use_timeout = (timeout.count() >= 0); + auto timeout = std::chrono::milliseconds{timeout_ms}; auto start_time = std::chrono::steady_clock::now(); - while (!isTableExist(table_name, restorer.getContext())) + while (!isTableExist(table_name, local_context)) { waitForReplicaToProcessAllEntries(50); - if (use_timeout) - { - auto elapsed = std::chrono::steady_clock::now() - start_time; - if (elapsed > timeout) - throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, - "Couldn't restore table {}.{} on other node or sync it (elapsed {})", - backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(table_name), to_string(elapsed)); - } + auto elapsed = std::chrono::steady_clock::now() - start_time; + if (elapsed > timeout) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, + "Couldn't restore table {}.{} on other node or sync it (elapsed {})", + backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(table_name), to_string(elapsed)); } } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 3aa2aa378b7..958ee3f133f 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -72,7 +72,7 @@ public: void shutdown() override; - void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) override; + void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; friend struct DatabaseReplicatedTask; friend class DatabaseReplicatedDDLWorker; diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 5dd17789e60..4ab0ed4792e 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -322,22 +322,45 @@ StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name backQuote(database_name), backQuote(table_name)); } -DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIteratorForBackup(const BackupEntriesCollector & backup_entries_collector) const +std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const { - /// Backup all the tables in this database. - /// Here we skip inner tables of materialized views. - auto skip_internal_tables = [](const String & table_name) { return !table_name.starts_with(".inner_id."); }; - return getTablesIterator(backup_entries_collector.getContext(), skip_internal_tables); + std::vector> res; + + for (auto it = getTablesIterator(local_context, filter); it->isValid(); it->next()) + { + bool ok = false; + + if (auto create_table_query = tryGetCreateTableQuery(it->name(), local_context)) + { + const auto & create = create_table_query->as(); + if (create.getTable() == it->name()) + { + auto storage = it->table(); + storage->adjustCreateQueryForBackup(create_table_query, consistency); + if (consistency) + { + res.emplace_back(create_table_query, storage); + ok = true; + } + } + } + + if (!ok) + { + consistency = false; + return {}; + } + } + + return res; } -void DatabaseWithOwnTablesBase::checkCreateTableQueryForBackup(const ASTPtr &, const BackupEntriesCollector &) const -{ -} - -void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) +void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr, UInt64) { /// Creates a table by executing a "CREATE TABLE" query. - restorer.executeCreateQuery(create_table_query); + InterpreterCreateQuery interpreter{create_table_query, local_context}; + interpreter.setInternal(true); + interpreter.execute(); } } diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index c960d295529..2b320349e2d 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -36,9 +36,8 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; - DatabaseTablesIteratorPtr getTablesIteratorForBackup(const BackupEntriesCollector & backup_entries_collector) const override; - void checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector & backup_entries_collector) const override; - void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer) override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const override; + void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; void shutdown() override; diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index 3adba0d85c8..e09eb5186ec 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -26,29 +25,22 @@ ASTPtr IDatabase::getCreateDatabaseQueryForBackup() const { auto query = getCreateDatabaseQuery(); - /// We don't want to see any UUIDs in backup (after RESTORE the table will have another UUID anyway). + /// We don't want to see any UUIDs in backup (after RESTORE the database will have another UUID anyway). auto & create = query->as(); create.uuid = UUIDHelpers::Nil; return query; } -DatabaseTablesIteratorPtr IDatabase::getTablesIteratorForBackup(const BackupEntriesCollector &) const -{ - /// IDatabase doesn't own any tables. - return std::make_unique(Tables{}, getDatabaseName()); -} - -void IDatabase::checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector &) const +std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &, bool &) const { /// Cannot restore any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_BACKUP_TABLE, - "Database engine {} does not support backups, cannot backup table {}.{}", - getEngineName(), backQuoteIfNeed(getDatabaseName()), - backQuoteIfNeed(create_table_query->as().getTable())); + "Database engine {} does not support backups, cannot backup tables in database {}", + getEngineName(), backQuoteIfNeed(getDatabaseName())); } -void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup &) +void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr, std::shared_ptr, UInt64) { /// Cannot restore any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 2223d657f7f..c8c9ff9d9a5 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -30,8 +30,7 @@ class SettingsChanges; using DictionariesWithID = std::vector>; struct ParsedTablesMetadata; struct QualifiedTableName; -class BackupEntriesCollector; -class RestorerFromBackup; +class IRestoreCoordination; namespace ErrorCodes { @@ -333,17 +332,14 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } - /// Returns a slightly changed version of the CREATE DATABASE query which must be written to a backup. + /// Returns a CREATE DATABASE query prepared for writing to a backup. virtual ASTPtr getCreateDatabaseQueryForBackup() const; - /// Returns an iterator that passes through all the tables when an user wants to backup the whole database. - virtual DatabaseTablesIteratorPtr getTablesIteratorForBackup(const BackupEntriesCollector & restorer) const; - - /// Checks a CREATE TABLE query before it will be written to a backup. Called by IStorage::getCreateQueryForBackup(). - virtual void checkCreateTableQueryForBackup(const ASTPtr & create_table_query, const BackupEntriesCollector & backup_entries_collector) const; + /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. + virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context, bool & consistency) const; /// Creates a table restored from backup. - virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, const RestorerFromBackup & restorer); + virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr context, std::shared_ptr restore_coordination, UInt64 timeout_ms); virtual ~IDatabase() = default; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 0fcf48b9afc..5f0fe303f27 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -248,40 +248,21 @@ bool IStorage::isStaticStorage() const return false; } -ASTPtr IStorage::getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const +void IStorage::adjustCreateQueryForBackup(ASTPtr & create_query, bool &) const { - auto table_id = getStorageID(); - auto db = DatabaseCatalog::instance().tryGetDatabase(table_id.getDatabaseName()); - if (!db) - throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", table_id.database_name, table_id.table_name); - ASTPtr query = db->tryGetCreateTableQuery(table_id.getTableName(), context); - if (!query) - throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", table_id.database_name, table_id.table_name); + create_query = create_query->clone(); /// We don't want to see any UUIDs in backup (after RESTORE the table will have another UUID anyway). - auto & create = query->as(); + auto & create = create_query->as(); create.uuid = UUIDHelpers::Nil; create.to_inner_uuid = UUIDHelpers::Nil; - /// If this is a definition of a system table we'll remove columns and comment because they're excessive for backups. - if (create.storage && create.storage->engine && create.storage->engine->name.starts_with("System")) + /// If this is a definition of a system table we'll remove columns and comment because they're reduntant for backups. + if (isSystemStorage()) { create.reset(create.columns_list); create.reset(create.comment); } - - if (database) - *database = db; - - return query; -} - -ASTPtr IStorage::getCreateQueryForBackup(const BackupEntriesCollector & backup_entries_collector) const -{ - DatabasePtr database; - auto query = getCreateQueryForBackup(backup_entries_collector.getContext(), &database); - database->checkCreateTableQueryForBackup(query, backup_entries_collector); - return query; } void IStorage::backupData(BackupEntriesCollector &, const String &, const std::optional &) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6dd329db02b..952f7bacbd3 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -223,10 +223,8 @@ public: /// Initially reserved virtual column name may be shadowed by real column. bool isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const; - /// Returns a slightly changed version of the CREATE TABLE query which must be written to a backup. - /// The function can throw `TABLE_IS_DROPPED` if this storage is not attached to a database. - virtual ASTPtr getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const; - virtual ASTPtr getCreateQueryForBackup(const BackupEntriesCollector & backup_entries_collector) const; + /// Modify a CREATE TABLE query to make a variant which must be written to a backup. + virtual void adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const; /// Makes backup entries to backup the data of this storage. virtual void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fac11db2ab9..e2f82603702 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8253,44 +8253,58 @@ void StorageReplicatedMergeTree::createAndStoreFreezeMetadata(DiskPtr disk, Data } -ASTPtr StorageReplicatedMergeTree::getCreateQueryForBackup(const ContextPtr & local_context, DatabasePtr * database) const +void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const { - ASTPtr query = MergeTreeData::getCreateQueryForBackup(local_context, database); + MergeTreeData::adjustCreateQueryForBackup(create_query, consistency); /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. - auto & create = query->as(); - if (create.storage && create.storage->engine && (create.uuid != UUIDHelpers::Nil)) + auto & create = create_query->as(); + + if (!create.storage || !create.storage->engine) { - auto & engine = *(create.storage->engine); - if (auto * engine_args_ast = typeid_cast(engine.arguments.get())) - { - auto & engine_args = engine_args_ast->children; - if (engine_args.size() >= 2) - { - auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); - auto * replica_name_ast = typeid_cast(engine_args[1].get()); - if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && - replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) - { - String & zookeeper_path_arg = zookeeper_path_ast->value.get(); - String & replica_name_arg = replica_name_ast->value.get(); - String table_uuid_str = toString(create.uuid); - if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) - zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); - const auto & config = getContext()->getConfigRef(); - if ((zookeeper_path_arg == getDefaultZooKeeperPath(config)) && (replica_name_arg == getDefaultReplicaName(config)) - && ((engine_args.size() == 2) || !engine_args[2]->as())) - { - engine_args.erase(engine_args.begin(), engine_args.begin() + 2); - } - } - } - } + /// The CREATE query doesn't correspond to this storage. + consistency = false; + return; } - return query; + auto & engine = *(create.storage->engine); + if (!engine.name.starts_with("Replicated") || !engine.name.ends_with("MergeTree")) + { + /// The CREATE query doesn't correspond to this storage. + consistency = false; + return; + } + + if (create.uuid == UUIDHelpers::Nil) + return; + + auto * engine_args_ast = typeid_cast(engine.arguments.get()); + if (!engine_args_ast) + return; + + auto & engine_args = engine_args_ast->children; + if (engine_args.size() < 2) + return; + + auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); + auto * replica_name_ast = typeid_cast(engine_args[1].get()); + if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && + replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) + { + String & zookeeper_path_arg = zookeeper_path_ast->value.get(); + String & replica_name_arg = replica_name_ast->value.get(); + String table_uuid_str = toString(create.uuid); + if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) + zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); + const auto & config = getContext()->getConfigRef(); + if ((zookeeper_path_arg == getDefaultZooKeeperPath(config)) && (replica_name_arg == getDefaultReplicaName(config)) + && ((engine_args.size() == 2) || !engine_args[2]->as())) + { + engine_args.erase(engine_args.begin(), engine_args.begin() + 2); + } + } } void StorageReplicatedMergeTree::backupData( @@ -8370,7 +8384,7 @@ void StorageReplicatedMergeTree::backupData( backup_entries_collector.addBackupEntry(data_path / relative_path, backup_entry); } }; - backup_entries_collector.addPostCollectingTask(post_collecting_task); + backup_entries_collector.addPostTask(post_collecting_task); } void StorageReplicatedMergeTree::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 73a08a2b921..f3bb4786cca 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -232,8 +232,8 @@ public: int getMetadataVersion() const { return metadata_version; } - /// Returns a slightly changed version of the CREATE TABLE query which must be written to a backup. - ASTPtr getCreateQueryForBackup(const ContextPtr & context, DatabasePtr * database) const override; + /// Modify a CREATE TABLE query to make a variant which must be written to a backup. + void adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const override; /// Makes backup entries to backup the data of the storage. void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 35545e95537..0a39576ce10 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -138,7 +138,7 @@ def test_backup_table_under_another_name(): assert instance.query("SELECT count(), sum(x) FROM test.table2") == "100\t4950\n" -def test_materialized_view(): +def test_materialized_view_select_1(): backup_name = new_backup_name() instance.query( "CREATE MATERIALIZED VIEW mv_1(x UInt8) ENGINE=MergeTree ORDER BY tuple() POPULATE AS SELECT 1 AS x" @@ -456,18 +456,32 @@ def test_temporary_table(): ) == TSV([["e"], ["q"], ["w"]]) -# "BACKUP DATABASE _temporary_and_external_tables" is allowed but the backup must not contain these tables. -def test_temporary_tables_database(): +# The backup created by "BACKUP DATABASE _temporary_and_external_tables" must not contain tables from other sessions. +def test_temporary_database(): session_id = new_session_id() instance.http_query( "CREATE TEMPORARY TABLE temp_tbl(s String)", params={"session_id": session_id} ) - backup_name = new_backup_name() - instance.query(f"BACKUP DATABASE _temporary_and_external_tables TO {backup_name}") + other_session_id = new_session_id() + instance.http_query( + "CREATE TEMPORARY TABLE other_temp_tbl(s String)", + params={"session_id": other_session_id}, + ) - assert os.listdir(os.path.join(get_path_to_backup(backup_name), "metadata/")) == [ - "_temporary_and_external_tables.sql" # database metadata only + backup_name = new_backup_name() + instance.http_query( + f"BACKUP DATABASE _temporary_and_external_tables TO {backup_name}", + params={"session_id": session_id}, + ) + + assert os.listdir( + os.path.join(get_path_to_backup(backup_name), "temporary_tables/metadata") + ) == ["temp_tbl.sql"] + + assert sorted(os.listdir(get_path_to_backup(backup_name))) == [ + ".backup", + "temporary_tables", ] From 461a31f237dff13ede4f7cc4b764d1a99e37f593 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 23 Jun 2022 12:17:54 +0200 Subject: [PATCH 290/408] Improve gathering metadata for backup - part 2. --- src/Backups/BackupEntriesCollector.cpp | 164 +++++++++++--------- src/Backups/BackupEntriesCollector.h | 3 +- src/Backups/RestorerFromBackup.cpp | 3 +- src/Common/ErrorCodes.cpp | 2 +- src/Databases/DatabaseMemory.cpp | 45 +++--- src/Databases/DatabaseMemory.h | 2 +- src/Databases/DatabasesCommon.cpp | 32 ++-- src/Databases/DatabasesCommon.h | 2 +- src/Databases/IDatabase.cpp | 2 +- src/Databases/IDatabase.h | 2 +- src/Storages/IStorage.cpp | 10 +- src/Storages/IStorage.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 17 +- src/Storages/StorageReplicatedMergeTree.h | 2 +- 14 files changed, 146 insertions(+), 142 deletions(-) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 9ee57cb4fd5..21b2741e237 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -21,7 +21,7 @@ namespace DB namespace ErrorCodes { - extern const int CANNOT_COLLECT_OBJECTS_FOR_BACKUP; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; extern const int CANNOT_BACKUP_TABLE; extern const int TABLE_IS_DROPPED; extern const int LOGICAL_ERROR; @@ -162,37 +162,37 @@ void BackupEntriesCollector::gatherMetadataAndCheckConsistency() bool use_timeout = (timeout.count() >= 0); auto start_time = std::chrono::steady_clock::now(); - int pass = 1; - for (;;) + for (size_t pass = 1;; ++pass) { - consistency = true; - - /// Collect information about databases and tables specified in the BACKUP query. - gatherDatabasesMetadata(); - gatherTablesMetadata(); - - /// We have to check consistency of collected information to protect from the case when some table or database is - /// renamed during this collecting making the collected information invalid. - checkConsistency(); - - if (consistency) - break; - - /// Two passes is absolute minimum (see `previous_table_names` & `previous_database_names`). - auto elapsed = std::chrono::steady_clock::now() - start_time; - if ((pass >= 2) && use_timeout) + try { - if (elapsed > timeout) - throw Exception( - ErrorCodes::CANNOT_COLLECT_OBJECTS_FOR_BACKUP, - "Couldn't collect tables and databases to make a backup (pass #{}, elapsed {})", - pass, - to_string(elapsed)); - } + /// Collect information about databases and tables specified in the BACKUP query. + database_infos.clear(); + table_infos.clear(); + gatherDatabasesMetadata(); + gatherTablesMetadata(); - if (pass >= 2) - LOG_WARNING(log, "Couldn't collect tables and databases to make a backup (pass #{}, elapsed {})", pass, to_string(elapsed)); - ++pass; + /// We have to check consistency of collected information to protect from the case when some table or database is + /// renamed during this collecting making the collected information invalid. + auto comparing_error = compareWithPrevious(); + if (!comparing_error) + break; /// no error, everything's fine + + if (pass >= 2) /// Two passes is minimum (we need to compare with table names with previous ones to be sure we don't miss anything). + throw *comparing_error; + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP) + throw; + + auto elapsed = std::chrono::steady_clock::now() - start_time; + e.addMessage("Couldn't gather tables and databases to make a backup (pass #{}, elapsed {})", pass, to_string(elapsed)); + if (use_timeout && (elapsed > timeout)) + throw; + else + LOG_WARNING(log, "{}", e.displayText()); + } } LOG_INFO(log, "Will backup {} databases and {} tables", database_infos.size(), table_infos.size()); @@ -200,8 +200,6 @@ void BackupEntriesCollector::gatherMetadataAndCheckConsistency() void BackupEntriesCollector::gatherDatabasesMetadata() { - database_infos.clear(); - /// Collect information about databases and tables specified in the BACKUP query. for (const auto & element : backup_query_elements) { @@ -264,16 +262,11 @@ void BackupEntriesCollector::gatherDatabasesMetadata() /* partitions= */ {}, /* all_tables= */ true, /* except_table_names= */ element.except_tables); - if (!consistency) - return; } } break; } } - - if (!consistency) - return; } } @@ -318,20 +311,14 @@ void BackupEntriesCollector::gatherDatabaseMetadata( } catch (...) { - /// The database has been dropped recently. - consistency = false; - return; + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for database {}", database_name); } database_info.create_database_query = create_database_query; const auto & create = create_database_query->as(); if (create.getDatabase() != database_name) - { - /// The database has been renamed recently. - consistency = false; - return; - } + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for database {}", backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(database_name)); } if (table_name) @@ -358,9 +345,6 @@ void BackupEntriesCollector::gatherDatabaseMetadata( void BackupEntriesCollector::gatherTablesMetadata() { - if (!consistency) - return; - table_infos.clear(); for (const auto & [database_name, database_info] : database_infos) { @@ -382,12 +366,8 @@ void BackupEntriesCollector::gatherTablesMetadata() return false; }; - auto db_tables = database->getTablesForBackup(filter_by_table_name, context, consistency); + auto db_tables = database->getTablesForBackup(filter_by_table_name, context); - if (!consistency) - return; - - /// Check that all tables were found. std::unordered_set found_table_names; for (const auto & db_table : db_tables) { @@ -395,13 +375,14 @@ void BackupEntriesCollector::gatherTablesMetadata() const auto & create = create_table_query->as(); found_table_names.emplace(create.getTable()); - if ((is_temporary_database && !create.temporary) || (!is_temporary_database && (create.getDatabase() != database_name))) - { - consistency = false; - return; - } + if (is_temporary_database && !create.temporary) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a non-temporary create query for {}", tableNameWithTypeToString(database_name, create.getTable(), false)); + + if (!is_temporary_database && (create.getDatabase() != database_name)) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected database name {} for {}", backQuoteIfNeed(create.getDatabase()), tableNameWithTypeToString(database_name, create.getTable(), false)); } + /// Check that all tables were found. for (const auto & [table_name, table_info] : database_info.tables) { if (table_info.throw_if_table_not_found && !found_table_names.contains(table_name)) @@ -443,10 +424,7 @@ void BackupEntriesCollector::gatherTablesMetadata() void BackupEntriesCollector::lockTablesForReading() { - if (!consistency) - return; - - for (auto & table_info : table_infos | boost::adaptors::map_values) + for (auto & [table_name, table_info] : table_infos) { auto storage = table_info.storage; TableLockHolder table_lock; @@ -460,19 +438,15 @@ void BackupEntriesCollector::lockTablesForReading() { if (e.code() != ErrorCodes::TABLE_IS_DROPPED) throw; - consistency = false; - return; + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} is dropped", tableNameWithTypeToString(table_name.database, table_name.table, true)); } } } } -/// Check for consistency of collected information about databases and tables. -void BackupEntriesCollector::checkConsistency() +/// Check consistency of collected information about databases and tables. +std::optional BackupEntriesCollector::compareWithPrevious() { - if (!consistency) - return; /// Already inconsistent, no more checks necessary - /// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed /// while we were scanning. std::set database_names; @@ -480,12 +454,62 @@ void BackupEntriesCollector::checkConsistency() boost::range::copy(database_infos | boost::adaptors::map_keys, std::inserter(database_names, database_names.end())); boost::range::copy(table_infos | boost::adaptors::map_keys, std::inserter(table_names, table_names.end())); - if ((previous_database_names != database_names) || (previous_table_names != table_names)) + if (previous_database_names != database_names) { + std::optional comparing_error; + for (const auto & database_name : database_names) + { + if (!previous_database_names.contains(database_name)) + { + comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were added during scanning", backQuoteIfNeed(database_name)}; + break; + } + } + if (!comparing_error) + { + for (const auto & database_name : previous_database_names) + { + if (!database_names.contains(database_name)) + { + comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were removed during scanning", backQuoteIfNeed(database_name)}; + break; + } + } + } + assert(comparing_error); previous_database_names = std::move(database_names); previous_table_names = std::move(table_names); - consistency = false; + return comparing_error; } + + if (previous_table_names != table_names) + { + std::optional comparing_error; + for (const auto & table_name : table_names) + { + if (!previous_table_names.contains(table_name)) + { + comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were added during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + break; + } + } + if (!comparing_error) + { + for (const auto & table_name : previous_table_names) + { + if (!table_names.contains(table_name)) + { + comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were removed during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + break; + } + } + } + assert(comparing_error); + previous_table_names = std::move(table_names); + return comparing_error; + } + + return {}; } /// Make backup entries for all the definitions of all the databases found. diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index c34c6204abb..03b7a968650 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -98,7 +98,7 @@ private: void gatherTablesMetadata(); void lockTablesForReading(); - void checkConsistency(); + std::optional compareWithPrevious(); void makeBackupEntriesForDatabasesDefs(); void makeBackupEntriesForTablesDefs(); @@ -147,7 +147,6 @@ private: std::map table_infos; std::set previous_database_names; std::set previous_table_names; - bool consistency = false; BackupEntries backup_entries; std::queue> post_tasks; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 16ffead3976..84cc1dec1fb 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -691,8 +691,7 @@ void RestorerFromBackup::createTables() if (!restore_settings.allow_different_table_def) { ASTPtr create_table_query = database->getCreateTableQuery(resolved_id.table_name, context); - bool consistency = true; - storage->adjustCreateQueryForBackup(create_table_query, consistency); + storage->adjustCreateQueryForBackup(create_table_query); ASTPtr expected_create_query = table_info.create_table_query; if (serializeAST(*create_table_query) != serializeAST(*expected_create_query)) { diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6f2ac41cc08..8e7eaf4c6e6 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -631,7 +631,7 @@ M(660, HDFS_ERROR) \ M(661, CANNOT_SEND_SIGNAL) \ M(662, FS_METADATA_ERROR) \ - M(663, CANNOT_COLLECT_OBJECTS_FOR_BACKUP) \ + M(663, INCONSISTENT_METADATA_FOR_BACKUP) \ M(664, ACCESS_STORAGE_DOESNT_ALLOW_BACKUP) \ \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 62cee31bbad..8540c785419 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int UNKNOWN_TABLE; extern const int LOGICAL_ERROR; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_) @@ -145,11 +146,11 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl DatabaseCatalog::instance().updateLoadingDependencies(table_id, std::move(new_dependencies)); } -std::vector> DatabaseMemory::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const +std::vector> DatabaseMemory::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const { /// We need a special processing for the temporary database. if (getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE) - return DatabaseWithOwnTablesBase::getTablesForBackup(filter, local_context, consistency); + return DatabaseWithOwnTablesBase::getTablesForBackup(filter, local_context); std::vector> res; @@ -162,32 +163,22 @@ std::vector> DatabaseMemory::getTablesForBackup(co if (!filter(table_name)) continue; - bool ok = false; - - if (auto storage_id = local_context->tryResolveStorageID(StorageID{"", table_name}, Context::ResolveExternal)) - { - /// Here `storage_id.table_name` looks like looks like "_tmp_ab9b15a3-fb43-4670-abec-14a0e9eb70f1" - /// it's not the real name of the table. - if (auto create_table_query = tryGetCreateTableQuery(storage_id.table_name, local_context)) - { - const auto & create = create_table_query->as(); - if (create.getTable() == table_name) - { - storage->adjustCreateQueryForBackup(create_table_query, consistency); - if (consistency) - { - res.emplace_back(create_table_query, storage); - ok = true; - } - } - } - } + auto storage_id = local_context->tryResolveStorageID(StorageID{"", table_name}, Context::ResolveExternal); + if (!storage_id) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't resolve the name of temporary table {}", backQuoteIfNeed(table_name)); - if (!ok) - { - consistency = false; - return {}; - } + /// Here `storage_id.table_name` looks like looks like "_tmp_ab9b15a3-fb43-4670-abec-14a0e9eb70f1" + /// it's not the real name of the table. + auto create_table_query = tryGetCreateTableQuery(storage_id.table_name, local_context); + if (!create_table_query) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for temporary table {}", backQuoteIfNeed(table_name)); + + const auto & create = create_table_query->as(); + if (create.getTable() != table_name) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for temporary table {}", backQuoteIfNeed(create.getTable()), backQuoteIfNeed(table_name)); + + storage->adjustCreateQueryForBackup(create_table_query); + res.emplace_back(create_table_query, storage); } return res; diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index 8ec216b165d..6262543b0c1 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -50,7 +50,7 @@ public: void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; - std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; private: const String data_path; diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 4ab0ed4792e..93a9523d115 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata) @@ -322,34 +323,23 @@ StoragePtr DatabaseWithOwnTablesBase::getTableUnlocked(const String & table_name backQuote(database_name), backQuote(table_name)); } -std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const +std::vector> DatabaseWithOwnTablesBase::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const { std::vector> res; for (auto it = getTablesIterator(local_context, filter); it->isValid(); it->next()) { - bool ok = false; + auto create_table_query = tryGetCreateTableQuery(it->name(), local_context); + if (!create_table_query) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for table {}.{}", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(it->name())); - if (auto create_table_query = tryGetCreateTableQuery(it->name(), local_context)) - { - const auto & create = create_table_query->as(); - if (create.getTable() == it->name()) - { - auto storage = it->table(); - storage->adjustCreateQueryForBackup(create_table_query, consistency); - if (consistency) - { - res.emplace_back(create_table_query, storage); - ok = true; - } - } - } + const auto & create = create_table_query->as(); + if (create.getTable() != it->name()) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for table {}.{}", backQuoteIfNeed(create.getTable()), backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(it->name())); - if (!ok) - { - consistency = false; - return {}; - } + auto storage = it->table(); + storage->adjustCreateQueryForBackup(create_table_query); + res.emplace_back(create_table_query, storage); } return res; diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index 2b320349e2d..c5842d7dac3 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -36,7 +36,7 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; - std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context, bool & consistency) const override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; void shutdown() override; diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index e09eb5186ec..a75f213a6bb 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -32,7 +32,7 @@ ASTPtr IDatabase::getCreateDatabaseQueryForBackup() const return query; } -std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &, bool &) const +std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const { /// Cannot restore any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_BACKUP_TABLE, diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index c8c9ff9d9a5..cdea03aa1cb 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -336,7 +336,7 @@ public: virtual ASTPtr getCreateDatabaseQueryForBackup() const; /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. - virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context, bool & consistency) const; + virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context) const; /// Creates a table restored from backup. virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr context, std::shared_ptr restore_coordination, UInt64 timeout_ms); diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 5f0fe303f27..a3f35ccc0f8 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -24,6 +24,7 @@ namespace ErrorCodes extern const int TABLE_IS_DROPPED; extern const int NOT_IMPLEMENTED; extern const int DEADLOCK_AVOIDED; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } bool IStorage::isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const @@ -248,7 +249,7 @@ bool IStorage::isStaticStorage() const return false; } -void IStorage::adjustCreateQueryForBackup(ASTPtr & create_query, bool &) const +void IStorage::adjustCreateQueryForBackup(ASTPtr & create_query) const { create_query = create_query->clone(); @@ -260,6 +261,13 @@ void IStorage::adjustCreateQueryForBackup(ASTPtr & create_query, bool &) const /// If this is a definition of a system table we'll remove columns and comment because they're reduntant for backups. if (isSystemStorage()) { + if (!create.storage || !create.storage->engine) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query without table engine for a system table {}", getStorageID().getFullTableName()); + + auto & engine = *(create.storage->engine); + if (!engine.name.starts_with("System")) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with an unexpected table engine {} for a system table {}", engine.name, getStorageID().getFullTableName()); + create.reset(create.columns_list); create.reset(create.comment); } diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 952f7bacbd3..34170785896 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -224,7 +224,7 @@ public: bool isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const; /// Modify a CREATE TABLE query to make a variant which must be written to a backup. - virtual void adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const; + virtual void adjustCreateQueryForBackup(ASTPtr & create_query) const; /// Makes backup entries to backup the data of this storage. virtual void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e2f82603702..66fb2a64a50 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -158,6 +158,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int CONCURRENT_ACCESS_NOT_SUPPORTED; extern const int CHECKSUM_DOESNT_MATCH; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; } namespace ActionLocks @@ -8253,9 +8254,9 @@ void StorageReplicatedMergeTree::createAndStoreFreezeMetadata(DiskPtr disk, Data } -void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const +void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_query) const { - MergeTreeData::adjustCreateQueryForBackup(create_query, consistency); + MergeTreeData::adjustCreateQueryForBackup(create_query); /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. @@ -8263,19 +8264,11 @@ void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_quer auto & create = create_query->as(); if (!create.storage || !create.storage->engine) - { - /// The CREATE query doesn't correspond to this storage. - consistency = false; - return; - } + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query without table engine for a replicated table {}", getStorageID().getFullTableName()); auto & engine = *(create.storage->engine); if (!engine.name.starts_with("Replicated") || !engine.name.ends_with("MergeTree")) - { - /// The CREATE query doesn't correspond to this storage. - consistency = false; - return; - } + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with an unexpected table engine {} for a replicated table {}", engine.name, getStorageID().getFullTableName()); if (create.uuid == UUIDHelpers::Nil) return; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index f3bb4786cca..86120b354bd 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -233,7 +233,7 @@ public: int getMetadataVersion() const { return metadata_version; } /// Modify a CREATE TABLE query to make a variant which must be written to a backup. - void adjustCreateQueryForBackup(ASTPtr & create_query, bool & consistency) const override; + void adjustCreateQueryForBackup(ASTPtr & create_query) const override; /// Makes backup entries to backup the data of the storage. void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; From 44db346fea82a35880eebba05b950954be3a8c07 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 23 Jun 2022 14:24:45 +0200 Subject: [PATCH 291/408] Improve gathering metadata for backup - part 3. --- src/Access/AccessControl.cpp | 2 +- src/Backups/BackupEntriesCollector.cpp | 40 +++--- src/Backups/BackupEntriesCollector.h | 2 + src/Backups/BackupUtils.cpp | 2 +- src/Backups/RestorerFromBackup.cpp | 162 +++++++++++-------------- src/Backups/RestorerFromBackup.h | 30 ++--- src/Databases/DDLRenamingVisitor.cpp | 68 +++++------ src/Databases/DDLRenamingVisitor.h | 3 - 8 files changed, 135 insertions(+), 174 deletions(-) diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index aa58044a6b0..b5b22caa400 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -467,7 +467,7 @@ void AccessControl::backup(BackupEntriesCollector & backup_entries_collector, Ac void AccessControl::restore(RestorerFromBackup & restorer, const String & data_path_in_backup) { /// The restorer must already know about `data_path_in_backup`, but let's check. - restorer.checkPathInBackupToRestoreAccess(data_path_in_backup); + restorer.checkPathInBackupIsRegisteredToRestoreAccess(data_path_in_backup); } void AccessControl::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 21b2741e237..feed9be4e92 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -36,7 +36,6 @@ namespace else return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); } - } std::string_view BackupEntriesCollector::toString(Stage stage) @@ -319,6 +318,9 @@ void BackupEntriesCollector::gatherDatabaseMetadata( if (create.getDatabase() != database_name) throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected name {} for database {}", backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(database_name)); + + String new_database_name = renaming_map.getNewDatabaseName(database_name); + database_info.metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql"); } if (table_name) @@ -395,17 +397,19 @@ void BackupEntriesCollector::gatherTablesMetadata() const auto & create = create_table_query->as(); String table_name = create.getTable(); - fs::path data_path_in_backup; - if (is_temporary_database) + fs::path metadata_path_in_backup, data_path_in_backup; + auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name}); + if (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE) { - auto table_name_in_backup = renaming_map.getNewTemporaryTableName(table_name); - data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup); + metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(table_name_in_backup.table) + ".sql"); + data_path_in_backup = root_path_in_backup / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table); } else { - auto table_name_in_backup = renaming_map.getNewTableName({database_name, table_name}); - data_path_in_backup - = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); + metadata_path_in_backup + = root_path_in_backup / "metadata" / escapeForFileName(table_name_in_backup.database) / (escapeForFileName(table_name_in_backup.table) + ".sql"); + data_path_in_backup = root_path_in_backup / "data" / escapeForFileName(table_name_in_backup.database) + / escapeForFileName(table_name_in_backup.table); } /// Add information to `table_infos`. @@ -413,6 +417,7 @@ void BackupEntriesCollector::gatherTablesMetadata() res_table_info.database = database; res_table_info.storage = db_table.second; res_table_info.create_table_query = create_table_query; + res_table_info.metadata_path_in_backup = metadata_path_in_backup; res_table_info.data_path_in_backup = data_path_in_backup; auto partitions_it = database_info.tables.find(table_name); @@ -525,9 +530,7 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() ASTPtr new_create_query = database_info.create_database_query; renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); - String new_database_name = renaming_map.getNewDatabaseName(database_name); - auto metadata_path_in_backup = root_path_in_backup / "metadata" / (escapeForFileName(new_database_name) + ".sql"); - + const String & metadata_path_in_backup = database_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); } } @@ -538,24 +541,11 @@ void BackupEntriesCollector::makeBackupEntriesForTablesDefs() for (const auto & [table_name, table_info] : table_infos) { LOG_TRACE(log, "Adding definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); - bool is_temporary_database = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE); ASTPtr new_create_query = table_info.create_table_query; renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); - fs::path metadata_path_in_backup; - if (is_temporary_database) - { - auto new_name = renaming_map.getNewTemporaryTableName(table_name.table); - metadata_path_in_backup = root_path_in_backup / "temporary_tables" / "metadata" / (escapeForFileName(new_name) + ".sql"); - } - else - { - auto new_name = renaming_map.getNewTableName({table_name.database, table_name.table}); - metadata_path_in_backup - = root_path_in_backup / "metadata" / escapeForFileName(new_name.database) / (escapeForFileName(new_name.table) + ".sql"); - } - + const String & metadata_path_in_backup = table_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); } } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 03b7a968650..5e37e268fc4 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -120,6 +120,7 @@ private: { DatabasePtr database; ASTPtr create_database_query; + String metadata_path_in_backup; struct TableParams { @@ -139,6 +140,7 @@ private: StoragePtr storage; TableLockHolder table_lock; ASTPtr create_table_query; + String metadata_path_in_backup; std::filesystem::path data_path_in_backup; std::optional partitions; }; diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index c5de4bd7e67..9ff91050177 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -39,7 +39,7 @@ DDLRenamingMap makeRenamingMapFromBackupQuery(const ASTBackupQuery::Elements & e const String & new_table_name = element.new_table_name; assert(!table_name.empty()); assert(!new_table_name.empty()); - map.setNewTemporaryTableName(table_name, new_table_name); + map.setNewTableName({DatabaseCatalog::TEMPORARY_DATABASE, table_name}, {DatabaseCatalog::TEMPORARY_DATABASE, new_table_name}); break; } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 84cc1dec1fb..74d4de631e3 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -39,7 +39,13 @@ namespace ErrorCodes namespace { - constexpr const std::string_view sql_ext = ".sql"; + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_char_uppercase) + { + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + return fmt::format("{}emporary table {}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(table_name)); + else + return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + } String tryGetTableEngine(const IAST & ast) { @@ -64,16 +70,6 @@ namespace } } -bool RestorerFromBackup::TableKey::operator ==(const TableKey & right) const -{ - return (name == right.name) && (is_temporary == right.is_temporary); -} - -bool RestorerFromBackup::TableKey::operator <(const TableKey & right) const -{ - return (name < right.name) || ((name == right.name) && (is_temporary < right.is_temporary)); -} - std::string_view RestorerFromBackup::toString(Stage stage) { switch (stage) @@ -135,10 +131,11 @@ void RestorerFromBackup::run(bool only_check_access) /// Find all the databases and tables which we will read from the backup. setStage(Stage::kFindingTablesInBackup); - collectDatabaseAndTableInfos(); + findDatabasesAndTablesInBackup(); /// Check access rights. - checkAccessForCollectedInfos(); + checkAccessForObjectsFoundInBackup(); + if (only_check_access) return; @@ -303,7 +300,7 @@ void RestorerFromBackup::findRootPathsInBackup() ", ")); } -void RestorerFromBackup::collectDatabaseAndTableInfos() +void RestorerFromBackup::findDatabasesAndTablesInBackup() { database_infos.clear(); table_infos.clear(); @@ -313,22 +310,22 @@ void RestorerFromBackup::collectDatabaseAndTableInfos() { case ASTBackupQuery::ElementType::TABLE: { - collectTableInfo({element.database_name, element.table_name}, false, element.partitions); + findTableInBackup({element.database_name, element.table_name}, element.partitions); break; } case ASTBackupQuery::ElementType::TEMPORARY_TABLE: { - collectTableInfo({element.database_name, element.table_name}, true, element.partitions); + findTableInBackup({DatabaseCatalog::TEMPORARY_DATABASE, element.table_name}, element.partitions); break; } case ASTBackupQuery::ElementType::DATABASE: { - collectDatabaseInfo(element.database_name, element.except_tables, /* throw_if_no_database_metadata_in_backup= */ true); + findDatabaseInBackup(element.database_name, element.except_tables); break; } case ASTBackupQuery::ElementType::ALL: { - collectAllDatabasesInfo(element.except_databases, element.except_tables); + findEverythingInBackup(element.except_databases, element.except_tables); break; } } @@ -337,9 +334,9 @@ void RestorerFromBackup::collectDatabaseAndTableInfos() LOG_INFO(log, "Will restore {} databases and {} tables", database_infos.size(), table_infos.size()); } -void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional & partitions) +void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional & partitions) { - String database_name_in_backup = is_temporary_table ? DatabaseCatalog::TEMPORARY_DATABASE : table_name_in_backup.database; + bool is_temporary_table = (table_name_in_backup.database == DatabaseCatalog::TEMPORARY_DATABASE); std::optional metadata_path; std::optional root_path_in_use; @@ -366,21 +363,20 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ } if (!metadata_path) - throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Table {} not found in backup", table_name_in_backup.getFullName()); + throw Exception( + ErrorCodes::BACKUP_ENTRY_NOT_FOUND, + "{} not found in backup", + tableNameWithTypeToString(table_name_in_backup.database, table_name_in_backup.table, true)); - TableKey table_key; fs::path data_path_in_backup; if (is_temporary_table) { data_path_in_backup = *root_path_in_use / "temporary_tables" / "data" / escapeForFileName(table_name_in_backup.table); - table_key.name.table = renaming_map.getNewTemporaryTableName(table_name_in_backup.table); - table_key.is_temporary = true; } else { data_path_in_backup = *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); - table_key.name = renaming_map.getNewTableName(table_name_in_backup); } auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); @@ -391,25 +387,26 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_table_query); - if (auto it = table_infos.find(table_key); it != table_infos.end()) + QualifiedTableName table_name = renaming_map.getNewTableName(table_name_in_backup); + + if (auto it = table_infos.find(table_name); it != table_infos.end()) { const TableInfo & table_info = it->second; if (table_info.create_table_query && (serializeAST(*table_info.create_table_query) != serializeAST(*create_table_query))) { throw Exception( ErrorCodes::CANNOT_RESTORE_TABLE, - "Extracted two different create queries for the same {}table {}: {} and {}", - (is_temporary_table ? "temporary " : ""), - table_key.name.getFullName(), + "Extracted two different create queries for the same {}: {} and {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*table_info.create_table_query), serializeAST(*create_table_query)); } } - TableInfo & res_table_info = table_infos[table_key]; + TableInfo & res_table_info = table_infos[table_name]; res_table_info.create_table_query = create_table_query; res_table_info.data_path_in_backup = data_path_in_backup; - res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_key.name, create_table_query); + res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_name, create_table_query); if (partitions) { @@ -426,27 +423,37 @@ void RestorerFromBackup::collectTableInfo(const QualifiedTableName & table_name_ } } -void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_backup, const std::set & except_table_names, bool throw_if_no_database_metadata_in_backup) +void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names) { std::optional metadata_path; std::unordered_set table_names_in_backup; for (const auto & root_path_in_backup : root_paths_in_backup) { - fs::path try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql"); - if (!metadata_path && backup->fileExists(try_metadata_path)) + fs::path try_metadata_path, try_tables_metadata_path; + if (database_name_in_backup == DatabaseCatalog::TEMPORARY_DATABASE) + { + try_tables_metadata_path = root_path_in_backup / "temporary_tables" / "metadata"; + } + else + { + try_metadata_path = root_path_in_backup / "metadata" / (escapeForFileName(database_name_in_backup) + ".sql"); + try_tables_metadata_path = root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup); + } + + if (!metadata_path && !try_metadata_path.empty() && backup->fileExists(try_metadata_path)) metadata_path = try_metadata_path; - Strings file_names = backup->listFiles(root_path_in_backup / "metadata" / escapeForFileName(database_name_in_backup)); + Strings file_names = backup->listFiles(try_tables_metadata_path); for (const String & file_name : file_names) { - if (!file_name.ends_with(sql_ext)) + if (!file_name.ends_with(".sql")) continue; - String file_name_without_ext = file_name.substr(0, file_name.length() - sql_ext.length()); + String file_name_without_ext = file_name.substr(0, file_name.length() - strlen(".sql")); table_names_in_backup.insert(unescapeForFileName(file_name_without_ext)); } } - if (!metadata_path && throw_if_no_database_metadata_in_backup) + if (!metadata_path && table_names_in_backup.empty()) throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Database {} not found in backup", backQuoteIfNeed(database_name_in_backup)); if (metadata_path) @@ -480,33 +487,26 @@ void RestorerFromBackup::collectDatabaseInfo(const String & database_name_in_bac if (except_table_names.contains({database_name_in_backup, table_name_in_backup})) continue; - collectTableInfo({database_name_in_backup, table_name_in_backup}, /* is_temporary_table= */ false, /* partitions= */ {}); + findTableInBackup({database_name_in_backup, table_name_in_backup}, /* partitions= */ {}); } } -void RestorerFromBackup::collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names) +void RestorerFromBackup::findEverythingInBackup(const std::set & except_database_names, const std::set & except_table_names) { std::unordered_set database_names_in_backup; - std::unordered_set temporary_table_names_in_backup; for (const auto & root_path_in_backup : root_paths_in_backup) { Strings file_names = backup->listFiles(root_path_in_backup / "metadata"); for (String & file_name : file_names) { - if (file_name.ends_with(sql_ext)) - file_name.resize(file_name.length() - sql_ext.length()); + if (file_name.ends_with(".sql")) + file_name.resize(file_name.length() - strlen(".sql")); database_names_in_backup.emplace(unescapeForFileName(file_name)); } - file_names = backup->listFiles(root_path_in_backup / "temporary_tables" / "metadata"); - for (String & file_name : file_names) - { - if (!file_name.ends_with(sql_ext)) - continue; - file_name.resize(file_name.length() - sql_ext.length()); - temporary_table_names_in_backup.emplace(unescapeForFileName(file_name)); - } + if (backup->hasFiles(root_path_in_backup / "temporary_tables" / "metadata")) + database_names_in_backup.emplace(DatabaseCatalog::TEMPORARY_DATABASE); } for (const String & database_name_in_backup : database_names_in_backup) @@ -514,14 +514,11 @@ void RestorerFromBackup::collectAllDatabasesInfo(const std::set & except if (except_database_names.contains(database_name_in_backup)) continue; - collectDatabaseInfo(database_name_in_backup, except_table_names, /* throw_if_no_database_metadata_in_backup= */ false); + findDatabaseInBackup(database_name_in_backup, except_table_names); } - - for (const String & temporary_table_name_in_backup : temporary_table_names_in_backup) - collectTableInfo({"", temporary_table_name_in_backup}, /* is_temporary_table= */ true, /* partitions= */ {}); } -void RestorerFromBackup::checkAccessForCollectedInfos() const +void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const { AccessRightsElements required_access; for (const auto & database_name : database_infos | boost::adaptors::map_keys) @@ -545,7 +542,7 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const if (hasSystemTableEngine(*table_info.create_table_query)) continue; - if (table_name.is_temporary) + if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) { if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); @@ -579,7 +576,7 @@ void RestorerFromBackup::checkAccessForCollectedInfos() const flags = AccessType::SHOW_TABLES; } - required_access.emplace_back(flags, table_name.name.database, table_name.name.table); + required_access.emplace_back(flags, table_name.database, table_name.table); } if (access_restore_task) @@ -611,7 +608,9 @@ void RestorerFromBackup::createDatabases() create_database_query->as().if_not_exists = true; } LOG_TRACE(log, "Creating database {}: {}", backQuoteIfNeed(database_name), serializeAST(*create_database_query)); - executeCreateQuery(create_database_query); + InterpreterCreateQuery interpreter{create_database_query, context}; + interpreter.setInternal(true); + interpreter.execute(); } DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); @@ -644,15 +643,11 @@ void RestorerFromBackup::createTables() if (tables_to_create.empty()) break; /// We've already created all the tables. - for (const auto & table_key : tables_to_create) + for (const auto & table_name : tables_to_create) { - auto & table_info = table_infos.at(table_key); + auto & table_info = table_infos.at(table_name); - DatabasePtr database; - if (table_key.is_temporary) - database = DatabaseCatalog::instance().getDatabaseForTemporaryTables(); - else - database = DatabaseCatalog::instance().getDatabase(table_key.name.database); + DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.database); bool need_create_table = (restore_settings.create_table != RestoreTableCreationMode::kMustExist); if (need_create_table && hasSystemTableEngine(*table_info.create_table_query)) @@ -670,9 +665,8 @@ void RestorerFromBackup::createTables() } LOG_TRACE( log, - "Creating {}table {}: {}", - (table_key.is_temporary ? "temporary " : ""), - table_key.name.getFullName(), + "Creating {}: {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*create_table_query)); database->createTableRestoredFromBackup(create_table_query, context, restore_coordination, create_table_timeout_ms); @@ -680,9 +674,9 @@ void RestorerFromBackup::createTables() table_info.created = true; - auto resolved_id = table_key.is_temporary - ? context->resolveStorageID(StorageID{"", table_key.name.table}, Context::ResolveExternal) - : context->resolveStorageID(StorageID{table_key.name.database, table_key.name.table}, Context::ResolveGlobal); + auto resolved_id = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + ? context->resolveStorageID(StorageID{"", table_name.table}, Context::ResolveExternal) + : context->resolveStorageID(StorageID{table_name.database, table_name.table}, Context::ResolveGlobal); auto storage = database->getTable(resolved_id.table_name, context); table_info.storage = storage; @@ -697,10 +691,9 @@ void RestorerFromBackup::createTables() { throw Exception( ErrorCodes::CANNOT_RESTORE_TABLE, - "The {}table {} has a different definition: {} " + "{} has a different definition: {} " "comparing to its definition in the backup: {}", - (table_key.is_temporary ? "temporary " : ""), - table_key.name.getFullName(), + tableNameWithTypeToString(table_name.database, table_name.table, true), serializeAST(*create_table_query), serializeAST(*expected_create_query)); } @@ -717,9 +710,9 @@ void RestorerFromBackup::createTables() } /// Returns the list of tables without dependencies or those which dependencies have been created before. -std::vector RestorerFromBackup::findTablesWithoutDependencies() const +std::vector RestorerFromBackup::findTablesWithoutDependencies() const { - std::vector tables_without_dependencies; + std::vector tables_without_dependencies; bool all_tables_created = true; for (const auto & [key, table_info] : table_infos) @@ -734,7 +727,7 @@ std::vector RestorerFromBackup::findTablesWithoutD bool all_dependencies_met = true; for (const auto & dependency : table_info.dependencies) { - auto it = table_infos.find(TableKey{dependency, false}); + auto it = table_infos.find(dependency); if ((it != table_infos.end()) && !it->second.created) { all_dependencies_met = false; @@ -753,7 +746,7 @@ std::vector RestorerFromBackup::findTablesWithoutD return {}; /// Cyclic dependency? We'll try to create those tables anyway but probably it's going to fail. - std::vector tables_with_cyclic_dependencies; + std::vector tables_with_cyclic_dependencies; for (const auto & [key, table_info] : table_infos) { if (!table_info.created) @@ -766,7 +759,7 @@ std::vector RestorerFromBackup::findTablesWithoutD "Some tables have cyclic dependency from each other: {}", boost::algorithm::join( tables_with_cyclic_dependencies - | boost::adaptors::transformed([](const TableKey & key) -> String { return key.name.getFullName(); }), + | boost::adaptors::transformed([](const QualifiedTableName & table_name) -> String { return table_name.getFullName(); }), ", ")); return tables_with_cyclic_dependencies; @@ -786,19 +779,12 @@ void RestorerFromBackup::addDataRestoreTasks(DataRestoreTasks && new_tasks) insertAtEnd(data_restore_tasks, std::move(new_tasks)); } -void RestorerFromBackup::checkPathInBackupToRestoreAccess(const String & path) +void RestorerFromBackup::checkPathInBackupIsRegisteredToRestoreAccess(const String & path) { if (!access_restore_task || !access_restore_task->hasDataPath(path)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Path to restore access was not added"); } -void RestorerFromBackup::executeCreateQuery(const ASTPtr & create_query) const -{ - InterpreterCreateQuery interpreter{create_query, context}; - interpreter.setInternal(true); - interpreter.execute(); -} - void RestorerFromBackup::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) { throw Exception( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 86edf08b484..3bdbafe844c 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -48,15 +48,14 @@ public: std::shared_ptr getRestoreCoordination() const { return restore_coordination; } std::chrono::seconds getTimeout() const { return timeout; } ContextMutablePtr getContext() const { return context; } - void executeCreateQuery(const ASTPtr & create_query) const; /// Adds a data restore task which will be later returned by getDataRestoreTasks(). /// This function can be called by implementations of IStorage::restoreFromBackup() in inherited storage classes. void addDataRestoreTask(DataRestoreTask && new_task); void addDataRestoreTasks(DataRestoreTasks && new_tasks); - /// Adds a new data path to restore access control. - void checkPathInBackupToRestoreAccess(const String & path); + /// Checks that a specified path is already registered to be used for restoring access control. + void checkPathInBackupIsRegisteredToRestoreAccess(const String & path); /// Reading a backup includes a few stages: enum class Stage @@ -104,11 +103,14 @@ private: void run(bool only_check_access); void setStage(Stage new_stage, const String & error_message = {}); void findRootPathsInBackup(); - void collectDatabaseAndTableInfos(); - void collectTableInfo(const QualifiedTableName & table_name_in_backup, bool is_temporary_table, const std::optional & partitions); - void collectDatabaseInfo(const String & database_name_in_backup, const std::set & except_table_names, bool throw_if_no_database_metadata_in_backup); - void collectAllDatabasesInfo(const std::set & except_database_names, const std::set & except_table_names); - void checkAccessForCollectedInfos() const; + + void findDatabasesAndTablesInBackup(); + void findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional & partitions); + void findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names); + void findEverythingInBackup(const std::set & except_database_names, const std::set & except_table_names); + + void checkAccessForObjectsFoundInBackup() const; + void createDatabases(); void createTables(); @@ -128,18 +130,10 @@ private: TableLockHolder table_lock; }; - struct TableKey - { - QualifiedTableName name; - bool is_temporary = false; - bool operator ==(const TableKey & right) const; - bool operator <(const TableKey & right) const; - }; - - std::vector findTablesWithoutDependencies() const; + std::vector findTablesWithoutDependencies() const; std::unordered_map database_infos; - std::map table_infos; + std::map table_infos; std::vector data_restore_tasks; std::shared_ptr access_restore_task; }; diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index caedfc55f3d..fc14d7abbd9 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -19,7 +19,6 @@ namespace DB namespace ErrorCodes { extern const int WRONG_DDL_RENAMING_SETTINGS; - extern const int LOGICAL_ERROR; } namespace @@ -31,24 +30,41 @@ namespace { /// CREATE TEMPORARY TABLE String table_name = create.getTable(); - const auto & new_table_name = data.renaming_map.getNewTemporaryTableName(table_name); - if (new_table_name != table_name) - create.setTable(new_table_name); + QualifiedTableName full_table_name{DatabaseCatalog::TEMPORARY_DATABASE, table_name}; + const auto & new_table_name = data.renaming_map.getNewTableName(full_table_name); + if (new_table_name != full_table_name) + { + create.setTable(new_table_name.table); + if (new_table_name.database != DatabaseCatalog::TEMPORARY_DATABASE) + { + create.temporary = false; + create.setDatabase(new_table_name.database); + } + } + } else if (create.table) { /// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW - QualifiedTableName qualified_name; - qualified_name.table = create.getTable(); - qualified_name.database = create.getDatabase(); + QualifiedTableName full_name; + full_name.table = create.getTable(); + full_name.database = create.getDatabase(); - if (!qualified_name.database.empty() && !qualified_name.table.empty()) + if (!full_name.database.empty() && !full_name.table.empty()) { - auto new_qualified_name = data.renaming_map.getNewTableName(qualified_name); - if (new_qualified_name != qualified_name) + auto new_table_name = data.renaming_map.getNewTableName(full_name); + if (new_table_name != full_name) { - create.setTable(new_qualified_name.table); - create.setDatabase(new_qualified_name.database); + create.setTable(new_table_name.table); + if (new_table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + { + create.temporary = true; + create.setDatabase(""); + } + else + { + create.setDatabase(new_table_name.database); + } } } } @@ -301,7 +317,7 @@ void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context, void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name) { if (old_table_name.table.empty() || old_table_name.database.empty() || new_table_name.table.empty() || new_table_name.database.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); + throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed"); auto it = old_to_new_table_names.find(old_table_name); if ((it != old_to_new_table_names.end())) @@ -321,7 +337,7 @@ void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, void DDLRenamingMap::setNewDatabaseName(const String & old_database_name, const String & new_database_name) { if (old_database_name.empty() || new_database_name.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); + throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Empty names are not allowed"); auto it = old_to_new_database_names.find(old_database_name); if ((it != old_to_new_database_names.end())) @@ -351,28 +367,4 @@ QualifiedTableName DDLRenamingMap::getNewTableName(const QualifiedTableName & ol return {getNewDatabaseName(old_table_name.database), old_table_name.table}; } -void DDLRenamingMap::setNewTemporaryTableName(const String & old_table_name, const String & new_table_name) -{ - if (old_table_name.empty() || new_table_name.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty names are not allowed"); - - auto it = old_to_new_temporary_table_names.find(old_table_name); - if ((it != old_to_new_temporary_table_names.end())) - { - if (it->second == new_table_name) - return; - throw Exception(ErrorCodes::WRONG_DDL_RENAMING_SETTINGS, "Wrong renaming: it's specified that temporary table {} should be renamed to {} and to {} at the same time", - backQuoteIfNeed(old_table_name), backQuoteIfNeed(it->second), backQuoteIfNeed(new_table_name)); - } - old_to_new_temporary_table_names[old_table_name] = new_table_name; -} - -const String & DDLRenamingMap::getNewTemporaryTableName(const String & old_table_name) const -{ - auto it = old_to_new_temporary_table_names.find(old_table_name); - if (it != old_to_new_temporary_table_names.end()) - return it->second; - return old_table_name; -} - } diff --git a/src/Databases/DDLRenamingVisitor.h b/src/Databases/DDLRenamingVisitor.h index 9d0f770d105..72b578b9fcb 100644 --- a/src/Databases/DDLRenamingVisitor.h +++ b/src/Databases/DDLRenamingVisitor.h @@ -25,16 +25,13 @@ class DDLRenamingMap public: void setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name); void setNewDatabaseName(const String & old_database_name, const String & new_database_name); - void setNewTemporaryTableName(const String & old_table_name, const String & new_table_name); QualifiedTableName getNewTableName(const QualifiedTableName & old_table_name) const; const String & getNewDatabaseName(const String & old_database_name) const; - const String & getNewTemporaryTableName(const String & old_table_name) const; private: std::unordered_map old_to_new_table_names; std::unordered_map old_to_new_database_names; - std::unordered_map old_to_new_temporary_table_names; }; /// Visits ASTCreateQuery and changes names of databases or tables. From aaf7f665498393f0cf051e78e825122bb105bf3a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 23 Jun 2022 19:45:36 +0200 Subject: [PATCH 292/408] Improve gathering metadata for backup - part 4. --- src/Backups/BackupCoordinationDistributed.cpp | 15 ++- src/Backups/BackupCoordinationDistributed.h | 7 +- src/Backups/BackupCoordinationHelpers.cpp | 95 +++++++++++-------- src/Backups/BackupCoordinationHelpers.h | 10 +- src/Backups/BackupCoordinationLocal.cpp | 8 +- src/Backups/BackupCoordinationLocal.h | 5 +- src/Backups/BackupEntriesCollector.cpp | 93 ++++++++++-------- src/Backups/BackupEntriesCollector.h | 29 +----- src/Backups/IBackupCoordination.h | 9 +- src/Backups/IRestoreCoordination.h | 9 +- .../RestoreCoordinationDistributed.cpp | 15 ++- src/Backups/RestoreCoordinationDistributed.h | 11 +-- src/Backups/RestoreCoordinationLocal.cpp | 8 +- src/Backups/RestoreCoordinationLocal.h | 9 +- src/Backups/RestorerFromBackup.cpp | 94 ++++++++++-------- src/Backups/RestorerFromBackup.h | 29 +----- 16 files changed, 232 insertions(+), 214 deletions(-) diff --git a/src/Backups/BackupCoordinationDistributed.cpp b/src/Backups/BackupCoordinationDistributed.cpp index 945239482fc..77377194012 100644 --- a/src/Backups/BackupCoordinationDistributed.cpp +++ b/src/Backups/BackupCoordinationDistributed.cpp @@ -131,7 +131,7 @@ namespace BackupCoordinationDistributed::BackupCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , stage_sync(zookeeper_path_ + "/stage", get_zookeeper_, &Poco::Logger::get("BackupCoordination")) + , status_sync(zookeeper_path_ + "/status", get_zookeeper_, &Poco::Logger::get("BackupCoordination")) { createRootNodes(); } @@ -157,14 +157,19 @@ void BackupCoordinationDistributed::removeAllNodes() } -void BackupCoordinationDistributed::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void BackupCoordinationDistributed::setStatus(const String & current_host, const String & new_status) { - stage_sync.syncStage(current_host, new_stage, wait_hosts, timeout); + status_sync.set(current_host, new_status); } -void BackupCoordinationDistributed::syncStageError(const String & current_host, const String & error_message) +void BackupCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) { - stage_sync.syncStageError(current_host, error_message); + status_sync.setAndWait(current_host, new_status, other_hosts); +} + +void BackupCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +{ + status_sync.setAndWaitFor(current_host, new_status, other_hosts, timeout_ms); } diff --git a/src/Backups/BackupCoordinationDistributed.h b/src/Backups/BackupCoordinationDistributed.h index 2872e1f3ae4..03da567bf07 100644 --- a/src/Backups/BackupCoordinationDistributed.h +++ b/src/Backups/BackupCoordinationDistributed.h @@ -14,8 +14,9 @@ public: BackupCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_); ~BackupCoordinationDistributed() override; - void syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - void syncStageError(const String & current_host, const String & error_message) override; + void setStatus(const String & current_host, const String & new_status) override; + void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; + void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames( const String & table_zk_path, @@ -51,7 +52,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - BackupCoordinationStageSync stage_sync; + BackupCoordinationStatusSync status_sync; mutable std::mutex mutex; mutable std::optional replicated_part_names; diff --git a/src/Backups/BackupCoordinationHelpers.cpp b/src/Backups/BackupCoordinationHelpers.cpp index 9528f888770..7c77e488119 100644 --- a/src/Backups/BackupCoordinationHelpers.cpp +++ b/src/Backups/BackupCoordinationHelpers.cpp @@ -243,7 +243,7 @@ void BackupCoordinationReplicatedPartNames::preparePartNames() const /// Helps to wait until all hosts come to a specified stage. -BackupCoordinationStageSync::BackupCoordinationStageSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_) +BackupCoordinationStatusSync::BackupCoordinationStatusSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) , log(log_) @@ -251,20 +251,46 @@ BackupCoordinationStageSync::BackupCoordinationStageSync(const String & zookeepe createRootNodes(); } -void BackupCoordinationStageSync::createRootNodes() +void BackupCoordinationStatusSync::createRootNodes() { auto zookeeper = get_zookeeper(); zookeeper->createAncestors(zookeeper_path); zookeeper->createIfNotExists(zookeeper_path, ""); } -void BackupCoordinationStageSync::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void BackupCoordinationStatusSync::set(const String & current_host, const String & new_status) { - /// Put new stage to ZooKeeper. - auto zookeeper = get_zookeeper(); - zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + std::to_string(new_stage), ""); + setImpl(current_host, new_status, {}, {}); +} - if (wait_hosts.empty() || ((wait_hosts.size() == 1) && (wait_hosts.front() == current_host))) +void BackupCoordinationStatusSync::setAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) +{ + setImpl(current_host, new_status, other_hosts, {}); +} + +void BackupCoordinationStatusSync::setAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +{ + setImpl(current_host, new_status, other_hosts, timeout_ms); +} + +void BackupCoordinationStatusSync::setImpl(const String & current_host, const String & new_status, const Strings & other_hosts, const std::optional & timeout_ms) +{ + /// Put new status to ZooKeeper. + auto zookeeper = get_zookeeper(); + + String result_status = new_status; + String message; + std::string_view error_prefix = "error: "; + bool is_error_status = new_status.starts_with(error_prefix); + if (is_error_status) + { + message = new_status.substr(error_prefix.length()); + result_status = "error"; + } + + zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + result_status, message); + + if (other_hosts.empty() || ((other_hosts.size() == 1) && (other_hosts.front() == current_host)) || is_error_status) return; /// Wait for other hosts. @@ -273,41 +299,35 @@ void BackupCoordinationStageSync::syncStage(const String & current_host, int new std::optional host_with_error; std::optional error_message; - std::map> unready_hosts; - for (const String & host : wait_hosts) - unready_hosts.emplace(host, std::optional{}); + std::map unready_hosts; + for (const String & host : other_hosts) + unready_hosts.emplace(host, ""); /// Process ZooKeeper's nodes and set `all_hosts_ready` or `unready_host` or `error_message`. auto process_zk_nodes = [&](const Strings & zk_nodes) { for (const String & zk_node : zk_nodes) { - if (zk_node == "error") + if (zk_node.starts_with("remove_watch-")) + continue; + + size_t separator_pos = zk_node.find('|'); + if (separator_pos == String::npos) + throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected zk node {}", zookeeper_path + "/" + zk_node); + String host = zk_node.substr(0, separator_pos); + String status = zk_node.substr(separator_pos + 1); + if (status == "error") { - String str = zookeeper->get(zookeeper_path + "/" + zk_node); - size_t separator_pos = str.find('|'); - if (separator_pos == String::npos) - throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected value of zk node {}: {}", zookeeper_path + "/" + zk_node, str); - host_with_error = str.substr(0, separator_pos); - error_message = str.substr(separator_pos + 1); - return; + host_with_error = host; + error_message = zookeeper->get(zookeeper_path + "/" + zk_node); + return; } - else if (!zk_node.starts_with("remove_watch-")) + auto it = unready_hosts.find(host); + if (it != unready_hosts.end()) { - size_t separator_pos = zk_node.find('|'); - if (separator_pos == String::npos) - throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected zk node {}", zookeeper_path + "/" + zk_node); - String host = zk_node.substr(0, separator_pos); - int found_stage = parseFromString(zk_node.substr(separator_pos + 1)); - auto it = unready_hosts.find(host); - if (it != unready_hosts.end()) - { - auto & stage = it->second; - if (!stage || (stage < found_stage)) - stage = found_stage; - if (stage >= new_stage) - unready_hosts.erase(it); - } + it->second = status; + if (status == result_status) + unready_hosts.erase(it); } } }; @@ -324,7 +344,8 @@ void BackupCoordinationStageSync::syncStage(const String & current_host, int new auto watch_triggered = [&] { return !watch_set; }; - bool use_timeout = (timeout.count() >= 0); + bool use_timeout = timeout_ms.has_value(); + std::chrono::milliseconds timeout{timeout_ms.value_or(0)}; std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); std::chrono::steady_clock::duration elapsed; std::mutex dummy_mutex; @@ -371,10 +392,4 @@ void BackupCoordinationStageSync::syncStage(const String & current_host, int new } } -void BackupCoordinationStageSync::syncStageError(const String & current_host, const String & error_message) -{ - auto zookeeper = get_zookeeper(); - zookeeper->createIfNotExists(zookeeper_path + "/error", current_host + "|" + error_message); -} - } diff --git a/src/Backups/BackupCoordinationHelpers.h b/src/Backups/BackupCoordinationHelpers.h index b0cd0440b98..ea07543ecb8 100644 --- a/src/Backups/BackupCoordinationHelpers.h +++ b/src/Backups/BackupCoordinationHelpers.h @@ -58,16 +58,18 @@ private: /// Helps to wait until all hosts come to a specified stage. -class BackupCoordinationStageSync +class BackupCoordinationStatusSync { public: - BackupCoordinationStageSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_); + BackupCoordinationStatusSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_); - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout); - void syncStageError(const String & current_host, const String & error_message); + void set(const String & current_host, const String & new_status); + void setAndWait(const String & current_host, const String & new_status, const Strings & other_hosts); + void setAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms); private: void createRootNodes(); + void setImpl(const String & current_host, const String & new_status, const Strings & other_hosts, const std::optional & timeout_ms); String zookeeper_path; zkutil::GetZooKeeper get_zookeeper; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 55a3c671a6e..bace1d800e0 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -13,11 +13,15 @@ using FileInfo = IBackupCoordination::FileInfo; BackupCoordinationLocal::BackupCoordinationLocal() = default; BackupCoordinationLocal::~BackupCoordinationLocal() = default; -void BackupCoordinationLocal::syncStage(const String &, int, const Strings &, std::chrono::seconds) +void BackupCoordinationLocal::setStatus(const String &, const String &) { } -void BackupCoordinationLocal::syncStageError(const String &, const String &) +void BackupCoordinationLocal::setStatusAndWait(const String &, const String &, const Strings &) +{ +} + +void BackupCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const Strings &, UInt64) { } diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 6529184c61a..090c5653f04 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -19,8 +19,9 @@ public: BackupCoordinationLocal(); ~BackupCoordinationLocal() override; - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - void syncStageError(const String & current_host, const String & error_message) override; + void setStatus(const String & current_host, const String & new_status) override; + void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; + void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index feed9be4e92..63f6d75170c 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -29,27 +29,44 @@ namespace ErrorCodes namespace { - String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_char_uppercase) - { - if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) - return fmt::format("{}emporary table {}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(table_name)); - else - return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); - } -} + /// Initial status. + constexpr const char kPreparingStatus[] = "preparing"; -std::string_view BackupEntriesCollector::toString(Stage stage) -{ - switch (stage) + /// Finding all tables and databases which we're going to put to the backup and collecting their metadata. + constexpr const char kGatheringMetadataStatus[] = "gathering metadata"; + + /// Making temporary hard links and prepare backup entries. + constexpr const char kExtractingDataFromTablesStatus[] = "extracting data from tables"; + + /// Running special tasks for replicated tables which can also prepare some backup entries. + constexpr const char kRunningPostTasksStatus[] = "running post-tasks"; + + /// Writing backup entries to the backup and removing temporary hard links. + constexpr const char kWritingBackupStatus[] = "writing backup"; + + /// Prefix for error statuses. + constexpr const char kErrorStatus[] = "error: "; + + /// Uppercases the first character of a passed string. + String toUpperFirst(const String & str) { - case Stage::kPreparing: return "Preparing"; - case Stage::kFindingTables: return "Finding tables"; - case Stage::kExtractingDataFromTables: return "Extracting data from tables"; - case Stage::kRunningPostTasks: return "Running post tasks"; - case Stage::kWritingBackup: return "Writing backup"; - case Stage::kError: return "Error"; + String res = str; + res[0] = std::toupper(res[0]); + return res; + } + + /// Outputs "table " or "temporary table " + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_upper) + { + String str; + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + str = fmt::format("temporary table {}", backQuoteIfNeed(table_name)); + else + str = fmt::format("table {}.{}", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + if (first_upper) + str[0] = std::toupper(str[0]); + return str; } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown backup stage: {}", static_cast(stage)); } @@ -65,6 +82,7 @@ BackupEntriesCollector::BackupEntriesCollector( , context(context_) , timeout(timeout_) , log(&Poco::Logger::get("BackupEntriesCollector")) + , current_status(kPreparingStatus) { } @@ -75,7 +93,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() try { /// getBackupEntries() must not be called multiple times. - if (current_stage != Stage::kPreparing) + if (current_status != kPreparingStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already making backup entries"); /// Calculate the root path for collecting backup entries, it's either empty or has the format "shards//replicas//". @@ -85,7 +103,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() renaming_map = makeRenamingMapFromBackupQuery(backup_query_elements); /// Find databases and tables which we're going to put to the backup. - setStage(Stage::kFindingTables); + setStatus(kGatheringMetadataStatus); gatherMetadataAndCheckConsistency(); /// Make backup entries for the definitions of the found databases. @@ -95,15 +113,15 @@ BackupEntries BackupEntriesCollector::getBackupEntries() makeBackupEntriesForTablesDefs(); /// Make backup entries for the data of the found tables. - setStage(Stage::kExtractingDataFromTables); + setStatus(kExtractingDataFromTablesStatus); makeBackupEntriesForTablesData(); /// Run all the tasks added with addPostCollectingTask(). - setStage(Stage::kRunningPostTasks); + setStatus(kRunningPostTasksStatus); runPostTasks(); /// No more backup entries or tasks are allowed after this point. - setStage(Stage::kWritingBackup); + setStatus(kWritingBackupStatus); return std::move(backup_entries); } @@ -111,7 +129,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() { try { - setStage(Stage::kError, getCurrentExceptionMessage(false)); + setStatus(kErrorStatus + getCurrentExceptionMessage(false)); } catch (...) { @@ -120,24 +138,21 @@ BackupEntries BackupEntriesCollector::getBackupEntries() } } -void BackupEntriesCollector::setStage(Stage new_stage, const String & error_message) +void BackupEntriesCollector::setStatus(const String & new_status) { - if (new_stage == Stage::kError) - LOG_ERROR(log, "{} failed with error: {}", toString(current_stage), error_message); - else - LOG_TRACE(log, "{}", toString(new_stage)); - - current_stage = new_stage; - - if (new_stage == Stage::kError) + bool is_error_status = new_status.starts_with(kErrorStatus); + if (is_error_status) { - backup_coordination->syncStageError(backup_settings.host_id, error_message); + LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), new_status); + backup_coordination->setStatus(backup_settings.host_id, new_status); } else { + LOG_TRACE(log, "{}", toUpperFirst(new_status)); + current_status = new_status; auto all_hosts = BackupSettings::Util::filterHostIDs(backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); - backup_coordination->syncStage(backup_settings.host_id, static_cast(new_stage), all_hosts, timeout); + backup_coordination->setStatusAndWait(backup_settings.host_id, new_status, all_hosts); } } @@ -575,28 +590,28 @@ void BackupEntriesCollector::makeBackupEntriesForTablesData() void BackupEntriesCollector::addBackupEntry(const String & file_name, BackupEntryPtr backup_entry) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); backup_entries.emplace_back(file_name, backup_entry); } void BackupEntriesCollector::addBackupEntries(const BackupEntries & backup_entries_) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); insertAtEnd(backup_entries, backup_entries_); } void BackupEntriesCollector::addBackupEntries(BackupEntries && backup_entries_) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding backup entries is not allowed"); insertAtEnd(backup_entries, std::move(backup_entries_)); } void BackupEntriesCollector::addPostTask(std::function task) { - if (current_stage == Stage::kWritingBackup) + if (current_status == kWritingBackupStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding post tasks is not allowed"); post_tasks.push(std::move(task)); } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 5e37e268fc4..4afca3f4cf9 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -51,35 +51,10 @@ public: /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. void addPostTask(std::function task); - /// Writing a backup includes a few stages: - enum class Stage - { - /// Initial stage. - kPreparing, - - /// Finding all tables and databases which we're going to put to the backup. - kFindingTables, - - /// Making temporary hard links and prepare backup entries. - kExtractingDataFromTables, - - /// Running special tasks for replicated databases or tables which can also prepare some backup entries. - kRunningPostTasks, - - /// Writing backup entries to the backup and removing temporary hard links. - kWritingBackup, - - /// An error happens during any of the stages above, the backup won't be written. - kError, - }; - static std::string_view toString(Stage stage); - /// Throws an exception that a specified table engine doesn't support partitions. [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); private: - void setStage(Stage new_stage, const String & error_message = {}); - void calculateRootPathInBackup(); void gatherMetadataAndCheckConsistency(); @@ -105,6 +80,8 @@ private: void makeBackupEntriesForTablesData(); void runPostTasks(); + void setStatus(const String & new_status); + const ASTBackupQuery::Elements backup_query_elements; const BackupSettings backup_settings; std::shared_ptr backup_coordination; @@ -112,7 +89,7 @@ private: std::chrono::seconds timeout; Poco::Logger * log; - Stage current_stage = Stage::kPreparing; + String current_status; std::filesystem::path root_path_in_backup; DDLRenamingMap renaming_map; diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 92b7139ed5f..58ed45e810d 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -13,11 +13,10 @@ class IBackupCoordination public: virtual ~IBackupCoordination() = default; - /// Sets the current stage and waits for other hosts to come to this stage too. - virtual void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) = 0; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - virtual void syncStageError(const String & current_host, const String & error_message) = 0; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + virtual void setStatus(const String & current_host, const String & new_status) = 0; + virtual void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) = 0; + virtual void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) = 0; struct PartNameAndChecksum { diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index fd9a67e1b96..b2ef32c2588 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -13,11 +13,10 @@ class IRestoreCoordination public: virtual ~IRestoreCoordination() = default; - /// Sets the current stage and waits for other hosts to come to this stage too. - virtual void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) = 0; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - virtual void syncStageError(const String & current_host, const String & error_message) = 0; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + virtual void setStatus(const String & current_host, const String & new_status) = 0; + virtual void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) = 0; + virtual void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) = 0; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. virtual bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) = 0; diff --git a/src/Backups/RestoreCoordinationDistributed.cpp b/src/Backups/RestoreCoordinationDistributed.cpp index e131ce7fe24..0b21f7367d8 100644 --- a/src/Backups/RestoreCoordinationDistributed.cpp +++ b/src/Backups/RestoreCoordinationDistributed.cpp @@ -9,7 +9,7 @@ namespace DB RestoreCoordinationDistributed::RestoreCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_) : zookeeper_path(zookeeper_path_) , get_zookeeper(get_zookeeper_) - , stage_sync(zookeeper_path_ + "/stage", get_zookeeper_, &Poco::Logger::get("RestoreCoordination")) + , status_sync(zookeeper_path_ + "/status", get_zookeeper_, &Poco::Logger::get("RestoreCoordination")) { createRootNodes(); } @@ -26,14 +26,19 @@ void RestoreCoordinationDistributed::createRootNodes() zookeeper->createIfNotExists(zookeeper_path + "/repl_access_storages_acquired", ""); } -void RestoreCoordinationDistributed::syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) +void RestoreCoordinationDistributed::setStatus(const String & current_host, const String & new_status) { - stage_sync.syncStage(current_host, new_stage, wait_hosts, timeout); + status_sync.set(current_host, new_status); } -void RestoreCoordinationDistributed::syncStageError(const String & current_host, const String & error_message) +void RestoreCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) { - stage_sync.syncStageError(current_host, error_message); + status_sync.setAndWait(current_host, new_status, other_hosts); +} + +void RestoreCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +{ + status_sync.setAndWaitFor(current_host, new_status, other_hosts, timeout_ms); } bool RestoreCoordinationDistributed::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationDistributed.h b/src/Backups/RestoreCoordinationDistributed.h index 0ea5db3f062..ecc8910bb9e 100644 --- a/src/Backups/RestoreCoordinationDistributed.h +++ b/src/Backups/RestoreCoordinationDistributed.h @@ -14,11 +14,10 @@ public: RestoreCoordinationDistributed(const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper); ~RestoreCoordinationDistributed() override; - /// Sets the current stage and waits for other hosts to come to this stage too. - void syncStage(const String & current_host, int new_stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - void syncStageError(const String & current_host, const String & error_message) override; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + void setStatus(const String & current_host, const String & new_status) override; + void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; + void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; @@ -42,7 +41,7 @@ private: const String zookeeper_path; const zkutil::GetZooKeeper get_zookeeper; - BackupCoordinationStageSync stage_sync; + BackupCoordinationStatusSync status_sync; }; } diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 9cecc3f90c9..e654ace454d 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -7,11 +7,15 @@ namespace DB RestoreCoordinationLocal::RestoreCoordinationLocal() = default; RestoreCoordinationLocal::~RestoreCoordinationLocal() = default; -void RestoreCoordinationLocal::syncStage(const String &, int, const Strings &, std::chrono::seconds) +void RestoreCoordinationLocal::setStatus(const String &, const String &) { } -void RestoreCoordinationLocal::syncStageError(const String &, const String &) +void RestoreCoordinationLocal::setStatusAndWait(const String &, const String &, const Strings &) +{ +} + +void RestoreCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const Strings &, UInt64) { } diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index b73f345df47..f38e1f94b9c 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -17,11 +17,10 @@ public: RestoreCoordinationLocal(); ~RestoreCoordinationLocal() override; - /// Sets the current stage and waits for other hosts to come to this stage too. - void syncStage(const String & current_host, int stage, const Strings & wait_hosts, std::chrono::seconds timeout) override; - - /// Sets that the current host encountered an error, so other hosts should know that and stop waiting in syncStage(). - void syncStageError(const String & current_host, const String & error_message) override; + /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. + void setStatus(const String & current_host, const String & new_status) override; + void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; + void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 74d4de631e3..bbe11525c58 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -39,12 +39,43 @@ namespace ErrorCodes namespace { - String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_char_uppercase) + /// Initial status. + constexpr const char kPreparingStatus[] = "preparing"; + + /// Finding databases and tables in the backup which we're going to restore. + constexpr const char kFindingTablesInBackupStatus[] = "finding tables in backup"; + + /// Creating databases or finding them and checking their definitions. + constexpr const char kCreatingDatabasesStatus[] = "creating databases"; + + /// Creating tables or finding them and checking their definition. + constexpr const char kCreatingTablesStatus[] = "creating tables"; + + /// Inserting restored data to tables. + constexpr const char kInsertingDataToTablesStatus[] = "inserting data to tables"; + + /// Prefix for error statuses. + constexpr const char kErrorStatus[] = "error: "; + + /// Uppercases the first character of a passed string. + String toUpperFirst(const String & str) { + String res = str; + res[0] = std::toupper(res[0]); + return res; + } + + /// Outputs "table " or "temporary table " + String tableNameWithTypeToString(const String & database_name, const String & table_name, bool first_upper) + { + String str; if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) - return fmt::format("{}emporary table {}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(table_name)); + str = fmt::format("temporary table {}", backQuoteIfNeed(table_name)); else - return fmt::format("{}able {}.{}", first_char_uppercase ? 'T' : 't', backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + str = fmt::format("table {}.{}", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); + if (first_upper) + str[0] = std::toupper(str[0]); + return str; } String tryGetTableEngine(const IAST & ast) @@ -70,20 +101,6 @@ namespace } } -std::string_view RestorerFromBackup::toString(Stage stage) -{ - switch (stage) - { - case Stage::kPreparing: return "Preparing"; - case Stage::kFindingTablesInBackup: return "Finding tables in backup"; - case Stage::kCreatingDatabases: return "Creating databases"; - case Stage::kCreatingTables: return "Creating tables"; - case Stage::kInsertingDataToTables: return "Inserting data to tables"; - case Stage::kError: return "Error"; - } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown restore stage: {}", static_cast(stage)); -} - RestorerFromBackup::RestorerFromBackup( const ASTBackupQuery::Elements & restore_query_elements_, @@ -100,6 +117,7 @@ RestorerFromBackup::RestorerFromBackup( , timeout(timeout_) , create_table_timeout_ms(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(&Poco::Logger::get("RestorerFromBackup")) + , current_status(kPreparingStatus) { } @@ -120,7 +138,7 @@ void RestorerFromBackup::run(bool only_check_access) try { /// restoreMetadata() must not be called multiple times. - if (current_stage != Stage::kPreparing) + if (current_status != kPreparingStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already restoring"); /// Calculate the root path in the backup for restoring, it's either empty or has the format "shards//replicas//". @@ -130,7 +148,7 @@ void RestorerFromBackup::run(bool only_check_access) renaming_map = makeRenamingMapFromBackupQuery(restore_query_elements); /// Find all the databases and tables which we will read from the backup. - setStage(Stage::kFindingTablesInBackup); + setStatus(kFindingTablesInBackupStatus); findDatabasesAndTablesInBackup(); /// Check access rights. @@ -140,23 +158,23 @@ void RestorerFromBackup::run(bool only_check_access) return; /// Create databases using the create queries read from the backup. - setStage(Stage::kCreatingDatabases); + setStatus(kCreatingDatabasesStatus); createDatabases(); /// Create tables using the create queries read from the backup. - setStage(Stage::kCreatingTables); + setStatus(kCreatingTablesStatus); createTables(); /// All what's left is to insert data to tables. /// No more data restoring tasks are allowed after this point. - setStage(Stage::kInsertingDataToTables); + setStatus(kInsertingDataToTablesStatus); } catch (...) { try { /// Other hosts should know that we've encountered an error. - setStage(Stage::kError, getCurrentExceptionMessage(false)); + setStatus(kErrorStatus + getCurrentExceptionMessage(false)); } catch (...) { @@ -168,7 +186,7 @@ void RestorerFromBackup::run(bool only_check_access) RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() { - if (current_stage != Stage::kInsertingDataToTables) + if (current_status != kInsertingDataToTablesStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Metadata wasn't restored"); if (data_restore_tasks.empty() && !access_restore_task) @@ -197,27 +215,23 @@ RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() return res_tasks; } -void RestorerFromBackup::setStage(Stage new_stage, const String & error_message) +void RestorerFromBackup::setStatus(const String & new_status) { - if (new_stage == Stage::kError) - LOG_ERROR(log, "{} failed with error: {}", toString(current_stage), error_message); - else - LOG_TRACE(log, "{}", toString(new_stage)); - - current_stage = new_stage; - - if (!restore_coordination) - return; - - if (new_stage == Stage::kError) + bool is_error_status = new_status.starts_with(kErrorStatus); + if (is_error_status) { - restore_coordination->syncStageError(restore_settings.host_id, error_message); + LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), new_status); + if (restore_coordination) + restore_coordination->setStatus(restore_settings.host_id, new_status); } else { + LOG_TRACE(log, "{}", toUpperFirst(new_status)); + current_status = new_status; auto all_hosts = BackupSettings::Util::filterHostIDs(restore_settings.cluster_host_ids, restore_settings.shard_num, restore_settings.replica_num); - restore_coordination->syncStage(restore_settings.host_id, static_cast(new_stage), all_hosts, timeout); + if (restore_coordination) + restore_coordination->setStatusAndWait(restore_settings.host_id, new_status, all_hosts); } } @@ -767,14 +781,14 @@ std::vector RestorerFromBackup::findTablesWithoutDependencie void RestorerFromBackup::addDataRestoreTask(DataRestoreTask && new_task) { - if (current_stage == Stage::kInsertingDataToTables) + if (current_status == kInsertingDataToTablesStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding data-restoring tasks is not allowed"); data_restore_tasks.push_back(std::move(new_task)); } void RestorerFromBackup::addDataRestoreTasks(DataRestoreTasks && new_tasks) { - if (current_stage == Stage::kInsertingDataToTables) + if (current_status == kInsertingDataToTablesStatus) throw Exception(ErrorCodes::LOGICAL_ERROR, "Adding data-restoring tasks is not allowed"); insertAtEnd(data_restore_tasks, std::move(new_tasks)); } diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 3bdbafe844c..07258837aab 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -57,29 +57,6 @@ public: /// Checks that a specified path is already registered to be used for restoring access control. void checkPathInBackupIsRegisteredToRestoreAccess(const String & path); - /// Reading a backup includes a few stages: - enum class Stage - { - /// Initial stage. - kPreparing, - - /// Finding databases and tables in the backup which we're going to restore. - kFindingTablesInBackup, - - /// Creating databases or finding them and checking their definitions. - kCreatingDatabases, - - /// Creating tables or finding them and checking their definition. - kCreatingTables, - - /// Inserting restored data to tables. - kInsertingDataToTables, - - /// An error happens during any of the stages above, the backup is not restored properly. - kError = -1, - }; - static std::string_view toString(Stage stage); - /// Throws an exception that a specified table engine doesn't support partitions. [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); @@ -96,12 +73,12 @@ private: UInt64 create_table_timeout_ms; Poco::Logger * log; - Stage current_stage = Stage::kPreparing; + String current_status; std::vector root_paths_in_backup; DDLRenamingMap renaming_map; void run(bool only_check_access); - void setStage(Stage new_stage, const String & error_message = {}); + void findRootPathsInBackup(); void findDatabasesAndTablesInBackup(); @@ -114,6 +91,8 @@ private: void createDatabases(); void createTables(); + void setStatus(const String & new_status); + struct DatabaseInfo { ASTPtr create_database_query; From 6ca400fd89ef5e5bbaf067a2b013a58b0fcf7039 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 23 Jun 2022 20:49:44 +0200 Subject: [PATCH 293/408] Improve gathering metadata for backup - part 5. --- src/Backups/BackupCoordinationDistributed.cpp | 12 +- src/Backups/BackupCoordinationDistributed.h | 6 +- src/Backups/BackupCoordinationHelpers.cpp | 61 +++-- src/Backups/BackupCoordinationHelpers.h | 10 +- src/Backups/BackupCoordinationLocal.cpp | 8 +- src/Backups/BackupCoordinationLocal.h | 6 +- src/Backups/BackupEntriesCollector.cpp | 210 ++++++++++++------ src/Backups/BackupEntriesCollector.h | 29 +-- src/Backups/BackupsWorker.cpp | 15 +- src/Backups/IBackupCoordination.h | 6 +- src/Backups/IRestoreCoordination.h | 6 +- .../RestoreCoordinationDistributed.cpp | 12 +- src/Backups/RestoreCoordinationDistributed.h | 6 +- src/Backups/RestoreCoordinationLocal.cpp | 8 +- src/Backups/RestoreCoordinationLocal.h | 6 +- src/Backups/RestorerFromBackup.cpp | 134 +++++------ src/Backups/RestorerFromBackup.h | 34 +-- 17 files changed, 317 insertions(+), 252 deletions(-) diff --git a/src/Backups/BackupCoordinationDistributed.cpp b/src/Backups/BackupCoordinationDistributed.cpp index 77377194012..9612b62dcdb 100644 --- a/src/Backups/BackupCoordinationDistributed.cpp +++ b/src/Backups/BackupCoordinationDistributed.cpp @@ -157,19 +157,19 @@ void BackupCoordinationDistributed::removeAllNodes() } -void BackupCoordinationDistributed::setStatus(const String & current_host, const String & new_status) +void BackupCoordinationDistributed::setStatus(const String & current_host, const String & new_status, const String & message) { - status_sync.set(current_host, new_status); + status_sync.set(current_host, new_status, message); } -void BackupCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) +Strings BackupCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) { - status_sync.setAndWait(current_host, new_status, other_hosts); + return status_sync.setAndWait(current_host, new_status, message, all_hosts); } -void BackupCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +Strings BackupCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) { - status_sync.setAndWaitFor(current_host, new_status, other_hosts, timeout_ms); + return status_sync.setAndWaitFor(current_host, new_status, message, all_hosts, timeout_ms); } diff --git a/src/Backups/BackupCoordinationDistributed.h b/src/Backups/BackupCoordinationDistributed.h index 03da567bf07..84cd4b3dddb 100644 --- a/src/Backups/BackupCoordinationDistributed.h +++ b/src/Backups/BackupCoordinationDistributed.h @@ -14,9 +14,9 @@ public: BackupCoordinationDistributed(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_); ~BackupCoordinationDistributed() override; - void setStatus(const String & current_host, const String & new_status) override; - void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; - void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames( const String & table_zk_path, diff --git a/src/Backups/BackupCoordinationHelpers.cpp b/src/Backups/BackupCoordinationHelpers.cpp index 7c77e488119..d86e8cff52c 100644 --- a/src/Backups/BackupCoordinationHelpers.cpp +++ b/src/Backups/BackupCoordinationHelpers.cpp @@ -258,51 +258,45 @@ void BackupCoordinationStatusSync::createRootNodes() zookeeper->createIfNotExists(zookeeper_path, ""); } -void BackupCoordinationStatusSync::set(const String & current_host, const String & new_status) +void BackupCoordinationStatusSync::set(const String & current_host, const String & new_status, const String & message) { - setImpl(current_host, new_status, {}, {}); + setImpl(current_host, new_status, message, {}, {}); } -void BackupCoordinationStatusSync::setAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) +Strings BackupCoordinationStatusSync::setAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) { - setImpl(current_host, new_status, other_hosts, {}); + return setImpl(current_host, new_status, message, all_hosts, {}); } -void BackupCoordinationStatusSync::setAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +Strings BackupCoordinationStatusSync::setAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) { - setImpl(current_host, new_status, other_hosts, timeout_ms); + return setImpl(current_host, new_status, message, all_hosts, timeout_ms); } -void BackupCoordinationStatusSync::setImpl(const String & current_host, const String & new_status, const Strings & other_hosts, const std::optional & timeout_ms) +Strings BackupCoordinationStatusSync::setImpl(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, const std::optional & timeout_ms) { - /// Put new status to ZooKeeper. + /// Put new status to ZooKeeper. auto zookeeper = get_zookeeper(); + zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + new_status, message); - String result_status = new_status; - String message; - std::string_view error_prefix = "error: "; - bool is_error_status = new_status.starts_with(error_prefix); - if (is_error_status) - { - message = new_status.substr(error_prefix.length()); - result_status = "error"; - } - - zookeeper->createIfNotExists(zookeeper_path + "/" + current_host + "|" + result_status, message); - - if (other_hosts.empty() || ((other_hosts.size() == 1) && (other_hosts.front() == current_host)) || is_error_status) - return; + if (all_hosts.empty() || (new_status == kErrorStatus)) + return {}; + if ((all_hosts.size() == 1) && (all_hosts.front() == current_host)) + return {message}; + /// Wait for other hosts. - /// Current stages of all hosts. + Strings ready_hosts_results; + ready_hosts_results.resize(all_hosts.size()); + + std::map /* index in `ready_hosts_results` */> unready_hosts; + for (size_t i = 0; i != all_hosts.size(); ++i) + unready_hosts[all_hosts[i]].push_back(i); + std::optional host_with_error; std::optional error_message; - std::map unready_hosts; - for (const String & host : other_hosts) - unready_hosts.emplace(host, ""); - /// Process ZooKeeper's nodes and set `all_hosts_ready` or `unready_host` or `error_message`. auto process_zk_nodes = [&](const Strings & zk_nodes) { @@ -316,18 +310,19 @@ void BackupCoordinationStatusSync::setImpl(const String & current_host, const St throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Unexpected zk node {}", zookeeper_path + "/" + zk_node); String host = zk_node.substr(0, separator_pos); String status = zk_node.substr(separator_pos + 1); - if (status == "error") + if (status == kErrorStatus) { host_with_error = host; error_message = zookeeper->get(zookeeper_path + "/" + zk_node); return; } auto it = unready_hosts.find(host); - if (it != unready_hosts.end()) + if ((it != unready_hosts.end()) && (status == new_status)) { - it->second = status; - if (status == result_status) - unready_hosts.erase(it); + String result = zookeeper->get(zookeeper_path + "/" + zk_node); + for (size_t i : it->second) + ready_hosts_results[i] = result; + unready_hosts.erase(it); } } }; @@ -390,6 +385,8 @@ void BackupCoordinationStatusSync::setImpl(const String & current_host, const St unready_hosts.begin()->first, to_string(elapsed)); } + + return ready_hosts_results; } } diff --git a/src/Backups/BackupCoordinationHelpers.h b/src/Backups/BackupCoordinationHelpers.h index ea07543ecb8..7d343edd3d0 100644 --- a/src/Backups/BackupCoordinationHelpers.h +++ b/src/Backups/BackupCoordinationHelpers.h @@ -63,13 +63,15 @@ class BackupCoordinationStatusSync public: BackupCoordinationStatusSync(const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, Poco::Logger * log_); - void set(const String & current_host, const String & new_status); - void setAndWait(const String & current_host, const String & new_status, const Strings & other_hosts); - void setAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms); + void set(const String & current_host, const String & new_status, const String & message); + Strings setAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts); + Strings setAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms); + + static constexpr const char * kErrorStatus = "error"; private: void createRootNodes(); - void setImpl(const String & current_host, const String & new_status, const Strings & other_hosts, const std::optional & timeout_ms); + Strings setImpl(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, const std::optional & timeout_ms); String zookeeper_path; zkutil::GetZooKeeper get_zookeeper; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index bace1d800e0..6ff9800797e 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -13,16 +13,18 @@ using FileInfo = IBackupCoordination::FileInfo; BackupCoordinationLocal::BackupCoordinationLocal() = default; BackupCoordinationLocal::~BackupCoordinationLocal() = default; -void BackupCoordinationLocal::setStatus(const String &, const String &) +void BackupCoordinationLocal::setStatus(const String &, const String &, const String &) { } -void BackupCoordinationLocal::setStatusAndWait(const String &, const String &, const Strings &) +Strings BackupCoordinationLocal::setStatusAndWait(const String &, const String &, const String &, const Strings &) { + return {}; } -void BackupCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const Strings &, UInt64) +Strings BackupCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const String &, const Strings &, UInt64) { + return {}; } void BackupCoordinationLocal::addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 090c5653f04..7fdd88d37cb 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -19,9 +19,9 @@ public: BackupCoordinationLocal(); ~BackupCoordinationLocal() override; - void setStatus(const String & current_host, const String & new_status) override; - void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; - void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 63f6d75170c..1ba94552589 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -29,23 +31,20 @@ namespace ErrorCodes namespace { - /// Initial status. - constexpr const char kPreparingStatus[] = "preparing"; - /// Finding all tables and databases which we're going to put to the backup and collecting their metadata. - constexpr const char kGatheringMetadataStatus[] = "gathering metadata"; + constexpr const char * kGatheringMetadataStatus = "gathering metadata"; /// Making temporary hard links and prepare backup entries. - constexpr const char kExtractingDataFromTablesStatus[] = "extracting data from tables"; + constexpr const char * kExtractingDataFromTablesStatus = "extracting data from tables"; /// Running special tasks for replicated tables which can also prepare some backup entries. - constexpr const char kRunningPostTasksStatus[] = "running post-tasks"; + constexpr const char * kRunningPostTasksStatus = "running post-tasks"; /// Writing backup entries to the backup and removing temporary hard links. - constexpr const char kWritingBackupStatus[] = "writing backup"; + constexpr const char * kWritingBackupStatus = "writing backup"; - /// Prefix for error statuses. - constexpr const char kErrorStatus[] = "error: "; + /// Error status. + constexpr const char * kErrorStatus = BackupCoordinationStatusSync::kErrorStatus; /// Uppercases the first character of a passed string. String toUpperFirst(const String & str) @@ -67,6 +66,19 @@ namespace str[0] = std::toupper(str[0]); return str; } + + /// How long we should sleep after finding an inconsistency error. + std::chrono::milliseconds getSleepTimeAfterInconsistencyError(size_t pass) + { + size_t ms; + if (pass == 1) /* pass is 1-based */ + ms = 0; + else if ((pass % 10) != 1) + ms = 0; + else + ms = 1000; + return std::chrono::milliseconds{ms}; + } } @@ -74,36 +86,37 @@ BackupEntriesCollector::BackupEntriesCollector( const ASTBackupQuery::Elements & backup_query_elements_, const BackupSettings & backup_settings_, std::shared_ptr backup_coordination_, - const ContextPtr & context_, - std::chrono::seconds timeout_) + const ContextPtr & context_) : backup_query_elements(backup_query_elements_) , backup_settings(backup_settings_) , backup_coordination(backup_coordination_) , context(context_) - , timeout(timeout_) + , consistent_metadata_snapshot_timeout(context->getConfigRef().getUInt64("backups.consistent_metadata_snapshot_timeout", 300000)) , log(&Poco::Logger::get("BackupEntriesCollector")) - , current_status(kPreparingStatus) { } BackupEntriesCollector::~BackupEntriesCollector() = default; -BackupEntries BackupEntriesCollector::getBackupEntries() +BackupEntries BackupEntriesCollector::run() { try { - /// getBackupEntries() must not be called multiple times. - if (current_status != kPreparingStatus) + /// run() can be called onle once. + if (!current_status.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already making backup entries"); - /// Calculate the root path for collecting backup entries, it's either empty or has the format "shards//replicas//". - calculateRootPathInBackup(); + /// Find other hosts working along with us to execute this ON CLUSTER query. + all_hosts + = BackupSettings::Util::filterHostIDs(backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); /// Do renaming in the create queries according to the renaming config. renaming_map = makeRenamingMapFromBackupQuery(backup_query_elements); + /// Calculate the root path for collecting backup entries, it's either empty or has the format "shards//replicas//". + calculateRootPathInBackup(); + /// Find databases and tables which we're going to put to the backup. - setStatus(kGatheringMetadataStatus); gatherMetadataAndCheckConsistency(); /// Make backup entries for the definitions of the found databases. @@ -129,7 +142,7 @@ BackupEntries BackupEntriesCollector::getBackupEntries() { try { - setStatus(kErrorStatus + getCurrentExceptionMessage(false)); + setStatus(kErrorStatus, getCurrentExceptionMessage(false)); } catch (...) { @@ -138,21 +151,34 @@ BackupEntries BackupEntriesCollector::getBackupEntries() } } -void BackupEntriesCollector::setStatus(const String & new_status) +Strings BackupEntriesCollector::setStatus(const String & new_status, const String & message) { - bool is_error_status = new_status.starts_with(kErrorStatus); - if (is_error_status) + if (new_status == kErrorStatus) { - LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), new_status); - backup_coordination->setStatus(backup_settings.host_id, new_status); + LOG_ERROR(log, "{} failed with error: {}", toUpperFirst(current_status), message); + backup_coordination->setStatus(backup_settings.host_id, new_status, message); + return {}; } else { LOG_TRACE(log, "{}", toUpperFirst(new_status)); current_status = new_status; - auto all_hosts - = BackupSettings::Util::filterHostIDs(backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); - backup_coordination->setStatusAndWait(backup_settings.host_id, new_status, all_hosts); + if (new_status.starts_with(kGatheringMetadataStatus)) + { + auto now = std::chrono::steady_clock::now(); + auto end_of_timeout = std::max(now, consistent_metadata_snapshot_start_time + consistent_metadata_snapshot_timeout); + + return backup_coordination->setStatusAndWaitFor( + backup_settings.host_id, + new_status, + message, + all_hosts, + std::chrono::duration_cast(end_of_timeout - now).count()); + } + else + { + return backup_coordination->setStatusAndWait(backup_settings.host_id, new_status, message, all_hosts); + } } } @@ -173,45 +199,87 @@ void BackupEntriesCollector::calculateRootPathInBackup() /// Finds databases and tables which we will put to the backup. void BackupEntriesCollector::gatherMetadataAndCheckConsistency() { - bool use_timeout = (timeout.count() >= 0); - auto start_time = std::chrono::steady_clock::now(); + consistent_metadata_snapshot_start_time = std::chrono::steady_clock::now(); + auto end_of_timeout = consistent_metadata_snapshot_start_time + consistent_metadata_snapshot_timeout; + setStatus(fmt::format("{} ({})", kGatheringMetadataStatus, 1)); for (size_t pass = 1;; ++pass) { - try + String new_status = fmt::format("{} ({})", kGatheringMetadataStatus, pass + 1); + std::optional inconsistency_error; + if (tryGatherMetadataAndCompareWithPrevious(inconsistency_error)) { - /// Collect information about databases and tables specified in the BACKUP query. - database_infos.clear(); - table_infos.clear(); - gatherDatabasesMetadata(); - gatherTablesMetadata(); - - /// We have to check consistency of collected information to protect from the case when some table or database is - /// renamed during this collecting making the collected information invalid. - auto comparing_error = compareWithPrevious(); - if (!comparing_error) - break; /// no error, everything's fine + /// Gathered metadata and checked consistency, cool! But we have to check that other hosts cope with that too. + auto all_hosts_results = setStatus(new_status, "consistent"); - if (pass >= 2) /// Two passes is minimum (we need to compare with table names with previous ones to be sure we don't miss anything). - throw *comparing_error; - } - catch (Exception & e) - { - if (e.code() != ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP) - throw; + std::optional host_with_inconsistency; + std::optional inconsistency_error_on_other_host; + for (size_t i = 0; i != all_hosts.size(); ++i) + { + if ((i < all_hosts_results.size()) && (all_hosts_results[i] != "consistent")) + { + host_with_inconsistency = all_hosts[i]; + inconsistency_error_on_other_host = all_hosts_results[i]; + break; + } + } - auto elapsed = std::chrono::steady_clock::now() - start_time; - e.addMessage("Couldn't gather tables and databases to make a backup (pass #{}, elapsed {})", pass, to_string(elapsed)); - if (use_timeout && (elapsed > timeout)) - throw; - else - LOG_WARNING(log, "{}", e.displayText()); + if (!host_with_inconsistency) + break; /// All hosts managed to gather metadata and everything is consistent, so we can go further to writing the backup. + + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Found inconsistency on host {}: {}", + *host_with_inconsistency, + *inconsistency_error_on_other_host}; } + else + { + /// Failed to gather metadata or something wasn't consistent. We'll let other hosts know that and try again. + setStatus(new_status, inconsistency_error->displayText()); + } + + /// Two passes is minimum (we need to compare with table names with previous ones to be sure we don't miss anything). + if (pass >= 2) + { + if (std::chrono::steady_clock::now() > end_of_timeout) + inconsistency_error->rethrow(); + else + LOG_WARNING(log, "{}", inconsistency_error->displayText()); + } + + auto sleep_time = getSleepTimeAfterInconsistencyError(pass); + if (sleep_time.count() > 0) + sleepForNanoseconds(std::chrono::duration_cast(sleep_time).count()); } LOG_INFO(log, "Will backup {} databases and {} tables", database_infos.size(), table_infos.size()); } +bool BackupEntriesCollector::tryGatherMetadataAndCompareWithPrevious(std::optional & inconsistency_error) +{ + try + { + /// Collect information about databases and tables specified in the BACKUP query. + database_infos.clear(); + table_infos.clear(); + gatherDatabasesMetadata(); + gatherTablesMetadata(); + } + catch (Exception & e) + { + if (e.code() != ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP) + throw; + + inconsistency_error = e; + return false; + } + + /// We have to check consistency of collected information to protect from the case when some table or database is + /// renamed during this collecting making the collected information invalid. + return compareWithPrevious(inconsistency_error); +} + void BackupEntriesCollector::gatherDatabasesMetadata() { /// Collect information about databases and tables specified in the BACKUP query. @@ -465,7 +533,7 @@ void BackupEntriesCollector::lockTablesForReading() } /// Check consistency of collected information about databases and tables. -std::optional BackupEntriesCollector::compareWithPrevious() +bool BackupEntriesCollector::compareWithPrevious(std::optional & inconsistency_error) { /// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed /// while we were scanning. @@ -476,60 +544,64 @@ std::optional BackupEntriesCollector::compareWithPrevious() if (previous_database_names != database_names) { - std::optional comparing_error; + bool error_message_ready = false; for (const auto & database_name : database_names) { if (!previous_database_names.contains(database_name)) { - comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were added during scanning", backQuoteIfNeed(database_name)}; + inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were added during scanning", backQuoteIfNeed(database_name)}; + error_message_ready = true; break; } } - if (!comparing_error) + if (!error_message_ready) { for (const auto & database_name : previous_database_names) { if (!database_names.contains(database_name)) { - comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were removed during scanning", backQuoteIfNeed(database_name)}; + inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were removed during scanning", backQuoteIfNeed(database_name)}; + error_message_ready = true; break; } } } - assert(comparing_error); + assert(error_message_ready); previous_database_names = std::move(database_names); previous_table_names = std::move(table_names); - return comparing_error; + return false; } if (previous_table_names != table_names) { - std::optional comparing_error; + bool error_message_ready = false; for (const auto & table_name : table_names) { if (!previous_table_names.contains(table_name)) { - comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were added during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were added during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + error_message_ready = true; break; } } - if (!comparing_error) + if (!error_message_ready) { for (const auto & table_name : previous_table_names) { if (!table_names.contains(table_name)) { - comparing_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were removed during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were removed during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; + error_message_ready = true; break; } } } - assert(comparing_error); + assert(error_message_ready); previous_table_names = std::move(table_names); - return comparing_error; + return false; } - return {}; + return true; } /// Make backup entries for all the definitions of all the databases found. diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 4afca3f4cf9..49901295003 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -27,26 +27,25 @@ public: BackupEntriesCollector(const ASTBackupQuery::Elements & backup_query_elements_, const BackupSettings & backup_settings_, std::shared_ptr backup_coordination_, - const ContextPtr & context_, - std::chrono::seconds timeout_ = std::chrono::seconds(-1) /* no timeout */); + const ContextPtr & context_); ~BackupEntriesCollector(); /// Collects backup entries and returns the result. - /// This function first generates a list of databases and then call IDatabase::backup() for each database from this list. - /// At this moment IDatabase::backup() calls IStorage::backup() and they both call addBackupEntry() to build a list of backup entries. - BackupEntries getBackupEntries(); + /// This function first generates a list of databases and then call IDatabase::getTablesForBackup() for each database from this list. + /// Then it calls IStorage::backupData() to build a list of backup entries. + BackupEntries run(); const BackupSettings & getBackupSettings() const { return backup_settings; } std::shared_ptr getBackupCoordination() const { return backup_coordination; } ContextPtr getContext() const { return context; } - /// Adds a backup entry which will be later returned by getBackupEntries(). - /// These function can be called by implementations of IStorage::backup() in inherited storage classes. + /// Adds a backup entry which will be later returned by run(). + /// These function can be called by implementations of IStorage::backupData() in inherited storage classes. void addBackupEntry(const String & file_name, BackupEntryPtr backup_entry); void addBackupEntries(const BackupEntries & backup_entries_); void addBackupEntries(BackupEntries && backup_entries_); - /// Adds a function which must be called after all IStorage::backup() have finished their work on all hosts. + /// Adds a function which must be called after all IStorage::backupData() have finished their work on all hosts. /// This function is designed to help making a consistent in some complex cases like /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. void addPostTask(std::function task); @@ -59,6 +58,8 @@ private: void gatherMetadataAndCheckConsistency(); + bool tryGatherMetadataAndCompareWithPrevious(std::optional & inconsistency_error); + void gatherDatabasesMetadata(); void gatherDatabaseMetadata( @@ -73,25 +74,25 @@ private: void gatherTablesMetadata(); void lockTablesForReading(); - std::optional compareWithPrevious(); + bool compareWithPrevious(std::optional & inconsistency_error); void makeBackupEntriesForDatabasesDefs(); void makeBackupEntriesForTablesDefs(); void makeBackupEntriesForTablesData(); void runPostTasks(); - void setStatus(const String & new_status); + Strings setStatus(const String & new_status, const String & message = ""); const ASTBackupQuery::Elements backup_query_elements; const BackupSettings backup_settings; std::shared_ptr backup_coordination; ContextPtr context; - std::chrono::seconds timeout; + std::chrono::milliseconds consistent_metadata_snapshot_timeout; Poco::Logger * log; - String current_status; - std::filesystem::path root_path_in_backup; + Strings all_hosts; DDLRenamingMap renaming_map; + std::filesystem::path root_path_in_backup; struct DatabaseInfo { @@ -122,6 +123,8 @@ private: std::optional partitions; }; + String current_status; + std::chrono::steady_clock::time_point consistent_metadata_snapshot_start_time; std::unordered_map database_infos; std::map table_infos; std::set previous_database_names; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 84dc63b4f9f..635b2810941 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -166,9 +166,8 @@ UUID BackupsWorker::startMakingBackup(const ASTPtr & query, const ContextPtr & c BackupEntries backup_entries; { - auto timeout = std::chrono::seconds{context_in_use->getConfigRef().getInt("backups.backup_prepare_timeout", -1)}; - BackupEntriesCollector backup_entries_collector{backup_query->elements, backup_settings, backup_coordination, context_in_use, timeout}; - backup_entries = backup_entries_collector.getBackupEntries(); + BackupEntriesCollector backup_entries_collector{backup_query->elements, backup_settings, backup_coordination, context_in_use}; + backup_entries = backup_entries_collector.run(); } writeBackupEntries(backup, std::move(backup_entries), backups_thread_pool); @@ -272,8 +271,8 @@ UUID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr conte String addr_database = address->default_database.empty() ? current_database : address->default_database; for (auto & element : restore_elements) element.setCurrentDatabase(addr_database); - RestorerFromBackup dummy_restorer{restore_elements, restore_settings, nullptr, backup, context_in_use, {}}; - dummy_restorer.checkAccessOnly(); + RestorerFromBackup dummy_restorer{restore_elements, restore_settings, nullptr, backup, context_in_use}; + dummy_restorer.run(RestorerFromBackup::CHECK_ACCESS_ONLY); } } @@ -325,11 +324,9 @@ UUID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePtr conte DataRestoreTasks data_restore_tasks; { - auto timeout = std::chrono::seconds{context_in_use->getConfigRef().getInt("backups.restore_metadata_timeout", -1)}; RestorerFromBackup restorer{restore_query->elements, restore_settings, restore_coordination, - backup, context_in_use, timeout}; - restorer.restoreMetadata(); - data_restore_tasks = restorer.getDataRestoreTasks(); + backup, context_in_use}; + data_restore_tasks = restorer.run(RestorerFromBackup::RESTORE); } restoreTablesData(std::move(data_restore_tasks), restores_thread_pool); diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 58ed45e810d..9df5b9efdc4 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -14,9 +14,9 @@ public: virtual ~IBackupCoordination() = default; /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. - virtual void setStatus(const String & current_host, const String & new_status) = 0; - virtual void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) = 0; - virtual void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) = 0; + virtual void setStatus(const String & current_host, const String & new_status, const String & message) = 0; + virtual Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts) = 0; + virtual Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts, UInt64 timeout_ms) = 0; struct PartNameAndChecksum { diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index b2ef32c2588..ba76a6e0c99 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -14,9 +14,9 @@ public: virtual ~IRestoreCoordination() = default; /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. - virtual void setStatus(const String & current_host, const String & new_status) = 0; - virtual void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) = 0; - virtual void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) = 0; + virtual void setStatus(const String & current_host, const String & new_status, const String & message) = 0; + virtual Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts) = 0; + virtual Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & other_hosts, UInt64 timeout_ms) = 0; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. virtual bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) = 0; diff --git a/src/Backups/RestoreCoordinationDistributed.cpp b/src/Backups/RestoreCoordinationDistributed.cpp index 0b21f7367d8..8cbaa01810d 100644 --- a/src/Backups/RestoreCoordinationDistributed.cpp +++ b/src/Backups/RestoreCoordinationDistributed.cpp @@ -26,19 +26,19 @@ void RestoreCoordinationDistributed::createRootNodes() zookeeper->createIfNotExists(zookeeper_path + "/repl_access_storages_acquired", ""); } -void RestoreCoordinationDistributed::setStatus(const String & current_host, const String & new_status) +void RestoreCoordinationDistributed::setStatus(const String & current_host, const String & new_status, const String & message) { - status_sync.set(current_host, new_status); + status_sync.set(current_host, new_status, message); } -void RestoreCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) +Strings RestoreCoordinationDistributed::setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) { - status_sync.setAndWait(current_host, new_status, other_hosts); + return status_sync.setAndWait(current_host, new_status, message, all_hosts); } -void RestoreCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) +Strings RestoreCoordinationDistributed::setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) { - status_sync.setAndWaitFor(current_host, new_status, other_hosts, timeout_ms); + return status_sync.setAndWaitFor(current_host, new_status, message, all_hosts, timeout_ms); } bool RestoreCoordinationDistributed::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationDistributed.h b/src/Backups/RestoreCoordinationDistributed.h index ecc8910bb9e..52b961cf0ef 100644 --- a/src/Backups/RestoreCoordinationDistributed.h +++ b/src/Backups/RestoreCoordinationDistributed.h @@ -15,9 +15,9 @@ public: ~RestoreCoordinationDistributed() override; /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. - void setStatus(const String & current_host, const String & new_status) override; - void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; - void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index e654ace454d..a999cc13195 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -7,16 +7,18 @@ namespace DB RestoreCoordinationLocal::RestoreCoordinationLocal() = default; RestoreCoordinationLocal::~RestoreCoordinationLocal() = default; -void RestoreCoordinationLocal::setStatus(const String &, const String &) +void RestoreCoordinationLocal::setStatus(const String &, const String &, const String &) { } -void RestoreCoordinationLocal::setStatusAndWait(const String &, const String &, const Strings &) +Strings RestoreCoordinationLocal::setStatusAndWait(const String &, const String &, const String &, const Strings &) { + return {}; } -void RestoreCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const Strings &, UInt64) +Strings RestoreCoordinationLocal::setStatusAndWaitFor(const String &, const String &, const String &, const Strings &, UInt64) { + return {}; } bool RestoreCoordinationLocal::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index f38e1f94b9c..68624481a7a 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -18,9 +18,9 @@ public: ~RestoreCoordinationLocal() override; /// Sets the current status and waits for other hosts to come to this status too. If status starts with "error:" it'll stop waiting on all the hosts. - void setStatus(const String & current_host, const String & new_status) override; - void setStatusAndWait(const String & current_host, const String & new_status, const Strings & other_hosts) override; - void setStatusAndWaitFor(const String & current_host, const String & new_status, const Strings & other_hosts, UInt64 timeout_ms) override; + void setStatus(const String & current_host, const String & new_status, const String & message) override; + Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; + Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index bbe11525c58..12151eff08f 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -39,23 +40,20 @@ namespace ErrorCodes namespace { - /// Initial status. - constexpr const char kPreparingStatus[] = "preparing"; - /// Finding databases and tables in the backup which we're going to restore. - constexpr const char kFindingTablesInBackupStatus[] = "finding tables in backup"; + constexpr const char * kFindingTablesInBackupStatus = "finding tables in backup"; /// Creating databases or finding them and checking their definitions. - constexpr const char kCreatingDatabasesStatus[] = "creating databases"; + constexpr const char * kCreatingDatabasesStatus = "creating databases"; /// Creating tables or finding them and checking their definition. - constexpr const char kCreatingTablesStatus[] = "creating tables"; + constexpr const char * kCreatingTablesStatus = "creating tables"; /// Inserting restored data to tables. - constexpr const char kInsertingDataToTablesStatus[] = "inserting data to tables"; + constexpr const char * kInsertingDataToTablesStatus = "inserting data to tables"; - /// Prefix for error statuses. - constexpr const char kErrorStatus[] = "error: "; + /// Error status. + constexpr const char * kErrorStatus = BackupCoordinationStatusSync::kErrorStatus; /// Uppercases the first character of a passed string. String toUpperFirst(const String & str) @@ -107,46 +105,37 @@ RestorerFromBackup::RestorerFromBackup( const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_, const BackupPtr & backup_, - const ContextMutablePtr & context_, - std::chrono::seconds timeout_) + const ContextMutablePtr & context_) : restore_query_elements(restore_query_elements_) , restore_settings(restore_settings_) , restore_coordination(restore_coordination_) , backup(backup_) , context(context_) - , timeout(timeout_) - , create_table_timeout_ms(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) + , create_table_timeout(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(&Poco::Logger::get("RestorerFromBackup")) - , current_status(kPreparingStatus) { } RestorerFromBackup::~RestorerFromBackup() = default; -void RestorerFromBackup::restoreMetadata() -{ - run(/* only_check_access= */ false); -} - -void RestorerFromBackup::checkAccessOnly() -{ - run(/* only_check_access= */ true); -} - -void RestorerFromBackup::run(bool only_check_access) +RestorerFromBackup::DataRestoreTasks RestorerFromBackup::run(Mode mode) { try { - /// restoreMetadata() must not be called multiple times. - if (current_status != kPreparingStatus) + /// run() can be called onle once. + if (!current_status.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Already restoring"); - /// Calculate the root path in the backup for restoring, it's either empty or has the format "shards//replicas//". - findRootPathsInBackup(); + /// Find other hosts working along with us to execute this ON CLUSTER query. + all_hosts = BackupSettings::Util::filterHostIDs( + restore_settings.cluster_host_ids, restore_settings.shard_num, restore_settings.replica_num); /// Do renaming in the create queries according to the renaming config. renaming_map = makeRenamingMapFromBackupQuery(restore_query_elements); + /// Calculate the root path in the backup for restoring, it's either empty or has the format "shards//replicas//". + findRootPathsInBackup(); + /// Find all the databases and tables which we will read from the backup. setStatus(kFindingTablesInBackupStatus); findDatabasesAndTablesInBackup(); @@ -154,8 +143,8 @@ void RestorerFromBackup::run(bool only_check_access) /// Check access rights. checkAccessForObjectsFoundInBackup(); - if (only_check_access) - return; + if (mode == Mode::CHECK_ACCESS_ONLY) + return {}; /// Create databases using the create queries read from the backup. setStatus(kCreatingDatabasesStatus); @@ -168,13 +157,14 @@ void RestorerFromBackup::run(bool only_check_access) /// All what's left is to insert data to tables. /// No more data restoring tasks are allowed after this point. setStatus(kInsertingDataToTablesStatus); + return getDataRestoreTasks(); } catch (...) { try { /// Other hosts should know that we've encountered an error. - setStatus(kErrorStatus + getCurrentExceptionMessage(false)); + setStatus(kErrorStatus, getCurrentExceptionMessage(false)); } catch (...) { @@ -183,55 +173,20 @@ void RestorerFromBackup::run(bool only_check_access) } } - -RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() +void RestorerFromBackup::setStatus(const String & new_status, const String & message) { - if (current_status != kInsertingDataToTablesStatus) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Metadata wasn't restored"); - - if (data_restore_tasks.empty() && !access_restore_task) - return {}; - - LOG_TRACE(log, "Will insert data to tables"); - - /// Storages and table locks must exist while we're executing data restoring tasks. - auto storages = std::make_shared>(); - auto table_locks = std::make_shared>(); - storages->reserve(table_infos.size()); - table_locks->reserve(table_infos.size()); - for (const auto & table_info : table_infos | boost::adaptors::map_values) + if (new_status == kErrorStatus) { - storages->push_back(table_info.storage); - table_locks->push_back(table_info.table_lock); - } - - DataRestoreTasks res_tasks; - for (const auto & task : data_restore_tasks) - res_tasks.push_back([task, storages, table_locks] { task(); }); - - if (access_restore_task) - res_tasks.push_back([task = access_restore_task, access_control = &context->getAccessControl()] { task->restore(*access_control); }); - - return res_tasks; -} - -void RestorerFromBackup::setStatus(const String & new_status) -{ - bool is_error_status = new_status.starts_with(kErrorStatus); - if (is_error_status) - { - LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), new_status); + LOG_ERROR(log, "{} failed with {}", toUpperFirst(current_status), message); if (restore_coordination) - restore_coordination->setStatus(restore_settings.host_id, new_status); + restore_coordination->setStatus(restore_settings.host_id, new_status, message); } else { LOG_TRACE(log, "{}", toUpperFirst(new_status)); current_status = new_status; - auto all_hosts - = BackupSettings::Util::filterHostIDs(restore_settings.cluster_host_ids, restore_settings.shard_num, restore_settings.replica_num); if (restore_coordination) - restore_coordination->setStatusAndWait(restore_settings.host_id, new_status, all_hosts); + restore_coordination->setStatusAndWait(restore_settings.host_id, new_status, message, all_hosts); } } @@ -677,13 +632,18 @@ void RestorerFromBackup::createTables() create_table_query = create_table_query->clone(); create_table_query->as().if_not_exists = true; } + LOG_TRACE( log, "Creating {}: {}", tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*create_table_query)); - database->createTableRestoredFromBackup(create_table_query, context, restore_coordination, create_table_timeout_ms); + database->createTableRestoredFromBackup( + create_table_query, + context, + restore_coordination, + std::chrono::duration_cast(create_table_timeout).count()); } table_info.created = true; @@ -799,6 +759,34 @@ void RestorerFromBackup::checkPathInBackupIsRegisteredToRestoreAccess(const Stri throw Exception(ErrorCodes::LOGICAL_ERROR, "Path to restore access was not added"); } +RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() +{ + if (data_restore_tasks.empty() && !access_restore_task) + return {}; + + LOG_TRACE(log, "Will insert data to tables"); + + /// Storages and table locks must exist while we're executing data restoring tasks. + auto storages = std::make_shared>(); + auto table_locks = std::make_shared>(); + storages->reserve(table_infos.size()); + table_locks->reserve(table_infos.size()); + for (const auto & table_info : table_infos | boost::adaptors::map_values) + { + storages->push_back(table_info.storage); + table_locks->push_back(table_info.table_lock); + } + + DataRestoreTasks res_tasks; + for (const auto & task : data_restore_tasks) + res_tasks.push_back([task, storages, table_locks] { task(); }); + + if (access_restore_task) + res_tasks.push_back([task = access_restore_task, access_control = &context->getAccessControl()] { task->restore(*access_control); }); + + return res_tasks; +} + void RestorerFromBackup::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) { throw Exception( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 07258837aab..f4d19a10cf6 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -26,27 +26,29 @@ public: const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_, const BackupPtr & backup_, - const ContextMutablePtr & context_, - std::chrono::seconds timeout_); + const ContextMutablePtr & context_); ~RestorerFromBackup(); - /// Restores the definition of databases and tables and prepares tasks to restore the data of the tables. - /// restoreMetadata() checks access rights internally so checkAccessRightsOnly() shouldn't be called first. - void restoreMetadata(); + enum Mode + { + /// Restores databases and tables. + RESTORE, - /// Only checks access rights without restoring anything. - void checkAccessOnly(); + /// Only checks access rights without restoring anything. + CHECK_ACCESS_ONLY + }; using DataRestoreTask = std::function; using DataRestoreTasks = std::vector; - DataRestoreTasks getDataRestoreTasks(); + + /// Restores the metadata of databases and tables and returns tasks to restore the data of tables. + DataRestoreTasks run(Mode mode); BackupPtr getBackup() const { return backup; } const RestoreSettings & getRestoreSettings() const { return restore_settings; } bool isNonEmptyTableAllowed() const { return getRestoreSettings().allow_non_empty_tables; } std::shared_ptr getRestoreCoordination() const { return restore_coordination; } - std::chrono::seconds getTimeout() const { return timeout; } ContextMutablePtr getContext() const { return context; } /// Adds a data restore task which will be later returned by getDataRestoreTasks(). @@ -69,15 +71,12 @@ private: std::shared_ptr restore_coordination; BackupPtr backup; ContextMutablePtr context; - std::chrono::seconds timeout; - UInt64 create_table_timeout_ms; + std::chrono::milliseconds create_table_timeout; Poco::Logger * log; - String current_status; - std::vector root_paths_in_backup; + Strings all_hosts; DDLRenamingMap renaming_map; - - void run(bool only_check_access); + std::vector root_paths_in_backup; void findRootPathsInBackup(); @@ -91,7 +90,9 @@ private: void createDatabases(); void createTables(); - void setStatus(const String & new_status); + DataRestoreTasks getDataRestoreTasks(); + + void setStatus(const String & new_status, const String & message = ""); struct DatabaseInfo { @@ -111,6 +112,7 @@ private: std::vector findTablesWithoutDependencies() const; + String current_status; std::unordered_map database_infos; std::map table_infos; std::vector data_restore_tasks; From 7689e0c36f1cb9e89faeb12694560d5fc350cdf9 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 24 Jun 2022 21:29:38 +0200 Subject: [PATCH 294/408] Improve gathering metadata for backup - part 6. --- src/Backups/BackupCoordinationDistributed.cpp | 22 +-- src/Backups/BackupCoordinationDistributed.h | 8 +- src/Backups/BackupCoordinationHelpers.cpp | 8 +- src/Backups/BackupCoordinationHelpers.h | 6 +- src/Backups/BackupCoordinationLocal.cpp | 16 +- src/Backups/BackupCoordinationLocal.h | 8 +- src/Backups/BackupEntriesCollector.cpp | 152 ++++++++++-------- src/Backups/BackupEntriesCollector.h | 7 +- src/Backups/DDLAdjustingForBackupVisitor.cpp | 114 +++++++++++++ src/Backups/DDLAdjustingForBackupVisitor.h | 36 +++++ src/Backups/IBackupCoordination.h | 8 +- src/Backups/RestorerFromBackup.cpp | 10 +- src/Databases/DDLRenamingVisitor.cpp | 15 +- src/Databases/DDLRenamingVisitor.h | 6 +- src/Databases/DatabaseReplicated.cpp | 40 +++++ src/Databases/DatabaseReplicated.h | 1 + src/Databases/IDatabase.cpp | 13 +- src/Databases/IDatabase.h | 3 - src/Storages/IStorage.cpp | 22 +-- .../extractZkPathFromCreateQuery.cpp | 60 +++++++ .../MergeTree/extractZkPathFromCreateQuery.h | 19 +++ src/Storages/StorageReplicatedMergeTree.cpp | 73 ++++----- src/Storages/StorageReplicatedMergeTree.h | 3 + .../test_backup_restore_on_cluster/test.py | 41 +++++ 24 files changed, 490 insertions(+), 201 deletions(-) create mode 100644 src/Backups/DDLAdjustingForBackupVisitor.cpp create mode 100644 src/Backups/DDLAdjustingForBackupVisitor.h create mode 100644 src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp create mode 100644 src/Storages/MergeTree/extractZkPathFromCreateQuery.h diff --git a/src/Backups/BackupCoordinationDistributed.cpp b/src/Backups/BackupCoordinationDistributed.cpp index 9612b62dcdb..9df17bf434e 100644 --- a/src/Backups/BackupCoordinationDistributed.cpp +++ b/src/Backups/BackupCoordinationDistributed.cpp @@ -174,7 +174,7 @@ Strings BackupCoordinationDistributed::setStatusAndWaitFor(const String & curren void BackupCoordinationDistributed::addReplicatedPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) @@ -186,39 +186,39 @@ void BackupCoordinationDistributed::addReplicatedPartNames( } auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_part_names/" + escapeForFileName(table_shared_id); zookeeper->createIfNotExists(path, ""); path += "/" + escapeForFileName(replica_name); zookeeper->create(path, ReplicatedPartNames::serialize(part_names_and_checksums, table_name_for_logs), zkutil::CreateMode::Persistent); } -Strings BackupCoordinationDistributed::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationDistributed::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const { std::lock_guard lock{mutex}; prepareReplicatedPartNames(); - return replicated_part_names->getPartNames(table_zk_path, replica_name); + return replicated_part_names->getPartNames(table_shared_id, replica_name); } void BackupCoordinationDistributed::addReplicatedDataPath( - const String & table_zk_path, const String & data_path) + const String & table_shared_id, const String & data_path) { auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_shared_id); + zookeeper->createIfNotExists(path, ""); + path += "/" + escapeForFileName(data_path); zookeeper->createIfNotExists(path, ""); - path += "/"; - zookeeper->create(path, data_path, zkutil::CreateMode::PersistentSequential); } -Strings BackupCoordinationDistributed::getReplicatedDataPaths(const String & table_zk_path) const +Strings BackupCoordinationDistributed::getReplicatedDataPaths(const String & table_shared_id) const { auto zookeeper = get_zookeeper(); - String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_zk_path); + String path = zookeeper_path + "/repl_data_paths/" + escapeForFileName(table_shared_id); Strings children = zookeeper->getChildren(path); Strings data_paths; data_paths.reserve(children.size()); for (const String & child : children) - data_paths.push_back(zookeeper->get(path + "/" + child)); + data_paths.push_back(unescapeForFileName(child)); return data_paths; } diff --git a/src/Backups/BackupCoordinationDistributed.h b/src/Backups/BackupCoordinationDistributed.h index 84cd4b3dddb..172c69edb20 100644 --- a/src/Backups/BackupCoordinationDistributed.h +++ b/src/Backups/BackupCoordinationDistributed.h @@ -19,15 +19,15 @@ public: Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; void addReplicatedPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; - Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override; + Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override; - void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override; - Strings getReplicatedDataPaths(const String & table_zk_path) const override; + void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; + Strings getReplicatedDataPaths(const String & table_shared_id) const override; void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; diff --git a/src/Backups/BackupCoordinationHelpers.cpp b/src/Backups/BackupCoordinationHelpers.cpp index d86e8cff52c..cca66f03aac 100644 --- a/src/Backups/BackupCoordinationHelpers.cpp +++ b/src/Backups/BackupCoordinationHelpers.cpp @@ -157,7 +157,7 @@ BackupCoordinationReplicatedPartNames::BackupCoordinationReplicatedPartNames() = BackupCoordinationReplicatedPartNames::~BackupCoordinationReplicatedPartNames() = default; void BackupCoordinationReplicatedPartNames::addPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) @@ -165,7 +165,7 @@ void BackupCoordinationReplicatedPartNames::addPartNames( if (part_names_prepared) throw Exception(ErrorCodes::LOGICAL_ERROR, "addPartNames() must not be called after getPartNames()"); - auto & table_info = table_infos[table_zk_path]; + auto & table_info = table_infos[table_shared_id]; if (!table_info.covered_parts_finder) table_info.covered_parts_finder = std::make_unique(table_name_for_logs); @@ -207,10 +207,10 @@ void BackupCoordinationReplicatedPartNames::addPartNames( } } -Strings BackupCoordinationReplicatedPartNames::getPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationReplicatedPartNames::getPartNames(const String & table_shared_id, const String & replica_name) const { preparePartNames(); - auto it = table_infos.find(table_zk_path); + auto it = table_infos.find(table_shared_id); if (it == table_infos.end()) return {}; const auto & replicas_parts = it->second.replicas_parts; diff --git a/src/Backups/BackupCoordinationHelpers.h b/src/Backups/BackupCoordinationHelpers.h index 7d343edd3d0..2e9e4b3cbde 100644 --- a/src/Backups/BackupCoordinationHelpers.h +++ b/src/Backups/BackupCoordinationHelpers.h @@ -24,7 +24,7 @@ public: /// getPartNames(). /// Checksums are used only to control that parts under the same names on different replicas are the same. void addPartNames( - const String & table_zk_path, + const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums); @@ -32,7 +32,7 @@ public: /// Returns the names of the parts which a specified replica of a replicated table should put to the backup. /// This is the same list as it was added by call of the function addPartNames() but without duplications and without /// parts covered by another parts. - Strings getPartNames(const String & table_zk_path, const String & replica_name) const; + Strings getPartNames(const String & table_shared_id, const String & replica_name) const; private: void preparePartNames() const; @@ -52,7 +52,7 @@ private: std::unique_ptr covered_parts_finder; }; - std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica. + std::map table_infos; /// Should be ordered because we need this map to be in the same order on every replica. mutable bool part_names_prepared = false; }; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 6ff9800797e..7fd6fec6c33 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -27,29 +27,29 @@ Strings BackupCoordinationLocal::setStatusAndWaitFor(const String &, const Strin return {}; } -void BackupCoordinationLocal::addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) +void BackupCoordinationLocal::addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) { std::lock_guard lock{mutex}; - replicated_part_names.addPartNames(table_zk_path, table_name_for_logs, replica_name, part_names_and_checksums); + replicated_part_names.addPartNames(table_shared_id, table_name_for_logs, replica_name, part_names_and_checksums); } -Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const +Strings BackupCoordinationLocal::getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const { std::lock_guard lock{mutex}; - return replicated_part_names.getPartNames(table_zk_path, replica_name); + return replicated_part_names.getPartNames(table_shared_id, replica_name); } -void BackupCoordinationLocal::addReplicatedDataPath(const String & table_zk_path, const String & data_path) +void BackupCoordinationLocal::addReplicatedDataPath(const String & table_shared_id, const String & data_path) { std::lock_guard lock{mutex}; - replicated_data_paths[table_zk_path].push_back(data_path); + replicated_data_paths[table_shared_id].push_back(data_path); } -Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_zk_path) const +Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_shared_id) const { std::lock_guard lock{mutex}; - auto it = replicated_data_paths.find(table_zk_path); + auto it = replicated_data_paths.find(table_shared_id); if (it == replicated_data_paths.end()) return {}; return it->second; diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 7fdd88d37cb..519c721c208 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -23,12 +23,12 @@ public: Strings setStatusAndWait(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts) override; Strings setStatusAndWaitFor(const String & current_host, const String & new_status, const String & message, const Strings & all_hosts, UInt64 timeout_ms) override; - void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, + void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; - Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const override; + Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const override; - void addReplicatedDataPath(const String & table_zk_path, const String & data_path) override; - Strings getReplicatedDataPaths(const String & table_zk_path) const override; + void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; + Strings getReplicatedDataPaths(const String & table_shared_id) const override; void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 1ba94552589..8104e363a68 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -389,7 +390,7 @@ void BackupEntriesCollector::gatherDatabaseMetadata( ASTPtr create_database_query; try { - create_database_query = database_info.database->getCreateDatabaseQueryForBackup(); + create_database_query = database_info.database->getCreateDatabaseQuery(); } catch (...) { @@ -537,68 +538,86 @@ bool BackupEntriesCollector::compareWithPrevious(std::optional & inco { /// We need to scan tables at least twice to be sure that we haven't missed any table which could be renamed /// while we were scanning. - std::set database_names; - std::set table_names; - boost::range::copy(database_infos | boost::adaptors::map_keys, std::inserter(database_names, database_names.end())); - boost::range::copy(table_infos | boost::adaptors::map_keys, std::inserter(table_names, table_names.end())); + std::vector> databases_metadata; + std::vector> tables_metadata; + databases_metadata.reserve(database_infos.size()); + tables_metadata.reserve(table_infos.size()); + for (const auto & [database_name, database_info] : database_infos) + databases_metadata.emplace_back(database_name, database_info.create_database_query ? serializeAST(*database_info.create_database_query) : ""); + for (const auto & [table_name, table_info] : table_infos) + tables_metadata.emplace_back(table_name, serializeAST(*table_info.create_table_query)); - if (previous_database_names != database_names) + /// We need to sort the lists to make the comparison below correct. + ::sort(databases_metadata.begin(), databases_metadata.end()); + ::sort(tables_metadata.begin(), tables_metadata.end()); + + SCOPE_EXIT({ + previous_databases_metadata = std::move(databases_metadata); + previous_tables_metadata = std::move(tables_metadata); + }); + + /// Databases must be the same as during the previous scan. + if (databases_metadata != previous_databases_metadata) { - bool error_message_ready = false; - for (const auto & database_name : database_names) + std::vector> difference; + difference.reserve(databases_metadata.size()); + std::set_difference(databases_metadata.begin(), databases_metadata.end(), previous_databases_metadata.begin(), + previous_databases_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) { - if (!previous_database_names.contains(database_name)) - { - inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were added during scanning", backQuoteIfNeed(database_name)}; - error_message_ready = true; - break; - } + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Database {} were created or changed its definition during scanning", + backQuoteIfNeed(difference[0].first)}; + return false; } - if (!error_message_ready) + + difference.clear(); + difference.reserve(previous_databases_metadata.size()); + std::set_difference(previous_databases_metadata.begin(), previous_databases_metadata.end(), databases_metadata.begin(), + databases_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) { - for (const auto & database_name : previous_database_names) - { - if (!database_names.contains(database_name)) - { - inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Database {} were removed during scanning", backQuoteIfNeed(database_name)}; - error_message_ready = true; - break; - } - } + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "Database {} were removed or changed its definition during scanning", + backQuoteIfNeed(difference[0].first)}; + return false; } - assert(error_message_ready); - previous_database_names = std::move(database_names); - previous_table_names = std::move(table_names); - return false; } - if (previous_table_names != table_names) + /// Tables must be the same as during the previous scan. + if (tables_metadata != previous_tables_metadata) { - bool error_message_ready = false; - for (const auto & table_name : table_names) + std::vector> difference; + difference.reserve(tables_metadata.size()); + std::set_difference(tables_metadata.begin(), tables_metadata.end(), previous_tables_metadata.begin(), + previous_tables_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) { - if (!previous_table_names.contains(table_name)) - { - inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were added during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; - error_message_ready = true; - break; - } + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "{} were created or changed its definition during scanning", + tableNameWithTypeToString(difference[0].first.database, difference[0].first.table, true)}; + return false; } - if (!error_message_ready) + + difference.clear(); + difference.reserve(previous_tables_metadata.size()); + std::set_difference(previous_tables_metadata.begin(), previous_tables_metadata.end(), tables_metadata.begin(), + tables_metadata.end(), std::back_inserter(difference)); + + if (!difference.empty()) { - for (const auto & table_name : previous_table_names) - { - if (!table_names.contains(table_name)) - { - inconsistency_error = Exception{ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "{} were removed during scanning", tableNameWithTypeToString(table_name.database, table_name.table, true)}; - error_message_ready = true; - break; - } - } + inconsistency_error = Exception{ + ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, + "{} were removed or changed its definition during scanning", + tableNameWithTypeToString(difference[0].first.database, difference[0].first.table, true)}; + return false; } - assert(error_message_ready); - previous_table_names = std::move(table_names); - return false; } return true; @@ -615,7 +634,8 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() LOG_TRACE(log, "Adding definition of database {}", backQuoteIfNeed(database_name)); ASTPtr new_create_query = database_info.create_database_query; - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); + adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), nullptr); + renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext()); const String & metadata_path_in_backup = database_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); @@ -625,12 +645,13 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() /// Calls IDatabase::backupTable() for all the tables found to make backup entries for tables. void BackupEntriesCollector::makeBackupEntriesForTablesDefs() { - for (const auto & [table_name, table_info] : table_infos) + for (auto & [table_name, table_info] : table_infos) { LOG_TRACE(log, "Adding definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); ASTPtr new_create_query = table_info.create_table_query; - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, new_create_query); + adjustCreateQueryForBackup(new_create_query, context->getGlobalContext(), &table_info.replicated_table_shared_id); + renameDatabaseAndTableNameInCreateQuery(new_create_query, renaming_map, context->getGlobalContext()); const String & metadata_path_in_backup = table_info.metadata_path_in_backup; backup_entries.emplace_back(metadata_path_in_backup, std::make_shared(serializeAST(*new_create_query))); @@ -645,18 +666,21 @@ void BackupEntriesCollector::makeBackupEntriesForTablesData() for (const auto & [table_name, table_info] : table_infos) { const auto & storage = table_info.storage; - if (!storage) - { - /// This storage exists on other replica and has not been created on this replica yet. - /// We store metadata only for such tables. - /// TODO: Need special processing if it's a ReplicatedMergeTree. - continue; - } - - LOG_TRACE(log, "Adding data of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); const auto & data_path_in_backup = table_info.data_path_in_backup; - const auto & partitions = table_info.partitions; - storage->backupData(*this, data_path_in_backup, partitions); + if (storage) + { + LOG_TRACE(log, "Adding data of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); + storage->backupData(*this, data_path_in_backup, table_info.partitions); + } + else + { + /// Storage == null means this storage exists on other replicas but it has not been created on this replica yet. + /// If this table is replicated in this case we call IBackupCoordination::addReplicatedDataPath() which will cause + /// other replicas to fill the storage's data in the backup. + /// If this table is not replicated we'll do nothing leaving the storage's data empty in the backup. + if (table_info.replicated_table_shared_id) + backup_coordination->addReplicatedDataPath(*table_info.replicated_table_shared_id, data_path_in_backup); + } } } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 49901295003..46a2bd1863a 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -120,15 +120,16 @@ private: ASTPtr create_table_query; String metadata_path_in_backup; std::filesystem::path data_path_in_backup; + std::optional replicated_table_shared_id; std::optional partitions; }; String current_status; std::chrono::steady_clock::time_point consistent_metadata_snapshot_start_time; std::unordered_map database_infos; - std::map table_infos; - std::set previous_database_names; - std::set previous_table_names; + std::unordered_map table_infos; + std::vector> previous_databases_metadata; + std::vector> previous_tables_metadata; BackupEntries backup_entries; std::queue> post_tasks; diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp new file mode 100644 index 00000000000..e3fc3ac5552 --- /dev/null +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace +{ + void visitStorageSystemTableEngine(ASTStorage &, const DDLAdjustingForBackupVisitor::Data & data) + { + /// Precondition: storage.engine && storage.engine->name.starts_with("System")) + + /// If this is a definition of a system table we'll remove columns and comment because they're reduntant for backups. + auto & create = data.create_query->as(); + create.reset(create.columns_list); + create.reset(create.comment); + } + + void visitStorageReplicatedTableEngine(ASTStorage & storage, const DDLAdjustingForBackupVisitor::Data & data) + { + /// Precondition: engine_name.starts_with("Replicated") && engine_name.ends_with("MergeTree") + + if (data.replicated_table_shared_id) + *data.replicated_table_shared_id = StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQuery(*data.create_query, data.global_context); + + /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there + /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. + /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. + auto & create = data.create_query->as(); + if (create.uuid == UUIDHelpers::Nil) + return; + + auto & engine = *storage.engine; + + auto * engine_args_ast = typeid_cast(engine.arguments.get()); + if (!engine_args_ast) + return; + + auto & engine_args = engine_args_ast->children; + if (engine_args.size() < 2) + return; + + auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); + auto * replica_name_ast = typeid_cast(engine_args[1].get()); + if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && + replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) + { + String & zookeeper_path_arg = zookeeper_path_ast->value.get(); + String & replica_name_arg = replica_name_ast->value.get(); + String table_uuid_str = toString(create.uuid); + if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) + zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); + const auto & config = data.global_context->getConfigRef(); + if ((zookeeper_path_arg == StorageReplicatedMergeTree::getDefaultZooKeeperPath(config)) + && (replica_name_arg == StorageReplicatedMergeTree::getDefaultReplicaName(config)) + && ((engine_args.size() == 2) || !engine_args[2]->as())) + { + engine_args.erase(engine_args.begin(), engine_args.begin() + 2); + } + } + } + + void visitStorage(ASTStorage & storage, const DDLAdjustingForBackupVisitor::Data & data) + { + if (!storage.engine) + return; + + const String & engine_name = storage.engine->name; + if (engine_name.starts_with("System")) + visitStorageSystemTableEngine(storage, data); + else if (engine_name.starts_with("Replicated") && engine_name.ends_with("MergeTree")) + visitStorageReplicatedTableEngine(storage, data); + } + + void visitCreateQuery(ASTCreateQuery & create, const DDLAdjustingForBackupVisitor::Data & data) + { + create.uuid = UUIDHelpers::Nil; + create.to_inner_uuid = UUIDHelpers::Nil; + + if (create.storage) + visitStorage(*create.storage, data); + } +} + + +bool DDLAdjustingForBackupVisitor::needChildVisit(const ASTPtr &, const ASTPtr &) +{ + return false; +} + +void DDLAdjustingForBackupVisitor::visit(ASTPtr ast, const Data & data) +{ + if (auto * create = ast->as()) + visitCreateQuery(*create, data); +} + +void adjustCreateQueryForBackup(ASTPtr & ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id) +{ + ast = ast->clone(); + if (replicated_table_shared_id) + *replicated_table_shared_id = {}; + + DDLAdjustingForBackupVisitor::Data data{ast, global_context, replicated_table_shared_id}; + DDLAdjustingForBackupVisitor::Visitor{data}.visit(ast); +} + +} diff --git a/src/Backups/DDLAdjustingForBackupVisitor.h b/src/Backups/DDLAdjustingForBackupVisitor.h new file mode 100644 index 00000000000..87498471cc4 --- /dev/null +++ b/src/Backups/DDLAdjustingForBackupVisitor.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class IAST; +using ASTPtr = std::shared_ptr; +class Context; +using ContextPtr = std::shared_ptr; + +/// Changes a create query to a form which is appropriate or suitable for saving in a backup. +/// Also extracts a replicated table's shared ID from the create query if this is a create query for a replicated table. +/// `replicated_table_shared_id` can be null if you don't need that. +void adjustCreateQueryForBackup(ASTPtr & ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id); + +/// Visits ASTCreateQuery and changes it to a form which is appropriate or suitable for saving in a backup. +class DDLAdjustingForBackupVisitor +{ +public: + struct Data + { + ASTPtr create_query; + ContextPtr global_context; + std::optional * replicated_table_shared_id = nullptr; + }; + + using Visitor = InDepthNodeVisitor; + + static bool needChildVisit(const ASTPtr & ast, const ASTPtr & child); + static void visit(ASTPtr ast, const Data & data); +}; + +} diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 9df5b9efdc4..0ae150c2b47 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -28,21 +28,21 @@ public: /// Multiple replicas of the replicated table call this function and then the added part names can be returned by call of the function /// getReplicatedPartNames(). /// Checksums are used only to control that parts under the same names on different replicas are the same. - virtual void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, + virtual void addReplicatedPartNames(const String & table_shared_id, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) = 0; /// Returns the names of the parts which a specified replica of a replicated table should put to the backup. /// This is the same list as it was added by call of the function addReplicatedPartNames() but without duplications and without /// parts covered by another parts. - virtual Strings getReplicatedPartNames(const String & table_zk_path, const String & replica_name) const = 0; + virtual Strings getReplicatedPartNames(const String & table_shared_id, const String & replica_name) const = 0; /// Adds a data path in backup for a replicated table. /// Multiple replicas of the replicated table call this function and then all the added paths can be returned by call of the function /// getReplicatedDataPaths(). - virtual void addReplicatedDataPath(const String & table_zk_path, const String & data_path) = 0; + virtual void addReplicatedDataPath(const String & table_shared_id, const String & data_path) = 0; /// Returns all the data paths in backup added for a replicated table (see also addReplicatedDataPath()). - virtual Strings getReplicatedDataPaths(const String & table_zk_path) const = 0; + virtual Strings getReplicatedDataPaths(const String & table_shared_id) const = 0; struct FileInfo { diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 12151eff08f..90c17ef0427 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -354,7 +355,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name read_buffer.reset(); ParserCreateQuery create_parser; ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_table_query); + renameDatabaseAndTableNameInCreateQuery(create_table_query, renaming_map, context->getGlobalContext()); QualifiedTableName table_name = renaming_map.getNewTableName(table_name_in_backup); @@ -433,7 +434,7 @@ void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_ba read_buffer.reset(); ParserCreateQuery create_parser; ASTPtr create_database_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - renameDatabaseAndTableNameInCreateQuery(context->getGlobalContext(), renaming_map, create_database_query); + renameDatabaseAndTableNameInCreateQuery(create_database_query, renaming_map, context->getGlobalContext()); String database_name = renaming_map.getNewDatabaseName(database_name_in_backup); DatabaseInfo & database_info = database_infos[database_name]; @@ -587,7 +588,8 @@ void RestorerFromBackup::createDatabases() if (!restore_settings.allow_different_database_def) { /// Check that the database's definition is the same as expected. - ASTPtr create_database_query = database->getCreateDatabaseQueryForBackup(); + ASTPtr create_database_query = database->getCreateDatabaseQuery(); + adjustCreateQueryForBackup(create_database_query, context->getGlobalContext(), nullptr); ASTPtr expected_create_query = database_info.create_database_query; if (serializeAST(*create_database_query) != serializeAST(*expected_create_query)) { @@ -659,7 +661,7 @@ void RestorerFromBackup::createTables() if (!restore_settings.allow_different_table_def) { ASTPtr create_table_query = database->getCreateTableQuery(resolved_id.table_name, context); - storage->adjustCreateQueryForBackup(create_table_query); + adjustCreateQueryForBackup(create_table_query, context->getGlobalContext(), nullptr); ASTPtr expected_create_query = table_info.create_table_query; if (serializeAST(*create_table_query) != serializeAST(*expected_create_query)) { diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index fc14d7abbd9..c8958fa06d4 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -307,13 +307,6 @@ void DDLRenamingVisitor::visit(ASTPtr ast, const Data & data) bool DDLRenamingVisitor::needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } -void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context, const DDLRenamingMap & renaming_map, ASTPtr & ast) -{ - DDLRenamingVisitor::Data data{global_context, renaming_map, ast}; - DDLRenamingVisitor::Visitor{data}.visit(ast); -} - - void DDLRenamingMap::setNewTableName(const QualifiedTableName & old_table_name, const QualifiedTableName & new_table_name) { if (old_table_name.table.empty() || old_table_name.database.empty() || new_table_name.table.empty() || new_table_name.database.empty()) @@ -367,4 +360,12 @@ QualifiedTableName DDLRenamingMap::getNewTableName(const QualifiedTableName & ol return {getNewDatabaseName(old_table_name.database), old_table_name.table}; } + +void renameDatabaseAndTableNameInCreateQuery(ASTPtr & ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context) +{ + ast = ast->clone(); + DDLRenamingVisitor::Data data{ast, renaming_map, global_context}; + DDLRenamingVisitor::Visitor{data}.visit(ast); +} + } diff --git a/src/Databases/DDLRenamingVisitor.h b/src/Databases/DDLRenamingVisitor.h index 72b578b9fcb..11e8c4676e0 100644 --- a/src/Databases/DDLRenamingVisitor.h +++ b/src/Databases/DDLRenamingVisitor.h @@ -17,7 +17,7 @@ class DDLRenamingMap; /// Changes names of databases or tables in a create query according to a specified renaming map. /// Does not validate AST, works a best-effort way. -void renameDatabaseAndTableNameInCreateQuery(const ContextPtr & global_context, const DDLRenamingMap & renaming_map, ASTPtr & ast); +void renameDatabaseAndTableNameInCreateQuery(ASTPtr & ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context); /// Renaming map keeps information about new names of databases or tables. class DDLRenamingMap @@ -40,9 +40,9 @@ class DDLRenamingVisitor public: struct Data { - ContextPtr global_context; - const DDLRenamingMap & renaming_map; ASTPtr create_query; + const DDLRenamingMap & renaming_map; + ContextPtr global_context; }; using Visitor = InDepthNodeVisitor; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 6286723aaa3..5a0eec10abb 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -44,6 +44,7 @@ namespace ErrorCodes extern const int INCORRECT_QUERY; extern const int ALL_CONNECTION_TRIES_FAILED; extern const int NO_ACTIVE_REPLICAS; + extern const int INCONSISTENT_METADATA_FOR_BACKUP; extern const int CANNOT_RESTORE_TABLE; } @@ -923,6 +924,45 @@ String DatabaseReplicated::readMetadataFile(const String & table_name) const } +std::vector> +DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr &) const +{ + /// Here we read metadata from ZooKeeper. We could do that by simple call of DatabaseAtomic::getTablesForBackup() however + /// reading from ZooKeeper is better because thus we won't be dependant on how fast the replication queue of this database is. + std::vector> res; + auto zookeeper = getContext()->getZooKeeper(); + auto escaped_table_names = zookeeper->getChildren(zookeeper_path + "/metadata"); + for (const auto & escaped_table_name : escaped_table_names) + { + String table_name = unescapeForFileName(escaped_table_name); + if (!filter(table_name)) + continue; + String zk_metadata; + if (!zookeeper->tryGet(zookeeper_path + "/metadata/" + escaped_table_name, zk_metadata)) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Metadata for table {} was not found in ZooKeeper", table_name); + + ParserCreateQuery parser; + auto create_table_query = parseQuery(parser, zk_metadata, 0, getContext()->getSettingsRef().max_parser_depth); + + auto & create = create_table_query->as(); + create.attach = false; + create.setTable(table_name); + create.setDatabase(getDatabaseName()); + + StoragePtr storage; + if (create.uuid != UUIDHelpers::Nil) + { + storage = DatabaseCatalog::instance().tryGetByUUID(create.uuid).second; + if (storage) + storage->adjustCreateQueryForBackup(create_table_query); + } + res.emplace_back(create_table_query, storage); + } + + return res; +} + + void DatabaseReplicated::createTableRestoredFromBackup( const ASTPtr & create_table_query, ContextMutablePtr local_context, diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 958ee3f133f..07014702067 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -72,6 +72,7 @@ public: void shutdown() override; + std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override; friend struct DatabaseReplicatedTask; diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index a75f213a6bb..9e33548b0dd 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -21,20 +21,9 @@ StoragePtr IDatabase::getTable(const String & name, ContextPtr context) const throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name)); } -ASTPtr IDatabase::getCreateDatabaseQueryForBackup() const -{ - auto query = getCreateDatabaseQuery(); - - /// We don't want to see any UUIDs in backup (after RESTORE the database will have another UUID anyway). - auto & create = query->as(); - create.uuid = UUIDHelpers::Nil; - - return query; -} - std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const { - /// Cannot restore any table because IDatabase doesn't own any tables. + /// Cannot backup any table because IDatabase doesn't own any tables. throw Exception(ErrorCodes::CANNOT_BACKUP_TABLE, "Database engine {} does not support backups, cannot backup tables in database {}", getEngineName(), backQuoteIfNeed(getDatabaseName())); diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index cdea03aa1cb..72155bc818c 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -332,9 +332,6 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } - /// Returns a CREATE DATABASE query prepared for writing to a backup. - virtual ASTPtr getCreateDatabaseQueryForBackup() const; - /// Returns CREATE TABLE queries and corresponding tables prepared for writing to a backup. virtual std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & context) const; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index a3f35ccc0f8..0d0a242e4fb 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -249,28 +249,8 @@ bool IStorage::isStaticStorage() const return false; } -void IStorage::adjustCreateQueryForBackup(ASTPtr & create_query) const +void IStorage::adjustCreateQueryForBackup(ASTPtr &) const { - create_query = create_query->clone(); - - /// We don't want to see any UUIDs in backup (after RESTORE the table will have another UUID anyway). - auto & create = create_query->as(); - create.uuid = UUIDHelpers::Nil; - create.to_inner_uuid = UUIDHelpers::Nil; - - /// If this is a definition of a system table we'll remove columns and comment because they're reduntant for backups. - if (isSystemStorage()) - { - if (!create.storage || !create.storage->engine) - throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query without table engine for a system table {}", getStorageID().getFullTableName()); - - auto & engine = *(create.storage->engine); - if (!engine.name.starts_with("System")) - throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with an unexpected table engine {} for a system table {}", engine.name, getStorageID().getFullTableName()); - - create.reset(create.columns_list); - create.reset(create.comment); - } } void IStorage::backupData(BackupEntriesCollector &, const String &, const std::optional &) diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp new file mode 100644 index 00000000000..45d667047af --- /dev/null +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +std::optional tryExtractZkPathFromCreateQuery(const IAST & create_query, const ContextPtr & global_context) +{ + const auto * create = create_query.as(); + if (!create || !create->storage || !create->storage->engine) + return {}; + + /// Check if the table engine is one of the ReplicatedMergeTree family. + const auto & ast_engine = *create->storage->engine; + if (!ast_engine.name.starts_with("Replicated") || !ast_engine.name.ends_with("MergeTree")) + return {}; + + /// Get the first argument. + const auto * ast_arguments = typeid_cast(ast_engine.arguments.get()); + if (!ast_arguments || ast_arguments->children.empty()) + return {}; + + auto * ast_zk_path = typeid_cast(ast_arguments->children[0].get()); + if (!ast_zk_path || (ast_zk_path->value.getType() != Field::Types::String)) + return {}; + + String zk_path = ast_zk_path->value.safeGet(); + + /// Expand macros. + Macros::MacroExpansionInfo info; + info.table_id.table_name = create->getTable(); + info.table_id.database_name = create->getDatabase(); + info.table_id.uuid = create->uuid; + auto database = DatabaseCatalog::instance().tryGetDatabase(info.table_id.database_name); + if (database && database->getEngineName() == "Replicated") + { + info.shard = getReplicatedDatabaseShardName(database); + info.replica = getReplicatedDatabaseReplicaName(database); + } + + try + { + zk_path = global_context->getMacros()->expand(zk_path, info); + } + catch (...) + { + return {}; /// Couldn't expand macros. + } + + return zk_path; +} + +} diff --git a/src/Storages/MergeTree/extractZkPathFromCreateQuery.h b/src/Storages/MergeTree/extractZkPathFromCreateQuery.h new file mode 100644 index 00000000000..e22f76d2cd5 --- /dev/null +++ b/src/Storages/MergeTree/extractZkPathFromCreateQuery.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +class IAST; +class Context; +using ContextPtr = std::shared_ptr; + +/// Extracts a zookeeper path from a specified CREATE TABLE query. Returns std::nullopt if fails. +/// The function takes the first argument of the ReplicatedMergeTree table engine and expands macros in it. +/// It works like a part of what the create() function in registerStorageMergeTree.cpp does but in a simpler manner. +std::optional tryExtractZkPathFromCreateQuery(const IAST & create_query, const ContextPtr & global_context); + +} diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 66fb2a64a50..0a544ebad02 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -7504,6 +7505,24 @@ void StorageReplicatedMergeTree::createTableSharedID() } +std::optional StorageReplicatedMergeTree::tryGetTableSharedIDFromCreateQuery(const IAST & create_query, const ContextPtr & global_context) +{ + auto zk_path = tryExtractZkPathFromCreateQuery(create_query, global_context); + if (!zk_path) + return {}; + + String zk_name = zkutil::extractZooKeeperName(*zk_path); + zk_path = zkutil::extractZooKeeperPath(*zk_path, false, nullptr); + zkutil::ZooKeeperPtr zookeeper = (zk_name == getDefaultZooKeeperName()) ? global_context->getZooKeeper() : global_context->getAuxiliaryZooKeeper(zk_name); + + String id; + if (!zookeeper->tryGet(fs::path(*zk_path) / "table_shared_id", id)) + return {}; + + return id; +} + + void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const { auto settings = getSettings(); @@ -8258,46 +8277,8 @@ void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_quer { MergeTreeData::adjustCreateQueryForBackup(create_query); - /// Before storing the metadata in a backup we have to find a zookeeper path in its definition and turn the table's UUID in there - /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. - /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. - auto & create = create_query->as(); - - if (!create.storage || !create.storage->engine) - throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query without table engine for a replicated table {}", getStorageID().getFullTableName()); - - auto & engine = *(create.storage->engine); - if (!engine.name.starts_with("Replicated") || !engine.name.ends_with("MergeTree")) - throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with an unexpected table engine {} for a replicated table {}", engine.name, getStorageID().getFullTableName()); - - if (create.uuid == UUIDHelpers::Nil) - return; - - auto * engine_args_ast = typeid_cast(engine.arguments.get()); - if (!engine_args_ast) - return; - - auto & engine_args = engine_args_ast->children; - if (engine_args.size() < 2) - return; - - auto * zookeeper_path_ast = typeid_cast(engine_args[0].get()); - auto * replica_name_ast = typeid_cast(engine_args[1].get()); - if (zookeeper_path_ast && (zookeeper_path_ast->value.getType() == Field::Types::String) && - replica_name_ast && (replica_name_ast->value.getType() == Field::Types::String)) - { - String & zookeeper_path_arg = zookeeper_path_ast->value.get(); - String & replica_name_arg = replica_name_ast->value.get(); - String table_uuid_str = toString(create.uuid); - if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) - zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); - const auto & config = getContext()->getConfigRef(); - if ((zookeeper_path_arg == getDefaultZooKeeperPath(config)) && (replica_name_arg == getDefaultReplicaName(config)) - && ((engine_args.size() == 2) || !engine_args[2]->as())) - { - engine_args.erase(engine_args.begin(), engine_args.begin() + 2); - } - } + if (getTableSharedID() != tryGetTableSharedIDFromCreateQuery(*create_query, getContext())) + throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Table {} has its shared ID to be different from one from the create query"); } void StorageReplicatedMergeTree::backupData( @@ -8309,8 +8290,8 @@ void StorageReplicatedMergeTree::backupData( auto backup_entries = backupParts(backup_entries_collector.getContext(), "", partitions); auto coordination = backup_entries_collector.getBackupCoordination(); - String full_zk_path = getZooKeeperName() + getZooKeeperPath(); - coordination->addReplicatedDataPath(full_zk_path, data_path_in_backup); + String shared_id = getTableSharedID(); + coordination->addReplicatedDataPath(shared_id, data_path_in_backup); std::unordered_map part_names_with_hashes_calculating; for (auto & [relative_path, backup_entry] : backup_entries) @@ -8348,23 +8329,23 @@ void StorageReplicatedMergeTree::backupData( } /// Send our list of part names to the coordination (to compare with other replicas). - coordination->addReplicatedPartNames(full_zk_path, getStorageID().getFullTableName(), getReplicaName(), part_names_with_hashes); + coordination->addReplicatedPartNames(shared_id, getStorageID().getFullTableName(), getReplicaName(), part_names_with_hashes); /// This task will be executed after all replicas have collected their parts and the coordination is ready to /// give us the final list of parts to add to the BackupEntriesCollector. - auto post_collecting_task = [full_zk_path, + auto post_collecting_task = [shared_id, replica_name = getReplicaName(), coordination, backup_entries = std::move(backup_entries), &backup_entries_collector]() { - Strings data_paths = coordination->getReplicatedDataPaths(full_zk_path); + Strings data_paths = coordination->getReplicatedDataPaths(shared_id); std::vector data_paths_fs; data_paths_fs.reserve(data_paths.size()); for (const auto & data_path : data_paths) data_paths_fs.push_back(data_path); - Strings part_names = coordination->getReplicatedPartNames(full_zk_path, replica_name); + Strings part_names = coordination->getReplicatedPartNames(shared_id, replica_name); std::unordered_set part_names_set{part_names.begin(), part_names.end()}; for (const auto & [relative_path, backup_entry] : backup_entries) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 86120b354bd..18b9ef54777 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -312,6 +312,9 @@ public: // Return table id, common for different replicas String getTableSharedID() const override; + /// Returns the same as getTableSharedID(), but extracts it from a create query. + static std::optional tryGetTableSharedIDFromCreateQuery(const IAST & create_query, const ContextPtr & global_context); + static String getDefaultZooKeeperName() { return default_zookeeper_name; } /// Check if there are new broken disks and enqueue part recovery tasks. diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 63198f40af9..885a0d851c2 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -557,3 +557,44 @@ def test_projection(): ) == "2\n" ) + + +def test_replicated_database_with_not_synced_tables(): + node1.query( + "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + node1.query("CREATE TABLE mydb.tbl(x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY x") + + backup_name = new_backup_name() + node2.query(f"BACKUP DATABASE mydb TO {backup_name}") + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + + node1.query(f"RESTORE DATABASE mydb FROM {backup_name}") + assert node1.query("EXISTS mydb.tbl") == "1\n" + + +def test_replicated_table_with_not_synced_def(): + node1.query( + "CREATE TABLE tbl (" + "x UInt8, y String" + ") ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')" + "ORDER BY tuple()" + ) + + node2.query( + "CREATE TABLE tbl (" + "x UInt8, y String" + ") ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')" + "ORDER BY tuple()" + ) + + node2.query("SYSTEM STOP REPLICATION QUEUES tbl") + node1.query("ALTER TABLE tbl MODIFY COLUMN x String") + + backup_name = new_backup_name() + node2.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") + + #node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + #node1.query(f"RESTORE TABLE tbl FROM {backup_name}") From 01921ce9a3322bd719d13a244c94c37594fe36c0 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 26 Jun 2022 17:17:43 +0200 Subject: [PATCH 295/408] Move most code from setTableStructure() to a separate function. --- .../ReplicatedMergeTreeTableMetadata.cpp | 120 ++++++++++++++++++ .../ReplicatedMergeTreeTableMetadata.h | 2 + src/Storages/StorageReplicatedMergeTree.cpp | 117 +---------------- 3 files changed, 123 insertions(+), 116 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 7dee7b8d0f8..ea90179caa3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -353,4 +354,123 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl return diff; } +StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(const ColumnsDescription & new_columns, ContextPtr context, const StorageInMemoryMetadata & old_metadata) const +{ + StorageInMemoryMetadata new_metadata = old_metadata; + new_metadata.columns = new_columns; + + if (!empty()) + { + auto parse_key_expr = [] (const String & key_expr) + { + ParserNotEmptyExpressionList parser(false); + auto new_sorting_key_expr_list = parseQuery(parser, key_expr, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + + ASTPtr order_by_ast; + if (new_sorting_key_expr_list->children.size() == 1) + order_by_ast = new_sorting_key_expr_list->children[0]; + else + { + auto tuple = makeASTFunction("tuple"); + tuple->arguments->children = new_sorting_key_expr_list->children; + order_by_ast = tuple; + } + return order_by_ast; + }; + + if (sorting_key_changed) + { + auto order_by_ast = parse_key_expr(new_sorting_key); + + new_metadata.sorting_key.recalculateWithNewAST(order_by_ast, new_metadata.columns, context); + + if (new_metadata.primary_key.definition_ast == nullptr) + { + /// Primary and sorting key become independent after this ALTER so we have to + /// save the old ORDER BY expression as the new primary key. + auto old_sorting_key_ast = old_metadata.getSortingKey().definition_ast; + new_metadata.primary_key = KeyDescription::getKeyFromAST( + old_sorting_key_ast, new_metadata.columns, context); + } + } + + if (sampling_expression_changed) + { + if (!new_sampling_expression.empty()) + { + auto sample_by_ast = parse_key_expr(new_sampling_expression); + new_metadata.sampling_key.recalculateWithNewAST(sample_by_ast, new_metadata.columns, context); + } + else /// SAMPLE BY was removed + { + new_metadata.sampling_key = {}; + } + } + + if (skip_indices_changed) + new_metadata.secondary_indices = IndicesDescription::parse(new_skip_indices, new_columns, context); + + if (constraints_changed) + new_metadata.constraints = ConstraintsDescription::parse(new_constraints); + + if (projections_changed) + new_metadata.projections = ProjectionsDescription::parse(new_projections, new_columns, context); + + if (ttl_table_changed) + { + if (!new_ttl_table.empty()) + { + ParserTTLExpressionList parser; + auto ttl_for_table_ast = parseQuery(parser, new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + ttl_for_table_ast, new_metadata.columns, context, new_metadata.primary_key); + } + else /// TTL was removed + { + new_metadata.table_ttl = TTLTableDescription{}; + } + } + } + + /// Changes in columns may affect following metadata fields + new_metadata.column_ttls_by_name.clear(); + for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) + { + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, context, new_metadata.primary_key); + new_metadata.column_ttls_by_name[name] = new_ttl_entry; + } + + if (new_metadata.partition_key.definition_ast != nullptr) + new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, context); + + if (!sorting_key_changed) /// otherwise already updated + new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, context); + + /// Primary key is special, it exists even if not defined + if (new_metadata.primary_key.definition_ast != nullptr) + { + new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, context); + } + else + { + new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, context); + new_metadata.primary_key.definition_ast = nullptr; + } + + if (!sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) + new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, context); + + if (!skip_indices_changed) /// otherwise already updated + { + for (auto & index : new_metadata.secondary_indices) + index.recalculateWithNewColumns(new_metadata.columns, context); + } + + if (!ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + new_metadata.table_ttl.definition_ast, new_metadata.columns, context, new_metadata.primary_key); + + return new_metadata; +} + } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 6d510d20304..eb2d087e988 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -66,6 +66,8 @@ struct ReplicatedMergeTreeTableMetadata return !sorting_key_changed && !sampling_expression_changed && !skip_indices_changed && !projections_changed && !ttl_table_changed && !constraints_changed; } + + StorageInMemoryMetadata getNewMetadata(const ColumnsDescription & new_columns, ContextPtr context, const StorageInMemoryMetadata & old_metadata) const; }; void checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0a544ebad02..8a93c813bae 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1094,123 +1094,8 @@ void StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr void StorageReplicatedMergeTree::setTableStructure( ColumnsDescription new_columns, const ReplicatedMergeTreeTableMetadata::Diff & metadata_diff) { - StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - - new_metadata.columns = new_columns; - - if (!metadata_diff.empty()) - { - auto parse_key_expr = [] (const String & key_expr) - { - ParserNotEmptyExpressionList parser(false); - auto new_sorting_key_expr_list = parseQuery(parser, key_expr, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - - ASTPtr order_by_ast; - if (new_sorting_key_expr_list->children.size() == 1) - order_by_ast = new_sorting_key_expr_list->children[0]; - else - { - auto tuple = makeASTFunction("tuple"); - tuple->arguments->children = new_sorting_key_expr_list->children; - order_by_ast = tuple; - } - return order_by_ast; - }; - - if (metadata_diff.sorting_key_changed) - { - auto order_by_ast = parse_key_expr(metadata_diff.new_sorting_key); - auto & sorting_key = new_metadata.sorting_key; - auto & primary_key = new_metadata.primary_key; - - sorting_key.recalculateWithNewAST(order_by_ast, new_metadata.columns, getContext()); - - if (primary_key.definition_ast == nullptr) - { - /// Primary and sorting key become independent after this ALTER so we have to - /// save the old ORDER BY expression as the new primary key. - auto old_sorting_key_ast = old_metadata.getSortingKey().definition_ast; - primary_key = KeyDescription::getKeyFromAST( - old_sorting_key_ast, new_metadata.columns, getContext()); - } - } - - if (metadata_diff.sampling_expression_changed) - { - if (!metadata_diff.new_sampling_expression.empty()) - { - auto sample_by_ast = parse_key_expr(metadata_diff.new_sampling_expression); - new_metadata.sampling_key.recalculateWithNewAST(sample_by_ast, new_metadata.columns, getContext()); - } - else /// SAMPLE BY was removed - { - new_metadata.sampling_key = {}; - } - } - - if (metadata_diff.skip_indices_changed) - new_metadata.secondary_indices = IndicesDescription::parse(metadata_diff.new_skip_indices, new_columns, getContext()); - - if (metadata_diff.constraints_changed) - new_metadata.constraints = ConstraintsDescription::parse(metadata_diff.new_constraints); - - if (metadata_diff.projections_changed) - new_metadata.projections = ProjectionsDescription::parse(metadata_diff.new_projections, new_columns, getContext()); - - if (metadata_diff.ttl_table_changed) - { - if (!metadata_diff.new_ttl_table.empty()) - { - ParserTTLExpressionList parser; - auto ttl_for_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - ttl_for_table_ast, new_metadata.columns, getContext(), new_metadata.primary_key); - } - else /// TTL was removed - { - new_metadata.table_ttl = TTLTableDescription{}; - } - } - } - - /// Changes in columns may affect following metadata fields - new_metadata.column_ttls_by_name.clear(); - for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, getContext(), new_metadata.primary_key); - new_metadata.column_ttls_by_name[name] = new_ttl_entry; - } - - if (new_metadata.partition_key.definition_ast != nullptr) - new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - if (!metadata_diff.sorting_key_changed) /// otherwise already updated - new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - /// Primary key is special, it exists even if not defined - if (new_metadata.primary_key.definition_ast != nullptr) - { - new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - } - else - { - new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, getContext()); - new_metadata.primary_key.definition_ast = nullptr; - } - - if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) - new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, getContext()); - - if (!metadata_diff.skip_indices_changed) /// otherwise already updated - { - for (auto & index : new_metadata.secondary_indices) - index.recalculateWithNewColumns(new_metadata.columns, getContext()); - } - - if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - new_metadata.table_ttl.definition_ast, new_metadata.columns, getContext(), new_metadata.primary_key); + StorageInMemoryMetadata new_metadata = metadata_diff.getNewMetadata(new_columns, getContext(), old_metadata); /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. From efbee5e7235af09192b5d387202cb6039fba2e6c Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 26 Jun 2022 17:53:20 +0200 Subject: [PATCH 296/408] Improve gathering metadata for backup - part 7. --- src/Storages/StorageReplicatedMergeTree.cpp | 15 +++- .../test_backup_restore_on_cluster/test.py | 90 +++++++++++++++---- 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8a93c813bae..c856786ffb3 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8160,10 +8160,19 @@ void StorageReplicatedMergeTree::createAndStoreFreezeMetadata(DiskPtr disk, Data void StorageReplicatedMergeTree::adjustCreateQueryForBackup(ASTPtr & create_query) const { - MergeTreeData::adjustCreateQueryForBackup(create_query); + /// Adjust the create query using values from ZooKeeper. + auto zookeeper = getZooKeeper(); + auto columns_from_entry = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_path) / "columns")); + auto metadata_from_entry = ReplicatedMergeTreeTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); - if (getTableSharedID() != tryGetTableSharedIDFromCreateQuery(*create_query, getContext())) - throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Table {} has its shared ID to be different from one from the create query"); + auto current_metadata = getInMemoryMetadataPtr(); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, current_metadata).checkAndFindDiff(metadata_from_entry, current_metadata->getColumns(), getContext()); + auto adjusted_metadata = metadata_diff.getNewMetadata(columns_from_entry, getContext(), *current_metadata); + applyMetadataChangesToCreateQuery(create_query, adjusted_metadata); + + /// Check that tryGetTableSharedIDFromCreateQuery() works for this storage. + if (tryGetTableSharedIDFromCreateQuery(*create_query, getContext()) != getTableSharedID()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Table {} has its shared ID to be different from one from the create query"); } void StorageReplicatedMergeTree::backupData( diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 885a0d851c2..02f855cf766 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -559,22 +559,6 @@ def test_projection(): ) -def test_replicated_database_with_not_synced_tables(): - node1.query( - "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" - ) - - node1.query("CREATE TABLE mydb.tbl(x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY x") - - backup_name = new_backup_name() - node2.query(f"BACKUP DATABASE mydb TO {backup_name}") - - node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") - - node1.query(f"RESTORE DATABASE mydb FROM {backup_name}") - assert node1.query("EXISTS mydb.tbl") == "1\n" - - def test_replicated_table_with_not_synced_def(): node1.query( "CREATE TABLE tbl (" @@ -593,8 +577,78 @@ def test_replicated_table_with_not_synced_def(): node2.query("SYSTEM STOP REPLICATION QUEUES tbl") node1.query("ALTER TABLE tbl MODIFY COLUMN x String") + # Not synced because the replication queue is stopped + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "UInt8"], ["y", "String"]]) + backup_name = new_backup_name() node2.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - #node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") - #node1.query(f"RESTORE TABLE tbl FROM {backup_name}") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + # But synced after RESTORE anyway + node1.query( + f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=1" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + + node2.query( + f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + +def test_table_in_replicated_database_with_not_synced_def(): + node1.query( + "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" + ) + + node1.query( + "CREATE TABLE mydb.tbl (x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY tuple()" + ) + + node1.query("ALTER TABLE mydb.tbl MODIFY COLUMN x String") + + backup_name = new_backup_name() + node2.query(f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name}") + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + + # But synced after RESTORE anyway + node1.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=1" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + + node2.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" + ) + assert node1.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) + assert node2.query( + "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" + ) == TSV([["x", "String"], ["y", "String"]]) From aa97bf512521486353012780116d8cdb9b2b6de4 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 28 Jun 2022 09:59:02 +0200 Subject: [PATCH 297/408] Improve handling predefined databases and tables. --- src/Backups/BackupEntriesCollector.cpp | 4 +- src/Backups/DDLAdjustingForBackupVisitor.cpp | 3 +- src/Backups/DDLAdjustingForBackupVisitor.h | 2 +- src/Backups/RestorerFromBackup.cpp | 49 ++++++++------------ src/Backups/RestorerFromBackup.h | 2 + src/Databases/DDLRenamingVisitor.cpp | 3 +- src/Databases/DDLRenamingVisitor.h | 2 +- src/Interpreters/DatabaseCatalog.cpp | 45 +++++++++++++++++- src/Interpreters/DatabaseCatalog.h | 9 +++- 9 files changed, 79 insertions(+), 40 deletions(-) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 8104e363a68..691e72a3f21 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -385,7 +385,7 @@ void BackupEntriesCollector::gatherDatabaseMetadata( DatabaseInfo & database_info = it->second; - if (backup_create_database_query && !database_info.create_database_query && !DatabaseCatalog::isPredefinedDatabaseName(database_name)) + if (backup_create_database_query && !database_info.create_database_query && (database_name != DatabaseCatalog::TEMPORARY_DATABASE)) { ASTPtr create_database_query; try @@ -629,7 +629,7 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() for (const auto & [database_name, database_info] : database_infos) { if (!database_info.create_database_query) - continue; /// We don't store CREATE queries for predefined databases (see DatabaseCatalog::isPredefinedDatabaseName()). + continue; /// We store CREATE DATABASE queries only if there was BACKUP DATABASE specified. LOG_TRACE(log, "Adding definition of database {}", backQuoteIfNeed(database_name)); diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp index e3fc3ac5552..7dd58629a49 100644 --- a/src/Backups/DDLAdjustingForBackupVisitor.cpp +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -101,9 +101,8 @@ void DDLAdjustingForBackupVisitor::visit(ASTPtr ast, const Data & data) visitCreateQuery(*create, data); } -void adjustCreateQueryForBackup(ASTPtr & ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id) +void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id) { - ast = ast->clone(); if (replicated_table_shared_id) *replicated_table_shared_id = {}; diff --git a/src/Backups/DDLAdjustingForBackupVisitor.h b/src/Backups/DDLAdjustingForBackupVisitor.h index 87498471cc4..63353dcc000 100644 --- a/src/Backups/DDLAdjustingForBackupVisitor.h +++ b/src/Backups/DDLAdjustingForBackupVisitor.h @@ -14,7 +14,7 @@ using ContextPtr = std::shared_ptr; /// Changes a create query to a form which is appropriate or suitable for saving in a backup. /// Also extracts a replicated table's shared ID from the create query if this is a create query for a replicated table. /// `replicated_table_shared_id` can be null if you don't need that. -void adjustCreateQueryForBackup(ASTPtr & ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id); +void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, std::optional * replicated_table_shared_id); /// Visits ASTCreateQuery and changes it to a form which is appropriate or suitable for saving in a backup. class DDLAdjustingForBackupVisitor diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 90c17ef0427..b5f81fe3b9f 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -77,26 +77,14 @@ namespace return str; } - String tryGetTableEngine(const IAST & ast) + /// Whether a specified name corresponds one of the tables backuping ACL. + bool isSystemAccessTableName(const QualifiedTableName & table_name) { - const ASTCreateQuery * create = ast.as(); - if (!create) - return {}; - if (!create->storage || !create->storage->engine) - return {}; - return create->storage->engine->name; - } + if (table_name.database != DatabaseCatalog::SYSTEM_DATABASE) + return false; - bool hasSystemTableEngine(const IAST & ast) - { - return tryGetTableEngine(ast).starts_with("System"); - } - - bool hasSystemAccessTableEngine(const IAST & ast) - { - String engine_name = tryGetTableEngine(ast); - return (engine_name == "SystemUsers") || (engine_name == "SystemRoles") || (engine_name == "SystemSettingsProfiles") - || (engine_name == "SystemRowPolicies") || (engine_name == "SystemQuotas"); + return (table_name.table == "users") || (table_name.table == "roles") || (table_name.table == "settings_profiles") + || (table_name.table == "row_policies") || (table_name.table == "quotas"); } } @@ -375,6 +363,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name TableInfo & res_table_info = table_infos[table_name]; res_table_info.create_table_query = create_table_query; + res_table_info.is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(StorageID{table_name.database, table_name.table}); res_table_info.data_path_in_backup = data_path_in_backup; res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_name, create_table_query); @@ -385,7 +374,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name insertAtEnd(*res_table_info.partitions, *partitions); } - if (hasSystemAccessTableEngine(*create_table_query)) + if (isSystemAccessTableName(table_name)) { if (!access_restore_task) access_restore_task = std::make_shared(backup, restore_settings, restore_coordination); @@ -450,6 +439,7 @@ void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_ba } database_info.create_database_query = create_database_query; + database_info.is_predefined_database = DatabaseCatalog::isPredefinedDatabase(database_name); } for (const String & table_name_in_backup : table_names_in_backup) @@ -491,9 +481,9 @@ void RestorerFromBackup::findEverythingInBackup(const std::set & except_ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const { AccessRightsElements required_access; - for (const auto & database_name : database_infos | boost::adaptors::map_keys) + for (const auto & [database_name, database_info] : database_infos) { - if (DatabaseCatalog::isPredefinedDatabaseName(database_name)) + if (database_info.is_predefined_database) continue; AccessFlags flags; @@ -509,7 +499,8 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const for (const auto & [table_name, table_info] : table_infos) { - if (hasSystemTableEngine(*table_info.create_table_query)) + /// Access required to restore ACL system tables is checked separately. + if (table_info.is_predefined_table) continue; if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) @@ -565,7 +556,7 @@ void RestorerFromBackup::createDatabases() for (const auto & [database_name, database_info] : database_infos) { bool need_create_database = (restore_settings.create_database != RestoreDatabaseCreationMode::kMustExist); - if (need_create_database && DatabaseCatalog::isPredefinedDatabaseName(database_name)) + if (database_info.is_predefined_database) need_create_database = false; /// Predefined databases always exist. if (need_create_database) @@ -585,7 +576,7 @@ void RestorerFromBackup::createDatabases() DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); - if (!restore_settings.allow_different_database_def) + if (!restore_settings.allow_different_database_def && !database_info.is_predefined_database) { /// Check that the database's definition is the same as expected. ASTPtr create_database_query = database->getCreateDatabaseQuery(); @@ -621,13 +612,11 @@ void RestorerFromBackup::createTables() DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_name.database); bool need_create_table = (restore_settings.create_table != RestoreTableCreationMode::kMustExist); - if (need_create_table && hasSystemTableEngine(*table_info.create_table_query)) - need_create_table = false; /// Tables with System* table engine already exist or can't be created by SQL anyway. + if (table_info.is_predefined_table) + need_create_table = false; /// Predefined tables always exist. if (need_create_table) { - /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some - /// database-specific things). auto create_table_query = table_info.create_table_query; if (restore_settings.create_table == RestoreTableCreationMode::kCreateIfNotExists) { @@ -641,6 +630,8 @@ void RestorerFromBackup::createTables() tableNameWithTypeToString(table_name.database, table_name.table, false), serializeAST(*create_table_query)); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some + /// database-specific things). database->createTableRestoredFromBackup( create_table_query, context, @@ -658,7 +649,7 @@ void RestorerFromBackup::createTables() table_info.storage = storage; table_info.table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); - if (!restore_settings.allow_different_table_def) + if (!restore_settings.allow_different_table_def && !table_info.is_predefined_table) { ASTPtr create_table_query = database->getCreateTableQuery(resolved_id.table_name, context); adjustCreateQueryForBackup(create_table_query, context->getGlobalContext(), nullptr); diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index f4d19a10cf6..d6f4eabd0dd 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -97,11 +97,13 @@ private: struct DatabaseInfo { ASTPtr create_database_query; + bool is_predefined_database = false; }; struct TableInfo { ASTPtr create_table_query; + bool is_predefined_table = false; std::optional partitions; std::filesystem::path data_path_in_backup; std::unordered_set dependencies; diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index c8958fa06d4..8dbcc2a24bb 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -361,9 +361,8 @@ QualifiedTableName DDLRenamingMap::getNewTableName(const QualifiedTableName & ol } -void renameDatabaseAndTableNameInCreateQuery(ASTPtr & ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context) +void renameDatabaseAndTableNameInCreateQuery(ASTPtr ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context) { - ast = ast->clone(); DDLRenamingVisitor::Data data{ast, renaming_map, global_context}; DDLRenamingVisitor::Visitor{data}.visit(ast); } diff --git a/src/Databases/DDLRenamingVisitor.h b/src/Databases/DDLRenamingVisitor.h index 11e8c4676e0..44146a8ee6b 100644 --- a/src/Databases/DDLRenamingVisitor.h +++ b/src/Databases/DDLRenamingVisitor.h @@ -17,7 +17,7 @@ class DDLRenamingMap; /// Changes names of databases or tables in a create query according to a specified renaming map. /// Does not validate AST, works a best-effort way. -void renameDatabaseAndTableNameInCreateQuery(ASTPtr & ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context); +void renameDatabaseAndTableNameInCreateQuery(ASTPtr ast, const DDLRenamingMap & renaming_map, const ContextPtr & global_context); /// Renaming map keeps information about new names of databases or tables. class DDLRenamingMap diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index a0579b813db..bd64b14624c 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -234,12 +234,13 @@ void DatabaseCatalog::shutdownImpl() view_dependencies.clear(); } -bool DatabaseCatalog::isPredefinedDatabaseName(const std::string_view & database_name) +bool DatabaseCatalog::isPredefinedDatabase(const std::string_view & database_name) { return database_name == TEMPORARY_DATABASE || database_name == SYSTEM_DATABASE || database_name == INFORMATION_SCHEMA || database_name == INFORMATION_SCHEMA_UPPERCASE; } + DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); @@ -328,6 +329,48 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( return {database, table}; } +bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const +{ + static const char * INFORMATION_SCHEMA_VIEWS[] = {"schemata", "tables", "views", "columns"}; + static const char * INFORMATION_SCHEMA_UPPERCASE_VIEWS[] = {"SCHEMATA", "TABLES", "VIEWS", "COLUMNS"}; + + auto checkDatabaseAndTableName = [&](const String & database_name, const String & table_name) + { + if (database_name == SYSTEM_DATABASE) + { + auto storage = getSystemDatabase()->tryGetTable(table_name, getContext()); + return storage && storage->isSystemStorage(); + } + if (database_name == INFORMATION_SCHEMA) + { + return std::find(std::begin(INFORMATION_SCHEMA_VIEWS), std::end(INFORMATION_SCHEMA_VIEWS), table_name) + != std::end(INFORMATION_SCHEMA_VIEWS); + } + if (database_name == INFORMATION_SCHEMA_UPPERCASE) + { + return std::find(std::begin(INFORMATION_SCHEMA_UPPERCASE_VIEWS), std::end(INFORMATION_SCHEMA_UPPERCASE_VIEWS), table_name) + != std::end(INFORMATION_SCHEMA_UPPERCASE_VIEWS); + } + return false; + }; + + if (table_id.hasUUID()) + { + if (auto storage = tryGetByUUID(table_id.uuid).second) + { + if (storage->isSystemStorage()) + return true; + auto res_id = storage->getStorageID(); + String database_name = res_id.getDatabaseName(); + if (database_name != SYSTEM_DATABASE) + return checkDatabaseAndTableName(database_name, res_id.getTableName()); + } + return false; + } + + return checkDatabaseAndTableName(table_id.getDatabaseName(), table_id.getTableName()); +} + void DatabaseCatalog::assertDatabaseExists(const String & database_name) const { std::lock_guard lock{databases_mutex}; diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 4468cc3a5d8..133cf0c5126 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -130,8 +130,8 @@ public: static constexpr const char * INFORMATION_SCHEMA = "information_schema"; static constexpr const char * INFORMATION_SCHEMA_UPPERCASE = "INFORMATION_SCHEMA"; - /// Returns true if a passed string is one of the predefined databases' names - static bool isPredefinedDatabaseName(const std::string_view & database_name); + /// Returns true if a passed name is one of the predefined databases' names. + static bool isPredefinedDatabase(const std::string_view & database_name); static DatabaseCatalog & init(ContextMutablePtr global_context_); static DatabaseCatalog & instance(); @@ -181,6 +181,11 @@ public: ContextPtr context, std::optional * exception = nullptr) const; + /// Returns true if a passed table_id refers to one of the predefined tables' names. + /// All tables in the "system" database with System* table engine are predefined. + /// Four views (tables, views, columns, schemata) in the "information_schema" database are predefined too. + bool isPredefinedTable(const StorageID & table_id) const; + void addDependency(const StorageID & from, const StorageID & where); void removeDependency(const StorageID & from, const StorageID & where); Dependencies getDependencies(const StorageID & from) const; From 11b51d2878d2f944fc86044ea5571b71ea9319b0 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 27 Jun 2022 15:39:12 +0200 Subject: [PATCH 298/408] Implement storing UDF in backups. --- src/Access/AccessBackup.cpp | 2 +- src/Backups/RestoreSettings.cpp | 7 +- src/Backups/RestoreSettings.h | 5 ++ src/Backups/RestorerFromBackup.cpp | 21 +++++- .../System/StorageSystemFunctions.cpp | 74 +++++++++++++++++++ src/Storages/System/StorageSystemFunctions.h | 3 + .../test_backup_restore_new/test.py | 22 ++++++ 7 files changed, 128 insertions(+), 6 deletions(-) diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 0322ad7457b..dd8cbbf269b 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -139,7 +139,7 @@ namespace } catch (Exception & e) { - e.addMessage("While parsing " + file_path); + e.addMessage("While parsing " + file_path + " from backup"); throw; } } diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp index 590d39f24f8..efa1fe2cfb8 100644 --- a/src/Backups/RestoreSettings.cpp +++ b/src/Backups/RestoreSettings.cpp @@ -74,7 +74,7 @@ namespace { case RestoreTableCreationMode::kCreate: return Field{true}; case RestoreTableCreationMode::kMustExist: return Field{false}; - case RestoreTableCreationMode::kCreateIfNotExists: return Field{"if not exists"}; + case RestoreTableCreationMode::kCreateIfNotExists: return Field{"if-not-exists"}; } throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected value of enum RestoreTableCreationMode: {}", static_cast(value)); } @@ -131,12 +131,14 @@ namespace switch (value) { case RestoreAccessCreationMode::kCreate: return Field{true}; - case RestoreAccessCreationMode::kCreateIfNotExists: return Field{"if not exists"}; + case RestoreAccessCreationMode::kCreateIfNotExists: return Field{"if-not-exists"}; case RestoreAccessCreationMode::kReplace: return Field{"replace"}; } throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected value of enum RestoreAccessCreationMode: {}", static_cast(value)); } }; + + using SettingFieldRestoreUDFCreationMode = SettingFieldRestoreAccessCreationMode; } /// List of restore settings except base_backup_name and cluster_host_ids. @@ -155,6 +157,7 @@ namespace M(Bool, allow_non_empty_tables) \ M(RestoreAccessCreationMode, create_access) \ M(Bool, allow_unresolved_access_dependencies) \ + M(RestoreUDFCreationMode, create_function) \ M(Bool, internal) \ M(String, host_id) \ M(String, coordination_zk_path) diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h index 5e941b79508..1bc5d867a37 100644 --- a/src/Backups/RestoreSettings.h +++ b/src/Backups/RestoreSettings.h @@ -36,6 +36,8 @@ enum class RestoreAccessCreationMode kReplace, }; +using RestoreUDFCreationMode = RestoreAccessCreationMode; + /// Settings specified in the "SETTINGS" clause of a RESTORE query. struct RestoreSettings { @@ -99,6 +101,9 @@ struct RestoreSettings /// For example, if an user has a profile assigned and that profile is not in the backup and doesn't exist locally. bool allow_unresolved_access_dependencies = false; + /// How the RESTORE command will handle if a user-defined function which it's going to restore already exists. + RestoreUDFCreationMode create_function = RestoreUDFCreationMode::kCreateIfNotExists; + /// Internal, should not be specified by user. bool internal = false; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index b5f81fe3b9f..ecd34b12742 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -86,7 +86,13 @@ namespace return (table_name.table == "users") || (table_name.table == "roles") || (table_name.table == "settings_profiles") || (table_name.table == "row_policies") || (table_name.table == "quotas"); } -} + + /// Whether a specified name corresponds one of the tables backuping ACL. + bool isSystemFunctionsTableName(const QualifiedTableName & table_name) + { + return (table_name.database == DatabaseCatalog::SYSTEM_DATABASE) && (table_name.table == "functions"); + } + } RestorerFromBackup::RestorerFromBackup( @@ -374,7 +380,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name insertAtEnd(*res_table_info.partitions, *partitions); } - if (isSystemAccessTableName(table_name)) + if (!restore_settings.structure_only && isSystemAccessTableName(table_name)) { if (!access_restore_task) access_restore_task = std::make_shared(backup, restore_settings, restore_coordination); @@ -499,9 +505,18 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const for (const auto & [table_name, table_info] : table_infos) { - /// Access required to restore ACL system tables is checked separately. if (table_info.is_predefined_table) + { + if (isSystemFunctionsTableName(table_name)) + { + /// CREATE_FUNCTION privilege is required to restore the "system.functions" table. + if (!restore_settings.structure_only && backup->hasFiles(table_info.data_path_in_backup)) + required_access.emplace_back(AccessType::CREATE_FUNCTION); + } + /// Privileges required to restore ACL system tables are checked separately + /// (see access_restore_task->getRequiredAccess() below). continue; + } if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) { diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index b3f1231bd1a..e346d8b427b 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -9,6 +9,16 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + namespace DB { @@ -20,6 +30,11 @@ enum class FunctionOrigin : Int8 EXECUTABLE_USER_DEFINED = 2 }; +namespace ErrorCodes +{ + extern const int CANNOT_RESTORE_TABLE; +} + namespace { template @@ -99,4 +114,63 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c fillRow(res_columns, function_name, UInt64(0), "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory); } } + +void StorageSystemFunctions::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional &) +{ + const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); + const auto & user_defined_sql_functions_names = user_defined_sql_functions_factory.getAllRegisteredNames(); + fs::path data_path_in_backup_fs{data_path_in_backup}; + for (const auto & function_name : user_defined_sql_functions_names) + { + auto ast = user_defined_sql_functions_factory.tryGet(function_name); + if (!ast) + continue; + backup_entries_collector.addBackupEntry( + data_path_in_backup_fs / (escapeForFileName(function_name) + ".sql"), + std::make_shared(queryToString(ast))); + } +} + +void StorageSystemFunctions::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional &) +{ + auto backup = restorer.getBackup(); + + Strings filenames = backup->listFiles(data_path_in_backup); + for (const auto & filename : filenames) + { + if (!filename.ends_with(".sql")) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore user-defined functions, expected *.sql files, got {}", filename); + } + + fs::path data_path_in_backup_fs{data_path_in_backup}; + auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); + const auto & restore_settings = restorer.getRestoreSettings(); + auto context = restorer.getContext(); + + for (const auto & filename : filenames) + { + String escaped_function_name = filename.substr(0, filename.length() - strlen(".sql")); + String function_name = unescapeForFileName(escaped_function_name); + + String filepath = data_path_in_backup_fs / filename; + auto function_def_entry = backup->readFile(filepath); + auto function_def_in = function_def_entry->getReadBuffer(); + String function_def; + readStringUntilEOF(function_def, *function_def_in); + + ParserCreateFunctionQuery parser; + ASTPtr ast = parseQuery( + parser, + function_def.data(), + function_def.data() + function_def.size(), + "in file " + filepath + " from backup " + backup->getName(), + 0, + context->getSettingsRef().max_parser_depth); + + bool replace = (restore_settings.create_function == RestoreUDFCreationMode::kReplace); + bool if_not_exists = (restore_settings.create_function == RestoreUDFCreationMode::kCreateIfNotExists); + user_defined_sql_functions_factory.registerFunction(context, function_name, ast, replace, if_not_exists, true); + } +} + } diff --git a/src/Storages/System/StorageSystemFunctions.h b/src/Storages/System/StorageSystemFunctions.h index fdbe79e29a2..606694a4c0b 100644 --- a/src/Storages/System/StorageSystemFunctions.h +++ b/src/Storages/System/StorageSystemFunctions.h @@ -19,6 +19,9 @@ public: static NamesAndTypesList getNamesAndTypes(); + void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; + void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 0a39576ce10..a94964bc8a3 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -777,3 +777,25 @@ def test_projection(): ) == "2\n" ) + + +def test_system_functions(): + instance.query("CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b;") + + instance.query("CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even');") + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE system.functions TO {backup_name}") + + instance.query("DROP FUNCTION linear_equation") + instance.query("DROP FUNCTION parity_str") + + instance.query(f"RESTORE TABLE system.functions FROM {backup_name}") + + assert instance.query( + "SELECT number, linear_equation(number, 2, 1) FROM numbers(3)" + ) == TSV([[0, 1], [1, 3], [2, 5]]) + + assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV( + [[0, "even"], [1, "odd"], [2, "even"]] + ) From 031ca28fdc13ad5a398a722d5dd2d6a48b291ffe Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 29 Jun 2022 14:42:23 +0200 Subject: [PATCH 299/408] Add test for partition clause. More checks for data compatibility on restore. --- src/Access/AccessBackup.cpp | 12 ++++++- src/Access/AccessBackup.h | 3 +- src/Backups/BackupEntriesCollector.cpp | 32 +++++++++++-------- src/Backups/BackupEntriesCollector.h | 3 -- src/Backups/DDLAdjustingForBackupVisitor.cpp | 12 +++---- src/Backups/RestorerFromBackup.cpp | 28 ++++++++-------- src/Backups/RestorerFromBackup.h | 8 ++--- src/Storages/IStorage.cpp | 12 +++++-- src/Storages/IStorage.h | 3 ++ src/Storages/MergeTree/MergeTreeData.cpp | 6 +++- src/Storages/MergeTree/MergeTreeData.h | 3 ++ src/Storages/StorageLog.cpp | 26 +++++++++------ src/Storages/StorageMaterializedView.cpp | 7 ++++ src/Storages/StorageMaterializedView.h | 1 + src/Storages/StorageMemory.cpp | 26 +++++++++------ src/Storages/StorageStripeLog.cpp | 26 +++++++++------ .../System/StorageSystemFunctions.cpp | 11 ++++--- src/Storages/System/StorageSystemQuotas.cpp | 10 ++---- src/Storages/System/StorageSystemRoles.cpp | 10 ++---- .../System/StorageSystemRowPolicies.cpp | 10 ++---- .../System/StorageSystemSettingsProfiles.cpp | 10 ++---- src/Storages/System/StorageSystemUsers.cpp | 10 ++---- .../test_backup_restore_new/test.py | 26 +++++++++++++++ 23 files changed, 177 insertions(+), 118 deletions(-) diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index dd8cbbf269b..180f17e3448 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -25,6 +25,7 @@ namespace DB namespace ErrorCodes { + extern const int CANNOT_RESTORE_TABLE; extern const int LOGICAL_ERROR; } @@ -317,12 +318,21 @@ AccessRestoreTask::AccessRestoreTask( AccessRestoreTask::~AccessRestoreTask() = default; -void AccessRestoreTask::addDataPath(const String & data_path) +void AccessRestoreTask::addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs) { if (!data_paths.emplace(data_path).second) return; + if (!backup->hasFiles(data_path)) + return; + String file_path = fs::path{data_path} / "access.txt"; + if (!backup->fileExists(file_path)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + table_name_for_logs.getFullName(), file_path); + } + auto backup_entry = backup->readFile(file_path); auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, file_path); diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 5c70e268eae..3eab9fa1494 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -17,6 +17,7 @@ class IRestoreCoordination; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; class AccessRightsElements; +struct QualifiedTableName; /// Makes a backup of access entities of a specified type. @@ -35,7 +36,7 @@ public: ~AccessRestoreTask(); /// Adds a data path to loads access entities from. - void addDataPath(const String & data_path); + void addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs); bool hasDataPath(const String & data_path) const; /// Checks that the current user can do restoring. diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 691e72a3f21..30f1ecd53cd 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -478,6 +478,7 @@ void BackupEntriesCollector::gatherTablesMetadata() for (const auto & db_table : db_tables) { const auto & create_table_query = db_table.first; + const auto & storage = db_table.second; const auto & create = create_table_query->as(); String table_name = create.getTable(); @@ -499,14 +500,28 @@ void BackupEntriesCollector::gatherTablesMetadata() /// Add information to `table_infos`. auto & res_table_info = table_infos[QualifiedTableName{database_name, table_name}]; res_table_info.database = database; - res_table_info.storage = db_table.second; + res_table_info.storage = storage; res_table_info.create_table_query = create_table_query; res_table_info.metadata_path_in_backup = metadata_path_in_backup; res_table_info.data_path_in_backup = data_path_in_backup; - auto partitions_it = database_info.tables.find(table_name); - if (partitions_it != database_info.tables.end()) - res_table_info.partitions = partitions_it->second.partitions; + if (!backup_settings.structure_only) + { + auto it = database_info.tables.find(table_name); + if (it != database_info.tables.end()) + { + const auto & partitions = it->second.partitions; + if (partitions && !storage->supportsBackupPartition()) + { + throw Exception( + ErrorCodes::CANNOT_BACKUP_TABLE, + "Table engine {} doesn't support partitions, cannot backup {}", + storage->getName(), + tableNameWithTypeToString(database_name, table_name, false)); + } + res_table_info.partitions = partitions; + } + } } } } @@ -724,13 +739,4 @@ void BackupEntriesCollector::runPostTasks() } } -void BackupEntriesCollector::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) -{ - throw Exception( - ErrorCodes::CANNOT_BACKUP_TABLE, - "Table engine {} doesn't support partitions, cannot backup table {}", - table_engine, - storage_id.getFullTableName()); -} - } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 46a2bd1863a..3b1260f6c99 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -50,9 +50,6 @@ public: /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. void addPostTask(std::function task); - /// Throws an exception that a specified table engine doesn't support partitions. - [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); - private: void calculateRootPathInBackup(); diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp index 7dd58629a49..2dedc677df8 100644 --- a/src/Backups/DDLAdjustingForBackupVisitor.cpp +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -34,9 +34,6 @@ namespace /// back into "{uuid}", and also we probably can remove the zookeeper path and replica name if they're default. /// So we're kind of reverting what we had done to the table's definition in registerStorageMergeTree.cpp before we created this table. auto & create = data.create_query->as(); - if (create.uuid == UUIDHelpers::Nil) - return; - auto & engine = *storage.engine; auto * engine_args_ast = typeid_cast(engine.arguments.get()); @@ -54,9 +51,12 @@ namespace { String & zookeeper_path_arg = zookeeper_path_ast->value.get(); String & replica_name_arg = replica_name_ast->value.get(); - String table_uuid_str = toString(create.uuid); - if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) - zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); + if (create.uuid != UUIDHelpers::Nil) + { + String table_uuid_str = toString(create.uuid); + if (size_t uuid_pos = zookeeper_path_arg.find(table_uuid_str); uuid_pos != String::npos) + zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}"); + } const auto & config = data.global_context->getConfigRef(); if ((zookeeper_path_arg == StorageReplicatedMergeTree::getDefaultZooKeeperPath(config)) && (replica_name_arg == StorageReplicatedMergeTree::getDefaultReplicaName(config)) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index ecd34b12742..247660bbce4 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -370,8 +370,9 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name TableInfo & res_table_info = table_infos[table_name]; res_table_info.create_table_query = create_table_query; res_table_info.is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(StorageID{table_name.database, table_name.table}); - res_table_info.data_path_in_backup = data_path_in_backup; res_table_info.dependencies = getDependenciesSetFromCreateQuery(context->getGlobalContext(), table_name, create_table_query); + res_table_info.has_data = backup->hasFiles(data_path_in_backup); + res_table_info.data_path_in_backup = data_path_in_backup; if (partitions) { @@ -384,7 +385,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name { if (!access_restore_task) access_restore_task = std::make_shared(backup, restore_settings, restore_coordination); - access_restore_task->addDataPath(data_path_in_backup); + access_restore_task->addDataPath(data_path_in_backup, table_name); } } @@ -510,7 +511,7 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const if (isSystemFunctionsTableName(table_name)) { /// CREATE_FUNCTION privilege is required to restore the "system.functions" table. - if (!restore_settings.structure_only && backup->hasFiles(table_info.data_path_in_backup)) + if (!restore_settings.structure_only && table_info.has_data) required_access.emplace_back(AccessType::CREATE_FUNCTION); } /// Privileges required to restore ACL system tables are checked separately @@ -538,8 +539,7 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const flags |= AccessType::CREATE_TABLE; } - if (!restore_settings.structure_only && !create.is_dictionary && !create.is_ordinary_view - && backup->hasFiles(table_info.data_path_in_backup)) + if (!restore_settings.structure_only && table_info.has_data) { flags |= AccessType::INSERT; } @@ -685,6 +685,15 @@ void RestorerFromBackup::createTables() { const auto & data_path_in_backup = table_info.data_path_in_backup; const auto & partitions = table_info.partitions; + if (partitions && !storage->supportsBackupPartition()) + { + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Table engine {} doesn't support partitions, cannot restore {}", + storage->getName(), + tableNameWithTypeToString(table_name.database, table_name.table, false)); + } + storage->restoreDataFromBackup(*this, data_path_in_backup, partitions); } } @@ -795,15 +804,6 @@ RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() return res_tasks; } -void RestorerFromBackup::throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine) -{ - throw Exception( - ErrorCodes::CANNOT_RESTORE_TABLE, - "Table engine {} doesn't support partitions, cannot table {}", - table_engine, - storage_id.getFullTableName()); -} - void RestorerFromBackup::throwTableIsNotEmpty(const StorageID & storage_id) { throw Exception( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index d6f4eabd0dd..e47aca0e69f 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -59,9 +59,6 @@ public: /// Checks that a specified path is already registered to be used for restoring access control. void checkPathInBackupIsRegisteredToRestoreAccess(const String & path); - /// Throws an exception that a specified table engine doesn't support partitions. - [[noreturn]] static void throwPartitionsNotSupported(const StorageID & storage_id, const String & table_engine); - /// Throws an exception that a specified table is already non-empty. [[noreturn]] static void throwTableIsNotEmpty(const StorageID & storage_id); @@ -104,9 +101,10 @@ private: { ASTPtr create_table_query; bool is_predefined_table = false; - std::optional partitions; - std::filesystem::path data_path_in_backup; std::unordered_set dependencies; + bool has_data = false; + std::filesystem::path data_path_in_backup; + std::optional partitions; bool created = false; StoragePtr storage; TableLockHolder table_lock; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 0d0a242e4fb..fc29769790d 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -14,7 +14,8 @@ #include #include #include -#include +#include +#include namespace DB @@ -24,7 +25,7 @@ namespace ErrorCodes extern const int TABLE_IS_DROPPED; extern const int NOT_IMPLEMENTED; extern const int DEADLOCK_AVOIDED; - extern const int INCONSISTENT_METADATA_FOR_BACKUP; + extern const int CANNOT_RESTORE_TABLE; } bool IStorage::isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const @@ -257,8 +258,13 @@ void IStorage::backupData(BackupEntriesCollector &, const String &, const std::o { } -void IStorage::restoreDataFromBackup(RestorerFromBackup &, const String &, const std::optional &) +void IStorage::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional &) { + /// If an inherited class doesn't override restoreDataFromBackup() that means it doesn't backup any data. + auto filenames = restorer.getBackup()->listFiles(data_path_in_backup); + if (!filenames.empty()) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: Folder {} in backup must be empty", + getStorageID().getFullTableName(), data_path_in_backup); } std::string PrewhereInfo::dump() const diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 34170785896..e265c94eb11 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -232,6 +232,9 @@ public: /// Extracts data from the backup and put it to the storage. virtual void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions); + /// Returns true if the storage supports backup/restore for specific partitions. + virtual bool supportsBackupPartition() const { return false; } + private: StorageID storage_id; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 02d4d9ebe8b..d4b2c8d177e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -150,6 +150,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; extern const int INCORRECT_QUERY; + extern const int CANNOT_RESTORE_TABLE; } static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key, bool check_sample_column_is_correct) @@ -4092,7 +4093,10 @@ void MergeTreeData::restorePartsFromBackup(RestorerFromBackup & restorer, const { const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, format_version); if (!part_info) - continue; + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't look like the name of a part", + getStorageID().getFullTableName(), String{data_path_in_backup_fs / part_name}); + } if (partition_ids && !partition_ids->contains(part_info->partition_id)) continue; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4fd7dd7d3cf..9aa14367f80 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -716,6 +716,9 @@ public: /// Extract data from the backup and put it to the storage. void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + /// Returns true if the storage supports backup/restore for specific partitions. + bool supportsBackupPartition() const override { return true; } + /// Moves partition to specified Disk void movePartitionToDisk(const ASTPtr & partition, const String & name, bool moving_part, ContextPtr context); diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 1324ebf5b28..ac6ead54016 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -51,6 +51,7 @@ namespace ErrorCodes extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; + extern const int CANNOT_RESTORE_TABLE; } /// NOTE: The lock `StorageLog::rwlock` is NOT kept locked while reading, @@ -921,11 +922,8 @@ std::optional StorageLog::totalBytes(const Settings &) const return total_bytes; } -void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto lock_timeout = getLockTimeout(backup_entries_collector.getContext()); loadMarks(lock_timeout); @@ -986,16 +984,16 @@ void StorageLog::backupData(BackupEntriesCollector & backup_entries_collector, c } } -void StorageLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); + auto backup = restorer.getBackup(); + if (!backup->hasFiles(data_path_in_backup)) + return; if (!num_data_files) return; - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_bytes && backup->hasFiles(data_path_in_backup)) + if (!restorer.isNonEmptyTableAllowed() && total_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); auto lock_timeout = getLockTimeout(restorer.getContext()); @@ -1024,6 +1022,11 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p for (const auto & data_file : data_files) { String file_path_in_backup = data_path_in_backup_fs / fileName(data_file.path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } auto backup_entry = backup->readFile(file_path_in_backup); auto in = backup_entry->getReadBuffer(); auto out = disk->writeFile(data_file.path, max_compress_block_size, WriteMode::Append); @@ -1035,6 +1038,11 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p /// Append marks. size_t num_extra_marks = 0; String file_path_in_backup = data_path_in_backup_fs / fileName(marks_file_path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } size_t file_size = backup->getFileSize(file_path_in_backup); if (file_size % (num_data_files * sizeof(Mark)) != 0) throw Exception("Size of marks file is inconsistent", ErrorCodes::SIZES_OF_MARKS_FILES_ARE_INCONSISTENT); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 2ece0af3359..b01415f9590 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -421,6 +421,13 @@ void StorageMaterializedView::restoreDataFromBackup(RestorerFromBackup & restore return getTargetTable()->restoreDataFromBackup(restorer, data_path_in_backup, partitions); } +bool StorageMaterializedView::supportsBackupPartition() const +{ + if (hasInnerTable()) + return getTargetTable()->supportsBackupPartition(); + return false; +} + std::optional StorageMaterializedView::totalRows(const Settings & settings) const { if (hasInnerTable()) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 0adf394876c..1d8808b302e 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -97,6 +97,7 @@ public: void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + bool supportsBackupPartition() const override; std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 5de8c3bda43..7baecaa594f 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -38,6 +38,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int CANNOT_RESTORE_TABLE; } @@ -479,24 +480,21 @@ namespace }; } -void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; backup_entries_collector.addBackupEntries( std::make_shared(getInMemoryMetadataPtr(), data.get(), data_path_in_backup, max_compress_block_size) ->getBackupEntries()); } -void StorageMemory::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageMemory::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_size_bytes && backup->hasFiles(data_path_in_backup)) + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.isNonEmptyTableAllowed() && total_size_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); restorer.addDataRestoreTask( @@ -514,6 +512,11 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat IndexForNativeFormat index; { String index_file_path = data_path_in_backup_fs / "index.mrk"; + if (!backup->fileExists(index_file_path)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), index_file_path); + } auto backup_entry = backup->readFile(index_file_path); auto in = backup_entry->getReadBuffer(); CompressedReadBuffer compressed_in{*in}; @@ -526,6 +529,11 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat size_t new_rows = 0; { String data_file_path = data_path_in_backup_fs / "data.bin"; + if (!backup->fileExists(data_file_path)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), data_file_path); + } auto backup_entry = backup->readFile(data_file_path); std::unique_ptr in = backup_entry->getReadBuffer(); std::optional temp_data_copy; diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index d569a81c4a7..2033d33a33d 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -55,6 +55,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; extern const int TIMEOUT_EXCEEDED; + extern const int CANNOT_RESTORE_TABLE; } @@ -527,11 +528,8 @@ std::optional StorageStripeLog::totalBytes(const Settings &) const } -void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) +void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - auto lock_timeout = getLockTimeout(backup_entries_collector.getContext()); loadIndices(lock_timeout); @@ -589,13 +587,13 @@ void StorageStripeLog::backupData(BackupEntriesCollector & backup_entries_collec data_path_in_backup_fs / "count.txt", std::make_unique(toString(num_rows))); } -void StorageStripeLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) +void StorageStripeLog::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto backup = restorer.getBackup(); - if (!restorer.isNonEmptyTableAllowed() && total_bytes && backup->hasFiles(data_path_in_backup)) + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.isNonEmptyTableAllowed() && total_bytes) RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); auto lock_timeout = getLockTimeout(restorer.getContext()); @@ -624,6 +622,11 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & auto old_data_size = file_checker.getFileSize(data_file_path); { String file_path_in_backup = data_path_in_backup_fs / fileName(data_file_path); + if (!backup->fileExists(file_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), file_path_in_backup); + } auto backup_entry = backup->readFile(file_path_in_backup); auto in = backup_entry->getReadBuffer(); auto out = disk->writeFile(data_file_path, max_compress_block_size, WriteMode::Append); @@ -634,6 +637,11 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & { String index_path_in_backup = data_path_in_backup_fs / fileName(index_file_path); IndexForNativeFormat extra_indices; + if (!backup->fileExists(index_path_in_backup)) + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", + getStorageID().getFullTableName(), index_path_in_backup); + } auto backup_entry = backup->readFile(index_path_in_backup); auto index_in = backup_entry->getReadBuffer(); CompressedReadBuffer index_compressed_in{*index_in}; diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp index e346d8b427b..e2bc699d3f1 100644 --- a/src/Storages/System/StorageSystemFunctions.cpp +++ b/src/Storages/System/StorageSystemFunctions.cpp @@ -115,7 +115,7 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c } } -void StorageSystemFunctions::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional &) +void StorageSystemFunctions::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); const auto & user_defined_sql_functions_names = user_defined_sql_functions_factory.getAllRegisteredNames(); @@ -131,18 +131,21 @@ void StorageSystemFunctions::backupData(BackupEntriesCollector & backup_entries_ } } -void StorageSystemFunctions::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional &) +void StorageSystemFunctions::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { auto backup = restorer.getBackup(); + fs::path data_path_in_backup_fs{data_path_in_backup}; Strings filenames = backup->listFiles(data_path_in_backup); for (const auto & filename : filenames) { if (!filename.ends_with(".sql")) - throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore user-defined functions, expected *.sql files, got {}", filename); + { + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't have the extension .sql", + getStorageID().getFullTableName(), String{data_path_in_backup_fs / filename}); + } } - fs::path data_path_in_backup_fs{data_path_in_backup}; auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance(); const auto & restore_settings = restorer.getRestoreSettings(); auto context = restorer.getContext(); diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index efe6b93fe57..27cf64cbcb4 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -122,21 +122,15 @@ void StorageSystemQuotas::fillData(MutableColumns & res_columns, ContextPtr cont } void StorageSystemQuotas::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); access_control.backup(backup_entries_collector, AccessEntityType::QUOTA, data_path_in_backup); } void StorageSystemQuotas::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); access_control.restore(restorer, data_path_in_backup); } diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp index ff3490ce8ba..22597530835 100644 --- a/src/Storages/System/StorageSystemRoles.cpp +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -60,21 +60,15 @@ void StorageSystemRoles::fillData(MutableColumns & res_columns, ContextPtr conte } void StorageSystemRoles::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); access_control.backup(backup_entries_collector, AccessEntityType::ROLE, data_path_in_backup); } void StorageSystemRoles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); access_control.restore(restorer, data_path_in_backup); } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 680f90adff7..1e13654d188 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -140,21 +140,15 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, ContextPtr } void StorageSystemRowPolicies::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); access_control.backup(backup_entries_collector, AccessEntityType::ROW_POLICY, data_path_in_backup); } void StorageSystemRowPolicies::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); access_control.restore(restorer, data_path_in_backup); } diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp index 7c3ccfe863a..aaf5bedadd0 100644 --- a/src/Storages/System/StorageSystemSettingsProfiles.cpp +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -87,21 +87,15 @@ void StorageSystemSettingsProfiles::fillData(MutableColumns & res_columns, Conte } void StorageSystemSettingsProfiles::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); access_control.backup(backup_entries_collector, AccessEntityType::SETTINGS_PROFILE, data_path_in_backup); } void StorageSystemSettingsProfiles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); access_control.restore(restorer, data_path_in_backup); } diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index f2cae638d45..d8dc1722a91 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -215,21 +215,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte } void StorageSystemUsers::backupData( - BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) + BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - BackupEntriesCollector::throwPartitionsNotSupported(getStorageID(), getName()); - const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); access_control.backup(backup_entries_collector, AccessEntityType::USER, data_path_in_backup); } void StorageSystemUsers::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) + RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) { - if (partitions) - RestorerFromBackup::throwPartitionsNotSupported(getStorageID(), getName()); - auto & access_control = restorer.getContext()->getAccessControl(); access_control.restore(restorer, data_path_in_backup); } diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index a94964bc8a3..7894daf5bad 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -799,3 +799,29 @@ def test_system_functions(): assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV( [[0, "even"], [1, "odd"], [2, "even"]] ) + + +def test_backup_partition(): + create_and_fill_table(n=30) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table PARTITIONS '1', '4' TO {backup_name}") + + instance.query("DROP TABLE test.table") + + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV([[1, '1'], [4, '4'], [11, '11'], [14, '14'], [21, '21'], [24, '24']]) + + +def test_restore_partition(): + create_and_fill_table(n=30) + + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + + instance.query("DROP TABLE test.table") + + instance.query(f"RESTORE TABLE test.table PARTITIONS '2', '3' FROM {backup_name}") + + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV([[2, '2'], [3, '3'], [12, '12'], [13, '13'], [22, '22'], [23, '23']]) From 5456bde4a2e6f40754d52de07b1c6115ef261079 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 29 Jun 2022 22:44:05 +0200 Subject: [PATCH 300/408] Improve gathering metadata for storing ACL in backups. --- src/Access/AccessBackup.cpp | 75 ++++++++++--------- src/Access/AccessBackup.h | 36 +++++---- src/Access/AccessControl.cpp | 15 +--- src/Access/AccessControl.h | 7 +- src/Access/DiskAccessStorage.cpp | 22 ++++-- src/Access/DiskAccessStorage.h | 2 +- src/Access/IAccessStorage.cpp | 30 +++++--- src/Access/IAccessStorage.h | 11 ++- src/Access/MemoryAccessStorage.cpp | 22 ++++-- src/Access/MemoryAccessStorage.h | 2 +- src/Access/MultipleAccessStorage.cpp | 22 +++--- src/Access/MultipleAccessStorage.h | 4 +- src/Access/ReplicatedAccessStorage.cpp | 62 +++++++++++++-- src/Access/ReplicatedAccessStorage.h | 3 +- src/Backups/BackupCoordinationDistributed.cpp | 43 +++++++++++ src/Backups/BackupCoordinationDistributed.h | 6 ++ src/Backups/BackupCoordinationLocal.cpp | 31 ++++++++ src/Backups/BackupCoordinationLocal.h | 8 ++ src/Backups/BackupEntriesCollector.cpp | 12 +++ src/Backups/BackupEntriesCollector.h | 6 ++ src/Backups/IBackupCoordination.h | 8 ++ src/Backups/RestorerFromBackup.cpp | 32 ++++---- src/Backups/RestorerFromBackup.h | 11 ++- src/Storages/System/StorageSystemQuotas.cpp | 6 +- src/Storages/System/StorageSystemRoles.cpp | 6 +- .../System/StorageSystemRowPolicies.cpp | 6 +- .../System/StorageSystemSettingsProfiles.cpp | 6 +- src/Storages/System/StorageSystemUsers.cpp | 6 +- 28 files changed, 337 insertions(+), 163 deletions(-) diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 180f17e3448..bd1344a6f14 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -226,7 +227,7 @@ namespace } } - AccessRightsElements getRequiredAccessToRestore(const std::unordered_map & entities) + AccessRightsElements getRequiredAccessToRestore(const std::vector> & entities) { AccessRightsElements res; for (const auto & entity : entities | boost::adaptors::map_values) @@ -295,74 +296,78 @@ namespace } } -void backupAccessEntities( - BackupEntriesCollector & backup_entries_collector, + +std::pair makeBackupEntryForAccess( + const std::vector> access_entities, const String & data_path_in_backup, - const AccessControl & access_control, - AccessEntityType type) + size_t counter, + const AccessControl & access_control) { - auto entities = access_control.readAllForBackup(type, backup_entries_collector.getBackupSettings()); - auto dependencies = readDependenciesNamesAndTypes(findDependencies(entities), access_control); + auto dependencies = readDependenciesNamesAndTypes(findDependencies(access_entities), access_control); AccessEntitiesInBackup ab; - boost::range::copy(entities, std::inserter(ab.entities, ab.entities.end())); + boost::range::copy(access_entities, std::inserter(ab.entities, ab.entities.end())); ab.dependencies = std::move(dependencies); - backup_entries_collector.addBackupEntry(fs::path{data_path_in_backup} / "access.txt", ab.toBackupEntry()); + String filename = fmt::format("access{:02}.txt", counter + 1); /// access01.txt, access02.txt, ... + String file_path_in_backup = fs::path{data_path_in_backup} / filename; + return {file_path_in_backup, ab.toBackupEntry()}; } -AccessRestoreTask::AccessRestoreTask( - const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_) - : backup(backup_), restore_settings(restore_settings_), restore_coordination(restore_coordination_) +AccessRestorerFromBackup::AccessRestorerFromBackup( + const BackupPtr & backup_, const RestoreSettings & restore_settings_) + : backup(backup_), allow_unresolved_access_dependencies(restore_settings_.allow_unresolved_access_dependencies) { } -AccessRestoreTask::~AccessRestoreTask() = default; +AccessRestorerFromBackup::~AccessRestorerFromBackup() = default; -void AccessRestoreTask::addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs) +void AccessRestorerFromBackup::addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs) { if (!data_paths.emplace(data_path).second) return; - if (!backup->hasFiles(data_path)) + fs::path data_path_in_backup_fs = data_path; + Strings filenames = backup->listFiles(data_path); + if (filenames.empty()) return; - String file_path = fs::path{data_path} / "access.txt"; - if (!backup->fileExists(file_path)) + for (const String & filename : filenames) { - throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File {} in backup is required", - table_name_for_logs.getFullName(), file_path); + if (!filename.starts_with("access") || !filename.ends_with(".txt")) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: File name {} doesn't match the wildcard \"access*.txt\"", + table_name_for_logs.getFullName(), String{data_path_in_backup_fs / filename}); } - auto backup_entry = backup->readFile(file_path); - auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, file_path); + ::sort(filenames.begin(), filenames.end()); + + for (const String & filename : filenames) + { + String filepath_in_backup = data_path_in_backup_fs / filename; + auto backup_entry = backup->readFile(filepath_in_backup); + auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, filepath_in_backup); + + boost::range::copy(ab.entities, std::back_inserter(entities)); + boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); + } - boost::range::copy(ab.entities, std::inserter(entities, entities.end())); - boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); for (const auto & id : entities | boost::adaptors::map_keys) dependencies.erase(id); } -bool AccessRestoreTask::hasDataPath(const String & data_path) const -{ - return data_paths.contains(data_path); -} - -AccessRightsElements AccessRestoreTask::getRequiredAccess() const +AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const { return getRequiredAccessToRestore(entities); } -void AccessRestoreTask::restore(AccessControl & access_control) const +std::vector> AccessRestorerFromBackup::getAccessEntities(const AccessControl & access_control) const { - auto old_to_new_ids = resolveDependencies(dependencies, access_control, restore_settings.allow_unresolved_access_dependencies); + auto new_entities = entities; - std::vector> new_entities; - boost::range::copy(entities, std::back_inserter(new_entities)); + auto old_to_new_ids = resolveDependencies(dependencies, access_control, allow_unresolved_access_dependencies); generateRandomIDs(new_entities, old_to_new_ids); - replaceDependencies(new_entities, old_to_new_ids); - access_control.insertFromBackup(new_entities, restore_settings, restore_coordination); + return new_entities; } } diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 3eab9fa1494..74f889e2c00 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -9,47 +9,45 @@ namespace DB { class AccessControl; enum class AccessEntityType; -class BackupEntriesCollector; -class RestorerFromBackup; -class IBackup; -using BackupPtr = std::shared_ptr; -class IRestoreCoordination; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; class AccessRightsElements; +class IBackup; +using BackupPtr = std::shared_ptr; +class IBackupEntry; +using BackupEntryPtr = std::shared_ptr; +struct RestoreSettings; struct QualifiedTableName; /// Makes a backup of access entities of a specified type. -void backupAccessEntities( - BackupEntriesCollector & backup_entries_collector, +std::pair makeBackupEntryForAccess( + const std::vector> access_entities, const String & data_path_in_backup, - const AccessControl & access_control, - AccessEntityType type); + size_t counter, + const AccessControl & access_control); + /// Restores access entities from a backup. -class AccessRestoreTask +class AccessRestorerFromBackup { public: - AccessRestoreTask( - const BackupPtr & backup_, const RestoreSettings & restore_settings_, std::shared_ptr restore_coordination_); - ~AccessRestoreTask(); + AccessRestorerFromBackup(const BackupPtr & backup_, const RestoreSettings & restore_settings_); + ~AccessRestorerFromBackup(); /// Adds a data path to loads access entities from. void addDataPath(const String & data_path, const QualifiedTableName & table_name_for_logs); - bool hasDataPath(const String & data_path) const; /// Checks that the current user can do restoring. AccessRightsElements getRequiredAccess() const; /// Inserts all access entities loaded from all the paths added by addDataPath(). - void restore(AccessControl & access_control) const; + std::vector> getAccessEntities(const AccessControl & access_control) const; private: BackupPtr backup; - RestoreSettings restore_settings; - std::shared_ptr restore_coordination; - std::unordered_map entities; + bool allow_unresolved_access_dependencies = false; + std::vector> entities; std::unordered_map> dependencies; std::unordered_set data_paths; }; diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index b5b22caa400..7152820b5bc 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -459,20 +459,9 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne } } -void AccessControl::backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const +void AccessControl::restoreFromBackup(RestorerFromBackup & restorer) { - backupAccessEntities(backup_entries_collector, data_path_in_backup, *this, type); -} - -void AccessControl::restore(RestorerFromBackup & restorer, const String & data_path_in_backup) -{ - /// The restorer must already know about `data_path_in_backup`, but let's check. - restorer.checkPathInBackupIsRegisteredToRestoreAccess(data_path_in_backup); -} - -void AccessControl::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) -{ - MultipleAccessStorage::insertFromBackup(entities_from_backup, restore_settings, restore_coordination); + MultipleAccessStorage::restoreFromBackup(restorer); changes_notifier->sendNotifications(); } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 90ad2895122..22ff0a488f7 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -42,8 +42,6 @@ class ClientInfo; class ExternalAuthenticators; class AccessChangesNotifier; struct Settings; -class BackupEntriesCollector; -class RestorerFromBackup; /// Manages access control entities. @@ -121,8 +119,7 @@ public: UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const; /// Makes a backup of access entities. - void backup(BackupEntriesCollector & backup_entries_collector, AccessEntityType type, const String & data_path_in_backup) const; - static void restore(RestorerFromBackup & restorer, const String & data_path_in_backup); + void restoreFromBackup(RestorerFromBackup & restorer) override; void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); @@ -198,8 +195,6 @@ public: /// Gets manager of notifications. AccessChangesNotifier & getChangesNotifier(); - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; - private: class ContextAccessCache; class CustomSettingsPrefixes; diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 231e325196d..994abc7b53a 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -650,19 +651,24 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const } -void DiskAccessStorage::insertFromBackup( - const std::vector> & entities_from_backup, - const RestoreSettings & restore_settings, - std::shared_ptr) +void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 1bdefbf82f9..d3bd61ff353 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -30,7 +30,7 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 476b1674ce1..230045c7749 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -2,9 +2,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -520,29 +523,34 @@ bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddr } -bool IAccessStorage::isRestoreAllowed() const -{ - return isBackupAllowed() && !isReadOnly(); -} - -std::vector> IAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings &) const +void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const { if (!isBackupAllowed()) throwBackupNotAllowed(); - auto res = readAllWithIDs(type); - boost::range::remove_erase_if(res, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); - return res; + auto entities = readAllWithIDs(type); + boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + + auto backup_entry = makeBackupEntryForAccess( + entities, + data_path_in_backup, + backup_entries_collector.getAccessCounter(type), + backup_entries_collector.getContext()->getAccessControl()); + + backup_entries_collector.addBackupEntry(backup_entry); } -void IAccessStorage::insertFromBackup(const std::vector> &, const RestoreSettings &, std::shared_ptr) + +void IAccessStorage::restoreFromBackup(RestorerFromBackup &) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertFromBackup() is not implemented in {}", getStorageType()); + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "restoreFromBackup() is not implemented in {}", getStorageType()); } + UUID IAccessStorage::generateRandomID() { static Poco::UUIDGenerator generator; diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 7b43309204d..394d3ed6358 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -18,9 +18,8 @@ struct User; class Credentials; class ExternalAuthenticators; enum class AuthenticationType; -struct BackupSettings; -struct RestoreSettings; -class IRestoreCoordination; +class BackupEntriesCollector; +class RestorerFromBackup; /// Contains entities, i.e. instances of classes derived from IAccessEntity. /// The implementations of this class MUST be thread-safe. @@ -158,11 +157,11 @@ public: /// Returns true if this storage can be stored to or restored from a backup. virtual bool isBackupAllowed() const { return false; } - virtual bool isRestoreAllowed() const; + virtual bool isRestoreAllowed() const { return isBackupAllowed() && !isReadOnly(); } /// Makes a backup of this access storage. - virtual std::vector> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const; - virtual void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination); + virtual void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const; + virtual void restoreFromBackup(RestorerFromBackup & restorer); protected: virtual std::optional findImpl(AccessEntityType type, const String & name) const = 0; diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index ad877e263ad..60669532e25 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -272,19 +273,24 @@ void MemoryAccessStorage::setAll(const std::vector> & entities_from_backup, - const RestoreSettings & restore_settings, - std::shared_ptr) +void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index aa4cd08252c..5c8d33ed443 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -29,7 +29,7 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 6f654f68e57..30c3865c1be 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -383,40 +383,38 @@ bool MultipleAccessStorage::isRestoreAllowed() const } -std::vector> MultipleAccessStorage::readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const +void MultipleAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const { - std::vector> res; auto storages = getStoragesInternal(); - size_t count = 0; + bool allowed = false; for (const auto & storage : *storages) { if (storage->isBackupAllowed()) { - insertAtEnd(res, storage->readAllForBackup(type, backup_settings)); - ++count; + storage->backup(backup_entries_collector, data_path_in_backup, type); + allowed = true; } } - if (!count) + if (!allowed) throwBackupNotAllowed(); - - return res; } - -void MultipleAccessStorage::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) +void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { auto storages = getStoragesInternal(); + for (const auto & storage : *storages) { if (storage->isRestoreAllowed()) { - storage->insertFromBackup(entities_from_backup, restore_settings, restore_coordination); + storage->restoreFromBackup(restorer); return; } } - throwRestoreNotAllowed(); + + throwBackupNotAllowed(); } } diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 2eacdafd3f3..58cf09fd0ff 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -45,8 +45,8 @@ public: bool isBackupAllowed() const override; bool isRestoreAllowed() const override; - std::vector> readAllForBackup(AccessEntityType type, const BackupSettings & backup_settings) const override; - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; + void restoreFromBackup(RestorerFromBackup & restorer) override; protected: std::optional findImpl(AccessEntityType type, const String & name) const override; diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 6a9d716c2f9..f6c8d0a7153 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -2,10 +2,14 @@ #include #include #include +#include +#include +#include #include +#include #include #include -#include +#include #include #include #include @@ -13,6 +17,7 @@ #include #include #include +#include namespace DB @@ -613,19 +618,64 @@ AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if return entry.entity; } -void ReplicatedAccessStorage::insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) + +void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const +{ + if (!isBackupAllowed()) + throwBackupNotAllowed(); + + auto entities = readAllWithIDs(type); + boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + + auto backup_entry_with_path = makeBackupEntryForAccess( + entities, + data_path_in_backup, + backup_entries_collector.getAccessCounter(type), + backup_entries_collector.getContext()->getAccessControl()); + + auto backup_coordination = backup_entries_collector.getBackupCoordination(); + backup_coordination->addReplicatedAccessPath(zookeeper_path, backup_entry_with_path.first); + String current_host_id = backup_entries_collector.getBackupSettings().host_id; + backup_coordination->setReplicatedAccessHost(zookeeper_path, current_host_id); + + backup_entries_collector.addPostTask( + [backup_entry = backup_entry_with_path.second, + zookeeper_path = zookeeper_path, + current_host_id, + &backup_entries_collector, + backup_coordination] + { + if (current_host_id != backup_coordination->getReplicatedAccessHost(zookeeper_path)) + return; + + for (const String & path : backup_coordination->getReplicatedAccessPaths(zookeeper_path)) + backup_entries_collector.addBackupEntry(path, backup_entry); + }); +} + + +void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) { if (!isRestoreAllowed()) throwRestoreNotAllowed(); + auto restore_coordination = restorer.getRestoreCoordination(); if (!restore_coordination->acquireReplicatedAccessStorage(zookeeper_path)) return; - bool replace_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kReplace); - bool throw_if_exists = (restore_settings.create_access == RestoreAccessCreationMode::kCreate); + auto entities = restorer.getAccessEntitiesToRestore(); + if (entities.empty()) + return; - for (const auto & [id, entity] : entities_from_backup) - insertWithID(id, entity, replace_if_exists, throw_if_exists); + auto create_access = restorer.getRestoreSettings().create_access; + bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace); + bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate); + + restorer.addDataRestoreTask([this, entities = std::move(entities), replace_if_exists, throw_if_exists] + { + for (const auto & [id, entity] : entities) + insertWithID(id, entity, replace_if_exists, throw_if_exists); + }); } } diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h index 7cccdc1793f..6311e2ac7c0 100644 --- a/src/Access/ReplicatedAccessStorage.h +++ b/src/Access/ReplicatedAccessStorage.h @@ -38,7 +38,8 @@ public: bool exists(const UUID & id) const override; bool isBackupAllowed() const override { return backup_allowed; } - void insertFromBackup(const std::vector> & entities_from_backup, const RestoreSettings & restore_settings, std::shared_ptr restore_coordination) override; + void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override; + void restoreFromBackup(RestorerFromBackup & restorer) override; private: String zookeeper_path; diff --git a/src/Backups/BackupCoordinationDistributed.cpp b/src/Backups/BackupCoordinationDistributed.cpp index 9df17bf434e..5b932229e71 100644 --- a/src/Backups/BackupCoordinationDistributed.cpp +++ b/src/Backups/BackupCoordinationDistributed.cpp @@ -145,6 +145,8 @@ void BackupCoordinationDistributed::createRootNodes() zookeeper->createIfNotExists(zookeeper_path, ""); zookeeper->createIfNotExists(zookeeper_path + "/repl_part_names", ""); zookeeper->createIfNotExists(zookeeper_path + "/repl_data_paths", ""); + zookeeper->createIfNotExists(zookeeper_path + "/repl_access_host", ""); + zookeeper->createIfNotExists(zookeeper_path + "/repl_access_paths", ""); zookeeper->createIfNotExists(zookeeper_path + "/file_names", ""); zookeeper->createIfNotExists(zookeeper_path + "/file_infos", ""); zookeeper->createIfNotExists(zookeeper_path + "/archive_suffixes", ""); @@ -245,6 +247,47 @@ void BackupCoordinationDistributed::prepareReplicatedPartNames() const } +void BackupCoordinationDistributed::addReplicatedAccessPath(const String & access_zk_path, const String & file_path) +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_paths/" + escapeForFileName(access_zk_path); + zookeeper->createIfNotExists(path, ""); + path += "/" + escapeForFileName(file_path); + zookeeper->createIfNotExists(path, ""); +} + +Strings BackupCoordinationDistributed::getReplicatedAccessPaths(const String & access_zk_path) const +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_paths/" + escapeForFileName(access_zk_path); + Strings children = zookeeper->getChildren(path); + Strings file_paths; + file_paths.reserve(children.size()); + for (const String & child : children) + file_paths.push_back(unescapeForFileName(child)); + return file_paths; +} + +void BackupCoordinationDistributed::setReplicatedAccessHost(const String & access_zk_path, const String & host_id) +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_host/" + escapeForFileName(access_zk_path); + auto code = zookeeper->tryCreate(path, host_id, zkutil::CreateMode::Persistent); + if ((code != Coordination::Error::ZOK) && (code != Coordination::Error::ZNODEEXISTS)) + throw zkutil::KeeperException(code, path); + + if (code == Coordination::Error::ZNODEEXISTS) + zookeeper->set(path, host_id); +} + +String BackupCoordinationDistributed::getReplicatedAccessHost(const String & access_zk_path) const +{ + auto zookeeper = get_zookeeper(); + String path = zookeeper_path + "/repl_access_host/" + escapeForFileName(access_zk_path); + return zookeeper->get(path); +} + + void BackupCoordinationDistributed::addFileInfo(const FileInfo & file_info, bool & is_data_file_required) { auto zookeeper = get_zookeeper(); diff --git a/src/Backups/BackupCoordinationDistributed.h b/src/Backups/BackupCoordinationDistributed.h index 172c69edb20..813132bd0b8 100644 --- a/src/Backups/BackupCoordinationDistributed.h +++ b/src/Backups/BackupCoordinationDistributed.h @@ -29,6 +29,12 @@ public: void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; Strings getReplicatedDataPaths(const String & table_shared_id) const override; + void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) override; + Strings getReplicatedAccessPaths(const String & access_zk_path) const override; + + void setReplicatedAccessHost(const String & access_zk_path, const String & host_id) override; + String getReplicatedAccessHost(const String & access_zk_path) const override; + void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 7fd6fec6c33..158988cf8b8 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -56,6 +56,37 @@ Strings BackupCoordinationLocal::getReplicatedDataPaths(const String & table_sha } +void BackupCoordinationLocal::addReplicatedAccessPath(const String & access_zk_path, const String & file_path) +{ + std::lock_guard lock{mutex}; + replicated_access_paths[access_zk_path].push_back(file_path); +} + +Strings BackupCoordinationLocal::getReplicatedAccessPaths(const String & access_zk_path) const +{ + std::lock_guard lock{mutex}; + auto it = replicated_access_paths.find(access_zk_path); + if (it == replicated_access_paths.end()) + return {}; + return it->second; +} + +void BackupCoordinationLocal::setReplicatedAccessHost(const String & access_zk_path, const String & host_id) +{ + std::lock_guard lock{mutex}; + replicated_access_hosts[access_zk_path] = host_id; +} + +String BackupCoordinationLocal::getReplicatedAccessHost(const String & access_zk_path) const +{ + std::lock_guard lock{mutex}; + auto it = replicated_access_hosts.find(access_zk_path); + if (it == replicated_access_hosts.end()) + return {}; + return it->second; +} + + void BackupCoordinationLocal::addFileInfo(const FileInfo & file_info, bool & is_data_file_required) { std::lock_guard lock{mutex}; diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 519c721c208..dcd6505a438 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -30,6 +30,12 @@ public: void addReplicatedDataPath(const String & table_shared_id, const String & data_path) override; Strings getReplicatedDataPaths(const String & table_shared_id) const override; + void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) override; + Strings getReplicatedAccessPaths(const String & access_zk_path) const override; + + void setReplicatedAccessHost(const String & access_zk_path, const String & host_id) override; + String getReplicatedAccessHost(const String & access_zk_path) const override; + void addFileInfo(const FileInfo & file_info, bool & is_data_file_required) override; void updateFileInfo(const FileInfo & file_info) override; @@ -48,6 +54,8 @@ private: mutable std::mutex mutex; BackupCoordinationReplicatedPartNames replicated_part_names TSA_GUARDED_BY(mutex); std::unordered_map replicated_data_paths TSA_GUARDED_BY(mutex); + std::unordered_map replicated_access_paths TSA_GUARDED_BY(mutex); + std::unordered_map replicated_access_hosts TSA_GUARDED_BY(mutex); std::map file_names TSA_GUARDED_BY(mutex); /// Should be ordered alphabetically, see listFiles(). For empty files we assume checksum = 0. std::map file_infos TSA_GUARDED_BY(mutex); /// Information about files. Without empty files. Strings archive_suffixes TSA_GUARDED_BY(mutex); diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 30f1ecd53cd..e237140cf2b 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -706,6 +707,11 @@ void BackupEntriesCollector::addBackupEntry(const String & file_name, BackupEntr backup_entries.emplace_back(file_name, backup_entry); } +void BackupEntriesCollector::addBackupEntry(const std::pair & backup_entry) +{ + addBackupEntry(backup_entry.first, backup_entry.second); +} + void BackupEntriesCollector::addBackupEntries(const BackupEntries & backup_entries_) { if (current_status == kWritingBackupStatus) @@ -739,4 +745,10 @@ void BackupEntriesCollector::runPostTasks() } } +size_t BackupEntriesCollector::getAccessCounter(AccessEntityType type) +{ + access_counters.resize(static_cast(AccessEntityType::MAX)); + return access_counters[static_cast(type)]++; +} + } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 3b1260f6c99..0772fe84b26 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -19,6 +19,7 @@ class IBackupCoordination; class IDatabase; using DatabasePtr = std::shared_ptr; struct StorageID; +enum class AccessEntityType; /// Collects backup entries for all databases and tables which should be put to a backup. class BackupEntriesCollector : private boost::noncopyable @@ -42,6 +43,7 @@ public: /// Adds a backup entry which will be later returned by run(). /// These function can be called by implementations of IStorage::backupData() in inherited storage classes. void addBackupEntry(const String & file_name, BackupEntryPtr backup_entry); + void addBackupEntry(const std::pair & backup_entry); void addBackupEntries(const BackupEntries & backup_entries_); void addBackupEntries(BackupEntries && backup_entries_); @@ -50,6 +52,9 @@ public: /// 1) we need to join (in a backup) the data of replicated tables gathered on different hosts. void addPostTask(std::function task); + /// Returns an incremental counter used to backup access control. + size_t getAccessCounter(AccessEntityType type); + private: void calculateRootPathInBackup(); @@ -130,6 +135,7 @@ private: BackupEntries backup_entries; std::queue> post_tasks; + std::vector access_counters; }; } diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 0ae150c2b47..b4c5c7b3d88 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -44,6 +44,14 @@ public: /// Returns all the data paths in backup added for a replicated table (see also addReplicatedDataPath()). virtual Strings getReplicatedDataPaths(const String & table_shared_id) const = 0; + /// Adds a path to access.txt file keeping access entities of a ReplicatedAccessStorage. + virtual void addReplicatedAccessPath(const String & access_zk_path, const String & file_path) = 0; + virtual Strings getReplicatedAccessPaths(const String & access_zk_path) const = 0; + + /// Sets the host id of a host storing access entities of a ReplicatedAccessStorage to backup. + virtual void setReplicatedAccessHost(const String & access_zk_path, const String & host) = 0; + virtual String getReplicatedAccessHost(const String & access_zk_path) const = 0; + struct FileInfo { String file_name; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 247660bbce4..6013eed7919 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -383,9 +383,9 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name if (!restore_settings.structure_only && isSystemAccessTableName(table_name)) { - if (!access_restore_task) - access_restore_task = std::make_shared(backup, restore_settings, restore_coordination); - access_restore_task->addDataPath(data_path_in_backup, table_name); + if (!access_restorer) + access_restorer = std::make_unique(backup, restore_settings); + access_restorer->addDataPath(data_path_in_backup, table_name); } } @@ -555,8 +555,8 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const required_access.emplace_back(flags, table_name.database, table_name.table); } - if (access_restore_task) - insertAtEnd(required_access, access_restore_task->getRequiredAccess()); + if (access_restorer) + insertAtEnd(required_access, access_restorer->getRequiredAccess()); /// We convert to AccessRights and back to check access rights in a predictable way /// (some elements could be duplicated or not sorted). @@ -770,15 +770,9 @@ void RestorerFromBackup::addDataRestoreTasks(DataRestoreTasks && new_tasks) insertAtEnd(data_restore_tasks, std::move(new_tasks)); } -void RestorerFromBackup::checkPathInBackupIsRegisteredToRestoreAccess(const String & path) -{ - if (!access_restore_task || !access_restore_task->hasDataPath(path)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Path to restore access was not added"); -} - RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() { - if (data_restore_tasks.empty() && !access_restore_task) + if (data_restore_tasks.empty()) return {}; LOG_TRACE(log, "Will insert data to tables"); @@ -798,12 +792,20 @@ RestorerFromBackup::DataRestoreTasks RestorerFromBackup::getDataRestoreTasks() for (const auto & task : data_restore_tasks) res_tasks.push_back([task, storages, table_locks] { task(); }); - if (access_restore_task) - res_tasks.push_back([task = access_restore_task, access_control = &context->getAccessControl()] { task->restore(*access_control); }); - return res_tasks; } +std::vector> RestorerFromBackup::getAccessEntitiesToRestore() +{ + if (!access_restorer || access_restored) + return {}; + + /// getAccessEntitiesToRestore() will return entities only when called first time (we don't want to restore the same entities again). + access_restored = true; + + return access_restorer->getAccessEntities(context->getAccessControl()); +} + void RestorerFromBackup::throwTableIsNotEmpty(const StorageID & storage_id) { throw Exception( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index e47aca0e69f..3d814f67713 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -15,7 +15,9 @@ class IBackup; using BackupPtr = std::shared_ptr; class IRestoreCoordination; struct StorageID; -class AccessRestoreTask; +class AccessRestorerFromBackup; +struct IAccessEntity; +using AccessEntityPtr = std::shared_ptr; /// Restores the definition of databases and tables and prepares tasks to restore the data of the tables. class RestorerFromBackup : private boost::noncopyable @@ -56,8 +58,8 @@ public: void addDataRestoreTask(DataRestoreTask && new_task); void addDataRestoreTasks(DataRestoreTasks && new_tasks); - /// Checks that a specified path is already registered to be used for restoring access control. - void checkPathInBackupIsRegisteredToRestoreAccess(const String & path); + /// Returns the list of access entities to restore. + std::vector> getAccessEntitiesToRestore(); /// Throws an exception that a specified table is already non-empty. [[noreturn]] static void throwTableIsNotEmpty(const StorageID & storage_id); @@ -116,7 +118,8 @@ private: std::unordered_map database_infos; std::map table_infos; std::vector data_restore_tasks; - std::shared_ptr access_restore_task; + std::unique_ptr access_restorer; + bool access_restored = false; }; } diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index 27cf64cbcb4..046db151684 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -125,14 +125,14 @@ void StorageSystemQuotas::backupData( BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::QUOTA, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::QUOTA); } void StorageSystemQuotas::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp index 22597530835..e5b8d53ce7e 100644 --- a/src/Storages/System/StorageSystemRoles.cpp +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -63,14 +63,14 @@ void StorageSystemRoles::backupData( BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::ROLE, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::ROLE); } void StorageSystemRoles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 1e13654d188..064f610730d 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -143,14 +143,14 @@ void StorageSystemRowPolicies::backupData( BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::ROW_POLICY, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::ROW_POLICY); } void StorageSystemRowPolicies::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp index aaf5bedadd0..d03848ba68b 100644 --- a/src/Storages/System/StorageSystemSettingsProfiles.cpp +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -90,14 +90,14 @@ void StorageSystemSettingsProfiles::backupData( BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::SETTINGS_PROFILE, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::SETTINGS_PROFILE); } void StorageSystemSettingsProfiles::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index d8dc1722a91..be56abfa3e8 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -218,14 +218,14 @@ void StorageSystemUsers::backupData( BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /* partitions */) { const auto & access_control = backup_entries_collector.getContext()->getAccessControl(); - access_control.backup(backup_entries_collector, AccessEntityType::USER, data_path_in_backup); + access_control.backup(backup_entries_collector, data_path_in_backup, AccessEntityType::USER); } void StorageSystemUsers::restoreDataFromBackup( - RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /* partitions */) + RestorerFromBackup & restorer, const String & /* data_path_in_backup */, const std::optional & /* partitions */) { auto & access_control = restorer.getContext()->getAccessControl(); - access_control.restore(restorer, data_path_in_backup); + access_control.restoreFromBackup(restorer); } } From 109c9bcbd54712654e141ab7b4e3e79477d699e5 Mon Sep 17 00:00:00 2001 From: loyispa Date: Thu, 30 Jun 2022 18:37:19 +0800 Subject: [PATCH 301/408] Fix typo --- src/Storages/LiveView/StorageLiveView.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 75b2f981389..6a079aa832f 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -328,7 +328,7 @@ StorageLiveView::StorageLiveView( blocks_metadata_ptr = std::make_shared(); active_ptr = std::make_shared(true); - periodic_refresh_task = getContext()->getSchedulePool().createTask("LieViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); + periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); periodic_refresh_task->deactivate(); } From c38841a044bdbffad927ab820b8959117739d5dc Mon Sep 17 00:00:00 2001 From: Evgeny Kruglov Date: Thu, 30 Jun 2022 12:47:35 +0200 Subject: [PATCH 302/408] Fixes for clickhouse/clickhouse-keeper docker image (#38462) --- docker/keeper/Dockerfile | 2 +- docker/keeper/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 068377e8f8c..282392bd98a 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -67,7 +67,7 @@ RUN arch=${TARGETARCH:-amd64} \ && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper -EXPOSE 2181 10181 44444 +EXPOSE 2181 10181 44444 9181 VOLUME /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 86e56e88aa9..939cd941aeb 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -31,7 +31,7 @@ else DO_CHOWN=0 fi -KEEPER_CONFIG="${KEEPER_CONFIG:-/etc/clickhouse-keeper/config.yaml}" +KEEPER_CONFIG="${KEEPER_CONFIG:-/etc/clickhouse-keeper/keeper_config.xml}" if [ -f "$KEEPER_CONFIG" ] && ! $gosu test -f "$KEEPER_CONFIG" -a -r "$KEEPER_CONFIG"; then echo "Configuration file '$KEEPER_CONFIG' isn't readable by user with id '$USER'" From f443cf66f0b69136d31a2ca9c6091c30f5557e3d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 29 Jun 2022 19:19:47 +0200 Subject: [PATCH 303/408] CacheDictionary simplify update queue --- .../CacheDictionaryUpdateQueue.cpp | 24 +++++++++++-------- src/Dictionaries/CacheDictionaryUpdateQueue.h | 6 ++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index aee1f0de2f6..1fdaf10c57c 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -68,9 +68,9 @@ void CacheDictionaryUpdateQueue::waitForCurrentUpdateFinish if (update_queue.isFinished()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "CacheDictionaryUpdateQueue finished"); - std::unique_lock update_lock(update_mutex); + std::unique_lock update_lock(update_unit_ptr->update_mutex); - bool result = is_update_finished.wait_for( + bool result = update_unit_ptr->is_update_finished.wait_for( update_lock, std::chrono::milliseconds(configuration.query_wait_timeout_milliseconds), [&] @@ -133,19 +133,23 @@ void CacheDictionaryUpdateQueue::updateThreadFunction() /// Update update_func(unit_to_update); - /// Notify thread about finished updating the bunch of ids - /// where their own ids were included. - std::lock_guard lock(update_mutex); + { + /// Notify thread about finished updating the bunch of ids + /// where their own ids were included. + std::lock_guard lock(unit_to_update->update_mutex); + unit_to_update->is_done = true; + } - unit_to_update->is_done = true; - is_update_finished.notify_all(); + unit_to_update->is_update_finished.notify_all(); } catch (...) { - std::lock_guard lock(update_mutex); + { + std::lock_guard lock(unit_to_update->update_mutex); + unit_to_update->current_exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing) + } - unit_to_update->current_exception = std::current_exception(); // NOLINT(bugprone-throw-keyword-missing) - is_update_finished.notify_all(); + unit_to_update->is_update_finished.notify_all(); } } } diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.h b/src/Dictionaries/CacheDictionaryUpdateQueue.h index d6a195ca7b8..48598cb4548 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.h +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.h @@ -77,6 +77,9 @@ private: std::atomic is_done{false}; std::exception_ptr current_exception{nullptr}; /// NOLINT + mutable std::mutex update_mutex; + mutable std::condition_variable is_update_finished; + /// While UpdateUnit is alive, it is accounted in update_queue size. CurrentMetrics::Increment alive_batch{CurrentMetrics::CacheDictionaryUpdateQueueBatches}; CurrentMetrics::Increment alive_keys; @@ -159,9 +162,6 @@ private: UpdateQueue update_queue; ThreadPool update_pool; - - mutable std::mutex update_mutex; - mutable std::condition_variable is_update_finished; }; extern template class CacheDictionaryUpdateQueue; From a47355877ef35c706603db8c1b251470233bf0cf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 30 Jun 2022 13:58:26 +0300 Subject: [PATCH 304/408] Add revision() function (#38555) It can be useful to match versions, since in some tables (system.trace_log) there is only revision column. P.S. came to this when was digging into stress reports from CI. P.P.S. case insensitive by analogy with version(). Signed-off-by: Azat Khuzhin --- src/Functions/registerFunctionsMiscellaneous.cpp | 2 ++ src/Functions/serverConstants.cpp | 15 +++++++++++++++ .../01773_case_sensitive_revision.reference | 1 + .../0_stateless/01773_case_sensitive_revision.sql | 1 + 4 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/01773_case_sensitive_revision.reference create mode 100644 tests/queries/0_stateless/01773_case_sensitive_revision.sql diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 9cd9c70da16..755d38409a6 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -40,6 +40,7 @@ void registerFunctionIsNaN(FunctionFactory &); void registerFunctionIfNotFinite(FunctionFactory &); void registerFunctionThrowIf(FunctionFactory &); void registerFunctionVersion(FunctionFactory &); +void registerFunctionRevision(FunctionFactory &); void registerFunctionBuildId(FunctionFactory &); void registerFunctionUptime(FunctionFactory &); void registerFunctionTimezone(FunctionFactory &); @@ -129,6 +130,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIfNotFinite(factory); registerFunctionThrowIf(factory); registerFunctionVersion(factory); + registerFunctionRevision(factory); registerFunctionBuildId(factory); registerFunctionUptime(factory); registerFunctionTimezone(factory); diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index 12134cf4e4c..e809ec7c298 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #if defined(OS_LINUX) # include @@ -88,6 +89,15 @@ namespace explicit FunctionVersion(ContextPtr context) : FunctionConstantBase(VERSION_STRING, context->isDistributed()) {} }; + /// revision() - returns the current revision. + class FunctionRevision : public FunctionConstantBase + { + public: + static constexpr auto name = "revision"; + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + explicit FunctionRevision(ContextPtr context) : FunctionConstantBase(ClickHouseRevision::getVersionRevision(), context->isDistributed()) {} + }; + class FunctionZooKeeperSessionUptime : public FunctionConstantBase { public: @@ -151,6 +161,11 @@ void registerFunctionVersion(FunctionFactory & factory) factory.registerFunction(FunctionFactory::CaseInsensitive); } +void registerFunctionRevision(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} + void registerFunctionZooKeeperSessionUptime(FunctionFactory & factory) { factory.registerFunction(); diff --git a/tests/queries/0_stateless/01773_case_sensitive_revision.reference b/tests/queries/0_stateless/01773_case_sensitive_revision.reference new file mode 100644 index 00000000000..72749c905a3 --- /dev/null +++ b/tests/queries/0_stateless/01773_case_sensitive_revision.reference @@ -0,0 +1 @@ +1 1 1 diff --git a/tests/queries/0_stateless/01773_case_sensitive_revision.sql b/tests/queries/0_stateless/01773_case_sensitive_revision.sql new file mode 100644 index 00000000000..16970daf61b --- /dev/null +++ b/tests/queries/0_stateless/01773_case_sensitive_revision.sql @@ -0,0 +1 @@ +SELECT revision()=Revision(), REVISION()=Revision(), revisiON()=reVision(); From 95687f2d01e9b17fd19815bc6f6b712ec3075504 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 30 Jun 2022 13:15:35 +0200 Subject: [PATCH 305/408] CacheDictionaryUpdateUnit make update state non atomic --- src/Dictionaries/CacheDictionaryUpdateQueue.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.h b/src/Dictionaries/CacheDictionaryUpdateQueue.h index 48598cb4548..8db5c4a59df 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.h +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.h @@ -74,12 +74,12 @@ private: template friend class CacheDictionaryUpdateQueue; - std::atomic is_done{false}; - std::exception_ptr current_exception{nullptr}; /// NOLINT - mutable std::mutex update_mutex; mutable std::condition_variable is_update_finished; + bool is_done{false}; + std::exception_ptr current_exception{nullptr}; /// NOLINT + /// While UpdateUnit is alive, it is accounted in update_queue size. CurrentMetrics::Increment alive_batch{CurrentMetrics::CacheDictionaryUpdateQueueBatches}; CurrentMetrics::Increment alive_keys; From 1e3b5bfcb75de0cff8bccc3f9342c6fe66a9cf5c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 30 Jun 2022 11:43:25 +0000 Subject: [PATCH 306/408] Fix test 00233_position_function_family --- src/Common/StringSearcher.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index 7d669ddd369..b8f8a9d3a88 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -853,7 +853,8 @@ struct StdLibASCIIStringSearcher : public StringSearcherBase else { return std::search( - haystack_start, haystack_end, needle_start, needle_end); + haystack_start, haystack_end, needle_start, needle_end, + [](char c1, char c2) {return c1 == c2;}); } } From 0d3904b788d48e8f2dc47a8b8c6b6255f55f4c9a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:11:43 +0200 Subject: [PATCH 307/408] Fix incorrect order of operations in disk transactions --- src/Disks/DiskDecorator.cpp | 5 ++ src/Disks/DiskDecorator.h | 2 + src/Disks/FakeDiskTransaction.h | 5 ++ src/Disks/IDiskTransaction.h | 3 + .../DiskObjectStorageTransaction.cpp | 55 ++++++++++++++++--- .../DiskObjectStorageTransaction.h | 4 +- 6 files changed, 64 insertions(+), 10 deletions(-) diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 8390ca589e5..44248cae841 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -8,6 +8,11 @@ DiskDecorator::DiskDecorator(const DiskPtr & delegate_) : delegate(delegate_) { } +DiskTransactionPtr DiskDecorator::createTransaction() +{ + return delegate->createTransaction(); +} + const String & DiskDecorator::getName() const { return delegate->getName(); diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index e17a5aff3c7..dfb1f654dfd 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -12,6 +12,8 @@ class DiskDecorator : public IDisk { public: explicit DiskDecorator(const DiskPtr & delegate_); + + DiskTransactionPtr createTransaction() override; const String & getName() const override; ReservationPtr reserve(UInt64 bytes) override; ~DiskDecorator() override = default; diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index 7f9247a0689..387361a9df8 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -125,6 +125,11 @@ public: disk.createHardLink(src_path, dst_path); } + std::string getUniqueId(const String & path) const override + { + return disk.getUniqueId(path); + } + bool isCommitedOrTriedToCommit() const override { return commit_called; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index c6d93f4cb37..b301d52a8ea 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -109,8 +109,11 @@ public: /// Create hardlink from `src_path` to `dst_path`. virtual void createHardLink(const std::string & src_path, const std::string & dst_path) = 0; + virtual std::string getUniqueId(const String & path) const = 0; + virtual bool isCommitedOrTriedToCommit() const = 0; + }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 4903168b76b..db932e644c6 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -277,6 +278,8 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio { std::string path; std::string blob_path; + size_t size; + std::function on_execute; WriteFileObjectStorageOperation( IObjectStorage & object_storage_, @@ -288,9 +291,14 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio , blob_path(blob_path_) {} - void execute(MetadataTransactionPtr) override + void setOnExecute(std::function && on_execute_) { + on_execute = on_execute_; + } + void execute(MetadataTransactionPtr tx) override + { + on_execute(tx); } void undo() override @@ -368,6 +376,7 @@ void DiskObjectStorageTransaction::createDirectory(const std::string & path) void DiskObjectStorageTransaction::createDirectories(const std::string & path) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "CREATE DIRECTORIES TRANSACTION FOR PATH {}", path); operations_to_execute.emplace_back( std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) { @@ -477,6 +486,14 @@ String revisionToString(UInt64 revision) } +std::string DiskObjectStorageTransaction::getUniqueId(const std::string & path) const +{ + auto it = unique_ids.find(path); + if (it != unique_ids.end()) + return it->second; + return ""; +} + std::unique_ptr DiskObjectStorageTransaction::writeFile( /// NOLINT const std::string & path, size_t buf_size, @@ -497,21 +514,41 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } + unique_ids[path] = blob_name; + auto blob_path = fs::path(remote_fs_root_path) / blob_name; + auto write_operation = std::make_unique(object_storage, metadata_storage, path, blob_path); + std::function create_metadata_callback; - auto create_metadata_callback = [tx = shared_from_this(), mode, path, blob_name, autocommit] (size_t count) + if (autocommit) { - if (mode == WriteMode::Rewrite) - tx->metadata_transaction->createMetadataFile(path, blob_name, count); - else - tx->metadata_transaction->addBlobToMetadata(path, blob_name, count); + create_metadata_callback = [tx = shared_from_this(), mode, path, blob_name] (size_t count) + { + if (mode == WriteMode::Rewrite) + tx->metadata_transaction->createMetadataFile(path, blob_name, count); + else + tx->metadata_transaction->addBlobToMetadata(path, blob_name, count); - if (autocommit) tx->metadata_transaction->commit(); - }; + }; + } + else + { + create_metadata_callback = [write_op = write_operation.get(), mode, path, blob_name] (size_t count) + { + write_op->setOnExecute([mode, path, blob_name, count](MetadataTransactionPtr tx) + { + if (mode == WriteMode::Rewrite) + tx->createMetadataFile(path, blob_name, count); + else + tx->addBlobToMetadata(path, blob_name, count); + }); + }; - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, path, blob_path)); + } + + operations_to_execute.emplace_back(std::move(write_operation)); /// We always use mode Rewrite because we simulate append using metadata and different files return object_storage.writeObject( diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 392152d1ce0..4312b380a04 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -56,6 +56,7 @@ private: /// TODO we can get rid of this params const std::string & remote_fs_root_path; DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; + std::unordered_map unique_ids; DiskObjectStorageOperations operations_to_execute; bool commit_called{false}; @@ -109,11 +110,12 @@ public: void setReadOnly(const std::string & path) override; void createHardLink(const std::string & src_path, const std::string & dst_path) override; + std::string getUniqueId(const std::string & path) const override; + bool isCommitedOrTriedToCommit() const override { return commit_called; } - }; using DiskObjectStorageTransactionPtr = std::shared_ptr; From b2b31103c508d81f8f093d6489fdc25c9f0af357 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Thu, 30 Jun 2022 13:39:31 +0200 Subject: [PATCH 308/408] Reuse common code for L2Squared and L2 --- src/Functions/array/arrayDistance.cpp | 18 +-- src/Functions/array/arrayNorm.cpp | 10 +- src/Functions/vectorFunctions.cpp | 154 ++++++++------------------ 3 files changed, 51 insertions(+), 131 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index fda93bceedd..21e05916a5c 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -70,26 +70,10 @@ struct L2Distance } }; -struct L2SquaredDistance +struct L2SquaredDistance : L2Distance { static inline String name = "L2Squared"; - struct ConstParams - { - }; - - template - struct State - { - FloatType sum = 0; - }; - - template - static void accumulate(State & state, ResultType x, ResultType y, const ConstParams &) - { - state.sum += (x - y) * (x - y); - } - template static ResultType finalize(const State & state, const ConstParams &) { diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index a9d1885c136..3ea16b23abd 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -57,18 +57,10 @@ struct L2Norm } }; -struct L2SquaredNorm +struct L2SquaredNorm : L2Norm { static inline String name = "L2Squared"; - struct ConstParams {}; - - template - inline static ResultType accumulate(ResultType result, ResultType value, const ConstParams &) - { - return result + value * value; - } - template inline static ResultType finalize(ResultType result, const ConstParams &) { diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp index d26eb40e1a8..7974a8bbaf4 100644 --- a/src/Functions/vectorFunctions.cpp +++ b/src/Functions/vectorFunctions.cpp @@ -521,111 +521,6 @@ public: }; using FunctionL1Norm = FunctionLNorm; -template <> -class FunctionLNorm : public ITupleFunction -{ -public: - static constexpr auto name = "L2Norm"; - - explicit FunctionLNorm(ContextPtr context_) : ITupleFunction(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } - - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 1; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override - { - const auto * cur_tuple = checkAndGetDataType(arguments[0].type.get()); - - if (!cur_tuple) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument 0 of function {} should be tuple, got {}", - getName(), arguments[0].type->getName()); - - const auto & cur_types = cur_tuple->getElements(); - - Columns cur_elements; - if (arguments[0].column) - cur_elements = getTupleElements(*arguments[0].column); - - size_t tuple_size = cur_types.size(); - if (tuple_size == 0) - return std::make_shared(); - - auto multiply = FunctionFactory::instance().get("multiply", context); - auto plus = FunctionFactory::instance().get("plus", context); - DataTypePtr res_type; - for (size_t i = 0; i < tuple_size; ++i) - { - try - { - ColumnWithTypeAndName cur{cur_elements.empty() ? nullptr : cur_elements[i], cur_types[i], {}}; - auto elem_multiply = multiply->build(ColumnsWithTypeAndName{cur, cur}); - - if (i == 0) - { - res_type = elem_multiply->getResultType(); - continue; - } - - ColumnWithTypeAndName left_type{res_type, {}}; - ColumnWithTypeAndName right_type{elem_multiply->getResultType(), {}}; - auto plus_elem = plus->build({left_type, right_type}); - res_type = plus_elem->getResultType(); - } - catch (DB::Exception & e) - { - e.addMessage("While executing function {} for tuple element {}", getName(), i); - throw; - } - } - - auto sqrt = FunctionFactory::instance().get("sqrt", context); - return sqrt->build({ColumnWithTypeAndName{res_type, {}}})->getResultType(); - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const auto * cur_tuple = checkAndGetDataType(arguments[0].type.get()); - const auto & cur_types = cur_tuple->getElements(); - auto cur_elements = getTupleElements(*arguments[0].column); - - size_t tuple_size = cur_elements.size(); - if (tuple_size == 0) - return DataTypeUInt8().createColumnConstWithDefaultValue(input_rows_count); - - auto multiply = FunctionFactory::instance().get("multiply", context); - auto plus = FunctionFactory::instance().get("plus", context); - ColumnWithTypeAndName res; - for (size_t i = 0; i < tuple_size; ++i) - { - ColumnWithTypeAndName cur{cur_elements[i], cur_types[i], {}}; - auto elem_multiply = multiply->build(ColumnsWithTypeAndName{cur, cur}); - - ColumnWithTypeAndName column; - column.type = elem_multiply->getResultType(); - column.column = elem_multiply->execute({cur, cur}, column.type, input_rows_count); - - if (i == 0) - { - res = std::move(column); - } - else - { - auto plus_elem = plus->build({res, column}); - auto res_type = plus_elem->getResultType(); - res.column = plus_elem->execute({res, column}, res_type, input_rows_count); - res.type = res_type; - } - } - - auto sqrt = FunctionFactory::instance().get("sqrt", context); - auto sqrt_elem = sqrt->build({res}); - return sqrt_elem->execute({res}, sqrt_elem->getResultType(), input_rows_count); - } -}; -using FunctionL2Norm = FunctionLNorm; - template <> class FunctionLNorm : public ITupleFunction { @@ -728,6 +623,55 @@ public: }; using FunctionL2SquaredNorm = FunctionLNorm; +template <> +class FunctionLNorm : public FunctionL2SquaredNorm +{ +private: + using Base = FunctionL2SquaredNorm; +public: + static constexpr auto name = "L2Norm"; + + explicit FunctionLNorm(ContextPtr context_) : Base(context_) {} + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + + String getName() const override { return name; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + const auto * cur_tuple = checkAndGetDataType(arguments[0].type.get()); + + if (!cur_tuple) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument 0 of function {} should be tuple, got {}", + getName(), arguments[0].type->getName()); + + const auto & cur_types = cur_tuple->getElements(); + size_t tuple_size = cur_types.size(); + if (tuple_size == 0) + return std::make_shared(); + + auto sqrt = FunctionFactory::instance().get("sqrt", context); + return sqrt->build({ColumnWithTypeAndName{Base::getReturnTypeImpl(arguments), {}}})->getResultType(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto cur_elements = getTupleElements(*arguments[0].column); + + size_t tuple_size = cur_elements.size(); + if (tuple_size == 0) + return DataTypeUInt8().createColumnConstWithDefaultValue(input_rows_count); + + ColumnWithTypeAndName squared_res; + squared_res.type = Base::getReturnTypeImpl(arguments); + squared_res.column = Base::executeImpl(arguments, squared_res.type, input_rows_count); + + auto sqrt = FunctionFactory::instance().get("sqrt", context); + auto sqrt_elem = sqrt->build({squared_res}); + return sqrt_elem->execute({squared_res}, sqrt_elem->getResultType(), input_rows_count); + } +}; +using FunctionL2Norm = FunctionLNorm; + template <> class FunctionLNorm : public ITupleFunction { From cb90aca2efe976e716b41d11806e8e2caf1a8c08 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:12:45 +0200 Subject: [PATCH 309/408] Some changes in merge tree --- .../MergeTree/DataPartStorageOnDisk.cpp | 6 ++++ .../MergeTree/DataPartStorageOnDisk.h | 2 ++ src/Storages/MergeTree/DataPartsExchange.cpp | 4 +-- src/Storages/MergeTree/IDataPartStorage.h | 5 ++++ src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 ---- src/Storages/MergeTree/IMergeTreeDataPart.h | 4 --- .../MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 28 +++++++++++-------- src/Storages/StorageReplicatedMergeTree.h | 4 +-- 14 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 347f43a03e4..833cc2498e8 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -847,6 +847,7 @@ std::string DataPartStorageBuilderOnDisk::getRelativePath() const void DataPartStorageBuilderOnDisk::createDirectories() { + LOG_INFO(&Poco::Logger::get("DEBUG"), "CREATING DIRECTORY {}", (fs::path(root_path) / part_dir).string()); transaction->createDirectories(fs::path(root_path) / part_dir); } @@ -874,6 +875,11 @@ DataPartStoragePtr DataPartStorageBuilderOnDisk::getStorage() const return std::make_shared(volume, root_path, part_dir); } +String DataPartStorageBuilderOnDisk::getUniqueId() const +{ + return transaction->getUniqueId(fs::path(getRelativePath()) / "checksums.txt"); +} + void DataPartStorageBuilderOnDisk::commit() { transaction->commit(); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index d7d34e96919..66bd009a55d 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -166,6 +166,8 @@ public: bool remove_new_dir_if_exists, bool fsync_part_dir) override; + String getUniqueId() const override; + void commit() override; private: diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 68e9f0a78d1..148d11d7a29 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -347,7 +347,7 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part /// Serialized metadatadatas with zero ref counts. auto metadatas = data_part_storage_on_disk->getSerializedMetadata(paths); - String part_id = part->getUniqueId(); + String part_id = data_part_storage_on_disk->getUniqueId(); writeStringBinary(part_id, out); writeBinary(checksums.files.size(), out); @@ -928,7 +928,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true, {}); + data.lockSharedData(*new_data_part, nullptr, /* replace_existing_lock = */ true, {}); LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", part_name, part_id, disk->getName()); diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 2b644abd184..d6087904c8c 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -250,6 +250,11 @@ public: bool remove_new_dir_if_exists, bool fsync_part_dir) = 0; + /// A leak of abstraction. + /// Return some uniq string for file. + /// Required for distinguish different copies of the same part on remote FS. + virtual String getUniqueId() const = 0; + virtual void commit() = 0; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 60941108f00..8ac4fe3a452 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1696,11 +1696,6 @@ bool IMergeTreeDataPart::checkAllTTLCalculated(const StorageMetadataPtr & metada return true; } -String IMergeTreeDataPart::getUniqueId() const -{ - return data_part_storage->getUniqueId(); -} - String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const { if (info.level != 0) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 7f3c41ce4c2..1f5a95e3cca 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -421,10 +421,6 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; - /// Return some uniq string for file. - /// Required for distinguish different copies of the same part on remote FS. - String getUniqueId() const; - /// Ensures that creation_tid was correctly set after part creation. void assertHasVersionMetadata(MergeTreeTransaction * txn) const; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 0e99d6ce04e..a70b329d57b 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -269,7 +269,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite try { - storage.checkPartChecksumsAndCommit(*transaction_ptr, part); + storage.checkPartChecksumsAndCommit(*transaction_ptr, part, builder); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 24840782a74..7ba451c7fee 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3448,7 +3448,7 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) if (original_active_part->data_part_storage->supportZeroCopyReplication() && part_copy->data_part_storage->supportZeroCopyReplication() && - original_active_part->getUniqueId() == part_copy->getUniqueId()) + original_active_part->data_part_storage->getUniqueId() == part_copy->data_part_storage->getUniqueId()) { /// May be when several volumes use the same S3/HDFS storage original_active_part->force_keep_shared_data = true; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index c8c6a03f4c0..94c8ba8b33d 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -961,7 +961,7 @@ public: /// Lock part in zookeeper for shared data in several nodes /// Overridden in StorageReplicatedMergeTree - virtual void lockSharedData(const IMergeTreeDataPart &, bool = false, std::optional = {}) const {} /// NOLINT + virtual void lockSharedData(const IMergeTreeDataPart &, DataPartStorageBuilderPtr = nullptr, bool = false, std::optional = {}) const {} /// NOLINT /// Unlock shared data part in zookeeper /// Overridden in StorageReplicatedMergeTree diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index fe5ec326790..89042e25a0e 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -410,7 +410,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( if (new_data_part->data_part_storage->exists()) { - LOG_WARNING(log, "Removing old temporary directory {}", new_data_part->data_part_storage->getFullPath()); + LOG_WARNING(log, "Removing old temporary directory {}", full_path); data_part_storage_builder->removeRecursive(); } diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index a51eb7854ab..19a11aa2385 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -179,7 +179,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit try { - storage.checkPartChecksumsAndCommit(*transaction_ptr, new_part, mutate_task->getHardlinkedFiles()); + storage.checkPartChecksumsAndCommit(*transaction_ptr, new_part, builder, mutate_task->getHardlinkedFiles()); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 65daa37bb41..0606f26747f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -502,7 +502,7 @@ void ReplicatedMergeTreeSink::commitPart( part->name); } - storage.lockSharedData(*part, false, {}); + storage.lockSharedData(*part, builder, false, {}); Coordination::Responses responses; Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a711ca803d4..31ad7160431 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1501,7 +1501,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(Transaction & transaction, - const DataPartPtr & part, std::optional hardlinked_files) + const DataPartPtr & part, DataPartStorageBuilderPtr builder, std::optional hardlinked_files) { auto zookeeper = getZooKeeper(); @@ -1511,7 +1511,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd Coordination::Requests ops; NameSet absent_part_paths_on_replicas; - lockSharedData(*part, false, hardlinked_files); + lockSharedData(*part, builder, false, hardlinked_files); /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`. checkPartChecksumsAndAddCommitOps(zookeeper, part, ops, part->name, &absent_part_paths_on_replicas); @@ -1659,7 +1659,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) part->version.setCreationTID(Tx::PrehistoricTID, nullptr); auto builder = part->data_part_storage->getBuilder(); renameTempPartAndReplace(part, transaction, builder); - checkPartChecksumsAndCommit(transaction, part); + checkPartChecksumsAndCommit(transaction, part, nullptr); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr); @@ -2347,7 +2347,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) renameTempPartAndReplace(part_desc->res_part, transaction, builder); getCommitPartOps(ops, part_desc->res_part); - lockSharedData(*part_desc->res_part, false, part_desc->hardlinked_files); + lockSharedData(*part_desc->res_part, nullptr, false, part_desc->hardlinked_files); } @@ -4089,7 +4089,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora Transaction transaction(*this, NO_TRANSACTION_RAW); renameTempPartAndReplace(part, transaction, builder); - replaced_parts = checkPartChecksumsAndCommit(transaction, part, hardlinked_files); + replaced_parts = checkPartChecksumsAndCommit(transaction, part, nullptr, hardlinked_files); /** If a quorum is tracked for this part, you must update it. * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. @@ -6624,7 +6624,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } for (size_t i = 0; i < dst_parts.size(); ++i) - lockSharedData(*dst_parts[i], false, hardlinked_files_for_parts[i]); + lockSharedData(*dst_parts[i], nullptr, false, hardlinked_files_for_parts[i]); Coordination::Error code = zookeeper->tryMulti(ops, op_results); if (code == Coordination::Error::ZOK) @@ -6860,8 +6860,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } for (size_t i = 0; i < dst_parts.size(); ++i) - dest_table_storage->lockSharedData(*dst_parts[i], false, hardlinked_files_for_parts[i]); - + dest_table_storage->lockSharedData(*dst_parts[i], nullptr, false, hardlinked_files_for_parts[i]); Coordination::Error code = zookeeper->tryMulti(ops, op_results); if (code == Coordination::Error::ZBADVERSION) continue; @@ -7546,7 +7545,7 @@ void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_nam } } -void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock, std::optional hardlinked_files) const +void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, DataPartStorageBuilderPtr builder, bool replace_existing_lock, std::optional hardlinked_files) const { auto settings = getSettings(); @@ -7560,7 +7559,12 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, if (!zookeeper) return; - String id = part.getUniqueId(); + String id; + if (builder) + id = builder->getUniqueId(); + else + id = part.data_part_storage->getUniqueId(); + boost::replace_all(id, "/", "_"); Strings zc_zookeeper_paths = getZeroCopyPartPath( @@ -7612,7 +7616,7 @@ std::pair StorageReplicatedMergeTree::unlockSharedData(const IMer return std::make_pair(true, NameSet{}); } - return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), part.name, replica_name, part.data_part_storage->getDiskType(), getZooKeeper(), *getSettings(), log, + return unlockSharedDataByID(part.data_part_storage->getUniqueId(), getTableSharedID(), part.name, replica_name, part.data_part_storage->getDiskType(), getZooKeeper(), *getSettings(), log, zookeeper_path); } @@ -8055,7 +8059,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP throw Exception(ErrorCodes::INCORRECT_DATA, "Tried to create empty part {}, but it replaces existing parts {}.", lost_part_name, fmt::join(part_names, ", ")); } - lockSharedData(*new_data_part, false, {}); + lockSharedData(*new_data_part, data_part_storage_builder, false, {}); while (true) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 73a08a2b921..3e7e1e0f4c4 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -261,7 +261,7 @@ public: DataPartStoragePtr executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); /// Lock part in zookeeper for use shared data in several nodes - void lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock, std::optional hardlinked_files) const override; + void lockSharedData(const IMergeTreeDataPart & part, DataPartStorageBuilderPtr builder, bool replace_existing_lock, std::optional hardlinked_files) const override; void lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; @@ -506,7 +506,7 @@ private: String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const; /// Accepts a PreActive part, atomically checks its checksums with ones on other replicas and commit the part - DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part, std::optional hardlinked_files = {}); + DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part, DataPartStorageBuilderPtr builder, std::optional hardlinked_files = {}); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; From 9a891543524069a8c04ce8acde064213a31fb94b Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:11:43 +0200 Subject: [PATCH 310/408] Fix incorrect order of operations in disk transactions (cherry picked from commit 0d3904b788d48e8f2dc47a8b8c6b6255f55f4c9a) --- src/Disks/DiskDecorator.cpp | 5 ++ src/Disks/DiskDecorator.h | 2 + src/Disks/IDiskTransaction.h | 1 - .../DiskObjectStorageTransaction.cpp | 55 ++++++++++++++++--- .../DiskObjectStorageTransaction.h | 1 + 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 8390ca589e5..44248cae841 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -8,6 +8,11 @@ DiskDecorator::DiskDecorator(const DiskPtr & delegate_) : delegate(delegate_) { } +DiskTransactionPtr DiskDecorator::createTransaction() +{ + return delegate->createTransaction(); +} + const String & DiskDecorator::getName() const { return delegate->getName(); diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index e17a5aff3c7..dfb1f654dfd 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -12,6 +12,8 @@ class DiskDecorator : public IDisk { public: explicit DiskDecorator(const DiskPtr & delegate_); + + DiskTransactionPtr createTransaction() override; const String & getName() const override; ReservationPtr reserve(UInt64 bytes) override; ~DiskDecorator() override = default; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index e7b1cf3f675..4b00a9bcefc 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -108,7 +108,6 @@ public: /// Create hardlink from `src_path` to `dst_path`. virtual void createHardLink(const std::string & src_path, const std::string & dst_path) = 0; - }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 1d16012437a..d3b8f162a7e 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -277,6 +278,8 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio { std::string path; std::string blob_path; + size_t size; + std::function on_execute; WriteFileObjectStorageOperation( IObjectStorage & object_storage_, @@ -288,9 +291,14 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio , blob_path(blob_path_) {} - void execute(MetadataTransactionPtr) override + void setOnExecute(std::function && on_execute_) { + on_execute = on_execute_; + } + void execute(MetadataTransactionPtr tx) override + { + on_execute(tx); } void undo() override @@ -368,6 +376,7 @@ void DiskObjectStorageTransaction::createDirectory(const std::string & path) void DiskObjectStorageTransaction::createDirectories(const std::string & path) { + LOG_DEBUG(&Poco::Logger::get("DEBUG"), "CREATE DIRECTORIES TRANSACTION FOR PATH {}", path); operations_to_execute.emplace_back( std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) { @@ -477,6 +486,14 @@ String revisionToString(UInt64 revision) } +std::string DiskObjectStorageTransaction::getUniqueId(const std::string & path) const +{ + auto it = unique_ids.find(path); + if (it != unique_ids.end()) + return it->second; + return ""; +} + std::unique_ptr DiskObjectStorageTransaction::writeFile( /// NOLINT const std::string & path, size_t buf_size, @@ -497,21 +514,41 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } + unique_ids[path] = blob_name; + auto blob_path = fs::path(remote_fs_root_path) / blob_name; + auto write_operation = std::make_unique(object_storage, metadata_storage, path, blob_path); + std::function create_metadata_callback; - auto create_metadata_callback = [tx = shared_from_this(), mode, path, blob_name, autocommit] (size_t count) + if (autocommit) { - if (mode == WriteMode::Rewrite) - tx->metadata_transaction->createMetadataFile(path, blob_name, count); - else - tx->metadata_transaction->addBlobToMetadata(path, blob_name, count); + create_metadata_callback = [tx = shared_from_this(), mode, path, blob_name] (size_t count) + { + if (mode == WriteMode::Rewrite) + tx->metadata_transaction->createMetadataFile(path, blob_name, count); + else + tx->metadata_transaction->addBlobToMetadata(path, blob_name, count); - if (autocommit) tx->metadata_transaction->commit(); - }; + }; + } + else + { + create_metadata_callback = [write_op = write_operation.get(), mode, path, blob_name] (size_t count) + { + write_op->setOnExecute([mode, path, blob_name, count](MetadataTransactionPtr tx) + { + if (mode == WriteMode::Rewrite) + tx->createMetadataFile(path, blob_name, count); + else + tx->addBlobToMetadata(path, blob_name, count); + }); + }; - operations_to_execute.emplace_back(std::make_unique(object_storage, metadata_storage, path, blob_path)); + } + + operations_to_execute.emplace_back(std::move(write_operation)); /// We always use mode Rewrite because we simulate append using metadata and different files return object_storage.writeObject( diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 362b5404707..19ce34cc06d 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -56,6 +56,7 @@ private: /// TODO we can get rid of this params const std::string & remote_fs_root_path; DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; + std::unordered_map unique_ids; DiskObjectStorageOperations operations_to_execute; public: From 1ff4222661188a962817721b4f1744d5d807834e Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:21:48 +0200 Subject: [PATCH 311/408] Remove redundant changes --- .../ObjectStorages/DiskObjectStorageTransaction.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index d3b8f162a7e..97885057833 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -486,14 +486,6 @@ String revisionToString(UInt64 revision) } -std::string DiskObjectStorageTransaction::getUniqueId(const std::string & path) const -{ - auto it = unique_ids.find(path); - if (it != unique_ids.end()) - return it->second; - return ""; -} - std::unique_ptr DiskObjectStorageTransaction::writeFile( /// NOLINT const std::string & path, size_t buf_size, @@ -514,8 +506,6 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } - unique_ids[path] = blob_name; - auto blob_path = fs::path(remote_fs_root_path) / blob_name; auto write_operation = std::make_unique(object_storage, metadata_storage, path, blob_path); From c5943d0384b65753879194be72719659ebabcffb Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:27:47 +0200 Subject: [PATCH 312/408] Better comment --- .../ObjectStorages/DiskObjectStorageTransaction.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 97885057833..3ac37af29a1 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -527,6 +527,16 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile { create_metadata_callback = [write_op = write_operation.get(), mode, path, blob_name] (size_t count) { + /// This callback called in WriteBuffer finalize method -- only there we actually know + /// how many bytes were written. We don't control when this finalize method will be called + /// so here we just modify operation itself, but don't execute anything (and don't modify metadata transaction). + /// Otherwise it's possible to get reorder of operations, like: + /// tx->createDirectory(xxx) -- will add metadata operation in execute + /// buf1 = tx->writeFile(xxx/yyy.bin) + /// buf2 = tx->writeFile(xxx/zzz.bin) + /// ... + /// buf1->finalize() // shouldn't do anything with metadata operations, just memoize what to do + /// tx->commit() write_op->setOnExecute([mode, path, blob_name, count](MetadataTransactionPtr tx) { if (mode == WriteMode::Rewrite) From dee62404949eff22ee643e3cb249b13db996a8c3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:34:32 +0200 Subject: [PATCH 313/408] Remove redundant --- src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp | 1 - src/Disks/ObjectStorages/DiskObjectStorageTransaction.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 3ac37af29a1..51f975943a4 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -376,7 +376,6 @@ void DiskObjectStorageTransaction::createDirectory(const std::string & path) void DiskObjectStorageTransaction::createDirectories(const std::string & path) { - LOG_DEBUG(&Poco::Logger::get("DEBUG"), "CREATE DIRECTORIES TRANSACTION FOR PATH {}", path); operations_to_execute.emplace_back( std::make_unique(object_storage, metadata_storage, [path](MetadataTransactionPtr tx) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 19ce34cc06d..362b5404707 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -56,7 +56,6 @@ private: /// TODO we can get rid of this params const std::string & remote_fs_root_path; DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; - std::unordered_map unique_ids; DiskObjectStorageOperations operations_to_execute; public: From 845264dd7bb961633c1f273f2a0339e7ae1beed2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 14:36:18 +0200 Subject: [PATCH 314/408] Followup --- src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 51f975943a4..6c9e5273fdb 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -298,7 +298,8 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio void execute(MetadataTransactionPtr tx) override { - on_execute(tx); + if (on_execute) + on_execute(tx); } void undo() override From d31ca4c4b6debe974f8f70272eb8e755cbbd1738 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 30 Jun 2022 14:49:11 +0200 Subject: [PATCH 315/408] Fixed tests --- src/Columns/ColumnVector.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 7cfb90d4371..60423e2b0fe 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -633,7 +633,8 @@ namespace while (sse_copy_counter) { - _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_copy), *(reinterpret_cast(data_copy_begin_ptr))); + __m128i copy_batch = _mm_loadu_si128(reinterpret_cast(data_copy_begin_ptr)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data_copy), copy_batch); result_data_copy += 4; data_copy_begin_ptr += 4; --sse_copy_counter; @@ -673,7 +674,8 @@ namespace while (sse_copy_counter) { - _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data), *(reinterpret_cast(data_copy_begin_ptr))); + __m128i copy_batch = _mm_loadu_si128(reinterpret_cast(data_copy_begin_ptr)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(result_data), copy_batch); result_data += 4; data_copy_begin_ptr += 4; --sse_copy_counter; From 488ee75fc4de884769d8e0e9d749f25c9700c02c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 30 Jun 2022 13:03:39 +0000 Subject: [PATCH 316/408] + use DistinctSorted for final distinct step + fix performance tests --- src/Core/Settings.h | 1 - src/Interpreters/InterpreterSelectQuery.cpp | 3 +- .../InterpreterSelectWithUnionQuery.cpp | 3 +- src/Processors/QueryPlan/DistinctStep.cpp | 24 +++++-- src/Processors/QueryPlan/DistinctStep.h | 4 +- .../DistinctSortedChunkTransform.cpp | 72 +++++++------------ .../Transforms/DistinctSortedChunkTransform.h | 4 +- tests/performance/distinct_in_order.xml | 40 +++++------ ...ct_in_order_optimization_explain.reference | 4 +- ..._distinct_in_order_optimization_explain.sh | 2 +- 10 files changed, 68 insertions(+), 89 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d8080fc427d..332b7364605 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -607,7 +607,6 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ - M(UInt64, distinct_in_order_range_search_step, 0, "Setting for DISTINCT in order optimization. TBD", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c7149cd30dc..28438a86e47 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2578,8 +2578,7 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before limit_for_distinct, columns, pre_distinct, - settings.optimize_distinct_in_order, - settings.distinct_in_order_range_search_step); + settings.optimize_distinct_in_order); if (pre_distinct) distinct_step->setStepDescription("Preliminary DISTINCT"); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 6a5e5658ccf..9f87a47fced 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -335,8 +335,7 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) 0, result_header.getNames(), false, - settings.optimize_distinct_in_order, - settings.distinct_in_order_range_search_step); + settings.optimize_distinct_in_order); query_plan.addStep(std::move(distinct_step)); } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index c182d336b35..946af9ca4d6 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -59,8 +59,7 @@ DistinctStep::DistinctStep( UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, - bool optimize_distinct_in_order_, - UInt64 distinct_in_order_range_search_step_) + bool optimize_distinct_in_order_) : ITransformingStep( input_stream_, input_stream_.header, @@ -68,7 +67,6 @@ DistinctStep::DistinctStep( , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) - , distinct_in_order_range_search_step(distinct_in_order_range_search_step_) , pre_distinct(pre_distinct_) , optimize_distinct_in_order(optimize_distinct_in_order_) { @@ -96,9 +94,8 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns); if (!distinct_sort_desc.empty()) { - /// pre-distinct for sorted chunks or - /// final distinct for sorted stream (sorting inside and among chunks) - if (pre_distinct || input_stream.has_single_port) + /// pre-distinct for sorted chunks + if (pre_distinct) { pipeline.addSimpleTransform( [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr @@ -107,7 +104,20 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil return nullptr; return std::make_shared( - header, set_size_limits, limit_hint, distinct_sort_desc, columns, distinct_in_order_range_search_step); + header, set_size_limits, limit_hint, distinct_sort_desc, columns); + }); + return; + } + /// final distinct for sorted stream (sorting inside and among chunks) + if (input_stream.has_single_port) + { + pipeline.addSimpleTransform( + [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipelineBuilder::StreamType::Main) + return nullptr; + + return std::make_shared(header, distinct_sort_desc, set_size_limits, limit_hint, columns); }); return; } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index 1d678ac3144..dc734a58704 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -15,8 +15,7 @@ public: UInt64 limit_hint_, const Names & columns_, bool pre_distinct_, /// If is enabled, execute distinct for separate streams. Otherwise, merge streams. - bool optimize_distinct_in_order_, - UInt64 distinct_in_order_range_search_step); + bool optimize_distinct_in_order_); String getName() const override { return "Distinct"; } @@ -31,7 +30,6 @@ private: SizeLimits set_size_limits; UInt64 limit_hint; Names columns; - UInt64 distinct_in_order_range_search_step = 0; bool pre_distinct; bool optimize_distinct_in_order; }; diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp index 89fa675dbc7..064c827a8cc 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.cpp @@ -13,13 +13,11 @@ DistinctSortedChunkTransform::DistinctSortedChunkTransform( const SizeLimits & output_size_limits_, UInt64 limit_hint_, const SortDescription & sorted_columns_descr_, - const Names & source_columns, - size_t range_search_step_) + const Names & source_columns) : ISimpleTransform(header_, header_, true) , limit_hint(limit_hint_) , output_size_limits(output_size_limits_) , sorted_columns_descr(sorted_columns_descr_) - , range_search_step(range_search_step_) { /// calculate sorted columns positions sorted_columns_pos.reserve(sorted_columns_descr.size()); @@ -124,57 +122,35 @@ bool DistinctSortedChunkTransform::isCurrentKey(const size_t row_pos) const return true; } -size_t DistinctSortedChunkTransform::getRangeEnd(size_t range_begin, size_t range_end) const +size_t DistinctSortedChunkTransform::getRangeEnd(size_t begin, size_t end) const { - assert(range_begin < range_end); + assert(begin < end); - // probe latest row - if (isCurrentKey(range_end-1)) { - return range_end; - } + const size_t linear_probe_threadhold = 16; + size_t linear_probe_end = begin + linear_probe_threadhold; + if (linear_probe_end > end) + linear_probe_end = end; - auto find_range_end = [this](size_t begin, size_t end) -> size_t + for (size_t pos = begin; pos < linear_probe_end; ++pos) { - const size_t linear_probe_threadhold = 32; - size_t linear_probe_end = begin + linear_probe_threadhold; - if (linear_probe_end > end) - linear_probe_end = end; - - for(size_t pos=begin; pos < linear_probe_end; ++pos) - { - if (!isCurrentKey(pos)) - return pos; - } - - size_t low = linear_probe_end; - size_t high = end - 1; - while (low <= high) - { - size_t mid = low + (high - low) / 2; - if (isCurrentKey(mid)) - low = mid + 1; - else - { - high = mid - 1; - end = mid; - } - } - return end; - }; - - const size_t step = range_search_step; - if (!step) - return find_range_end(range_begin, range_end); - - size_t begin = range_begin; - while (begin + step <= range_end) - { - const size_t pos = find_range_end(begin, begin + step); - if (pos < begin + step) + if (!isCurrentKey(pos)) return pos; - begin += step; } - return find_range_end(begin, range_end); + + size_t low = linear_probe_end; + size_t high = end - 1; + while (low <= high) + { + size_t mid = low + (high - low) / 2; + if (isCurrentKey(mid)) + low = mid + 1; + else + { + high = mid - 1; + end = mid; + } + } + return end; } std::pair DistinctSortedChunkTransform::continueWithPrevRange(const size_t chunk_rows, IColumn::Filter & filter) diff --git a/src/Processors/Transforms/DistinctSortedChunkTransform.h b/src/Processors/Transforms/DistinctSortedChunkTransform.h index 983b976c49f..2e21c36f7dc 100644 --- a/src/Processors/Transforms/DistinctSortedChunkTransform.h +++ b/src/Processors/Transforms/DistinctSortedChunkTransform.h @@ -32,8 +32,7 @@ public: const SizeLimits & output_size_limits_, UInt64 limit_hint_, const SortDescription & sorted_columns_descr_, - const Names & source_columns_, - size_t range_search_step); + const Names & source_columns_); String getName() const override { return "DistinctSortedChunkTransform"; } @@ -68,7 +67,6 @@ private: ColumnRawPtrs other_columns; // used during processing MutableColumns current_key; - const size_t range_search_step = 0; }; } diff --git a/tests/performance/distinct_in_order.xml b/tests/performance/distinct_in_order.xml index ea76fedcc34..c4d09aa825b 100644 --- a/tests/performance/distinct_in_order.xml +++ b/tests/performance/distinct_in_order.xml @@ -1,33 +1,33 @@ - DROP TABLE IF EXISTS distinct_cardinality_high - CREATE TABLE distinct_cardinality_high (high UInt64, medium UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, medium, low) - INSERT INTO distinct_cardinality_high SELECT number % 10000, number % 1000, number % 100 from numbers(1000000) + CREATE TABLE distinct_cardinality_high (high UInt64, medium UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, medium) + INSERT INTO distinct_cardinality_high SELECT number % 10000, number % 1000, number % 100 FROM numbers_mt(1e8) - select distinct high from distinct_cardinality_high - select distinct high, low from distinct_cardinality_high - select distinct high, medium from distinct_cardinality_high - select distinct high, medium, low from distinct_cardinality_high + SELECT DISTINCT high FROM distinct_cardinality_high FORMAT Null + SELECT DISTINCT high, low FROM distinct_cardinality_high FORMAT Null + SELECT DISTINCT high, medium FROM distinct_cardinality_high FORMAT Null + SELECT DISTINCT high, medium, low FROM distinct_cardinality_high FORMAT Null - select distinct high, medium from distinct_cardinality_high order by medium - select distinct high, low from distinct_cardinality_high order by low - select distinct high, medium, low from distinct_cardinality_high order by high + SELECT DISTINCT high, medium FROM distinct_cardinality_high ORDER BY medium FORMAT Null + SELECT DISTINCT high, medium FROM distinct_cardinality_high ORDER BY high FORMAT Null + SELECT DISTINCT high, low FROM distinct_cardinality_high ORDER BY low FORMAT Null + SELECT DISTINCT high, medium, low FROM distinct_cardinality_high ORDER BY low FORMAT Null DROP TABLE IF EXISTS distinct_cardinality_high - DROP TABLE IF EXISTS distinct_cardinality_low - CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium, high) - INSERT INTO distinct_cardinality_low SELECT number % 100, number % 1000, number % 10000 from numbers(1000000) + CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium) + INSERT INTO distinct_cardinality_low SELECT number % 100, number % 1000, number % 10000 FROM numbers_mt(1e8) - select distinct low from distinct_cardinality_low - select distinct low, medium from distinct_cardinality_low - select distinct low, high from distinct_cardinality_low - select distinct low, medium, high from distinct_cardinality_low + SELECT DISTINCT low FROM distinct_cardinality_low FORMAT Null + SELECT DISTINCT low, medium FROM distinct_cardinality_low FORMAT Null + SELECT DISTINCT low, high FROM distinct_cardinality_low FORMAT Null + SELECT DISTINCT low, medium, high FROM distinct_cardinality_low FORMAT Null - select distinct low, medium from distinct_cardinality_low order by medium - select distinct low, high from distinct_cardinality_low order by high - select distinct low, medium, high from distinct_cardinality_low order by low + SELECT DISTINCT low, medium FROM distinct_cardinality_low ORDER BY medium FORMAT Null + SELECT DISTINCT low, medium FROM distinct_cardinality_low ORDER BY low FORMAT Null + SELECT DISTINCT low, high FROM distinct_cardinality_low ORDER BY high FORMAT Null + SELECT DISTINCT low, medium, high FROM distinct_cardinality_low ORDER BY high FORMAT Null DROP TABLE IF EXISTS distinct_cardinality_low diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference index d3951859b1b..2dac69edc41 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference @@ -7,13 +7,13 @@ DistinctSortedChunkTransform -- distinct with primary key prefix -> pre-distinct optimization only DistinctSortedChunkTransform -- distinct with primary key prefix and order by on column in distinct -> pre-distinct and final distinct optimization -DistinctSortedChunkTransform +DistinctSortedTransform DistinctSortedChunkTransform -- distinct with primary key prefix and order by on column _not_ in distinct -> pre-distinct optimization only DistinctSortedChunkTransform -- distinct with non-primary key prefix -> no optimizations No optimizations -- distinct with non-primary key prefix and order by on column in distinct -> final distinct optimization only -DistinctSortedChunkTransform +DistinctSortedTransform -- distinct with non-primary key prefix and order by on column _not_ in distinct -> no optimizations No optimizations diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh index 33fb6f12110..21f50a147ac 100755 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) DISABLE_OPTIMIZATION="set optimize_distinct_in_order=0" ENABLE_OPTIMIZATION="set optimize_distinct_in_order=1" -GREP_OPTIMIZATIONS="grep 'DistinctSortedChunkTransform'" +GREP_OPTIMIZATIONS="grep 'DistinctSortedChunkTransform\|DistinctSortedTransform'" TRIM_LEADING_SPACES="sed -e 's/^[ \t]*//'" FIND_OPTIMIZATIONS="$GREP_OPTIMIZATIONS | $TRIM_LEADING_SPACES" From cbcd740dc19db2ba502af5422bd805188bd7eaaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 30 Jun 2022 15:10:09 +0200 Subject: [PATCH 317/408] Adapt some more nodes to avoid issues with pre-22.4 replicas --- src/Storages/StorageReplicatedMergeTree.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fac11db2ab9..e44013f39ca 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -598,11 +598,19 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() auto zookeeper = getZooKeeper(); std::vector futures; - /// We need to confirm /quorum exists here although it's called under createTableIfNotExists because in older CH releases (pre 22.4) - /// it was created here, so if metadata creation is done by an older replica the node might not exists when reaching this call - futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum", String(), zkutil::CreateMode::Persistent)); - futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); + /// These 4 nodes used to be created in createNewZookeeperNodes() and they were moved to createTable() + /// This means that if the first replica creating the table metadata has an older version of CH (22.3 or previous) + /// there will be a time between its calls to `createTable` and `createNewZookeeperNodes` where the nodes won't exists + /// and that will cause issues in newer replicas + /// See https://github.com/ClickHouse/ClickHouse/issues/38600 for example + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/last_part", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/failed_parts", String(), zkutil::CreateMode::Persistent)); + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/mutations", String(), zkutil::CreateMode::Persistent)); + + + futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); /// Nodes for remote fs zero-copy replication const auto settings = getSettings(); if (settings->allow_remote_fs_zero_copy_replication) From e367d96964e2630057a2752fab0fbb9ffa4d398a Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 30 Jun 2022 10:10:12 +0200 Subject: [PATCH 318/408] Fix style. --- src/Access/IAccessStorage.cpp | 1 - src/Access/MultipleAccessStorage.cpp | 2 +- src/Backups/BackupCoordinationHelpers.cpp | 4 +-- src/Backups/BackupCoordinationLocal.cpp | 2 +- src/Backups/BackupEntriesCollector.cpp | 25 ++++++++++--------- src/Backups/BackupEntriesCollector.h | 2 +- src/Backups/DDLAdjustingForBackupVisitor.cpp | 4 +-- src/Backups/RestorerFromBackup.cpp | 2 +- src/Backups/RestorerFromBackup.h | 6 ++--- src/Databases/DDLRenamingVisitor.cpp | 1 - src/Databases/DatabaseReplicated.cpp | 4 +-- src/Interpreters/DatabaseCatalog.cpp | 20 +++++++-------- .../MergeTree/DataPartStorageOnDisk.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 1 - .../test_backup_restore_new/test.py | 10 +++++--- 15 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 230045c7749..fa9c78816c7 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -550,7 +550,6 @@ void IAccessStorage::restoreFromBackup(RestorerFromBackup &) } - UUID IAccessStorage::generateRandomID() { static Poco::UUIDGenerator generator; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 30c3865c1be..e7151cc7b4b 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -414,7 +414,7 @@ void MultipleAccessStorage::restoreFromBackup(RestorerFromBackup & restorer) } } - throwBackupNotAllowed(); + throwBackupNotAllowed(); } } diff --git a/src/Backups/BackupCoordinationHelpers.cpp b/src/Backups/BackupCoordinationHelpers.cpp index cca66f03aac..7f570ba9c85 100644 --- a/src/Backups/BackupCoordinationHelpers.cpp +++ b/src/Backups/BackupCoordinationHelpers.cpp @@ -284,7 +284,7 @@ Strings BackupCoordinationStatusSync::setImpl(const String & current_host, const if ((all_hosts.size() == 1) && (all_hosts.front() == current_host)) return {message}; - + /// Wait for other hosts. Strings ready_hosts_results; @@ -314,7 +314,7 @@ Strings BackupCoordinationStatusSync::setImpl(const String & current_host, const { host_with_error = host; error_message = zookeeper->get(zookeeper_path + "/" + zk_node); - return; + return; } auto it = unready_hosts.find(host); if ((it != unready_hosts.end()) && (status == new_status)) diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 158988cf8b8..a7d5602ca30 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -74,7 +74,7 @@ Strings BackupCoordinationLocal::getReplicatedAccessPaths(const String & access_ void BackupCoordinationLocal::setReplicatedAccessHost(const String & access_zk_path, const String & host_id) { std::lock_guard lock{mutex}; - replicated_access_hosts[access_zk_path] = host_id; + replicated_access_hosts[access_zk_path] = host_id; } String BackupCoordinationLocal::getReplicatedAccessHost(const String & access_zk_path) const diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index e237140cf2b..d5ed9e0da2b 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes extern const int INCONSISTENT_METADATA_FOR_BACKUP; extern const int CANNOT_BACKUP_TABLE; extern const int TABLE_IS_DROPPED; + extern const int UNKNOWN_TABLE; extern const int LOGICAL_ERROR; } @@ -169,7 +170,7 @@ Strings BackupEntriesCollector::setStatus(const String & new_status, const Strin { auto now = std::chrono::steady_clock::now(); auto end_of_timeout = std::max(now, consistent_metadata_snapshot_start_time + consistent_metadata_snapshot_timeout); - + return backup_coordination->setStatusAndWaitFor( backup_settings.host_id, new_status, @@ -213,7 +214,7 @@ void BackupEntriesCollector::gatherMetadataAndCheckConsistency() { /// Gathered metadata and checked consistency, cool! But we have to check that other hosts cope with that too. auto all_hosts_results = setStatus(new_status, "consistent"); - + std::optional host_with_inconsistency; std::optional inconsistency_error_on_other_host; for (size_t i = 0; i != all_hosts.size(); ++i) @@ -397,7 +398,7 @@ void BackupEntriesCollector::gatherDatabaseMetadata( { throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Couldn't get a create query for database {}", database_name); } - + database_info.create_database_query = create_database_query; const auto & create = create_database_query->as(); @@ -420,7 +421,7 @@ void BackupEntriesCollector::gatherDatabaseMetadata( } database_info.except_table_names.emplace(*table_name); } - + if (all_tables) { database_info.all_tables = all_tables; @@ -437,13 +438,13 @@ void BackupEntriesCollector::gatherTablesMetadata() { const auto & database = database_info.database; bool is_temporary_database = (database_name == DatabaseCatalog::TEMPORARY_DATABASE); - + auto filter_by_table_name = [database_info = &database_info](const String & table_name) { /// We skip inner tables of materialized views. if (table_name.starts_with(".inner_id.")) return false; - + if (database_info->tables.contains(table_name)) return true; @@ -464,7 +465,7 @@ void BackupEntriesCollector::gatherTablesMetadata() if (is_temporary_database && !create.temporary) throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a non-temporary create query for {}", tableNameWithTypeToString(database_name, create.getTable(), false)); - + if (!is_temporary_database && (create.getDatabase() != database_name)) throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Got a create query with unexpected database name {} for {}", backQuoteIfNeed(create.getDatabase()), tableNameWithTypeToString(database_name, create.getTable(), false)); } @@ -579,7 +580,7 @@ bool BackupEntriesCollector::compareWithPrevious(std::optional & inco difference.reserve(databases_metadata.size()); std::set_difference(databases_metadata.begin(), databases_metadata.end(), previous_databases_metadata.begin(), previous_databases_metadata.end(), std::back_inserter(difference)); - + if (!difference.empty()) { inconsistency_error = Exception{ @@ -593,7 +594,7 @@ bool BackupEntriesCollector::compareWithPrevious(std::optional & inco difference.reserve(previous_databases_metadata.size()); std::set_difference(previous_databases_metadata.begin(), previous_databases_metadata.end(), databases_metadata.begin(), databases_metadata.end(), std::back_inserter(difference)); - + if (!difference.empty()) { inconsistency_error = Exception{ @@ -611,7 +612,7 @@ bool BackupEntriesCollector::compareWithPrevious(std::optional & inco difference.reserve(tables_metadata.size()); std::set_difference(tables_metadata.begin(), tables_metadata.end(), previous_tables_metadata.begin(), previous_tables_metadata.end(), std::back_inserter(difference)); - + if (!difference.empty()) { inconsistency_error = Exception{ @@ -625,7 +626,7 @@ bool BackupEntriesCollector::compareWithPrevious(std::optional & inco difference.reserve(previous_tables_metadata.size()); std::set_difference(previous_tables_metadata.begin(), previous_tables_metadata.end(), tables_metadata.begin(), tables_metadata.end(), std::back_inserter(difference)); - + if (!difference.empty()) { inconsistency_error = Exception{ @@ -646,7 +647,7 @@ void BackupEntriesCollector::makeBackupEntriesForDatabasesDefs() { if (!database_info.create_database_query) continue; /// We store CREATE DATABASE queries only if there was BACKUP DATABASE specified. - + LOG_TRACE(log, "Adding definition of database {}", backQuoteIfNeed(database_name)); ASTPtr new_create_query = database_info.create_database_query; diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 0772fe84b26..9a653ee7e4d 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -63,7 +63,7 @@ private: bool tryGatherMetadataAndCompareWithPrevious(std::optional & inconsistency_error); void gatherDatabasesMetadata(); - + void gatherDatabaseMetadata( const String & database_name, bool throw_if_database_not_found, diff --git a/src/Backups/DDLAdjustingForBackupVisitor.cpp b/src/Backups/DDLAdjustingForBackupVisitor.cpp index 2dedc677df8..8223e08f127 100644 --- a/src/Backups/DDLAdjustingForBackupVisitor.cpp +++ b/src/Backups/DDLAdjustingForBackupVisitor.cpp @@ -17,7 +17,7 @@ namespace { /// Precondition: storage.engine && storage.engine->name.starts_with("System")) - /// If this is a definition of a system table we'll remove columns and comment because they're reduntant for backups. + /// If this is a definition of a system table we'll remove columns and comment because they're redundant for backups. auto & create = data.create_query->as(); create.reset(create.columns_list); create.reset(create.comment); @@ -105,7 +105,7 @@ void adjustCreateQueryForBackup(ASTPtr ast, const ContextPtr & global_context, s { if (replicated_table_shared_id) *replicated_table_shared_id = {}; - + DDLAdjustingForBackupVisitor::Data data{ast, global_context, replicated_table_shared_id}; DDLAdjustingForBackupVisitor::Visitor{data}.visit(ast); } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 6013eed7919..5b211bc50a8 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -638,7 +638,7 @@ void RestorerFromBackup::createTables() create_table_query = create_table_query->clone(); create_table_query->as().if_not_exists = true; } - + LOG_TRACE( log, "Creating {}: {}", diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index 3d814f67713..ae2f0c76832 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -78,14 +78,14 @@ private: std::vector root_paths_in_backup; void findRootPathsInBackup(); - + void findDatabasesAndTablesInBackup(); void findTableInBackup(const QualifiedTableName & table_name_in_backup, const std::optional & partitions); void findDatabaseInBackup(const String & database_name_in_backup, const std::set & except_table_names); void findEverythingInBackup(const std::set & except_database_names, const std::set & except_table_names); - + void checkAccessForObjectsFoundInBackup() const; - + void createDatabases(); void createTables(); diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp index 8dbcc2a24bb..7ea5dbeda83 100644 --- a/src/Databases/DDLRenamingVisitor.cpp +++ b/src/Databases/DDLRenamingVisitor.cpp @@ -41,7 +41,6 @@ namespace create.setDatabase(new_table_name.database); } } - } else if (create.table) { diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 5a0eec10abb..a7ad632efff 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -928,7 +928,7 @@ std::vector> DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr &) const { /// Here we read metadata from ZooKeeper. We could do that by simple call of DatabaseAtomic::getTablesForBackup() however - /// reading from ZooKeeper is better because thus we won't be dependant on how fast the replication queue of this database is. + /// reading from ZooKeeper is better because thus we won't be dependent on how fast the replication queue of this database is. std::vector> res; auto zookeeper = getContext()->getZooKeeper(); auto escaped_table_names = zookeeper->getChildren(zookeeper_path + "/metadata"); @@ -940,7 +940,7 @@ DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, cons String zk_metadata; if (!zookeeper->tryGet(zookeeper_path + "/metadata/" + escaped_table_name, zk_metadata)) throw Exception(ErrorCodes::INCONSISTENT_METADATA_FOR_BACKUP, "Metadata for table {} was not found in ZooKeeper", table_name); - + ParserCreateQuery parser; auto create_table_query = parseQuery(parser, zk_metadata, 0, getContext()->getSettingsRef().max_parser_depth); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index bd64b14624c..bae2aed2cd5 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -331,10 +331,10 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const { - static const char * INFORMATION_SCHEMA_VIEWS[] = {"schemata", "tables", "views", "columns"}; - static const char * INFORMATION_SCHEMA_UPPERCASE_VIEWS[] = {"SCHEMATA", "TABLES", "VIEWS", "COLUMNS"}; + static const char * information_schema_views[] = {"schemata", "tables", "views", "columns"}; + static const char * information_schema_views_uppercase[] = {"SCHEMATA", "TABLES", "VIEWS", "COLUMNS"}; - auto checkDatabaseAndTableName = [&](const String & database_name, const String & table_name) + auto check_database_and_table_name = [&](const String & database_name, const String & table_name) { if (database_name == SYSTEM_DATABASE) { @@ -343,13 +343,13 @@ bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const } if (database_name == INFORMATION_SCHEMA) { - return std::find(std::begin(INFORMATION_SCHEMA_VIEWS), std::end(INFORMATION_SCHEMA_VIEWS), table_name) - != std::end(INFORMATION_SCHEMA_VIEWS); + return std::find(std::begin(information_schema_views), std::end(information_schema_views), table_name) + != std::end(information_schema_views); } if (database_name == INFORMATION_SCHEMA_UPPERCASE) { - return std::find(std::begin(INFORMATION_SCHEMA_UPPERCASE_VIEWS), std::end(INFORMATION_SCHEMA_UPPERCASE_VIEWS), table_name) - != std::end(INFORMATION_SCHEMA_UPPERCASE_VIEWS); + return std::find(std::begin(information_schema_views_uppercase), std::end(information_schema_views_uppercase), table_name) + != std::end(information_schema_views_uppercase); } return false; }; @@ -362,13 +362,13 @@ bool DatabaseCatalog::isPredefinedTable(const StorageID & table_id) const return true; auto res_id = storage->getStorageID(); String database_name = res_id.getDatabaseName(); - if (database_name != SYSTEM_DATABASE) - return checkDatabaseAndTableName(database_name, res_id.getTableName()); + if (database_name != SYSTEM_DATABASE) /// If (database_name == SYSTEM_DATABASE) then we have already checked it (see isSystemStorage() above). + return check_database_and_table_name(database_name, res_id.getTableName()); } return false; } - return checkDatabaseAndTableName(table_id.getDatabaseName(), table_id.getTableName()); + return check_database_and_table_name(table_id.getDatabaseName(), table_id.getTableName()); } void DatabaseCatalog::assertDatabaseExists(const String & database_name) const diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 68a2a0cbd15..dc80b0aafb6 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -646,7 +646,7 @@ void DataPartStorageOnDisk::backup( String filepath_on_disk = part_path_on_disk / filepath; String filepath_in_backup = part_path_in_backup / filepath; String hardlink_filepath = temp_part_dir / filepath; - + disk->createHardLink(filepath_on_disk, hardlink_filepath); UInt128 file_hash{checksum.file_hash.first, checksum.file_hash.second}; backup_entries.emplace_back( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c856786ffb3..6a5b9c2d8e6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -159,7 +159,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int CONCURRENT_ACCESS_NOT_SUPPORTED; extern const int CHECKSUM_DOESNT_MATCH; - extern const int INCONSISTENT_METADATA_FOR_BACKUP; } namespace ActionLocks diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 7894daf5bad..a930ddac7df 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -795,7 +795,7 @@ def test_system_functions(): assert instance.query( "SELECT number, linear_equation(number, 2, 1) FROM numbers(3)" ) == TSV([[0, 1], [1, 3], [2, 5]]) - + assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV( [[0, "even"], [1, "odd"], [2, "even"]] ) @@ -811,7 +811,9 @@ def test_backup_partition(): instance.query(f"RESTORE TABLE test.table FROM {backup_name}") - assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV([[1, '1'], [4, '4'], [11, '11'], [14, '14'], [21, '21'], [24, '24']]) + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[1, "1"], [4, "4"], [11, "11"], [14, "14"], [21, "21"], [24, "24"]] + ) def test_restore_partition(): @@ -824,4 +826,6 @@ def test_restore_partition(): instance.query(f"RESTORE TABLE test.table PARTITIONS '2', '3' FROM {backup_name}") - assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV([[2, '2'], [3, '3'], [12, '12'], [13, '13'], [22, '22'], [23, '23']]) + assert instance.query("SELECT * FROM test.table ORDER BY x") == TSV( + [[2, "2"], [3, "3"], [12, "12"], [13, "13"], [22, "22"], [23, "23"]] + ) From 4ba4e9b95177f3217d8a17ac8f8fcc6fd7d1910f Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Thu, 30 Jun 2022 09:53:30 -0400 Subject: [PATCH 319/408] Remove 404ing original article --- docs/en/engines/table-engines/special/null.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/en/engines/table-engines/special/null.md b/docs/en/engines/table-engines/special/null.md index 5e775227dab..ca02d8e300b 100644 --- a/docs/en/engines/table-engines/special/null.md +++ b/docs/en/engines/table-engines/special/null.md @@ -10,6 +10,3 @@ When writing to a `Null` table, data is ignored. When reading from a `Null` tabl :::note If you are wondering why this is useful, note that you can create a materialized view on a `Null` table. So the data written to the table will end up affecting the view, but original raw data will still be discarded. ::: - - -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/null/) From d2cbdc7c5340f1515972875b2f4ad0da53de4938 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jun 2022 17:23:53 +0300 Subject: [PATCH 320/408] Update ReplicatedMergeTreeQueue.cpp --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index add1ba875aa..f6c80baba05 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1103,7 +1103,9 @@ bool ReplicatedMergeTreeQueue::isCoveredByFuturePartsImpl(const LogEntry & entry continue; /// Parts are not disjoint, so new_part_name either contains or covers future_part. - chassert(future_part.contains(result_part) || result_part.contains(future_part)); + if (!(future_part.contains(result_part) || result_part.contains(future_part))) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected non-disjoint parts: {} and {}", future_part_elem.first, new_part_name); + /// We cannot execute `entry` (or upgrade its actual_part_name to `new_part_name`) /// while any covered or covering parts are processed. /// But we also cannot simply return true and postpone entry processing, because it may lead to kind of livelock. From e76ebdfb475401fd057f365f9e34cbb6df03da0a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 29 Jun 2022 12:33:20 +0200 Subject: [PATCH 321/408] Change clickhouse-{client,server} packages architecture --- docs/_includes/install/tgz.sh | 30 +++++-- docs/en/development/continuous-integration.md | 6 +- docs/en/getting-started/install.md | 30 +++++-- docs/ru/getting-started/install.md | 30 +++++-- docs/zh/development/continuous-integration.md | 7 +- docs/zh/getting-started/install.md | 30 +++++-- packages/clickhouse-client.yaml | 2 +- packages/clickhouse-server.yaml | 2 +- tests/ci/download_previous_release.py | 86 ++++++++----------- 9 files changed, 131 insertions(+), 92 deletions(-) diff --git a/docs/_includes/install/tgz.sh b/docs/_includes/install/tgz.sh index 4ba5890b32b..d6d7cd8bc36 100644 --- a/docs/_includes/install/tgz.sh +++ b/docs/_includes/install/tgz.sh @@ -1,20 +1,34 @@ LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index f8fcead3ca2..7b2da4416d6 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t - **Build log**: link to the building and files copying log, useful when build failed. - **Build time**. - **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests. diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 8505b040fa3..a5e6495d8d8 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 09e91530473..9f3eabc73ae 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 5bebb3aec2a..836c4a35433 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -86,13 +86,10 @@ git push - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. - **Build time**. - **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` + - `clickhouse-client_XXX_amd64.deb` -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` + - `clickhouse-server_XXX_amd64.deb` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index a8b803547a8..6a0b47607f5 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) export LATEST_VERSION -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" -curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +case $(uname -m) in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) echo "Unknown architecture $(uname -m)"; exit 1 ;; +esac + +for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client +do + curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \ + || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz" +done + +exit 0 + +tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \ + || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" ``` diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml index 5e53090b581..642d66f5475 100644 --- a/packages/clickhouse-client.yaml +++ b/packages/clickhouse-client.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-client" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index ed56eb27e54..28995689754 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -1,7 +1,7 @@ # package sources should be placed in ${PWD}/root # nfpm should run from the same directory with a config name: "clickhouse-server" -arch: "all" +arch: "${DEB_ARCH}" # amd64, arm64 platform: "linux" version: "${CLICKHOUSE_VERSION_STRING}" vendor: "ClickHouse Inc." diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index fa03d164f23..86beed35b5a 100755 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -4,25 +4,24 @@ import re import os import logging -import requests +import requests # type: ignore -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter # type: ignore +from urllib3.util.retry import Retry # type: ignore CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" -CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" -CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" -CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb" - - +DOWNLOAD_PREFIX = ( + "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/" +) CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = ( "clickhouse-common-static-dbg_{version}_amd64.deb" ) -CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb" +CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb" PACKETS_DIR = "previous_release_package_folder/" VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" @@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3): session.mount("http://", adapter) session.mount("https://", adapter) response = session.get(url) - print(url) - if response.ok: - open(out_path, "wb").write(response.content) + response.raise_for_status() + print(f"Download {url} to {out_path}") + with open(out_path, "wb") as fd: + fd.write(response.content) def download_packets(release, dest_path=PACKETS_DIR): @@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR): logging.info("Will download %s", release) - download_packet( - CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version), - ), - ) + def get_dest_path(pkg_name): + return os.path.join(dest_path, pkg_name) - download_packet( - CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, - CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version), - ), - ) + for pkg in ( + CLICKHOUSE_COMMON_STATIC_PACKET_NAME, + CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME, + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + download_packet(url, pkg_name) - download_packet( - CLICKHOUSE_SERVER_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version) - ), - ) - - download_packet( - CLICKHOUSE_CLIENT_DOWNLOAD_URL.format( - version=release.version, type=release.type - ), - out_path=os.path.join( - dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version) - ), - ) + for pkg, fallback in ( + (CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK), + (CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK), + ): + url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type) + pkg_name = get_dest_path(pkg.format(version=release.version)) + try: + download_packet(url, pkg_name) + except Exception: + url = (DOWNLOAD_PREFIX + fallback).format( + version=release.version, type=release.type + ) + pkg_name = get_dest_path(fallback.format(version=release.version)) + download_packet(url, pkg_name) def download_previous_release(dest_path): From 6925bd6a03c3f8822681c1aa8c4651cd3bca5eff Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 11:58:24 +0200 Subject: [PATCH 322/408] Fix hardcoded retries for get_with_retries --- tests/ci/build_download_helper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 67e1c6ee85d..f5eb72dddee 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -20,15 +20,17 @@ def get_with_retries( sleep: int = 3, **kwargs, ) -> requests.Response: - logging.info("Getting URL with %i and sleep %i in between: %s", retries, sleep, url) + logging.info( + "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url + ) exc = None # type: Optional[Exception] - for i in range(DOWNLOAD_RETRIES_COUNT): + for i in range(retries): try: response = requests.get(url, **kwargs) response.raise_for_status() break except Exception as e: - if i + 1 < DOWNLOAD_RETRIES_COUNT: + if i + 1 < retries: logging.info("Exception '%s' while getting, retry %i", e, i + 1) time.sleep(sleep) From 738769d1f7ef93a905a2b52e4adcc903a3c82178 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 15:41:50 +0200 Subject: [PATCH 323/408] Download arm64 packages and push to artifactory --- tests/ci/push_to_artifactory.py | 139 +++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 48 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index be977bdd907..98de315ddae 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -4,11 +4,12 @@ import argparse import logging import os import re -from typing import List, Tuple +from collections import namedtuple +from typing import Dict, List, Tuple from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import dowload_build_with_progress -from env_helper import RUNNER_TEMP +from env_helper import RUNNER_TEMP, S3_BUILDS_BUCKET from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix @@ -25,88 +26,144 @@ TEMP_PATH = os.path.join(RUNNER_TEMP, "push_to_artifactory") JFROG_API_KEY = getenv("JFROG_API_KEY", "") JFROG_TOKEN = getenv("JFROG_TOKEN", "") +CheckDesc = namedtuple("CheckDesc", ("check_name", "deb_arch", "rpm_arch")) + class Packages: - rpm_arch = dict(all="noarch", amd64="x86_64") + checks = ( + CheckDesc("package_release", "amd64", "x86_64"), + CheckDesc("package_aarch64", "arm64", "aarch64"), + ) packages = ( - ("clickhouse-client", "all"), - ("clickhouse-common-static", "amd64"), - ("clickhouse-common-static-dbg", "amd64"), - ("clickhouse-server", "all"), + "clickhouse-client", + "clickhouse-common-static", + "clickhouse-common-static-dbg", + "clickhouse-server", ) def __init__(self, version: str): - self.deb = tuple( - "_".join((name, version, arch + ".deb")) for name, arch in self.packages - ) + # Dicts of name: s3_path_suffix + self.deb = {} # type: Dict[str, str] + self.rpm = {} # type: Dict[str, str] + self.tgz = {} # type: Dict[str, str] + for check in self.checks: + for name in self.packages: + deb = f"{name}_{version}_{check.deb_arch}.deb" + self.deb[deb] = f"{check.check_name}/{deb}" - self.rpm = tuple( - "-".join((name, version + "." + self.rpm_arch[arch] + ".rpm")) - for name, arch in self.packages - ) + rpm = f"{name}-{version}.{check.rpm_arch}.rpm" + self.rpm[rpm] = f"{check.check_name}/{rpm}" - self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages) + tgz = f"{name}-{version}-{check.deb_arch}.tgz" + self.tgz[tgz] = f"{check.check_name}/{tgz}" def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: raise ValueError(f"{deb_pkg} not in {self.deb}") return removesuffix(deb_pkg, ".deb").split("_")[-1] + def replace_with_fallback(self, name: str): + if name.endswith(".deb"): + suffix = self.deb.pop(name) + self.deb[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + elif name.endswith(".rpm"): + suffix = self.rpm.pop(name) + self.rpm[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + elif name.endswith(".tgz"): + suffix = self.tgz.pop(name) + self.tgz[self.fallback_to_all(name)] = self.fallback_to_all(suffix) + else: + raise KeyError(f"unknown package type for {name}") + @staticmethod def path(package_file: str) -> str: return os.path.join(TEMP_PATH, package_file) + @staticmethod + def fallback_to_all(url_or_name: str): + """Until July 2022 we had clickhouse-server and clickhouse-client with + arch 'all'""" + # deb + if url_or_name.endswith("amd64.deb") or url_or_name.endswith("arm64.deb"): + return f"{url_or_name[:-9]}all.deb" + # rpm + if url_or_name.endswith("x86_64.rpm") or url_or_name.endswith("aarch64.rpm"): + new = removesuffix(removesuffix(url_or_name, "x86_64.rpm"), "aarch64.rpm") + return f"{new}noarch.rpm" + # tgz + if url_or_name.endswith("-amd64.tgz") or url_or_name.endswith("-arm64.tgz"): + return f"{url_or_name[:-10]}.tgz" + return url_or_name + class S3: template = ( "https://s3.amazonaws.com/" # "clickhouse-builds/" - "{bucket_name}/" + f"{S3_BUILDS_BUCKET}/" # "33333/" or "21.11/" from --release, if pull request is omitted "{pr}/" # "2bef313f75e4cacc6ea2ef2133e8849ecf0385ec/" "{commit}/" - # "package_release/" - "{check_name}/" - # "clickhouse-common-static_21.11.5.0_amd64.deb" - "{package}" + # "package_release/clickhouse-common-static_21.11.5.0_amd64.deb" + "{s3_path_suffix}" ) def __init__( self, - bucket_name: str, pr: int, commit: str, - check_name: str, version: str, force_download: bool, ): self._common = dict( - bucket_name=bucket_name, pr=pr, commit=commit, - check_name=check_name, ) self.force_download = force_download self.packages = Packages(version) - def download_package(self, package_file: str): - if not self.force_download and os.path.exists(Packages.path(package_file)): + def download_package(self, package_file: str, s3_path_suffix: str): + path = Packages.path(package_file) + fallback_path = Packages.fallback_to_all(path) + if not self.force_download and ( + os.path.exists(path) or os.path.exists(fallback_path) + ): + if os.path.exists(fallback_path): + self.packages.replace_with_fallback(package_file) + return - url = self.template.format_map({**self._common, "package": package_file}) - dowload_build_with_progress(url, Packages.path(package_file)) + url = self.template.format_map( + {**self._common, "s3_path_suffix": s3_path_suffix} + ) + try: + dowload_build_with_progress(url, path) + except Exception as e: + if "Cannot download dataset from" in e.args[0]: + new_url = Packages.fallback_to_all(url) + logging.warning( + "Fallback downloading %s for old release", fallback_path + ) + dowload_build_with_progress(new_url, fallback_path) + self.packages.replace_with_fallback(package_file) def download_deb(self): - for package_file in self.packages.deb: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.deb.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) def download_rpm(self): - for package_file in self.packages.rpm: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.rpm.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) def download_tgz(self): - for package_file in self.packages.tgz: - self.download_package(package_file) + # Copy to have a way to pop/add fallback packages + packages = self.packages.tgz.copy() + for package_file, s3_path_suffix in packages.items(): + self.download_package(package_file, s3_path_suffix) class Release: @@ -223,17 +280,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--commit", required=True, type=commit, help="commit hash for S3 bucket" ) - parser.add_argument( - "--bucket-name", - default="clickhouse-builds", - help="AWS S3 bucket name", - ) - parser.add_argument( - "--check-name", - default="package_release", - help="check name, a part of bucket path, " - "will be converted to lower case with spaces->underscore", - ) parser.add_argument( "--all", action="store_true", help="implies all deb, rpm and tgz" ) @@ -276,7 +322,6 @@ def parse_args() -> argparse.Namespace: args.deb = args.rpm = args.tgz = True if not (args.deb or args.rpm or args.tgz): parser.error("at least one of --deb, --rpm or --tgz should be specified") - args.check_name = args.check_name.lower().replace(" ", "_") if args.pull_request == 0: args.pull_request = ".".join(args.release.version_parts[:2]) return args @@ -305,10 +350,8 @@ def main(): args = parse_args() os.makedirs(TEMP_PATH, exist_ok=True) s3 = S3( - args.bucket_name, args.pull_request, args.commit, - args.check_name, args.release.version, args.force_download, ) From 95c3eff4c78c467dbb7c0b2eb2203b64b4f2261d Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 30 Jun 2022 11:17:38 -0400 Subject: [PATCH 324/408] remove 404ing original article URL --- docs/en/engines/database-engines/postgresql.md | 1 - docs/en/engines/table-engines/log-family/index.md | 1 - docs/en/engines/table-engines/special/file.md | 1 - docs/en/engines/table-engines/special/join.md | 1 - docs/en/engines/table-engines/special/merge.md | 1 - docs/en/engines/table-engines/special/set.md | 1 - docs/en/engines/table-engines/special/url.md | 1 - docs/en/operations/external-authenticators/index.md | 1 - docs/en/operations/system-tables/distributed_ddl_queue.md | 1 - docs/en/operations/system-tables/distribution_queue.md | 1 - docs/en/operations/system-tables/opentelemetry_span_log.md | 1 - docs/en/operations/system-tables/parts_columns.md | 1 - docs/en/operations/system-tables/replication_queue.md | 1 - .../en/sql-reference/aggregate-functions/reference/meanztest.md | 2 -- .../sql-reference/aggregate-functions/reference/welchttest.md | 1 - docs/en/sql-reference/data-types/domains/index.md | 1 - docs/en/sql-reference/data-types/geo.md | 1 - docs/en/sql-reference/data-types/map.md | 1 - docs/en/sql-reference/data-types/simpleaggregatefunction.md | 1 - docs/en/sql-reference/functions/encryption-functions.md | 1 - docs/en/sql-reference/table-functions/mysql.md | 1 - 21 files changed, 22 deletions(-) diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md index 07181cc23e8..969a326b701 100644 --- a/docs/en/engines/database-engines/postgresql.md +++ b/docs/en/engines/database-engines/postgresql.md @@ -136,4 +136,3 @@ DESCRIBE TABLE test_database.test_table; └────────┴───────────────────┘ ``` -[Original article](https://clickhouse.com/docs/en/database-engines/postgresql/) diff --git a/docs/en/engines/table-engines/log-family/index.md b/docs/en/engines/table-engines/log-family/index.md index 8e772341733..4ea2294554a 100644 --- a/docs/en/engines/table-engines/log-family/index.md +++ b/docs/en/engines/table-engines/log-family/index.md @@ -43,4 +43,3 @@ The `TinyLog` engine is the simplest in the family and provides the poorest func The `Log` and `StripeLog` engines support parallel data reading. When reading data, ClickHouse uses multiple threads. Each thread processes a separate data block. The `Log` engine uses a separate file for each column of the table. `StripeLog` stores all the data in one file. As a result, the `StripeLog` engine uses fewer file descriptors, but the `Log` engine provides higher efficiency when reading data. -[Original article](https://clickhouse.com/docs/en/operations/table_engines/log_family/) diff --git a/docs/en/engines/table-engines/special/file.md b/docs/en/engines/table-engines/special/file.md index 5f27bc73e1d..7a53670bebd 100644 --- a/docs/en/engines/table-engines/special/file.md +++ b/docs/en/engines/table-engines/special/file.md @@ -86,4 +86,3 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 - Indices - Replication -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/file/) diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index c95ebe19c31..4e628b8b9b0 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -151,4 +151,3 @@ ALTER TABLE id_val_join DELETE WHERE id = 3; └────┴─────┘ ``` -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/join/) diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index ab15ad8dc76..d32547a300c 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -86,4 +86,3 @@ SELECT * FROM WatchLog; - [Virtual columns](../../../engines/table-engines/special/index.md#table_engines-virtual_columns) - [merge](../../../sql-reference/table-functions/merge.md) table function -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/merge/) diff --git a/docs/en/engines/table-engines/special/set.md b/docs/en/engines/table-engines/special/set.md index 46e31af7ff1..f7114f04cea 100644 --- a/docs/en/engines/table-engines/special/set.md +++ b/docs/en/engines/table-engines/special/set.md @@ -20,4 +20,3 @@ When creating a table, the following settings are applied: - [persistent](../../../operations/settings/settings.md#persistent) -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/set/) diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md index 19246b82219..82617e9425d 100644 --- a/docs/en/engines/table-engines/special/url.md +++ b/docs/en/engines/table-engines/special/url.md @@ -89,4 +89,3 @@ SELECT * FROM url_engine_table - Indexes. - Replication. -[Original article](https://clickhouse.com/docs/en/operations/table_engines/special/url/) diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index af2ba713ec1..d358267c4f0 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -13,4 +13,3 @@ The following external authenticators and directories are supported: - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) - [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) -[Original article](https://clickhouse.com/docs/en/operations/external-authenticators/index/) diff --git a/docs/en/operations/system-tables/distributed_ddl_queue.md b/docs/en/operations/system-tables/distributed_ddl_queue.md index 5a2478b22d9..a35d4a2a5b7 100644 --- a/docs/en/operations/system-tables/distributed_ddl_queue.md +++ b/docs/en/operations/system-tables/distributed_ddl_queue.md @@ -61,4 +61,3 @@ exception_code: ZOK 2 rows in set. Elapsed: 0.025 sec. ``` -[Original article](https://clickhouse.com/docs/en/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) diff --git a/docs/en/operations/system-tables/distribution_queue.md b/docs/en/operations/system-tables/distribution_queue.md index 88d376c7553..896491a458b 100644 --- a/docs/en/operations/system-tables/distribution_queue.md +++ b/docs/en/operations/system-tables/distribution_queue.md @@ -47,4 +47,3 @@ last_exception: - [Distributed table engine](../../engines/table-engines/special/distributed.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/distribution_queue) diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index 1b3b97af019..a9ca32ae030 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -50,4 +50,3 @@ attribute.values: [] - [OpenTelemetry](../../operations/opentelemetry.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/opentelemetry_span_log) diff --git a/docs/en/operations/system-tables/parts_columns.md b/docs/en/operations/system-tables/parts_columns.md index 0439da79ab3..2f85b912f38 100644 --- a/docs/en/operations/system-tables/parts_columns.md +++ b/docs/en/operations/system-tables/parts_columns.md @@ -145,4 +145,3 @@ column_marks_bytes: 48 - [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/parts_columns) diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index cb22345c3a2..a7ac748ebbd 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -88,4 +88,3 @@ last_postpone_time: 1970-01-01 03:00:00 - [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md#query-language-system-replicated) -[Original article](https://clickhouse.com/docs/en/operations/system_tables/replication_queue) diff --git a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md index d129e5722bc..0752df05818 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/meanztest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/meanztest.md @@ -66,5 +66,3 @@ Result: └──────────────────────────────────────────────────────────────────────────────────┘ ``` - -[Original article](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/meanZTest/) diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 82c09ed606e..0a0278f970e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -69,4 +69,3 @@ Result: - [Welch's t-test](https://en.wikipedia.org/wiki/Welch%27s_t-test) - [studentTTest function](studentttest.md#studentttest) -[Original article](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/welchTTest/) diff --git a/docs/en/sql-reference/data-types/domains/index.md b/docs/en/sql-reference/data-types/domains/index.md index f9bd6eea07e..50599db2f47 100644 --- a/docs/en/sql-reference/data-types/domains/index.md +++ b/docs/en/sql-reference/data-types/domains/index.md @@ -27,4 +27,3 @@ You can use domains anywhere corresponding base type can be used, for example: - Can’t implicitly convert string values into domain values when inserting data from another column or table. - Domain adds no constrains on stored values. -[Original article](https://clickhouse.com/docs/en/data_types/domains/) diff --git a/docs/en/sql-reference/data-types/geo.md b/docs/en/sql-reference/data-types/geo.md index c8edf985582..22fc56dbcf5 100644 --- a/docs/en/sql-reference/data-types/geo.md +++ b/docs/en/sql-reference/data-types/geo.md @@ -104,4 +104,3 @@ Result: └─────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────┘ ``` -[Original article](https://clickhouse.com/docs/en/data-types/geo/) diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index e913a5f34e3..65a0f9cbc52 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -108,4 +108,3 @@ Result: - [map()](../../sql-reference/functions/tuple-map-functions.md#function-map) function - [CAST()](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function -[Original article](https://clickhouse.com/docs/en/data-types/map/) diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md index d0f604e8d8e..069e2e68671 100644 --- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md @@ -39,4 +39,3 @@ Values of the `SimpleAggregateFunction(func, Type)` look and stored the same way CREATE TABLE simple (id UInt64, val SimpleAggregateFunction(sum, Double)) ENGINE=AggregatingMergeTree ORDER BY id; ``` -[Original article](https://clickhouse.com/docs/en/data_types/simpleaggregatefunction/) diff --git a/docs/en/sql-reference/functions/encryption-functions.md b/docs/en/sql-reference/functions/encryption-functions.md index 5b3fe2201e6..75f6cf18766 100644 --- a/docs/en/sql-reference/functions/encryption-functions.md +++ b/docs/en/sql-reference/functions/encryption-functions.md @@ -355,4 +355,3 @@ Result: └───────────┘ ``` -[Original article](https://clickhouse.com/docs/en/sql-reference/functions/encryption_functions/) diff --git a/docs/en/sql-reference/table-functions/mysql.md b/docs/en/sql-reference/table-functions/mysql.md index 33ddaa89435..60d95b17c4c 100644 --- a/docs/en/sql-reference/table-functions/mysql.md +++ b/docs/en/sql-reference/table-functions/mysql.md @@ -111,4 +111,3 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); - [The ‘MySQL’ table engine](../../engines/table-engines/integrations/mysql.md) - [Using MySQL as a source of external dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) -[Original article](https://clickhouse.com/docs/en/sql-reference/table_functions/mysql/) From 7a1346ad22f3e04233098a30ffc7b0c4ac948a74 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jun 2022 18:27:28 +0300 Subject: [PATCH 325/408] Update 02067_lost_part_s3.sql --- tests/queries/0_stateless/02067_lost_part_s3.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02067_lost_part_s3.sql b/tests/queries/0_stateless/02067_lost_part_s3.sql index 87cbdca1d06..463f80348b2 100644 --- a/tests/queries/0_stateless/02067_lost_part_s3.sql +++ b/tests/queries/0_stateless/02067_lost_part_s3.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check:22.5.1 +-- Tags: no-backward-compatibility-check DROP TABLE IF EXISTS partslost_0; DROP TABLE IF EXISTS partslost_1; From 5c5f05dd4b7bf9d82f3dd9d1c3c4c7dcce4e4f9e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 17:38:10 +0200 Subject: [PATCH 326/408] Clean out randomized integration volumes each run --- tests/integration/runner | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/integration/runner b/tests/integration/runner index d82e73068af..cd07875ad1d 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -346,6 +346,17 @@ if __name__ == "__main__": ) except Exception as ex: print("Volume creationg failed, probably it already exists, exception", ex) + # TODO: this part cleans out stale volumes produced by container name + # randomizer, we should remove it after Sep 2022 + try: + subprocess.check_call( + "docker volume rm $(docker volume ls -q | " + f"grep '{VOLUME_NAME}_.*_volume')", + shell=True, + ) + except Exception as ex: + print("Probably, some stale volumes still there, just continue:", ex) + # TODO END dockerd_internal_volume = f"--volume={VOLUME_NAME}_volume:/var/lib/docker" # If enabled we kill and remove containers before pytest session run. @@ -392,7 +403,11 @@ if __name__ == "__main__": command=args.command, ) - containers = subprocess.check_output(f"docker ps -a -q --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", shell=True, universal_newlines=True).splitlines() + containers = subprocess.check_output( + f"docker ps -a -q --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", + shell=True, + universal_newlines=True, + ).splitlines() if containers: print(f"Trying to kill containers name={CONTAINER_NAME} ids={containers}") subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True) From aeaf16ae2ff5c02c8967723fc9bce310e3dc869e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 18:01:17 +0200 Subject: [PATCH 327/408] Use the script push_to_artifactory.py always from master --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83e22c0b7c6..4d57ae450c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ jobs: EOF - name: Check out repository code uses: actions/checkout@v2 + with: + # Always use the most recent script version + ref: master - name: Download packages and push to Artifactory run: | rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH" From 5b0fd31c64885b51870785489bc2375a318b098c Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 30 Jun 2022 16:14:30 +0000 Subject: [PATCH 328/408] Put column names in quotes --- src/Core/Settings.h | 1 + src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 4 +- .../Formats/Impl/SQLInsertRowOutputFormat.cpp | 54 ++++++++++++------- .../Formats/Impl/SQLInsertRowOutputFormat.h | 3 ++ .../02322_sql_insert_format.reference | 26 +++++---- .../0_stateless/02322_sql_insert_format.sql | 5 +- 7 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 50495700236..a7074f13bb0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -767,6 +767,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(String, output_format_sql_insert_table_name, "table", "The name of table in the output INSERT query", 0) \ M(Bool, output_format_sql_insert_include_column_names, true, "Include column names in INSERT query", 0) \ M(Bool, output_format_sql_insert_use_replace, false, "Use REPLACE statement instead of INSERT", 0) \ + M(Bool, output_format_sql_insert_quote_names, true, "Quote column names with '`' characters", 0) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index c1714279a9c..756b33d3eb2 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -162,6 +162,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.sql_insert.include_column_names = settings.output_format_sql_insert_include_column_names; format_settings.sql_insert.table_name = settings.output_format_sql_insert_table_name; format_settings.sql_insert.use_replace = settings.output_format_sql_insert_use_replace; + format_settings.sql_insert.quote_names = settings.output_format_sql_insert_quote_names; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context if (format_settings.schema.is_server) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 44e305e9eb4..70bf8979383 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -277,10 +278,11 @@ struct FormatSettings struct { - UInt64 max_batch_size = 65505; + UInt64 max_batch_size = DEFAULT_BLOCK_SIZE; String table_name = "table"; bool include_column_names = true; bool use_replace = false; + bool quote_names = true; } sql_insert; }; diff --git a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp index aa714af2716..749b4b40984 100644 --- a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp @@ -13,28 +13,44 @@ SQLInsertRowOutputFormat::SQLInsertRowOutputFormat(WriteBuffer & out_, const Blo void SQLInsertRowOutputFormat::writeRowStartDelimiter() { if (rows_in_line == 0) - { - if (format_settings.sql_insert.use_replace) - writeCString("REPLACE INTO ", out); - else - writeCString("INSERT INTO ", out); - writeString(format_settings.sql_insert.table_name, out); - if (format_settings.sql_insert.include_column_names) - { - writeCString(" (", out); - for (size_t i = 0; i != column_names.size(); ++i) - { - writeString(column_names[i], out); - if (i + 1 != column_names.size()) - writeCString(", ", out); - } - writeChar(')', out); - } - writeCString(" VALUES ", out); - } + printLineStart(); writeChar('(', out); } +void SQLInsertRowOutputFormat::printLineStart() +{ + if (format_settings.sql_insert.use_replace) + writeCString("REPLACE INTO ", out); + else + writeCString("INSERT INTO ", out); + + writeString(format_settings.sql_insert.table_name, out); + + if (format_settings.sql_insert.include_column_names) + printColumnNames(); + + writeCString(" VALUES ", out); +} + +void SQLInsertRowOutputFormat::printColumnNames() +{ + writeCString(" (", out); + for (size_t i = 0; i != column_names.size(); ++i) + { + if (format_settings.sql_insert.quote_names) + writeChar('`', out); + + writeString(column_names[i], out); + + if (format_settings.sql_insert.quote_names) + writeChar('`', out); + + if (i + 1 != column_names.size()) + writeCString(", ", out); + } + writeChar(')', out); +} + void SQLInsertRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { serialization.serializeTextQuoted(column, row_num, out, format_settings); diff --git a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h index de39c82abac..aaaf39a9e4d 100644 --- a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h +++ b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.h @@ -32,6 +32,9 @@ protected: virtual void writeRowBetweenDelimiter() override; virtual void writeSuffix() override; + void printLineStart(); + void printColumnNames(); + size_t rows_in_line = 0; Names column_names; const FormatSettings format_settings; diff --git a/tests/queries/0_stateless/02322_sql_insert_format.reference b/tests/queries/0_stateless/02322_sql_insert_format.reference index 220ee09e140..e64ef587fa7 100644 --- a/tests/queries/0_stateless/02322_sql_insert_format.reference +++ b/tests/queries/0_stateless/02322_sql_insert_format.reference @@ -1,17 +1,23 @@ -INSERT INTO table (x, y, z) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); -INSERT INTO table (x, y, z) VALUES (0, 0, 'Hello'); -INSERT INTO table (x, y, z) VALUES (1, 1, 'Hello'); -INSERT INTO table (x, y, z) VALUES (2, 2, 'Hello'); -INSERT INTO table (x, y, z) VALUES (3, 0, 'Hello'); -INSERT INTO table (x, y, z) VALUES (4, 1, 'Hello'); -INSERT INTO table (x, y, z) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'); -INSERT INTO table (x, y, z) VALUES (2, 2, 'Hello'), (3, 0, 'Hello'); -INSERT INTO table (x, y, z) VALUES (4, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (1, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (2, 2, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (3, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (4, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (2, 2, 'Hello'), (3, 0, 'Hello'); +INSERT INTO table (`x`, `y`, `z`) VALUES (4, 1, 'Hello'); INSERT INTO table VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); -REPLACE INTO table (x, y, z) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +REPLACE INTO table (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); +INSERT INTO test (`x`, `y`, `z`) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); INSERT INTO test (x, y, z) VALUES (0, 0, 'Hello'), (1, 1, 'Hello'), (2, 2, 'Hello'), (3, 0, 'Hello'), (4, 1, 'Hello'); 0 0 Hello 1 1 Hello 2 2 Hello 3 0 Hello 4 1 Hello +0 0 Hello +1 1 Hello +2 2 Hello +3 0 Hello +4 1 Hello diff --git a/tests/queries/0_stateless/02322_sql_insert_format.sql b/tests/queries/0_stateless/02322_sql_insert_format.sql index e00f1cd300b..adc28c1d01e 100644 --- a/tests/queries/0_stateless/02322_sql_insert_format.sql +++ b/tests/queries/0_stateless/02322_sql_insert_format.sql @@ -6,5 +6,8 @@ select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInse select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_include_column_names=0; select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_use_replace=1; select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test'; -insert into function file(02322_data.sql, 'SQLInsert') select number as x, number % 3 as y, 'Hello' as z from numbers(5) settings output_format_sql_insert_max_batch_size=2, engine_file_truncate_on_insert=1; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test', output_format_sql_quote_names=0; +insert into function file(02322_data.sql, 'SQLInsert') select number as x, number % 3 as y, 'Hello' as z from numbers(5) settings output_format_sql_insert_max_batch_size=2, output_format_sql_quote_names=0, engine_file_truncate_on_insert=1; +select * from file(02322_data.sql, 'MySQLDump'); +insert into function file(02322_data.sql, 'SQLInsert') select number, number % 3, 'Hello' from numbers(5) settings output_format_sql_insert_max_batch_size=2, engine_file_truncate_on_insert=1; select * from file(02322_data.sql, 'MySQLDump'); From 34a92383f18db527e376b9875d58d84f58d4f54f Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 30 Jun 2022 16:17:14 +0000 Subject: [PATCH 329/408] Update docs --- docs/en/interfaces/formats.md | 1 + docs/en/operations/settings/settings.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 27f42985fb3..5d8ed9cdacd 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -495,6 +495,7 @@ To read data output by this format ypu can use [MySQLDump](#mysqldump) input for - [output_format_sql_insert_table_name](../operations/settings/settings.md#output_format_sql_insert_table_name) - The name of table in the output INSERT query. Default value - `'table'`. - [output_format_sql_insert_include_column_names](../operations/settings/settings.md#output_format_sql_insert_include_column_names) - Include column names in INSERT query. Default value - `true`. - [output_format_sql_insert_use_replace](../operations/settings/settings.md#output_format_sql_insert_use_replace) - Use REPLACE statement instead of INSERT. Default value - `false`. +- [output_format_sql_insert_quote_names](../operations/settings/settings.md#output_format_sql_insert_quote_names) - Quote column names with "\`" characters . Default value - `true`. ## JSON {#json} diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2fc124f9bab..75c2aa57b32 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4663,3 +4663,9 @@ Default value: `true`. Use REPLACE keyword instead of INSERT. Default value: `false`. + +### output_format_sql_insert_quote_names {#output_format_sql_insert_quote_names} + +Quote column names with "`" characters + +Default value: `true`. From 26749c53fe15fcea8501a379ad9b48c822ea7684 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 30 Jun 2022 12:45:10 -0400 Subject: [PATCH 330/408] fix formatting of code clocks and lists --- .../mergetree-family/mergetree.md | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 20d9a14b194..3e5a0635339 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -68,40 +68,42 @@ For a description of parameters, see the [CREATE query description](../../../sql `ORDER BY` — The sorting key. - A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`. +A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`. - ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause. +ClickHouse uses the sorting key as a primary key if the primary key is not defined explicitly by the `PRIMARY KEY` clause. - Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). +Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). #### PARTITION BY `PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). - For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. +For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. #### PRIMARY KEY `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. - By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. +By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. #### SAMPLE BY `SAMPLE BY` — An expression for sampling. Optional. - If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. +If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. #### TTL `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. - Expression must have one `Date` or `DateTime` column as a result. Example: - `TTL date + INTERVAL 1 DAY` +Expression must have one `Date` or `DateTime` column as a result. Example: +``` +TTL date + INTERVAL 1 DAY +``` - Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule. +Type of the rule `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'|GROUP BY` specifies an action to be done with the part if the expression is satisfied (reaches current time): removal of expired rows, moving a part (if expression is satisfied for all rows in a part) to specified disk (`TO DISK 'xxx'`) or to volume (`TO VOLUME 'xxx'`), or aggregating values in expired rows. Default type of the rule is removal (`DELETE`). List of multiple rules can be specified, but there should be no more than one `DELETE` rule. - For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) +For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) ### SETTINGS Additional parameters that control the behavior of the `MergeTree` (optional): @@ -129,7 +131,6 @@ Additional parameters that control the behavior of the `MergeTree` (optional): #### min_merge_bytes_to_use_direct_io `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. - #### merge_with_ttl_timeout @@ -305,15 +306,29 @@ For `SELECT` queries, ClickHouse analyzes whether an index can be used. An index Thus, it is possible to quickly run queries on one or many ranges of the primary key. In this example, queries will be fast when run for a specific tracking tag, for a specific tag and date range, for a specific tag and date, for multiple tags with a date range, and so on. Let’s look at the engine configured as follows: - - ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 +```sql +ENGINE MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity=8192 +``` In this case, in queries: ``` sql -SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 -SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) -SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +SELECT count() FROM table +WHERE EventDate = toDate(now()) +AND CounterID = 34 + +SELECT count() FROM table +WHERE EventDate = toDate(now()) +AND (CounterID = 34 OR CounterID = 42) + +SELECT count() FROM table +WHERE ((EventDate >= toDate('2014-01-01') +AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) +AND CounterID IN (101500, 731962, 160656) +AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) ``` ClickHouse will use the primary key index to trim improper data and the monthly partitioning key to trim partitions that are in improper date ranges. @@ -376,36 +391,36 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 #### `minmax` - Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of data like the primary key. +Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of data like the primary key. #### `set(max_rows)` - Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means “no limits”). Uses the values to check if the `WHERE` expression is not satisfiable on a block of data. +Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means “no limits”). Uses the values to check if the `WHERE` expression is not satisfiable on a block of data. #### `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that contains all ngrams from a block of data. Works only with datatypes: [String](../../../sql-reference/data-types/string.md), [FixedString](../../../sql-reference/data-types/fixedstring.md) and [Map](../../../sql-reference/data-types/map.md). Can be used for optimization of `EQUALS`, `LIKE` and `IN` expressions. +Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that contains all ngrams from a block of data. Works only with datatypes: [String](../../../sql-reference/data-types/string.md), [FixedString](../../../sql-reference/data-types/fixedstring.md) and [Map](../../../sql-reference/data-types/map.md). Can be used for optimization of `EQUALS`, `LIKE` and `IN` expressions. - - `n` — ngram size, - - `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). - - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - - `random_seed` — The seed for Bloom filter hash functions. +- `n` — ngram size, +- `size_of_bloom_filter_in_bytes` — Bloom filter size in bytes (you can use large values here, for example, 256 or 512, because it can be compressed well). +- `number_of_hash_functions` — The number of hash functions used in the Bloom filter. +- `random_seed` — The seed for Bloom filter hash functions. #### `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. +The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. #### `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) for the specified columns. - The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. +The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. - Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`, `UUID`, `Map`. +Supported data types: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`, `UUID`, `Map`. - For `Map` data type client can specify if index should be created for keys or values using [mapKeys](../../../sql-reference/functions/tuple-map-functions.md#mapkeys) or [mapValues](../../../sql-reference/functions/tuple-map-functions.md#mapvalues) function. +For `Map` data type client can specify if index should be created for keys or values using [mapKeys](../../../sql-reference/functions/tuple-map-functions.md#mapkeys) or [mapValues](../../../sql-reference/functions/tuple-map-functions.md#mapvalues) function. - The following functions can use the filter: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall). +The following functions can use the filter: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions), [notIn](../../../sql-reference/functions/in-functions), [has](../../../sql-reference/functions/array-functions#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions#hasany), [hasAll](../../../sql-reference/functions/array-functions#hasall). - Example of index creation for `Map` data type +Example of index creation for `Map` data type ``` INDEX map_key_index mapKeys(map_column) TYPE bloom_filter GRANULARITY 1 From 987f6bc8ff29fbfa0f65109336682452f9c9238d Mon Sep 17 00:00:00 2001 From: Yuko Takagi <70714860+yukotakagi@users.noreply.github.com> Date: Thu, 30 Jun 2022 12:03:09 -0600 Subject: [PATCH 331/408] Update README.md Update upcoming meetup. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 153a0d5ce11..1d0146582a6 100644 --- a/README.md +++ b/README.md @@ -15,5 +15,8 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. ## Upcoming events -* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/286304312/) Please join us for an evening of talks (in English), food and discussion. Featuring talks of ClickHouse in production and at least one on the deep internals of ClickHouse itself. * [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. +* [ClickHouse Meetup at the Cloudflare office in London](https://www.meetup.com/clickhouse-london-user-group/events/286891586/) ClickHouse meetup at the Cloudflare office space in central London +* [ClickHouse Meetup at the Metoda office in Munich](https://www.meetup.com/clickhouse-meetup-munich/events/286891667/) ClickHouse meetup at the Metoda office in Munich + + From eb5046ab268a0d4fb03db2ae88896897948249d1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 22:51:27 +0200 Subject: [PATCH 332/408] Simplify everything --- .../ObjectStorages/DiskObjectStorage.cpp | 35 +++++++++++-------- src/Disks/ObjectStorages/DiskObjectStorage.h | 3 ++ .../MergeTree/DataPartStorageOnDisk.cpp | 5 --- .../MergeTree/DataPartStorageOnDisk.h | 2 -- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- src/Storages/MergeTree/IDataPartStorage.h | 5 --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 +++ src/Storages/MergeTree/IMergeTreeDataPart.h | 4 +++ .../MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 31 +++++++--------- src/Storages/StorageReplicatedMergeTree.h | 4 +-- .../test_s3_zero_copy_replication/test.py | 26 +++++++------- 15 files changed, 64 insertions(+), 66 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 540672d9b0a..a354ad27049 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace DB @@ -78,8 +79,12 @@ private: } - DiskTransactionPtr DiskObjectStorage::createTransaction() +{ + return std::make_shared(*this); +} + +DiskTransactionPtr DiskObjectStorage::createObjectStorageTransaction() { return std::make_shared( *object_storage, @@ -176,7 +181,7 @@ bool DiskObjectStorage::isFile(const String & path) const void DiskObjectStorage::createFile(const String & path) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->createFile(path); transaction->commit(); } @@ -201,7 +206,7 @@ void DiskObjectStorage::moveFile(const String & from_path, const String & to_pat metadata_helper->createFileOperationObject("rename", revision, object_metadata); } - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->moveFile(from_path, to_path); transaction->commit(); } @@ -215,7 +220,7 @@ void DiskObjectStorage::replaceFile(const String & from_path, const String & to_ { if (exists(to_path)) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->replaceFile(from_path, to_path); transaction->commit(); } @@ -225,7 +230,7 @@ void DiskObjectStorage::replaceFile(const String & from_path, const String & to_ void DiskObjectStorage::removeSharedFile(const String & path, bool delete_metadata_only) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->removeSharedFile(path, delete_metadata_only); transaction->commit(); } @@ -276,7 +281,7 @@ void DiskObjectStorage::createHardLink(const String & src_path, const String & d metadata_helper->createFileOperationObject("hardlink", revision, object_metadata); } - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->createHardLink(src_path, dst_path); transaction->commit(); } @@ -291,7 +296,7 @@ void DiskObjectStorage::setReadOnly(const String & path) { /// We should store read only flag inside metadata file (instead of using FS flag), /// because we modify metadata file when create hard-links from it. - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->setReadOnly(path); transaction->commit(); } @@ -305,7 +310,7 @@ bool DiskObjectStorage::isDirectory(const String & path) const void DiskObjectStorage::createDirectory(const String & path) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->createDirectory(path); transaction->commit(); } @@ -313,7 +318,7 @@ void DiskObjectStorage::createDirectory(const String & path) void DiskObjectStorage::createDirectories(const String & path) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->createDirectories(path); transaction->commit(); } @@ -321,7 +326,7 @@ void DiskObjectStorage::createDirectories(const String & path) void DiskObjectStorage::clearDirectory(const String & path) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->clearDirectory(path); transaction->commit(); } @@ -329,7 +334,7 @@ void DiskObjectStorage::clearDirectory(const String & path) void DiskObjectStorage::removeDirectory(const String & path) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->removeDirectory(path); transaction->commit(); } @@ -350,7 +355,7 @@ void DiskObjectStorage::listFiles(const String & path, std::vector & fil void DiskObjectStorage::setLastModified(const String & path, const Poco::Timestamp & timestamp) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->setLastModified(path, timestamp); transaction->commit(); } @@ -394,14 +399,14 @@ ReservationPtr DiskObjectStorage::reserve(UInt64 bytes) void DiskObjectStorage::removeSharedFileIfExists(const String & path, bool delete_metadata_only) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->removeSharedFileIfExists(path, delete_metadata_only); transaction->commit(); } void DiskObjectStorage::removeSharedRecursive(const String & path, bool keep_all_batch_data, const NameSet & file_names_remove_metadata_only) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); transaction->removeSharedRecursive(path, keep_all_batch_data, file_names_remove_metadata_only); transaction->commit(); } @@ -451,7 +456,7 @@ std::unique_ptr DiskObjectStorage::writeFile( WriteMode mode, const WriteSettings & settings) { - auto transaction = createTransaction(); + auto transaction = createObjectStorageTransaction(); auto result = transaction->writeFile(path, buf_size, mode, settings); return result; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index ef29dc8f071..fd9fa6bb110 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -163,6 +163,9 @@ public: UInt64 getRevision() const override; private: + + DiskTransactionPtr createObjectStorageTransaction(); + const String name; const String remote_fs_root_path; Poco::Logger * log; diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 833cc2498e8..4364d74e3b9 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -875,11 +875,6 @@ DataPartStoragePtr DataPartStorageBuilderOnDisk::getStorage() const return std::make_shared(volume, root_path, part_dir); } -String DataPartStorageBuilderOnDisk::getUniqueId() const -{ - return transaction->getUniqueId(fs::path(getRelativePath()) / "checksums.txt"); -} - void DataPartStorageBuilderOnDisk::commit() { transaction->commit(); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDisk.h index 66bd009a55d..d7d34e96919 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.h @@ -166,8 +166,6 @@ public: bool remove_new_dir_if_exists, bool fsync_part_dir) override; - String getUniqueId() const override; - void commit() override; private: diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 148d11d7a29..3eedac2888a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -928,7 +928,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - data.lockSharedData(*new_data_part, nullptr, /* replace_existing_lock = */ true, {}); + data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true, {}); LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", part_name, part_id, disk->getName()); diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index d6087904c8c..2b644abd184 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -250,11 +250,6 @@ public: bool remove_new_dir_if_exists, bool fsync_part_dir) = 0; - /// A leak of abstraction. - /// Return some uniq string for file. - /// Required for distinguish different copies of the same part on remote FS. - virtual String getUniqueId() const = 0; - virtual void commit() = 0; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8ac4fe3a452..60941108f00 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1696,6 +1696,11 @@ bool IMergeTreeDataPart::checkAllTTLCalculated(const StorageMetadataPtr & metada return true; } +String IMergeTreeDataPart::getUniqueId() const +{ + return data_part_storage->getUniqueId(); +} + String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const { if (info.level != 0) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 1f5a95e3cca..7f3c41ce4c2 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -421,6 +421,10 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + /// Return some uniq string for file. + /// Required for distinguish different copies of the same part on remote FS. + String getUniqueId() const; + /// Ensures that creation_tid was correctly set after part creation. void assertHasVersionMetadata(MergeTreeTransaction * txn) const; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index a70b329d57b..0e99d6ce04e 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -269,7 +269,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite try { - storage.checkPartChecksumsAndCommit(*transaction_ptr, part, builder); + storage.checkPartChecksumsAndCommit(*transaction_ptr, part); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 94c8ba8b33d..c8c6a03f4c0 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -961,7 +961,7 @@ public: /// Lock part in zookeeper for shared data in several nodes /// Overridden in StorageReplicatedMergeTree - virtual void lockSharedData(const IMergeTreeDataPart &, DataPartStorageBuilderPtr = nullptr, bool = false, std::optional = {}) const {} /// NOLINT + virtual void lockSharedData(const IMergeTreeDataPart &, bool = false, std::optional = {}) const {} /// NOLINT /// Unlock shared data part in zookeeper /// Overridden in StorageReplicatedMergeTree diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 19a11aa2385..a51eb7854ab 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -179,7 +179,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit try { - storage.checkPartChecksumsAndCommit(*transaction_ptr, new_part, builder, mutate_task->getHardlinkedFiles()); + storage.checkPartChecksumsAndCommit(*transaction_ptr, new_part, mutate_task->getHardlinkedFiles()); } catch (const Exception & e) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 0606f26747f..65daa37bb41 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -502,7 +502,7 @@ void ReplicatedMergeTreeSink::commitPart( part->name); } - storage.lockSharedData(*part, builder, false, {}); + storage.lockSharedData(*part, false, {}); Coordination::Responses responses; Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 09673f3b3e4..a711ca803d4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -598,9 +598,6 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() auto zookeeper = getZooKeeper(); std::vector futures; - /// We need to confirm /quorum exists here although it's called under createTableIfNotExists because in older CH releases (pre 22.4) - /// it was created here, so if metadata creation is done by an older replica the node might not exists when reaching this call - futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum", String(), zkutil::CreateMode::Persistent)); futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); /// Nodes for remote fs zero-copy replication @@ -1504,7 +1501,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(Transaction & transaction, - const DataPartPtr & part, DataPartStorageBuilderPtr builder, std::optional hardlinked_files) + const DataPartPtr & part, std::optional hardlinked_files) { auto zookeeper = getZooKeeper(); @@ -1514,7 +1511,7 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd Coordination::Requests ops; NameSet absent_part_paths_on_replicas; - lockSharedData(*part, builder, false, hardlinked_files); + lockSharedData(*part, false, hardlinked_files); /// Checksums are checked here and `ops` is filled. In fact, the part is added to ZK just below, when executing `multi`. checkPartChecksumsAndAddCommitOps(zookeeper, part, ops, part->name, &absent_part_paths_on_replicas); @@ -1662,7 +1659,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) part->version.setCreationTID(Tx::PrehistoricTID, nullptr); auto builder = part->data_part_storage->getBuilder(); renameTempPartAndReplace(part, transaction, builder); - checkPartChecksumsAndCommit(transaction, part, nullptr); + checkPartChecksumsAndCommit(transaction, part); writePartLog(PartLogElement::Type::NEW_PART, {}, 0 /** log entry is fake so we don't measure the time */, part->name, part, {} /** log entry is fake so there are no initial parts */, nullptr); @@ -2350,7 +2347,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) renameTempPartAndReplace(part_desc->res_part, transaction, builder); getCommitPartOps(ops, part_desc->res_part); - lockSharedData(*part_desc->res_part, nullptr, false, part_desc->hardlinked_files); + lockSharedData(*part_desc->res_part, false, part_desc->hardlinked_files); } @@ -4092,7 +4089,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora Transaction transaction(*this, NO_TRANSACTION_RAW); renameTempPartAndReplace(part, transaction, builder); - replaced_parts = checkPartChecksumsAndCommit(transaction, part, nullptr, hardlinked_files); + replaced_parts = checkPartChecksumsAndCommit(transaction, part, hardlinked_files); /** If a quorum is tracked for this part, you must update it. * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. @@ -6627,7 +6624,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } for (size_t i = 0; i < dst_parts.size(); ++i) - lockSharedData(*dst_parts[i], nullptr, false, hardlinked_files_for_parts[i]); + lockSharedData(*dst_parts[i], false, hardlinked_files_for_parts[i]); Coordination::Error code = zookeeper->tryMulti(ops, op_results); if (code == Coordination::Error::ZOK) @@ -6863,7 +6860,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } for (size_t i = 0; i < dst_parts.size(); ++i) - dest_table_storage->lockSharedData(*dst_parts[i], nullptr, false, hardlinked_files_for_parts[i]); + dest_table_storage->lockSharedData(*dst_parts[i], false, hardlinked_files_for_parts[i]); + Coordination::Error code = zookeeper->tryMulti(ops, op_results); if (code == Coordination::Error::ZBADVERSION) continue; @@ -7548,7 +7546,7 @@ void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_nam } } -void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, DataPartStorageBuilderPtr builder, bool replace_existing_lock, std::optional hardlinked_files) const +void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock, std::optional hardlinked_files) const { auto settings = getSettings(); @@ -7562,12 +7560,7 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, if (!zookeeper) return; - String id; - if (builder) - id = builder->getUniqueId(); - else - id = part.data_part_storage->getUniqueId(); - + String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); Strings zc_zookeeper_paths = getZeroCopyPartPath( @@ -7619,7 +7612,7 @@ std::pair StorageReplicatedMergeTree::unlockSharedData(const IMer return std::make_pair(true, NameSet{}); } - return unlockSharedDataByID(part.data_part_storage->getUniqueId(), getTableSharedID(), part.name, replica_name, part.data_part_storage->getDiskType(), getZooKeeper(), *getSettings(), log, + return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), part.name, replica_name, part.data_part_storage->getDiskType(), getZooKeeper(), *getSettings(), log, zookeeper_path); } @@ -8062,7 +8055,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP throw Exception(ErrorCodes::INCORRECT_DATA, "Tried to create empty part {}, but it replaces existing parts {}.", lost_part_name, fmt::join(part_names, ", ")); } - lockSharedData(*new_data_part, data_part_storage_builder, false, {}); + lockSharedData(*new_data_part, false, {}); while (true) { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 3e7e1e0f4c4..73a08a2b921 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -261,7 +261,7 @@ public: DataPartStoragePtr executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); /// Lock part in zookeeper for use shared data in several nodes - void lockSharedData(const IMergeTreeDataPart & part, DataPartStorageBuilderPtr builder, bool replace_existing_lock, std::optional hardlinked_files) const override; + void lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock, std::optional hardlinked_files) const override; void lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; @@ -506,7 +506,7 @@ private: String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const; /// Accepts a PreActive part, atomically checks its checksums with ones on other replicas and commit the part - DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part, DataPartStorageBuilderPtr builder, std::optional hardlinked_files = {}); + DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part, std::optional hardlinked_files = {}); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index b5354c0e01b..39be0d564df 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -110,7 +110,7 @@ def test_s3_zero_copy_replication(cluster, policy): ) node1.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") - node2.query("SYSTEM SYNC REPLICA s3_test") + node2.query("SYSTEM SYNC REPLICA s3_test", timeout=30) assert ( node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" @@ -124,7 +124,7 @@ def test_s3_zero_copy_replication(cluster, policy): assert get_large_objects_count(cluster) == 1 node2.query("INSERT INTO s3_test VALUES (2,'data'),(3,'data')") - node1.query("SYSTEM SYNC REPLICA s3_test") + node1.query("SYSTEM SYNC REPLICA s3_test", timeout=30) assert ( node2.query("SELECT * FROM s3_test order by id FORMAT Values") @@ -166,7 +166,7 @@ def test_s3_zero_copy_on_hybrid_storage(cluster): ) node1.query("INSERT INTO hybrid_test VALUES (0,'data'),(1,'data')") - node2.query("SYSTEM SYNC REPLICA hybrid_test") + node2.query("SYSTEM SYNC REPLICA hybrid_test", timeout=30) assert ( node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") @@ -292,7 +292,7 @@ def test_s3_zero_copy_with_ttl_move(cluster, storage_policy, large_data, iterati node1.query("INSERT INTO ttl_move_test VALUES (11, now() - INTERVAL 1 DAY)") node1.query("OPTIMIZE TABLE ttl_move_test FINAL") - node2.query("SYSTEM SYNC REPLICA ttl_move_test") + node2.query("SYSTEM SYNC REPLICA ttl_move_test", timeout=30) if large_data: assert ( @@ -362,8 +362,8 @@ def test_s3_zero_copy_with_ttl_delete(cluster, large_data, iterations): node1.query("OPTIMIZE TABLE ttl_delete_test FINAL") - node1.query("SYSTEM SYNC REPLICA ttl_delete_test") - node2.query("SYSTEM SYNC REPLICA ttl_delete_test") + node1.query("SYSTEM SYNC REPLICA ttl_delete_test", timeout=30) + node2.query("SYSTEM SYNC REPLICA ttl_delete_test", timeout=30) if large_data: assert ( @@ -445,7 +445,7 @@ def s3_zero_copy_unfreeze_base(cluster, unfreeze_query_template): check_objects_exisis(cluster, objects01) node1.query("TRUNCATE TABLE unfreeze_test") - node2.query("SYSTEM SYNC REPLICA unfreeze_test") + node2.query("SYSTEM SYNC REPLICA unfreeze_test", timeout=30) objects11 = node1.get_backuped_s3_objects("s31", "freeze_backup1") objects12 = node2.get_backuped_s3_objects("s31", "freeze_backup2") @@ -499,7 +499,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup1'") node1.query("INSERT INTO drop_detached_test VALUES (1)") node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup2'") - node2.query("SYSTEM SYNC REPLICA drop_detached_test") + node2.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) objects1 = node1.get_backuped_s3_objects("s31", "detach_backup1") objects2 = node1.get_backuped_s3_objects("s31", "detach_backup2") @@ -511,7 +511,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '0'") node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '1'") - node2.query("SYSTEM SYNC REPLICA drop_detached_test") + node2.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -523,7 +523,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}, ) - node1.query("SYSTEM SYNC REPLICA drop_detached_test") + node1.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -534,7 +534,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}, ) - node2.query("SYSTEM SYNC REPLICA drop_detached_test") + node2.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -545,7 +545,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}, ) - node2.query("SYSTEM SYNC REPLICA drop_detached_test") + node2.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -555,7 +555,7 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}, ) - node1.query("SYSTEM SYNC REPLICA drop_detached_test") + node1.query("SYSTEM SYNC REPLICA drop_detached_test", timeout=30) wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) From 3afd0c4ab6d5b7ed8e04886d6e0f5be2ffcd52ea Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 22:57:39 +0200 Subject: [PATCH 333/408] Remove redundant method --- src/Disks/FakeDiskTransaction.h | 5 ----- src/Disks/IDiskTransaction.h | 2 -- .../ObjectStorages/DiskObjectStorageTransaction.cpp | 10 ---------- .../ObjectStorages/DiskObjectStorageTransaction.h | 3 --- 4 files changed, 20 deletions(-) diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index 387361a9df8..7f9247a0689 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -125,11 +125,6 @@ public: disk.createHardLink(src_path, dst_path); } - std::string getUniqueId(const String & path) const override - { - return disk.getUniqueId(path); - } - bool isCommitedOrTriedToCommit() const override { return commit_called; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index fcfe4009aa3..e51a02267d1 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -109,8 +109,6 @@ public: /// Create hardlink from `src_path` to `dst_path`. virtual void createHardLink(const std::string & src_path, const std::string & dst_path) = 0; - virtual std::string getUniqueId(const String & path) const = 0; - virtual bool isCommitedOrTriedToCommit() const = 0; }; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 0669e49c9b3..e07b050dd1f 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -487,14 +487,6 @@ String revisionToString(UInt64 revision) } -std::string DiskObjectStorageTransaction::getUniqueId(const std::string & path) const -{ - auto it = unique_ids.find(path); - if (it != unique_ids.end()) - return it->second; - return ""; -} - std::unique_ptr DiskObjectStorageTransaction::writeFile( /// NOLINT const std::string & path, size_t buf_size, @@ -515,8 +507,6 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } - unique_ids[path] = blob_name; - auto blob_path = fs::path(remote_fs_root_path) / blob_name; auto write_operation = std::make_unique(object_storage, metadata_storage, path, blob_path); diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 4312b380a04..6ce0d2bcdd2 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -56,7 +56,6 @@ private: /// TODO we can get rid of this params const std::string & remote_fs_root_path; DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; - std::unordered_map unique_ids; DiskObjectStorageOperations operations_to_execute; bool commit_called{false}; @@ -110,8 +109,6 @@ public: void setReadOnly(const std::string & path) override; void createHardLink(const std::string & src_path, const std::string & dst_path) override; - std::string getUniqueId(const std::string & path) const override; - bool isCommitedOrTriedToCommit() const override { return commit_called; From b7c0d2552858644d694ee4a90e65e67d81bcaeee Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 23:01:55 +0200 Subject: [PATCH 334/408] Less code --- src/Disks/FakeDiskTransaction.h | 7 ------- src/Disks/IDiskTransaction.h | 2 -- src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp | 1 - src/Disks/ObjectStorages/DiskObjectStorageTransaction.h | 6 ------ 4 files changed, 16 deletions(-) diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index 7f9247a0689..fae43ae3d2a 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -17,7 +17,6 @@ public: void commit() override { - commit_called = true; } void createDirectory(const std::string & path) override @@ -125,14 +124,8 @@ public: disk.createHardLink(src_path, dst_path); } - bool isCommitedOrTriedToCommit() const override - { - return commit_called; - } - private: IDisk & disk; - bool commit_called{false}; }; } diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index e51a02267d1..4b00a9bcefc 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -108,8 +108,6 @@ public: /// Create hardlink from `src_path` to `dst_path`. virtual void createHardLink(const std::string & src_path, const std::string & dst_path) = 0; - - virtual bool isCommitedOrTriedToCommit() const = 0; }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index e07b050dd1f..224174ee5a9 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -602,7 +602,6 @@ void DiskObjectStorageTransaction::copyFile(const std::string & from_file_path, void DiskObjectStorageTransaction::commit() { - commit_called = true; for (size_t i = 0; i < operations_to_execute.size(); ++i) { try diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 6ce0d2bcdd2..362b5404707 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -58,7 +58,6 @@ private: DiskObjectStorageRemoteMetadataRestoreHelper * metadata_helper; DiskObjectStorageOperations operations_to_execute; - bool commit_called{false}; public: DiskObjectStorageTransaction( IObjectStorage & object_storage_, @@ -108,11 +107,6 @@ public: void setLastModified(const std::string & path, const Poco::Timestamp & timestamp) override; void setReadOnly(const std::string & path) override; void createHardLink(const std::string & src_path, const std::string & dst_path) override; - - bool isCommitedOrTriedToCommit() const override - { - return commit_called; - } }; using DiskObjectStorageTransactionPtr = std::shared_ptr; From 09faa1641e3237b3bc6897e68f7a61f20dc069e4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 23:09:32 +0200 Subject: [PATCH 335/408] Some comments --- src/Disks/ObjectStorages/DiskObjectStorage.h | 3 +++ src/Storages/MergeTree/IDataPartStorage.h | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index fd9fa6bb110..56a1f7b7a3f 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -37,6 +37,7 @@ public: bool send_metadata_, uint64_t thread_pool_size); + /// Create fake transaction DiskTransactionPtr createTransaction() override; DiskType getType() const override { return disk_type; } @@ -164,6 +165,8 @@ public: UInt64 getRevision() const override; private: + /// Create actual disk object storage transaction for operations + /// execution. DiskTransactionPtr createObjectStorageTransaction(); const String name; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 2b644abd184..49b38a6c703 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -199,6 +199,9 @@ public: /// Right now, this is needed for rename table query. virtual void changeRootPath(const std::string & from_root, const std::string & to_root) = 0; + /// Leak of abstraction as well. We should use builder as one-time object which allow + /// us to build parts, while storage should be read-only method to access part properties + /// related to disk. However our code is really tricky and sometimes we need ad-hoc builders. virtual DataPartStorageBuilderPtr getBuilder() const = 0; }; @@ -243,6 +246,8 @@ public: /// Ideally, new_root_path should be the same as current root (but it is not true). /// Examples are: 'all_1_2_1' -> 'detached/all_1_2_1' /// 'moving/tmp_all_1_2_1' -> 'all_1_2_1' + /// + /// To notify storage also call onRename for it with first two args virtual void rename( const std::string & new_root_path, const std::string & new_part_dir, From dfb0dfc4a758c99e53d366115f6ee785ba20c830 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jun 2022 23:11:20 +0200 Subject: [PATCH 336/408] Less diff --- src/Disks/FakeDiskTransaction.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index fae43ae3d2a..6d61ac752f2 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -15,9 +15,7 @@ public: : disk(disk_) {} - void commit() override - { - } + void commit() override {} void createDirectory(const std::string & path) override { From 12290dd644422becc0581f36a682fcb8e922321a Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Thu, 30 Jun 2022 17:35:42 -0400 Subject: [PATCH 337/408] upload from self-extracting directory --- tests/ci/build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 3976e2ba916..5dcc0e3e49f 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -231,7 +231,7 @@ def upload_master_static_binaries( return s3_path = "/".join((pr_info.base_ref, static_binary_name, "clickhouse")) - binary = os.path.join(build_output_path, "clickhouse") + binary = os.path.join(build_output_path, "self-extracting", "clickhouse") url = s3_helper.upload_build_file_to_s3(binary, s3_path) print(f"::notice ::Binary static URL: {url}") From bb358617e1e0ef6e0cf43f67b1eeea366986c7bd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 30 Jun 2022 23:35:44 +0200 Subject: [PATCH 338/408] Better naming for stuff related to splitted debug symbols The previous name was slightly misleading, e.g. it is not about "intalling stripped binaries" but about splitting debug symbols from the binary. --- CMakeLists.txt | 6 +++--- cmake/{strip_binary.cmake => split_debug_symbols.cmake} | 2 +- docker/packager/packager | 2 +- docs/en/development/cmake-in-clickhouse.md | 2 +- programs/CMakeLists.txt | 8 ++++---- programs/keeper/CMakeLists.txt | 6 +++--- programs/library-bridge/CMakeLists.txt | 8 ++++---- programs/odbc-bridge/CMakeLists.txt | 8 ++++---- 8 files changed, 21 insertions(+), 21 deletions(-) rename cmake/{strip_binary.cmake => split_debug_symbols.cmake} (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8bb1a2d1ca..4c7b732c68c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,10 +252,10 @@ else () endif () # Optionally split binaries and debug symbols. -option(INSTALL_STRIPPED_BINARIES "Split binaries and debug symbols" OFF) -if (INSTALL_STRIPPED_BINARIES) +option(SPLIT_DEBUG_SYMBOLS "Split binaries and debug symbols" OFF) +if (SPLIT_DEBUG_SYMBOLS) message(STATUS "Will split binaries and debug symbols") - set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information") + set(SPLITTED_DEBUG_SYMBOLS_DIR "stripped" CACHE STRING "A separate directory for stripped information") endif() cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd diff --git a/cmake/strip_binary.cmake b/cmake/split_debug_symbols.cmake similarity index 98% rename from cmake/strip_binary.cmake rename to cmake/split_debug_symbols.cmake index 6e38c86fc70..12182ed9c20 100644 --- a/cmake/strip_binary.cmake +++ b/cmake/split_debug_symbols.cmake @@ -1,4 +1,4 @@ -macro(clickhouse_strip_binary) +macro(clickhouse_split_debug_symbols) set(oneValueArgs TARGET DESTINATION_DIR BINARY_PATH) cmake_parse_arguments(STRIP "" "${oneValueArgs}" "" ${ARGN}) diff --git a/docker/packager/packager b/docker/packager/packager index 14147b8e069..7c0f046b76c 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -202,7 +202,7 @@ def parse_env_variables( cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") if is_release_build(build_type, package_type, sanitizer, split_binary): - cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") + cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON") result.append("WITH_PERFORMANCE=1") if is_cross_arm: cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1") diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md index 5625cf3657d..83279f5f69a 100644 --- a/docs/en/development/cmake-in-clickhouse.md +++ b/docs/en/development/cmake-in-clickhouse.md @@ -349,7 +349,7 @@ Note that ClickHouse uses forks of these libraries, see https://github.com/Click Only for Linux, x86_64 or aarch64. -INSTALL_STRIPPED_BINARIES +SPLIT_DEBUG_SYMBOLS OFF Build stripped binaries with debug info in separate directory diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a2c6eb1a27e..1639af163a9 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,7 +2,7 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) # The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. @@ -511,10 +511,10 @@ else () add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) endif() - if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${STRIPPED_BINARIES_OUTPUT} BINARY_PATH clickhouse) + if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR}) install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() endif() diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index c77b335b615..cf6c8a6e975 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -131,10 +131,10 @@ if (BUILD_STANDALONE_KEEPER) add_dependencies(clickhouse-keeper clickhouse_keeper_configs) set_target_properties(clickhouse-keeper PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../) - if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-keeper) + if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-keeper) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-keeper RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() endif() diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 90ce3d8be7f..a80f2568f04 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,4 +1,4 @@ -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES library-bridge.cpp @@ -24,9 +24,9 @@ target_link_libraries(clickhouse-library-bridge PRIVATE set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-library-bridge) +if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-library-bridge) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index b530e08ca26..f64bec9892f 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -1,4 +1,4 @@ -include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) +include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_ODBC_BRIDGE_SOURCES ColumnInfoHandler.cpp @@ -39,10 +39,10 @@ if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse-odbc-bridge POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} ../clickhouse-odbc-bridge COMMENT "Adding .gdb-index to clickhouse-odbc-bridge" VERBATIM) endif() -if (INSTALL_STRIPPED_BINARIES) - clickhouse_strip_binary(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-odbc-bridge) +if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-odbc-bridge) else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT}) + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR}) install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() From 9e544d38bd6385c8455573e9776c4e25bb5851f4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 30 Jun 2022 18:51:33 -0300 Subject: [PATCH 339/408] Doc. Fix a note about ClickHouse Keeper --- docs/en/operations/tips.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index f364bc85088..ea6a29177c0 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -128,7 +128,7 @@ You should never use manually written scripts to transfer data between different If you want to divide an existing ZooKeeper cluster into two, the correct way is to increase the number of its replicas and then reconfigure it as two independent clusters. -You can run ClickHouse Keeper on the same server as ClickHouse, but do not run ZooKeeper on the same servers as ClickHouse. Because ZooKeeper is very sensitive for latency and ClickHouse may utilize all available system resources. +You can run ClickHouse Keeper on the same server as ClickHouse in test environments, or in environments with low ingestion rate. Do not run ZooKeeper and ClickHouse Keeper on the same servers as ClickHouse in production environments using the same one disk. Because ZooKeeper/Keeper are very sensitive for latency and ClickHouse may utilize all available system resources. You can have ZooKeeper observers in an ensemble but ClickHouse servers should not interact with observers. From 66705eacb04e6ef054baff98d632fe9426bcd841 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 00:07:22 +0200 Subject: [PATCH 340/408] Add test for keeper mntr command --- src/Coordination/Keeper4LWInfo.h | 3 +- src/Coordination/KeeperDispatcher.cpp | 5 +- src/Coordination/KeeperDispatcher.h | 5 + src/Coordination/KeeperServer.cpp | 2 - .../test_keeper_mntr_pressure/__init__.py | 1 + .../config/enable_keeper1.xml | 41 +++++++++ .../config/enable_keeper2.xml | 41 +++++++++ .../config/enable_keeper3.xml | 40 ++++++++ .../test_keeper_mntr_pressure/test.py | 92 +++++++++++++++++++ 9 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_keeper_mntr_pressure/__init__.py create mode 100644 tests/integration/test_keeper_mntr_pressure/config/enable_keeper1.xml create mode 100644 tests/integration/test_keeper_mntr_pressure/config/enable_keeper2.xml create mode 100644 tests/integration/test_keeper_mntr_pressure/config/enable_keeper3.xml create mode 100644 tests/integration/test_keeper_mntr_pressure/test.py diff --git a/src/Coordination/Keeper4LWInfo.h b/src/Coordination/Keeper4LWInfo.h index bf7267a68e2..7d90152611e 100644 --- a/src/Coordination/Keeper4LWInfo.h +++ b/src/Coordination/Keeper4LWInfo.h @@ -3,6 +3,7 @@ #include #include +#include namespace DB { @@ -42,7 +43,7 @@ struct Keeper4LWInfo if (is_follower) return "follower"; - throw Exception(ErrorCodes::LOGICAL_ERROR, "RAFT server has undefined state state, it's a bug"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "RAFT server has undefined state, it's a bug"); } }; diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 9ad5fe9e8ed..623bc43f85e 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -652,10 +652,11 @@ uint64_t KeeperDispatcher::getSnapDirSize() const Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const { Keeper4LWInfo result; - result.is_follower = server->isFollower(); + result.is_follower = isFollower(); result.is_standalone = !result.is_follower && server->getFollowerCount() == 0; result.is_leader = isLeader(); - result.is_observer = server->isObserver(); + result.is_observer = isObserver(); + result.has_leader = hasLeader(); { std::lock_guard lock(push_request_mutex); diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index b632327a165..5e2701299f4 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -150,6 +150,11 @@ public: return server->isLeader(); } + bool isFollower() const + { + return server->isFollower(); + } + bool hasLeader() const { return server->isLeaderAlive(); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 8a46d8ee296..eba2e78d46a 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -444,14 +444,12 @@ bool KeeperServer::isLeader() const return raft_instance->is_leader(); } - bool KeeperServer::isObserver() const { auto srv_config = state_manager->get_srv_config(); return srv_config->is_learner(); } - bool KeeperServer::isFollower() const { return !isLeader() && !isObserver(); diff --git a/tests/integration/test_keeper_mntr_pressure/__init__.py b/tests/integration/test_keeper_mntr_pressure/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_keeper_mntr_pressure/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_keeper_mntr_pressure/config/enable_keeper1.xml b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper1.xml new file mode 100644 index 00000000000..17455ed12f5 --- /dev/null +++ b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper1.xml @@ -0,0 +1,41 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_mntr_pressure/config/enable_keeper2.xml b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper2.xml new file mode 100644 index 00000000000..03a23984cc2 --- /dev/null +++ b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper2.xml @@ -0,0 +1,41 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_mntr_pressure/config/enable_keeper3.xml b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper3.xml new file mode 100644 index 00000000000..a69cabf8c54 --- /dev/null +++ b/tests/integration/test_keeper_mntr_pressure/config/enable_keeper3.xml @@ -0,0 +1,40 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_mntr_pressure/test.py b/tests/integration/test_keeper_mntr_pressure/test.py new file mode 100644 index 00000000000..736b1648241 --- /dev/null +++ b/tests/integration/test_keeper_mntr_pressure/test.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +from helpers.cluster import ClickHouseCluster +import pytest +import random +import string +import os +import time +from io import StringIO +import socket +import threading + +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", main_configs=["config/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["config/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["config/enable_keeper3.xml"], stay_alive=True +) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_keeper_socket(node_name): + hosts = cluster.get_instance_ip(node_name) + client = socket.socket() + client.settimeout(10) + client.connect((hosts, 9181)) + return client + + +def close_keeper_socket(cli): + if cli is not None: + cli.close() + +def send_4lw_cmd(node_name, cmd="ruok"): + client = None + try: + client = get_keeper_socket(node_name) + client.send(cmd.encode()) + data = client.recv(100_000) + data = data.decode() + return data + finally: + if client is not None: + client.close() + + +def test_aggressive_mntr(started_cluster): + + def go_mntr(node_name): + for i in range(100000): + print(node_name, send_4lw_cmd(node_name, "mntr")) + + node1_thread = threading.Thread(target=lambda: go_mntr(node1.name)) + node2_thread = threading.Thread(target=lambda: go_mntr(node2.name)) + node3_thread = threading.Thread(target=lambda: go_mntr(node3.name)) + node1_thread.start() + node2_thread.start() + node3_thread.start() + + node2.stop_clickhouse() + node3.stop_clickhouse() + node1.stop_clickhouse() + starters = [] + for node in [node1, node2, node3]: + start_thread = threading.Thread(target=lambda: node.start_clickhouse()) + start_thread.start() + starters.append(start_thread) + + for start_thread in starters: + start_thread.join() + + node1_thread.join() + node2_thread.join() + node3_thread.join() + + for node in [node1, node2, node3]: + assert not node.contains_in_log("LOGICAL_ERROR") From 7378f0e30f4eb10d91c3d5a72182046ce469472a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 30 Jun 2022 22:33:56 +0000 Subject: [PATCH 341/408] Print stacktraces if test queue is full --- tests/clickhouse-test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 8744e8bf95b..f21d1734029 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1594,6 +1594,8 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel): queue.close() except Full: + print("Couldn't put test to the queue within timeout. Server probably hung.") + print_stacktraces() queue.close() pool.join() From 0888d6bdb519b6b99363d1dca60fd62056f8ddde Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 00:43:30 +0200 Subject: [PATCH 342/408] Black --- tests/integration/test_keeper_mntr_pressure/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_keeper_mntr_pressure/test.py b/tests/integration/test_keeper_mntr_pressure/test.py index 736b1648241..59ea64d0561 100644 --- a/tests/integration/test_keeper_mntr_pressure/test.py +++ b/tests/integration/test_keeper_mntr_pressure/test.py @@ -23,6 +23,7 @@ node3 = cluster.add_instance( "node3", main_configs=["config/enable_keeper3.xml"], stay_alive=True ) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -46,6 +47,7 @@ def close_keeper_socket(cli): if cli is not None: cli.close() + def send_4lw_cmd(node_name, cmd="ruok"): client = None try: @@ -60,7 +62,6 @@ def send_4lw_cmd(node_name, cmd="ruok"): def test_aggressive_mntr(started_cluster): - def go_mntr(node_name): for i in range(100000): print(node_name, send_4lw_cmd(node_name, "mntr")) From f62c5e46f7b96ff7dcbfc1ff71405dcebd5d8f2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 1 Jul 2022 02:47:15 +0300 Subject: [PATCH 343/408] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ea0ce2118..dfc51952250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,6 @@ * Add two new settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines` to allow skipping specified number of lines in the beginning of the file in CSV/TSV formats. [#37537](https://github.com/ClickHouse/ClickHouse/pull/37537) ([Kruglov Pavel](https://github.com/Avogar)). * `showCertificate` function shows current server's SSL certificate. [#37540](https://github.com/ClickHouse/ClickHouse/pull/37540) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * HTTP source for Data Dictionaries in Named Collections is supported. [#37581](https://github.com/ClickHouse/ClickHouse/pull/37581) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Added a new window function `nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL x SECOND])`. [#37628](https://github.com/ClickHouse/ClickHouse/pull/37628) ([Andrey Zvonov](https://github.com/zvonand)). * Implemented changing the comment for `ReplicatedMergeTree` tables. [#37416](https://github.com/ClickHouse/ClickHouse/pull/37416) ([Vasily Nemkov](https://github.com/Enmk)). * Added `SYSTEM UNFREEZE` query that deletes the whole backup regardless if the corresponding table is deleted or not. [#36424](https://github.com/ClickHouse/ClickHouse/pull/36424) ([Vadim Volodin](https://github.com/PolyProgrammist)). From 97009ab9a334d4c3ab4df48a7f8273af2cd7c7f4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 30 Jun 2022 21:12:08 -0300 Subject: [PATCH 344/408] Update tips.md --- docs/en/operations/tips.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md index ea6a29177c0..5325311a9e6 100644 --- a/docs/en/operations/tips.md +++ b/docs/en/operations/tips.md @@ -128,7 +128,8 @@ You should never use manually written scripts to transfer data between different If you want to divide an existing ZooKeeper cluster into two, the correct way is to increase the number of its replicas and then reconfigure it as two independent clusters. -You can run ClickHouse Keeper on the same server as ClickHouse in test environments, or in environments with low ingestion rate. Do not run ZooKeeper and ClickHouse Keeper on the same servers as ClickHouse in production environments using the same one disk. Because ZooKeeper/Keeper are very sensitive for latency and ClickHouse may utilize all available system resources. +You can run ClickHouse Keeper on the same server as ClickHouse in test environments, or in environments with low ingestion rate. +For production environments we suggest to use separate servers for ClickHouse and ZooKeeper/Keeper, or place ClickHouse files and Keeper files on to separate disks. Because ZooKeeper/Keeper are very sensitive for disk latency and ClickHouse may utilize all available system resources. You can have ZooKeeper observers in an ensemble but ClickHouse servers should not interact with observers. From bab954c461aa07a75180d6a34fa33a05346f7879 Mon Sep 17 00:00:00 2001 From: lgbo-ustc Date: Fri, 1 Jul 2022 10:15:35 +0800 Subject: [PATCH 345/408] update codes based on review comment --- src/Interpreters/TranslateQualifiedNamesVisitor.cpp | 6 +++--- src/Interpreters/TranslateQualifiedNamesVisitor.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index fd71dc01595..b58b90b6d47 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -31,12 +31,12 @@ namespace ErrorCodes extern const int UNSUPPORTED_JOIN_KEYS; extern const int LOGICAL_ERROR; } -bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const String & name, const String & column_name, DataTypePtr column_type) +bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type) { if (name.size() < column_name.size()) return false; - if (std::strncmp(name.data(), column_name.data(), column_name.size()) != 0) + if (!name.starts_with(column_name)) return false; if (name.size() == column_name.size()) @@ -49,7 +49,7 @@ bool TranslateQualifiedNamesMatcher::Data::matchColumnName(const String & name, { const Strings & names = type_tuple->getElementNames(); const DataTypes & element_types = type_tuple->getElements(); - String sub_name = name.substr(column_name.size() + 1, name.size() - column_name.size()); + std::string_view sub_name = name.substr(column_name.size() + 1); for (size_t i = 0; i < names.size(); ++i) { if (matchColumnName(sub_name, names[i], element_types[i])) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index b1d4d94d01c..e0c2f6b6bc0 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -39,7 +39,7 @@ public: bool hasTable() const { return !tables.empty(); } bool processAsterisks() const { return hasTable() && has_columns; } bool unknownColumn(size_t table_pos, const ASTIdentifier & identifier) const; - static bool matchColumnName(const String & name, const String & column_name, DataTypePtr column_type); + static bool matchColumnName(const std::string_view & name, const String & column_name, DataTypePtr column_type); }; static void visit(ASTPtr & ast, Data & data); From d73f9758492e9ab2d9090f6d392a14f653302d7d Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 12 May 2022 21:57:12 +0800 Subject: [PATCH 346/408] Fix intermediate header in aggregate projection --- src/Storages/MergeTree/MergeTreeData.cpp | 15 ++------------- .../01710_minmax_count_projection.reference | 1 + .../0_stateless/01710_minmax_count_projection.sql | 3 +++ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2c4dcfa05ee..e7f87c59d19 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5438,7 +5438,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (analysis_result.before_where) { candidate.where_column_name = analysis_result.where_column_name; - candidate.remove_where_filter = analysis_result.remove_where_filter; + candidate.remove_where_filter = !required_columns.contains(analysis_result.where_column_name); candidate.before_where = analysis_result.before_where->clone(); auto new_required_columns = candidate.before_where->foldActionsByProjection( @@ -5571,20 +5571,9 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg candidate.before_aggregation->reorderAggregationKeysForProjection(key_name_pos_map); candidate.before_aggregation->addAggregatesViaProjection(aggregates); - // minmax_count_projections only have aggregation actions - if (minmax_count_projection) - candidate.required_columns = {required_columns.begin(), required_columns.end()}; - if (rewrite_before_where(candidate, projection, required_columns, sample_block_for_keys, aggregates)) { - if (minmax_count_projection) - { - candidate.before_where = nullptr; - candidate.prewhere_info = nullptr; - } - else - candidate.required_columns = {required_columns.begin(), required_columns.end()}; - + candidate.required_columns = {required_columns.begin(), required_columns.end()}; for (const auto & aggregate : aggregates) candidate.required_columns.push_back(aggregate.name); candidates.push_back(std::move(candidate)); diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index 259d320a38a..3e24cac4f6d 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -3,6 +3,7 @@ 1 9999 5000 0 9998 5000 1 +0 9998 5000 0 1 0 9999 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index a6c04725583..4aea3cfd29e 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -15,6 +15,9 @@ select min(i), max(i), count() from d where _partition_value.1 = 10 group by _pa -- fuzz crash select min(i) from d where 1 = _partition_value.1; +-- fuzz crash https://github.com/ClickHouse/ClickHouse/issues/37151 +SELECT min(i), max(i), count() FROM d WHERE (_partition_value.1) = 0 GROUP BY ignore(bitTest(ignore(NULL), 65535), NULL, (_partition_value.1) = 7, '10.25', bitTest(NULL, -9223372036854775808), NULL, ignore(ignore(-2147483647, NULL)), 1024), _partition_id ORDER BY _partition_id ASC NULLS FIRST; + drop table d; drop table if exists has_final_mark; From 974e99fcfe9f7509585dc5891d1ca6c638381f9b Mon Sep 17 00:00:00 2001 From: santrancisco Date: Fri, 1 Jul 2022 19:07:42 +1000 Subject: [PATCH 347/408] Remove broken client library link --- docs/en/interfaces/third-party/client-libraries.md | 1 - docs/ru/interfaces/third-party/client-libraries.md | 1 - docs/zh/interfaces/third-party/client-libraries.md | 1 - 3 files changed, 3 deletions(-) diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index 705b9ef42c0..8067b18cc35 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -51,7 +51,6 @@ ClickHouse Inc does **not** maintain the libraries listed below and hasn’t don - [clickhouse-rs](https://github.com/suharev7/clickhouse-rs) - [Klickhouse](https://github.com/Protryon/klickhouse) - R - - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 702b53622da..ab2c9419b7f 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -45,7 +45,6 @@ sidebar_label: "Клиентские библиотеки от сторонни - [clickhouse-rs](https://github.com/suharev7/clickhouse-rs) - [Klickhouse](https://github.com/Protryon/klickhouse) - R - - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index 52f7d70c0bf..8ed482eee73 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -46,7 +46,6 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试 - [clickhouse-rs](https://github.com/suharev7/clickhouse-rs) - [Klickhouse](https://github.com/Protryon/klickhouse) - R - - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickHouse](https://github.com/IMSMWU/RClickHouse) - Java - [clickhouse-client-java](https://github.com/VirtusAI/clickhouse-client-java) From 16ccd3eccdd81371bdf92e6ef3400b68760c1b4e Mon Sep 17 00:00:00 2001 From: Filatenkov Artur <58165623+FArthur-cmd@users.noreply.github.com> Date: Fri, 1 Jul 2022 12:12:31 +0300 Subject: [PATCH 348/408] Update hardware.sh --- benchmark/hardware.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index e8c9c58aca3..0c3a1396440 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -40,10 +40,16 @@ if [[ $(./clickhouse client --query "EXISTS hits") == '1' && $(./clickhouse clie echo "Dataset already downloaded" else echo "Will download the dataset" + if [ "`uname`" = "Darwin" ] + then + ./clickhouse client --receive_timeout 1000 --max_insert_threads $(sysctl -n hw.ncpu) --progress --query " + CREATE OR REPLACE TABLE hits ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) + AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" + else ./clickhouse client --receive_timeout 1000 --max_insert_threads $(nproc || 4) --progress --query " CREATE OR REPLACE TABLE hits ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" - + fi ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM hits" fi @@ -63,8 +69,8 @@ QUERY_NUM=1 cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do sync - if [ "${OS}" = "Darwin" ] - then + if [ "`uname`" = "Darwin" ] + then sudo purge > /dev/null else echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null @@ -90,8 +96,8 @@ echo touch {cpu_model,cpu,df,memory,memory_total,blk,mdstat,instance}.txt -if [ "${OS}" = "Darwin" ] -then +if [ "`uname`" = "Darwin" ] +then echo '----Version, build id-----------' ./clickhouse local --query "SELECT format('Version: {}', version())" ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw From 9531bbf497076049388c2663357907121a00457e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 14 Jun 2022 12:14:45 +0200 Subject: [PATCH 349/408] Move clickhouse-diagnostics to programs --- {tools/clickhouse-diagnostics => programs/diagnostics}/.gitignore | 0 .../diagnostics}/CONTRIBUTION.md | 0 {tools/clickhouse-diagnostics => programs/diagnostics}/Makefile | 0 {tools/clickhouse-diagnostics => programs/diagnostics}/README.md | 0 .../diagnostics}/cmd/collect.go | 0 .../diagnostics}/cmd/convert.go | 0 .../clickhouse-diagnostics => programs/diagnostics}/cmd/help.go | 0 .../diagnostics}/cmd/params/params.go | 0 .../diagnostics}/cmd/params/params_test.go | 0 .../clickhouse-diagnostics => programs/diagnostics}/cmd/root.go | 0 .../diagnostics}/cmd/version.go | 0 {tools/clickhouse-diagnostics => programs/diagnostics}/go.mod | 0 {tools/clickhouse-diagnostics => programs/diagnostics}/go.sum | 0 .../diagnostics}/internal/collectors/clickhouse/config.go | 0 .../diagnostics}/internal/collectors/clickhouse/config_test.go | 0 .../diagnostics}/internal/collectors/clickhouse/db_logs.go | 0 .../diagnostics}/internal/collectors/clickhouse/db_logs_test.go | 0 .../diagnostics}/internal/collectors/clickhouse/logs.go | 0 .../diagnostics}/internal/collectors/clickhouse/logs_test.go | 0 .../diagnostics}/internal/collectors/clickhouse/queries.json | 0 .../diagnostics}/internal/collectors/clickhouse/summary.go | 0 .../diagnostics}/internal/collectors/clickhouse/summary_test.go | 0 .../diagnostics}/internal/collectors/clickhouse/system.go | 0 .../diagnostics}/internal/collectors/clickhouse/system_test.go | 0 .../diagnostics}/internal/collectors/clickhouse/zookeeper.go | 0 .../diagnostics}/internal/collectors/clickhouse/zookeeper_test.go | 0 .../diagnostics}/internal/collectors/registry.go | 0 .../diagnostics}/internal/collectors/registry_test.go | 0 .../diagnostics}/internal/collectors/system/command.go | 0 .../diagnostics}/internal/collectors/system/command_test.go | 0 .../diagnostics}/internal/collectors/system/file.go | 0 .../diagnostics}/internal/collectors/system/file_test.go | 0 .../diagnostics}/internal/collectors/system/system.go | 0 .../diagnostics}/internal/collectors/system/system_test.go | 0 .../diagnostics}/internal/outputs/file/simple.go | 0 .../diagnostics}/internal/outputs/file/simple_test.go | 0 .../diagnostics}/internal/outputs/registry.go | 0 .../diagnostics}/internal/outputs/registry_test.go | 0 .../diagnostics}/internal/outputs/terminal/report.go | 0 .../diagnostics}/internal/platform/config/models.go | 0 .../diagnostics}/internal/platform/config/models_test.go | 0 .../diagnostics}/internal/platform/config/utils.go | 0 .../diagnostics}/internal/platform/config/utils_test.go | 0 .../diagnostics}/internal/platform/data/bundle.go | 0 .../diagnostics}/internal/platform/data/bundle_test.go | 0 .../diagnostics}/internal/platform/data/database.go | 0 .../diagnostics}/internal/platform/data/database_test.go | 0 .../diagnostics}/internal/platform/data/field.go | 0 .../diagnostics}/internal/platform/data/file.go | 0 .../diagnostics}/internal/platform/data/file_test.go | 0 .../diagnostics}/internal/platform/data/frame.go | 0 .../diagnostics}/internal/platform/data/memory.go | 0 .../diagnostics}/internal/platform/data/memory_test.go | 0 .../diagnostics}/internal/platform/data/misc.go | 0 .../diagnostics}/internal/platform/database/native.go | 0 .../diagnostics}/internal/platform/database/native_test.go | 0 .../diagnostics}/internal/platform/manager.go | 0 .../diagnostics}/internal/platform/manager_test.go | 0 .../diagnostics}/internal/platform/test/data.go | 0 .../diagnostics}/internal/platform/test/env.go | 0 .../diagnostics}/internal/platform/utils/file.go | 0 .../diagnostics}/internal/platform/utils/file_test.go | 0 .../diagnostics}/internal/platform/utils/process.go | 0 .../diagnostics}/internal/platform/utils/process_test.go | 0 .../diagnostics}/internal/platform/utils/slices.go | 0 .../diagnostics}/internal/platform/utils/slices_test.go | 0 .../diagnostics}/internal/platform/utils/time.go | 0 .../diagnostics}/internal/runner.go | 0 .../diagnostics}/internal/runner_test.go | 0 {tools/clickhouse-diagnostics => programs/diagnostics}/main.go | 0 .../diagnostics}/testdata/configs/include/xml/server-include.xml | 0 .../diagnostics}/testdata/configs/include/xml/user-include.xml | 0 .../testdata/configs/include/yaml/server-include.yaml | 0 .../diagnostics}/testdata/configs/include/yaml/user-include.yaml | 0 .../diagnostics}/testdata/configs/xml/config.xml | 0 .../testdata/configs/xml/users.d/default-password.xml | 0 .../diagnostics}/testdata/configs/xml/users.xml | 0 .../diagnostics}/testdata/configs/yaml/config.yaml | 0 .../testdata/configs/yaml/users.d/default-password.yaml | 0 .../diagnostics}/testdata/configs/yaml/users.yaml | 0 .../diagnostics}/testdata/configs/yandex_xml/config.xml | 0 .../diagnostics}/testdata/docker/admin.xml | 0 .../diagnostics}/testdata/docker/custom.xml | 0 .../diagnostics}/testdata/logs/var/logs/clickhouse-server.err.log | 0 .../diagnostics}/testdata/logs/var/logs/clickhouse-server.log | 0 .../diagnostics}/testdata/logs/var/logs/clickhouse-server.log.gz | 0 86 files changed, 0 insertions(+), 0 deletions(-) rename {tools/clickhouse-diagnostics => programs/diagnostics}/.gitignore (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/CONTRIBUTION.md (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/Makefile (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/README.md (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/collect.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/convert.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/help.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/params/params.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/params/params_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/root.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/cmd/version.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/go.mod (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/go.sum (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/config.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/config_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/db_logs.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/db_logs_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/logs.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/logs_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/queries.json (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/summary.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/summary_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/system.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/system_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/zookeeper.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/clickhouse/zookeeper_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/registry.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/registry_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/command.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/command_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/file.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/file_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/system.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/collectors/system/system_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/outputs/file/simple.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/outputs/file/simple_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/outputs/registry.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/outputs/registry_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/outputs/terminal/report.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/config/models.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/config/models_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/config/utils.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/config/utils_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/bundle.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/bundle_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/database.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/database_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/field.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/file.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/file_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/frame.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/memory.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/memory_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/data/misc.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/database/native.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/database/native_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/manager.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/manager_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/test/data.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/test/env.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/file.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/file_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/process.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/process_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/slices.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/slices_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/platform/utils/time.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/runner.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/internal/runner_test.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/main.go (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/include/xml/server-include.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/include/xml/user-include.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/include/yaml/server-include.yaml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/include/yaml/user-include.yaml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/xml/config.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/xml/users.d/default-password.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/xml/users.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/yaml/config.yaml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/yaml/users.d/default-password.yaml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/yaml/users.yaml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/configs/yandex_xml/config.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/docker/admin.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/docker/custom.xml (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/logs/var/logs/clickhouse-server.err.log (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/logs/var/logs/clickhouse-server.log (100%) rename {tools/clickhouse-diagnostics => programs/diagnostics}/testdata/logs/var/logs/clickhouse-server.log.gz (100%) diff --git a/tools/clickhouse-diagnostics/.gitignore b/programs/diagnostics/.gitignore similarity index 100% rename from tools/clickhouse-diagnostics/.gitignore rename to programs/diagnostics/.gitignore diff --git a/tools/clickhouse-diagnostics/CONTRIBUTION.md b/programs/diagnostics/CONTRIBUTION.md similarity index 100% rename from tools/clickhouse-diagnostics/CONTRIBUTION.md rename to programs/diagnostics/CONTRIBUTION.md diff --git a/tools/clickhouse-diagnostics/Makefile b/programs/diagnostics/Makefile similarity index 100% rename from tools/clickhouse-diagnostics/Makefile rename to programs/diagnostics/Makefile diff --git a/tools/clickhouse-diagnostics/README.md b/programs/diagnostics/README.md similarity index 100% rename from tools/clickhouse-diagnostics/README.md rename to programs/diagnostics/README.md diff --git a/tools/clickhouse-diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/collect.go rename to programs/diagnostics/cmd/collect.go diff --git a/tools/clickhouse-diagnostics/cmd/convert.go b/programs/diagnostics/cmd/convert.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/convert.go rename to programs/diagnostics/cmd/convert.go diff --git a/tools/clickhouse-diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/help.go rename to programs/diagnostics/cmd/help.go diff --git a/tools/clickhouse-diagnostics/cmd/params/params.go b/programs/diagnostics/cmd/params/params.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/params/params.go rename to programs/diagnostics/cmd/params/params.go diff --git a/tools/clickhouse-diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/params/params_test.go rename to programs/diagnostics/cmd/params/params_test.go diff --git a/tools/clickhouse-diagnostics/cmd/root.go b/programs/diagnostics/cmd/root.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/root.go rename to programs/diagnostics/cmd/root.go diff --git a/tools/clickhouse-diagnostics/cmd/version.go b/programs/diagnostics/cmd/version.go similarity index 100% rename from tools/clickhouse-diagnostics/cmd/version.go rename to programs/diagnostics/cmd/version.go diff --git a/tools/clickhouse-diagnostics/go.mod b/programs/diagnostics/go.mod similarity index 100% rename from tools/clickhouse-diagnostics/go.mod rename to programs/diagnostics/go.mod diff --git a/tools/clickhouse-diagnostics/go.sum b/programs/diagnostics/go.sum similarity index 100% rename from tools/clickhouse-diagnostics/go.sum rename to programs/diagnostics/go.sum diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/config.go rename to programs/diagnostics/internal/collectors/clickhouse/config.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/config_test.go b/programs/diagnostics/internal/collectors/clickhouse/config_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/config_test.go rename to programs/diagnostics/internal/collectors/clickhouse/config_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/db_logs.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/db_logs.go rename to programs/diagnostics/internal/collectors/clickhouse/db_logs.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/db_logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/db_logs_test.go rename to programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/logs.go b/programs/diagnostics/internal/collectors/clickhouse/logs.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/logs.go rename to programs/diagnostics/internal/collectors/clickhouse/logs.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/logs_test.go rename to programs/diagnostics/internal/collectors/clickhouse/logs_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/queries.json b/programs/diagnostics/internal/collectors/clickhouse/queries.json similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/queries.json rename to programs/diagnostics/internal/collectors/clickhouse/queries.json diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/summary.go b/programs/diagnostics/internal/collectors/clickhouse/summary.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/summary.go rename to programs/diagnostics/internal/collectors/clickhouse/summary.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/summary_test.go b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/summary_test.go rename to programs/diagnostics/internal/collectors/clickhouse/summary_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/system.go b/programs/diagnostics/internal/collectors/clickhouse/system.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/system.go rename to programs/diagnostics/internal/collectors/clickhouse/system.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/system_test.go b/programs/diagnostics/internal/collectors/clickhouse/system_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/system_test.go rename to programs/diagnostics/internal/collectors/clickhouse/system_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/zookeeper.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/zookeeper.go rename to programs/diagnostics/internal/collectors/clickhouse/zookeeper.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/clickhouse/zookeeper_test.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/clickhouse/zookeeper_test.go rename to programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/registry.go b/programs/diagnostics/internal/collectors/registry.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/registry.go rename to programs/diagnostics/internal/collectors/registry.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/registry_test.go b/programs/diagnostics/internal/collectors/registry_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/registry_test.go rename to programs/diagnostics/internal/collectors/registry_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/command.go b/programs/diagnostics/internal/collectors/system/command.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/command.go rename to programs/diagnostics/internal/collectors/system/command.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/command_test.go b/programs/diagnostics/internal/collectors/system/command_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/command_test.go rename to programs/diagnostics/internal/collectors/system/command_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/file.go b/programs/diagnostics/internal/collectors/system/file.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/file.go rename to programs/diagnostics/internal/collectors/system/file.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/file_test.go b/programs/diagnostics/internal/collectors/system/file_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/file_test.go rename to programs/diagnostics/internal/collectors/system/file_test.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/system.go rename to programs/diagnostics/internal/collectors/system/system.go diff --git a/tools/clickhouse-diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/collectors/system/system_test.go rename to programs/diagnostics/internal/collectors/system/system_test.go diff --git a/tools/clickhouse-diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/outputs/file/simple.go rename to programs/diagnostics/internal/outputs/file/simple.go diff --git a/tools/clickhouse-diagnostics/internal/outputs/file/simple_test.go b/programs/diagnostics/internal/outputs/file/simple_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/outputs/file/simple_test.go rename to programs/diagnostics/internal/outputs/file/simple_test.go diff --git a/tools/clickhouse-diagnostics/internal/outputs/registry.go b/programs/diagnostics/internal/outputs/registry.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/outputs/registry.go rename to programs/diagnostics/internal/outputs/registry.go diff --git a/tools/clickhouse-diagnostics/internal/outputs/registry_test.go b/programs/diagnostics/internal/outputs/registry_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/outputs/registry_test.go rename to programs/diagnostics/internal/outputs/registry_test.go diff --git a/tools/clickhouse-diagnostics/internal/outputs/terminal/report.go b/programs/diagnostics/internal/outputs/terminal/report.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/outputs/terminal/report.go rename to programs/diagnostics/internal/outputs/terminal/report.go diff --git a/tools/clickhouse-diagnostics/internal/platform/config/models.go b/programs/diagnostics/internal/platform/config/models.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/config/models.go rename to programs/diagnostics/internal/platform/config/models.go diff --git a/tools/clickhouse-diagnostics/internal/platform/config/models_test.go b/programs/diagnostics/internal/platform/config/models_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/config/models_test.go rename to programs/diagnostics/internal/platform/config/models_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/config/utils.go b/programs/diagnostics/internal/platform/config/utils.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/config/utils.go rename to programs/diagnostics/internal/platform/config/utils.go diff --git a/tools/clickhouse-diagnostics/internal/platform/config/utils_test.go b/programs/diagnostics/internal/platform/config/utils_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/config/utils_test.go rename to programs/diagnostics/internal/platform/config/utils_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/bundle.go b/programs/diagnostics/internal/platform/data/bundle.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/bundle.go rename to programs/diagnostics/internal/platform/data/bundle.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/bundle_test.go b/programs/diagnostics/internal/platform/data/bundle_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/bundle_test.go rename to programs/diagnostics/internal/platform/data/bundle_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/database.go b/programs/diagnostics/internal/platform/data/database.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/database.go rename to programs/diagnostics/internal/platform/data/database.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/database_test.go b/programs/diagnostics/internal/platform/data/database_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/database_test.go rename to programs/diagnostics/internal/platform/data/database_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/field.go b/programs/diagnostics/internal/platform/data/field.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/field.go rename to programs/diagnostics/internal/platform/data/field.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/file.go rename to programs/diagnostics/internal/platform/data/file.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/file_test.go rename to programs/diagnostics/internal/platform/data/file_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/frame.go b/programs/diagnostics/internal/platform/data/frame.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/frame.go rename to programs/diagnostics/internal/platform/data/frame.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/memory.go b/programs/diagnostics/internal/platform/data/memory.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/memory.go rename to programs/diagnostics/internal/platform/data/memory.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/memory_test.go b/programs/diagnostics/internal/platform/data/memory_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/memory_test.go rename to programs/diagnostics/internal/platform/data/memory_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/data/misc.go b/programs/diagnostics/internal/platform/data/misc.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/data/misc.go rename to programs/diagnostics/internal/platform/data/misc.go diff --git a/tools/clickhouse-diagnostics/internal/platform/database/native.go b/programs/diagnostics/internal/platform/database/native.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/database/native.go rename to programs/diagnostics/internal/platform/database/native.go diff --git a/tools/clickhouse-diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/database/native_test.go rename to programs/diagnostics/internal/platform/database/native_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/manager.go rename to programs/diagnostics/internal/platform/manager.go diff --git a/tools/clickhouse-diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/manager_test.go rename to programs/diagnostics/internal/platform/manager_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/test/data.go b/programs/diagnostics/internal/platform/test/data.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/test/data.go rename to programs/diagnostics/internal/platform/test/data.go diff --git a/tools/clickhouse-diagnostics/internal/platform/test/env.go b/programs/diagnostics/internal/platform/test/env.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/test/env.go rename to programs/diagnostics/internal/platform/test/env.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/file.go b/programs/diagnostics/internal/platform/utils/file.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/file.go rename to programs/diagnostics/internal/platform/utils/file.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/file_test.go b/programs/diagnostics/internal/platform/utils/file_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/file_test.go rename to programs/diagnostics/internal/platform/utils/file_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/process.go b/programs/diagnostics/internal/platform/utils/process.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/process.go rename to programs/diagnostics/internal/platform/utils/process.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/process_test.go rename to programs/diagnostics/internal/platform/utils/process_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/slices.go b/programs/diagnostics/internal/platform/utils/slices.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/slices.go rename to programs/diagnostics/internal/platform/utils/slices.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/slices_test.go b/programs/diagnostics/internal/platform/utils/slices_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/slices_test.go rename to programs/diagnostics/internal/platform/utils/slices_test.go diff --git a/tools/clickhouse-diagnostics/internal/platform/utils/time.go b/programs/diagnostics/internal/platform/utils/time.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/platform/utils/time.go rename to programs/diagnostics/internal/platform/utils/time.go diff --git a/tools/clickhouse-diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/runner.go rename to programs/diagnostics/internal/runner.go diff --git a/tools/clickhouse-diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go similarity index 100% rename from tools/clickhouse-diagnostics/internal/runner_test.go rename to programs/diagnostics/internal/runner_test.go diff --git a/tools/clickhouse-diagnostics/main.go b/programs/diagnostics/main.go similarity index 100% rename from tools/clickhouse-diagnostics/main.go rename to programs/diagnostics/main.go diff --git a/tools/clickhouse-diagnostics/testdata/configs/include/xml/server-include.xml b/programs/diagnostics/testdata/configs/include/xml/server-include.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/include/xml/server-include.xml rename to programs/diagnostics/testdata/configs/include/xml/server-include.xml diff --git a/tools/clickhouse-diagnostics/testdata/configs/include/xml/user-include.xml b/programs/diagnostics/testdata/configs/include/xml/user-include.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/include/xml/user-include.xml rename to programs/diagnostics/testdata/configs/include/xml/user-include.xml diff --git a/tools/clickhouse-diagnostics/testdata/configs/include/yaml/server-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/server-include.yaml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/include/yaml/server-include.yaml rename to programs/diagnostics/testdata/configs/include/yaml/server-include.yaml diff --git a/tools/clickhouse-diagnostics/testdata/configs/include/yaml/user-include.yaml b/programs/diagnostics/testdata/configs/include/yaml/user-include.yaml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/include/yaml/user-include.yaml rename to programs/diagnostics/testdata/configs/include/yaml/user-include.yaml diff --git a/tools/clickhouse-diagnostics/testdata/configs/xml/config.xml b/programs/diagnostics/testdata/configs/xml/config.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/xml/config.xml rename to programs/diagnostics/testdata/configs/xml/config.xml diff --git a/tools/clickhouse-diagnostics/testdata/configs/xml/users.d/default-password.xml b/programs/diagnostics/testdata/configs/xml/users.d/default-password.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/xml/users.d/default-password.xml rename to programs/diagnostics/testdata/configs/xml/users.d/default-password.xml diff --git a/tools/clickhouse-diagnostics/testdata/configs/xml/users.xml b/programs/diagnostics/testdata/configs/xml/users.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/xml/users.xml rename to programs/diagnostics/testdata/configs/xml/users.xml diff --git a/tools/clickhouse-diagnostics/testdata/configs/yaml/config.yaml b/programs/diagnostics/testdata/configs/yaml/config.yaml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/yaml/config.yaml rename to programs/diagnostics/testdata/configs/yaml/config.yaml diff --git a/tools/clickhouse-diagnostics/testdata/configs/yaml/users.d/default-password.yaml b/programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/yaml/users.d/default-password.yaml rename to programs/diagnostics/testdata/configs/yaml/users.d/default-password.yaml diff --git a/tools/clickhouse-diagnostics/testdata/configs/yaml/users.yaml b/programs/diagnostics/testdata/configs/yaml/users.yaml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/yaml/users.yaml rename to programs/diagnostics/testdata/configs/yaml/users.yaml diff --git a/tools/clickhouse-diagnostics/testdata/configs/yandex_xml/config.xml b/programs/diagnostics/testdata/configs/yandex_xml/config.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/configs/yandex_xml/config.xml rename to programs/diagnostics/testdata/configs/yandex_xml/config.xml diff --git a/tools/clickhouse-diagnostics/testdata/docker/admin.xml b/programs/diagnostics/testdata/docker/admin.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/docker/admin.xml rename to programs/diagnostics/testdata/docker/admin.xml diff --git a/tools/clickhouse-diagnostics/testdata/docker/custom.xml b/programs/diagnostics/testdata/docker/custom.xml similarity index 100% rename from tools/clickhouse-diagnostics/testdata/docker/custom.xml rename to programs/diagnostics/testdata/docker/custom.xml diff --git a/tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.err.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log similarity index 100% rename from tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.err.log rename to programs/diagnostics/testdata/logs/var/logs/clickhouse-server.err.log diff --git a/tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.log b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log similarity index 100% rename from tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.log rename to programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log diff --git a/tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz b/programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz similarity index 100% rename from tools/clickhouse-diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz rename to programs/diagnostics/testdata/logs/var/logs/clickhouse-server.log.gz From 51556704e053c981f5a7b7c310adc8a74befc4cc Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 14 Jun 2022 12:57:04 +0200 Subject: [PATCH 350/408] Rename module to reflect the current state --- programs/diagnostics/Makefile | 3 ++- programs/diagnostics/README.md | 2 +- programs/diagnostics/cmd/collect.go | 23 +++++++++-------- programs/diagnostics/cmd/help.go | 11 ++++---- programs/diagnostics/cmd/params/params.go | 7 +++--- .../diagnostics/cmd/params/params_test.go | 9 ++++--- programs/diagnostics/cmd/root.go | 13 +++++----- programs/diagnostics/go.mod | 2 +- .../internal/collectors/clickhouse/config.go | 13 +++++----- .../collectors/clickhouse/config_test.go | 11 ++++---- .../internal/collectors/clickhouse/db_logs.go | 8 +++--- .../collectors/clickhouse/db_logs_test.go | 11 ++++---- .../internal/collectors/clickhouse/logs.go | 11 ++++---- .../collectors/clickhouse/logs_test.go | 13 +++++----- .../internal/collectors/clickhouse/summary.go | 13 +++++----- .../collectors/clickhouse/summary_test.go | 11 ++++---- .../internal/collectors/clickhouse/system.go | 11 ++++---- .../collectors/clickhouse/system_test.go | 13 +++++----- .../collectors/clickhouse/zookeeper.go | 11 ++++---- .../collectors/clickhouse/zookeeper_test.go | 13 +++++----- .../internal/collectors/registry.go | 5 ++-- .../internal/collectors/registry_test.go | 11 ++++---- .../internal/collectors/system/command.go | 11 ++++---- .../collectors/system/command_test.go | 11 ++++---- .../internal/collectors/system/file.go | 11 ++++---- .../internal/collectors/system/file_test.go | 11 ++++---- .../internal/collectors/system/system.go | 11 ++++---- .../internal/collectors/system/system_test.go | 11 ++++---- .../internal/outputs/file/simple.go | 15 +++++------ .../internal/outputs/file/simple_test.go | 11 ++++---- .../diagnostics/internal/outputs/registry.go | 5 ++-- .../internal/outputs/registry_test.go | 9 ++++--- .../internal/outputs/terminal/report.go | 9 ++++--- .../internal/platform/config/models.go | 3 ++- .../internal/platform/config/models_test.go | 5 ++-- .../internal/platform/config/utils.go | 3 ++- .../internal/platform/config/utils_test.go | 5 ++-- .../internal/platform/data/bundle_test.go | 5 ++-- .../internal/platform/data/database_test.go | 5 ++-- .../internal/platform/data/file.go | 7 +++--- .../internal/platform/data/file_test.go | 5 ++-- .../internal/platform/data/memory_test.go | 5 ++-- .../internal/platform/database/native.go | 5 ++-- .../internal/platform/database/native_test.go | 13 +++++----- .../diagnostics/internal/platform/manager.go | 5 ++-- .../internal/platform/manager_test.go | 11 ++++---- .../internal/platform/test/data.go | 7 +++--- .../internal/platform/utils/file_test.go | 5 ++-- .../internal/platform/utils/process_test.go | 11 ++++---- .../internal/platform/utils/slices_test.go | 5 ++-- programs/diagnostics/internal/runner.go | 10 ++++---- programs/diagnostics/internal/runner_test.go | 25 ++++++++++--------- programs/diagnostics/main.go | 2 +- 53 files changed, 265 insertions(+), 217 deletions(-) diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 10d77f6c44d..349ea751213 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -5,7 +5,8 @@ BUILD_DIR=dist TIMESTAMP := $(shell date +%Y%m%d-%H%M) COMMIT := $(shell git rev-parse --short HEAD) -DEVLDFLAGS = -ldflags "-X github.com/ClickHouse/clickhouse-diagnostics/cmd.Version=v.dev-${TIMESTAMP} -X github.com/ClickHouse/clickhouse-diagnostics/cmd.Commit=${COMMIT}" +MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics +DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=v.dev-${TIMESTAMP} -X ${MODULE}/cmd.Commit=${COMMIT}" # override with env variable to test other versions e.g. 21.11.10.1 CLICKHOUSE_VERSION ?= latest diff --git a/programs/diagnostics/README.md b/programs/diagnostics/README.md index 89de0afdf2c..b69a6e3a996 100644 --- a/programs/diagnostics/README.md +++ b/programs/diagnostics/README.md @@ -71,7 +71,7 @@ We currently support the following collectors. A `*` indicates this collector is - `config*` - Collects the ClickHouse configuration from the local filesystem. A best effort is made using process information if ClickHouse is not installed locally. `include_path` are also considered. - `db_logs*` - Collects the ClickHouse logs directly from the database. - `logs*` - Collects the ClickHouse logs directly from the database. -- `summary*` - Collects summary statistics on the database based on a set of known useful queries. This represents the easiest collector to extend - contributions are welcome to this set which can be found [here](https://github.com/ClickHouse/clickhouse-diagnostics/blob/main/internal/collectors/clickhouse/queries.json). +- `summary*` - Collects summary statistics on the database based on a set of known useful queries. This represents the easiest collector to extend - contributions are welcome to this set which can be found [here](https://github.com/ClickHouse/ClickHouse/blob/master/programs/diagnostics/internal/collectors/clickhouse/queries.json). - `file` - Collects files based on glob patterns. Does not collect directories. To preview files which will be collected try, `clickhouse-diagnostics collect --collectors=file --collector.file.file_pattern= --output report` - `command` - Collects the output of a user specified command. To preview output, `clickhouse-diagnostics collect --collectors=command --collector.command.command="" --output report` - `zookeeper_db` - Collects information about zookeeper using the `system.zookeeper` table, recursively iterating the zookeeper tree/table. Note: changing the default parameter values can cause extremely high load to be placed on the database. Use with caution. By default, uses the glob `/clickhouse/{task_queue}/**` to match zookeeper paths and iterates to a max depth of 8. diff --git a/programs/diagnostics/cmd/collect.go b/programs/diagnostics/cmd/collect.go index e2228407541..503d8e41fb7 100644 --- a/programs/diagnostics/cmd/collect.go +++ b/programs/diagnostics/cmd/collect.go @@ -2,21 +2,22 @@ package cmd import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/cmd/params" - "github.com/ClickHouse/clickhouse-diagnostics/internal" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/terminal" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" + "os" + "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" - "os" - "strings" ) var id string diff --git a/programs/diagnostics/cmd/help.go b/programs/diagnostics/cmd/help.go index ba15fb8e1b1..750576dda25 100644 --- a/programs/diagnostics/cmd/help.go +++ b/programs/diagnostics/cmd/help.go @@ -2,13 +2,14 @@ package cmd import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/cmd/params" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" + "os" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "os" ) var cHelp = params.StringOptionsVar{ diff --git a/programs/diagnostics/cmd/params/params.go b/programs/diagnostics/cmd/params/params.go index 5d2bdc5fbe8..c4464aab5d2 100644 --- a/programs/diagnostics/cmd/params/params.go +++ b/programs/diagnostics/cmd/params/params.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/csv" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/spf13/cobra" "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/spf13/cobra" ) type cliParamType uint8 diff --git a/programs/diagnostics/cmd/params/params_test.go b/programs/diagnostics/cmd/params/params_test.go index 80a8b039d36..7671506ba59 100644 --- a/programs/diagnostics/cmd/params/params_test.go +++ b/programs/diagnostics/cmd/params/params_test.go @@ -1,13 +1,14 @@ package params_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/cmd/params" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/spf13/cobra" - "github.com/stretchr/testify/require" "os" "sort" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd/params" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" ) var conf = map[string]config.Configuration{ diff --git a/programs/diagnostics/cmd/root.go b/programs/diagnostics/cmd/root.go index b6d860df76a..4cf329d5438 100644 --- a/programs/diagnostics/cmd/root.go +++ b/programs/diagnostics/cmd/root.go @@ -2,17 +2,18 @@ package cmd import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/spf13/viper" "net/http" _ "net/http/pprof" "os" "strings" "time" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) func enableDebug() { diff --git a/programs/diagnostics/go.mod b/programs/diagnostics/go.mod index 1672cb93817..19fc2ec8202 100644 --- a/programs/diagnostics/go.mod +++ b/programs/diagnostics/go.mod @@ -1,4 +1,4 @@ -module github.com/ClickHouse/clickhouse-diagnostics +module github.com/ClickHouse/ClickHouse/programs/diagnostics go 1.17 diff --git a/programs/diagnostics/internal/collectors/clickhouse/config.go b/programs/diagnostics/internal/collectors/clickhouse/config.go index f6f2d441ed2..92368bce6f3 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/config.go +++ b/programs/diagnostics/internal/collectors/clickhouse/config.go @@ -2,13 +2,14 @@ package clickhouse import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/pkg/errors" "path/filepath" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/pkg/errors" ) type ConfigCollector struct { diff --git a/programs/diagnostics/internal/collectors/clickhouse/config_test.go b/programs/diagnostics/internal/collectors/clickhouse/config_test.go index 67205fb9384..355cbb65620 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/config_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/config_test.go @@ -3,15 +3,16 @@ package clickhouse_test import ( "encoding/xml" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "io" "os" "path" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestConfigConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go index 23a47c33c7f..3253f504c1b 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs.go +++ b/programs/diagnostics/internal/collectors/clickhouse/db_logs.go @@ -1,10 +1,10 @@ package clickhouse import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" ) diff --git a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go index 9c403de281a..3fc585f3352 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/db_logs_test.go @@ -1,12 +1,13 @@ package clickhouse_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) func TestDbLogsConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs.go b/programs/diagnostics/internal/collectors/clickhouse/logs.go index 8d01c858947..8436a392c47 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/logs.go +++ b/programs/diagnostics/internal/collectors/clickhouse/logs.go @@ -2,12 +2,13 @@ package clickhouse import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" "path/filepath" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" ) // This collector collects logs diff --git a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go index dd94997c465..5f0be734445 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/logs_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/logs_test.go @@ -2,15 +2,16 @@ package clickhouse_test import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "os" "path" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) func TestLogsConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary.go b/programs/diagnostics/internal/collectors/clickhouse/summary.go index 603fc954642..0b6dd3aff20 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/summary.go +++ b/programs/diagnostics/internal/collectors/clickhouse/summary.go @@ -4,14 +4,15 @@ import ( "bytes" _ "embed" "encoding/json" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/Masterminds/semver" - "github.com/pkg/errors" "strings" "text/template" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/Masterminds/semver" + "github.com/pkg/errors" ) // This collector collects the system db from database diff --git a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go index 7c15cd58a1e..92945d987ed 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/summary_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/summary_test.go @@ -1,12 +1,13 @@ package clickhouse_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) func TestSummaryConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/clickhouse/system.go b/programs/diagnostics/internal/collectors/clickhouse/system.go index b370a3ab1df..d47cfd924f3 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/system.go +++ b/programs/diagnostics/internal/collectors/clickhouse/system.go @@ -2,11 +2,12 @@ package clickhouse import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" "github.com/pkg/errors" ) diff --git a/programs/diagnostics/internal/collectors/clickhouse/system_test.go b/programs/diagnostics/internal/collectors/clickhouse/system_test.go index a11bbd75843..d1b9a6e7859 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/system_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/system_test.go @@ -1,13 +1,14 @@ package clickhouse_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) func TestSystemConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go index cd7cd8bfc6c..78aefeaa0c1 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go +++ b/programs/diagnostics/internal/collectors/clickhouse/zookeeper.go @@ -2,14 +2,15 @@ package clickhouse import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/bmatcuk/doublestar/v4" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "strings" ) // This collector collects the system zookeeper db diff --git a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go index 8d53d044e76..3e56f6200f0 100644 --- a/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go +++ b/programs/diagnostics/internal/collectors/clickhouse/zookeeper_test.go @@ -1,13 +1,14 @@ package clickhouse_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) func TestZookeeperConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/registry.go b/programs/diagnostics/internal/collectors/registry.go index 29eb1f5939c..5611f947466 100644 --- a/programs/diagnostics/internal/collectors/registry.go +++ b/programs/diagnostics/internal/collectors/registry.go @@ -2,8 +2,9 @@ package collectors import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" "github.com/rs/zerolog/log" ) diff --git a/programs/diagnostics/internal/collectors/registry_test.go b/programs/diagnostics/internal/collectors/registry_test.go index 9f920f4b5c6..eccc5f2265d 100644 --- a/programs/diagnostics/internal/collectors/registry_test.go +++ b/programs/diagnostics/internal/collectors/registry_test.go @@ -1,12 +1,13 @@ package collectors_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/stretchr/testify/require" ) func TestGetCollectorNames(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/system/command.go b/programs/diagnostics/internal/collectors/system/command.go index 95a958ec6ce..ba4dd1e996c 100644 --- a/programs/diagnostics/internal/collectors/system/command.go +++ b/programs/diagnostics/internal/collectors/system/command.go @@ -2,13 +2,14 @@ package system import ( "bytes" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "os/exec" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/google/shlex" "github.com/pkg/errors" - "os/exec" ) // This collector runs a user specified command and collects it to a file diff --git a/programs/diagnostics/internal/collectors/system/command_test.go b/programs/diagnostics/internal/collectors/system/command_test.go index e3d04cebe21..7de00cdabf4 100644 --- a/programs/diagnostics/internal/collectors/system/command_test.go +++ b/programs/diagnostics/internal/collectors/system/command_test.go @@ -2,12 +2,13 @@ package system_test import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestCommandConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/system/file.go b/programs/diagnostics/internal/collectors/system/file.go index 68e96314d1d..cda91636c52 100644 --- a/programs/diagnostics/internal/collectors/system/file.go +++ b/programs/diagnostics/internal/collectors/system/file.go @@ -1,14 +1,15 @@ package system import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "os" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/yargevad/filepathx" - "os" ) // This collector collects arbitrary user files diff --git a/programs/diagnostics/internal/collectors/system/file_test.go b/programs/diagnostics/internal/collectors/system/file_test.go index f52c190d087..5b1d5b3a92f 100644 --- a/programs/diagnostics/internal/collectors/system/file_test.go +++ b/programs/diagnostics/internal/collectors/system/file_test.go @@ -1,12 +1,13 @@ package system_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestFileConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go index b2aaee976ba..0bf612b664c 100644 --- a/programs/diagnostics/internal/collectors/system/system.go +++ b/programs/diagnostics/internal/collectors/system/system.go @@ -1,15 +1,16 @@ package system import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/elastic/gosigar" "github.com/jaypipes/ghw" "github.com/matishsiao/goInfo" "github.com/pkg/errors" - "strings" ) // This collector collects the system overview diff --git a/programs/diagnostics/internal/collectors/system/system_test.go b/programs/diagnostics/internal/collectors/system/system_test.go index 35777f6a298..fb1e16bd1ed 100644 --- a/programs/diagnostics/internal/collectors/system/system_test.go +++ b/programs/diagnostics/internal/collectors/system/system_test.go @@ -1,12 +1,13 @@ package system_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestSystemConfiguration(t *testing.T) { diff --git a/programs/diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go index d19c1222fe1..f91ec9f74ee 100644 --- a/programs/diagnostics/internal/outputs/file/simple.go +++ b/programs/diagnostics/internal/outputs/file/simple.go @@ -4,18 +4,19 @@ import ( "context" "encoding/csv" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/mholt/archiver/v4" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" "os" "path" "path/filepath" "strconv" "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/mholt/archiver/v4" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" ) const OutputName = "simple" diff --git a/programs/diagnostics/internal/outputs/file/simple_test.go b/programs/diagnostics/internal/outputs/file/simple_test.go index dfa9bc6d80a..471a1c70cc1 100644 --- a/programs/diagnostics/internal/outputs/file/simple_test.go +++ b/programs/diagnostics/internal/outputs/file/simple_test.go @@ -4,16 +4,17 @@ import ( "bufio" "encoding/xml" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/file" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" "io" "os" "path" "strings" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" ) var clusterFrame = test.NewFakeDataFrame("clusters", []string{"cluster", "shard_num", "shard_weight", "replica_num", "host_name", "host_address", "port", "is_local", "user", "default_database", "errors_count", "slowdowns_count", "estimated_recovery_time"}, diff --git a/programs/diagnostics/internal/outputs/registry.go b/programs/diagnostics/internal/outputs/registry.go index 8782ecfda4f..0187cd9105d 100644 --- a/programs/diagnostics/internal/outputs/registry.go +++ b/programs/diagnostics/internal/outputs/registry.go @@ -2,8 +2,9 @@ package outputs import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" "github.com/rs/zerolog/log" ) diff --git a/programs/diagnostics/internal/outputs/registry_test.go b/programs/diagnostics/internal/outputs/registry_test.go index 0b0fe3597af..ba8408e5a59 100644 --- a/programs/diagnostics/internal/outputs/registry_test.go +++ b/programs/diagnostics/internal/outputs/registry_test.go @@ -1,11 +1,12 @@ package outputs_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/file" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/terminal" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/terminal" + "github.com/stretchr/testify/require" ) func TestGetOutputNames(t *testing.T) { diff --git a/programs/diagnostics/internal/outputs/terminal/report.go b/programs/diagnostics/internal/outputs/terminal/report.go index a601e10687a..8337f542457 100644 --- a/programs/diagnostics/internal/outputs/terminal/report.go +++ b/programs/diagnostics/internal/outputs/terminal/report.go @@ -3,12 +3,13 @@ package terminal import ( "bufio" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "os" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/olekukonko/tablewriter" "github.com/pkg/errors" - "os" ) const OutputName = "report" diff --git a/programs/diagnostics/internal/platform/config/models.go b/programs/diagnostics/internal/platform/config/models.go index 52b2489886d..6c76b8f149b 100644 --- a/programs/diagnostics/internal/platform/config/models.go +++ b/programs/diagnostics/internal/platform/config/models.go @@ -2,8 +2,9 @@ package config import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" ) type ConfigParam interface { diff --git a/programs/diagnostics/internal/platform/config/models_test.go b/programs/diagnostics/internal/platform/config/models_test.go index b87e66408ef..916d20ec28b 100644 --- a/programs/diagnostics/internal/platform/config/models_test.go +++ b/programs/diagnostics/internal/platform/config/models_test.go @@ -1,9 +1,10 @@ package config_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/stretchr/testify/require" ) var conf = config.Configuration{ diff --git a/programs/diagnostics/internal/platform/config/utils.go b/programs/diagnostics/internal/platform/config/utils.go index 127ff95570e..5f84c38d4f4 100644 --- a/programs/diagnostics/internal/platform/config/utils.go +++ b/programs/diagnostics/internal/platform/config/utils.go @@ -2,7 +2,8 @@ package config import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" ) func ReadStringListValues(conf Configuration, paramName string) ([]string, error) { diff --git a/programs/diagnostics/internal/platform/config/utils_test.go b/programs/diagnostics/internal/platform/config/utils_test.go index 0f9791eb60e..9e03e5e69d2 100644 --- a/programs/diagnostics/internal/platform/config/utils_test.go +++ b/programs/diagnostics/internal/platform/config/utils_test.go @@ -1,9 +1,10 @@ package config_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/stretchr/testify/require" ) func TestReadStringListValues(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/data/bundle_test.go b/programs/diagnostics/internal/platform/data/bundle_test.go index 5438a50ae0a..ff9cfc2cf56 100644 --- a/programs/diagnostics/internal/platform/data/bundle_test.go +++ b/programs/diagnostics/internal/platform/data/bundle_test.go @@ -1,10 +1,11 @@ package data_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" "github.com/stretchr/testify/require" - "testing" ) func TestBundleError(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/data/database_test.go b/programs/diagnostics/internal/platform/data/database_test.go index 7491f016a67..57d89e78efc 100644 --- a/programs/diagnostics/internal/platform/data/database_test.go +++ b/programs/diagnostics/internal/platform/data/database_test.go @@ -2,10 +2,11 @@ package data_test import ( "database/sql" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/require" - "testing" ) func TestString(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go index 8bdde8a6437..19bf5884876 100644 --- a/programs/diagnostics/internal/platform/data/file.go +++ b/programs/diagnostics/internal/platform/data/file.go @@ -3,14 +3,15 @@ package data import ( "bufio" "encoding/xml" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" "io/ioutil" "os" "path" "path/filepath" "regexp" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" ) type File interface { diff --git a/programs/diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go index 49f0bb9cf72..d273987d327 100644 --- a/programs/diagnostics/internal/platform/data/file_test.go +++ b/programs/diagnostics/internal/platform/data/file_test.go @@ -2,14 +2,15 @@ package data_test import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "io/ioutil" "os" "path" "path/filepath" "strings" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestNextFileDirectoryFrame(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/data/memory_test.go b/programs/diagnostics/internal/platform/data/memory_test.go index 3fc2f6822ef..fcc02e37d32 100644 --- a/programs/diagnostics/internal/platform/data/memory_test.go +++ b/programs/diagnostics/internal/platform/data/memory_test.go @@ -1,9 +1,10 @@ package data_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/stretchr/testify/require" ) func TestNextMemoryFrame(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/database/native.go b/programs/diagnostics/internal/platform/database/native.go index f167fb5540c..e512a634fbf 100644 --- a/programs/diagnostics/internal/platform/database/native.go +++ b/programs/diagnostics/internal/platform/database/native.go @@ -3,10 +3,11 @@ package database import ( "database/sql" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" _ "github.com/ClickHouse/clickhouse-go/v2" "github.com/pkg/errors" - "strings" ) type ClickhouseNativeClient struct { diff --git a/programs/diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go index 1e936fe2449..8c317ab7f12 100644 --- a/programs/diagnostics/internal/platform/database/native_test.go +++ b/programs/diagnostics/internal/platform/database/native_test.go @@ -3,16 +3,17 @@ package database_test import ( "context" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/database" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "os" "path" "strconv" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" ) func TestMain(m *testing.M) { diff --git a/programs/diagnostics/internal/platform/manager.go b/programs/diagnostics/internal/platform/manager.go index e23a534a6fc..b4435b62ea2 100644 --- a/programs/diagnostics/internal/platform/manager.go +++ b/programs/diagnostics/internal/platform/manager.go @@ -2,9 +2,10 @@ package platform import ( "errors" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/database" "sync" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/database" ) var once sync.Once diff --git a/programs/diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go index 09316052b53..e63ec6af6a7 100644 --- a/programs/diagnostics/internal/platform/manager_test.go +++ b/programs/diagnostics/internal/platform/manager_test.go @@ -3,15 +3,16 @@ package platform_test import ( "context" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "os" "path" "strconv" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" ) func TestMain(m *testing.M) { diff --git a/programs/diagnostics/internal/platform/test/data.go b/programs/diagnostics/internal/platform/test/data.go index 44ce7c70908..7710e9a69a1 100644 --- a/programs/diagnostics/internal/platform/test/data.go +++ b/programs/diagnostics/internal/platform/test/data.go @@ -2,11 +2,12 @@ package test import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/pkg/errors" "sort" "strings" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/pkg/errors" ) type fakeClickhouseClient struct { diff --git a/programs/diagnostics/internal/platform/utils/file_test.go b/programs/diagnostics/internal/platform/utils/file_test.go index 51c8ed2e9c5..8d0430090c9 100644 --- a/programs/diagnostics/internal/platform/utils/file_test.go +++ b/programs/diagnostics/internal/platform/utils/file_test.go @@ -2,11 +2,12 @@ package utils_test import ( "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" "os" "path" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/stretchr/testify/require" ) func TestFileExists(t *testing.T) { diff --git a/programs/diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go index 45bbc18bdef..0c7541f4abb 100644 --- a/programs/diagnostics/internal/platform/utils/process_test.go +++ b/programs/diagnostics/internal/platform/utils/process_test.go @@ -3,14 +3,15 @@ package utils_test import ( "context" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "os" "path" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" ) func TestMain(m *testing.M) { diff --git a/programs/diagnostics/internal/platform/utils/slices_test.go b/programs/diagnostics/internal/platform/utils/slices_test.go index 18965415013..ea5c1c81dcc 100644 --- a/programs/diagnostics/internal/platform/utils/slices_test.go +++ b/programs/diagnostics/internal/platform/utils/slices_test.go @@ -1,9 +1,10 @@ package utils_test import ( - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/stretchr/testify/require" ) func TestIntersection(t *testing.T) { diff --git a/programs/diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go index d2147cd1c65..6960cf1cd23 100644 --- a/programs/diagnostics/internal/runner.go +++ b/programs/diagnostics/internal/runner.go @@ -1,11 +1,11 @@ package internal import ( - c "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - o "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/data" + c "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + o "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data" "github.com/pkg/errors" "github.com/rs/zerolog/log" ) diff --git a/programs/diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go index a6e7dbe4cbc..81fe9b70a2b 100644 --- a/programs/diagnostics/internal/runner_test.go +++ b/programs/diagnostics/internal/runner_test.go @@ -3,23 +3,24 @@ package internal_test import ( "context" "fmt" - "github.com/ClickHouse/clickhouse-diagnostics/internal" - "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/clickhouse" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/collectors/system" - "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs" - _ "github.com/ClickHouse/clickhouse-diagnostics/internal/outputs/file" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/config" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/test" - "github.com/ClickHouse/clickhouse-diagnostics/internal/platform/utils" - "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "io/ioutil" "os" "path" "strconv" "testing" + + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/clickhouse" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/collectors/system" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs" + _ "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/outputs/file" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/config" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/test" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/utils" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" ) func TestMain(m *testing.M) { diff --git a/programs/diagnostics/main.go b/programs/diagnostics/main.go index 6187ed364ef..0a849a9f520 100644 --- a/programs/diagnostics/main.go +++ b/programs/diagnostics/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/ClickHouse/clickhouse-diagnostics/cmd" + "github.com/ClickHouse/ClickHouse/programs/diagnostics/cmd" ) func main() { From a70cda9670df1950bda07f85f6c75ac4d867ced5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 14 Jun 2022 13:05:36 +0200 Subject: [PATCH 351/408] Create a path for clickhouse-diagnostics installation --- programs/diagnostics/Makefile | 2 +- programs/diagnostics/{ => cmd/clickhouse-diagnostics}/main.go | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename programs/diagnostics/{ => cmd/clickhouse-diagnostics}/main.go (100%) diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 349ea751213..4097cdb7a74 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -27,7 +27,7 @@ release: ## Release is delegated to goreleaser ## Build: build: ## Build a binary for local use # timestamped version - $(GOCMD) build ${DEVLDFLAGS} -o $(BINARY_NAME) . + $(GOCMD) build ${DEVLDFLAGS} -o $(BINARY_NAME) ./cmd/clickhouse-diagnostics clean: ## Remove build related file rm ${BINARY_NAME} diff --git a/programs/diagnostics/main.go b/programs/diagnostics/cmd/clickhouse-diagnostics/main.go similarity index 100% rename from programs/diagnostics/main.go rename to programs/diagnostics/cmd/clickhouse-diagnostics/main.go From 1553f678098959fc7f571ebaae3e5dbce15eacfd Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 15 Jun 2022 11:52:37 +0200 Subject: [PATCH 352/408] Can set version in Makefile --- programs/diagnostics/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 4097cdb7a74..0a12630ca6d 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -6,7 +6,8 @@ BUILD_DIR=dist TIMESTAMP := $(shell date +%Y%m%d-%H%M) COMMIT := $(shell git rev-parse --short HEAD) MODULE := github.com/ClickHouse/ClickHouse/programs/diagnostics -DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=v.dev-${TIMESTAMP} -X ${MODULE}/cmd.Commit=${COMMIT}" +VERSION := v.dev-${TIMESTAMP} +DEVLDFLAGS = -ldflags "-X ${MODULE}/cmd.Version=${VERSION} -X ${MODULE}/cmd.Commit=${COMMIT}" # override with env variable to test other versions e.g. 21.11.10.1 CLICKHOUSE_VERSION ?= latest From 0aae60f8f13272fb49dee35bf80cf4fb09fe288c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 28 Jun 2022 19:12:05 +0200 Subject: [PATCH 353/408] Update nfpm version --- docker/packager/binary/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 995cecfebc7..ab603cd9ea4 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -97,7 +97,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH -ARG NFPM_VERSION=2.15.1 +ARG NFPM_VERSION=2.16.0 RUN arch=${TARGETARCH:-amd64} \ && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \ From 3513b4168308125685acc05e1e61a9e5a9065df3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 18:42:55 +0200 Subject: [PATCH 354/408] Fix linter issue --- programs/diagnostics/internal/collectors/system/system.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/diagnostics/internal/collectors/system/system.go b/programs/diagnostics/internal/collectors/system/system.go index 0bf612b664c..69d16f36b8b 100644 --- a/programs/diagnostics/internal/collectors/system/system.go +++ b/programs/diagnostics/internal/collectors/system/system.go @@ -26,7 +26,7 @@ func NewSystemCollector(m *platform.ResourceManager) *SystemCollector { } func (sc *SystemCollector) Collect(conf config.Configuration) (*data.DiagnosticBundle, error) { - conf, err := conf.ValidateConfig(sc.Configuration()) + _, err := conf.ValidateConfig(sc.Configuration()) if err != nil { return &data.DiagnosticBundle{}, err } From 27a76cb627d0692b74927a4b8046d366636b7816 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 11:44:51 +0200 Subject: [PATCH 355/408] Overwrite in encrypted disk --- src/Disks/DiskEncrypted.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h index 4b6fca7ab2a..1a714395f82 100644 --- a/src/Disks/DiskEncrypted.h +++ b/src/Disks/DiskEncrypted.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -238,6 +239,13 @@ public: SyncGuardPtr getDirectorySyncGuard(const String & path) const override; + DiskTransactionPtr createTransaction() override + { + /// Need to overwrite explicetly because this disk change + /// a lot of "delegate" methods. + return std::make_shared(*this); + } + private: String wrappedPath(const String & path) const { From ae2f586170b49e40d1f93688336c45e0fb1a712f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 1 Jul 2022 12:18:09 +0200 Subject: [PATCH 356/408] Fix crash when granting ALL on cluster. --- src/Access/ContextAccess.cpp | 22 ++++++++++++------- .../test_access_control_on_cluster/test.py | 10 +++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 221113cb425..995a46d07ca 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -412,15 +412,18 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg return false; }; + if (is_full_access) + return access_granted(); + + if (user_was_dropped) + return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER); + if (flags & AccessType::CLUSTER && !access_control->doesOnClusterQueriesRequireClusterGrant()) flags &= ~AccessType::CLUSTER; - if (!flags || is_full_access) + if (!flags) return access_granted(); - if (!tryGetUser()) - return access_denied("User has been dropped", ErrorCodes::UNKNOWN_USER); - /// Access to temporary tables is controlled in an unusual way, not like normal tables. /// Creating of temporary tables is controlled by AccessType::CREATE_TEMPORARY_TABLES grant, /// and other grants are considered as always given. @@ -600,9 +603,6 @@ void ContextAccess::checkGrantOption(const AccessRightsElements & elements) cons template bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const { - if (!std::size(role_ids) || is_full_access) - return true; - auto show_error = [this](const String & msg, int error_code [[maybe_unused]]) { UNUSED(this); @@ -610,12 +610,18 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const throw Exception(getUserName() + ": " + msg, error_code); }; - if (!tryGetUser()) + if (is_full_access) + return true; + + if (user_was_dropped) { show_error("User has been dropped", ErrorCodes::UNKNOWN_USER); return false; } + if (!std::size(role_ids)) + return true; + if (isGranted(AccessType::ROLE_ADMIN)) return true; diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 6c2331178e0..db76233a35f 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -49,3 +49,13 @@ def test_access_control_on_cluster(): assert "There is no user `Alex`" in ch1.query_and_get_error("SHOW CREATE USER Alex") assert "There is no user `Alex`" in ch2.query_and_get_error("SHOW CREATE USER Alex") assert "There is no user `Alex`" in ch3.query_and_get_error("SHOW CREATE USER Alex") + + +def test_grant_all_on_cluster(): + ch1.query("CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'") + ch1.query("GRANT ALL ON *.* TO Alex ON CLUSTER 'cluster'") + + assert ch1.query("SHOW GRANTS FOR Alex") == "GRANT ALL ON *.* TO Alex\n" + assert ch2.query("SHOW GRANTS FOR Alex") == "GRANT ALL ON *.* TO Alex\n" + + ch1.query("DROP USER Alex ON CLUSTER 'cluster'") From 7cc063a1d0e54aa159c6456f46b2849d2e7622c2 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 1 Jul 2022 12:08:53 +0200 Subject: [PATCH 357/408] Fix strange backport titles issues --- tests/ci/cherry_pick_utils/cherrypick.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/cherry_pick_utils/cherrypick.py b/tests/ci/cherry_pick_utils/cherrypick.py index 92c87800828..c844beaee88 100644 --- a/tests/ci/cherry_pick_utils/cherrypick.py +++ b/tests/ci/cherry_pick_utils/cherrypick.py @@ -165,7 +165,7 @@ class CherryPick: "user.name=robot-clickhouse", ] - title = (self._pr["title"].replace('"', r"\""),) + title = self._pr["title"].replace('"', r"\"") pr_title = f"Backport #{self._pr['number']} to {self.target_branch}: {title}" self._run(git_prefix + ["checkout", "-f", self.backport_branch]) From 2a23c39a789c581778e3e59d44431dbf22109515 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 1 Jul 2022 15:20:33 +0300 Subject: [PATCH 358/408] Remove outdated cmake documentation When it had been added, initially, it was autogenerated - #14711 Later, after documentation rework, it had been removed, in #37441. And this makes documentation out dated, and out dated documentation may provide more harm then help, so let's remove it. Also it has links to the code in github that does not contain commit SHA1, so those links may point in the wrong place. Signed-off-by: Azat Khuzhin --- docs/_includes/cmake_in_clickhouse_footer.md | 121 ---- docs/_includes/cmake_in_clickhouse_header.md | 27 - docs/en/development/cmake-in-clickhouse.md | 545 ------------------- 3 files changed, 693 deletions(-) delete mode 100644 docs/_includes/cmake_in_clickhouse_footer.md delete mode 100644 docs/_includes/cmake_in_clickhouse_header.md delete mode 100644 docs/en/development/cmake-in-clickhouse.md diff --git a/docs/_includes/cmake_in_clickhouse_footer.md b/docs/_includes/cmake_in_clickhouse_footer.md deleted file mode 100644 index bf8411ba815..00000000000 --- a/docs/_includes/cmake_in_clickhouse_footer.md +++ /dev/null @@ -1,121 +0,0 @@ - -## Developer's guide for adding new CMake options - -### Don't be obvious. Be informative. - -Bad: -```cmake -option (ENABLE_TESTS "Enables testing" OFF) -``` - -This description is quite useless as is neither gives the viewer any additional information nor explains the option purpose. - -Better: - -```cmake -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some -pre-conditions, leave a comment above the `option()` line and explain what it does. -The best way would be linking the docs page (if it exists). -The comment is parsed into a separate column (see below). - -Even better: - -```cmake -# implies ${TESTS_ARE_ENABLED} -# see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. - -Suppose you have an option that may strip debug symbols from the ClickHouse's part. -This can speed up the linking process, but produces a binary that cannot be debugged. -In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. -Also, such options should be disabled if applies. - -Bad: -```cmake -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() - -``` -Better: - -```cmake -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions." - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -### In the option's description, explain WHAT the option does rather than WHY it does something. - -The WHY explanation should be placed in the comment. -You may find that the option's name is self-descriptive. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better: - -```cmake -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -### Don't assume other developers know as much as you do. - -In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to -the tool's docs. It won't take much of your time. - -Bad: - -```cmake -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better (combined with the above hint): - -```cmake -# https://clang.llvm.org/docs/ThinLTO.html -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -Other example, bad: - -```cmake -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) -``` - -Better: - -```cmake -# https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) -``` - -### Prefer consistent default values. - -CMake allows you to pass a plethora of values representing boolean `true/false`, e.g. `1, ON, YES, ...`. -Prefer the `ON/OFF` values, if possible. diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md deleted file mode 100644 index 2f2e0421946..00000000000 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ /dev/null @@ -1,27 +0,0 @@ -# CMake in ClickHouse - -## TL; DR How to make ClickHouse compile and link faster? - -Minimal ClickHouse build example: - -```bash -cmake .. \ - -DCMAKE_C_COMPILER=$(which clang-14) \ - -DCMAKE_CXX_COMPILER=$(which clang++-14) \ - -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_UTILS=OFF \ - -DENABLE_TESTS=OFF -``` - -## CMake files types - -1. ClickHouse's source CMake files (located in the root directory and in `/src`). -2. Arch-dependent CMake files (located in `/cmake/*os_name*`). -3. Libraries finders (search for contrib libraries, located in `/contrib/*/CMakeLists.txt`). -3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) - -## List of CMake flags - -* This list is auto-generated by [this Python script](https://github.com/clickhouse/clickhouse/blob/master/docs/tools/cmake_in_clickhouse_generator.py). -* The flag name is a link to its position in the code. -* If an option's default value is itself an option, it's also a link to its position in this list. diff --git a/docs/en/development/cmake-in-clickhouse.md b/docs/en/development/cmake-in-clickhouse.md deleted file mode 100644 index 83279f5f69a..00000000000 --- a/docs/en/development/cmake-in-clickhouse.md +++ /dev/null @@ -1,545 +0,0 @@ ---- -sidebar_position: 69 -sidebar_label: CMake in ClickHouse -description: How to make ClickHouse compile and link faster ---- - -# CMake in ClickHouse - -How to make ClickHouse compile and link faster. Minimal ClickHouse build example: - -```bash -cmake .. \ - -DCMAKE_C_COMPILER=$(which clang-13) \ - -DCMAKE_CXX_COMPILER=$(which clang++-13) \ - -DCMAKE_BUILD_TYPE=Debug \ - -DENABLE_UTILS=OFF \ - -DENABLE_TESTS=OFF -``` - -## CMake files types - -1. ClickHouse source CMake files (located in the root directory and in /src). -2. Arch-dependent CMake files (located in /cmake/*os_name*). -3. Libraries finders (search for contrib libraries, located in /contrib/*/CMakeLists.txt). -4. Contrib build CMake files (used instead of libraries' own CMake files, located in /cmake/modules) - -## List of CMake flags -- The flag name is a link to its position in the code. -- If an option's default value is itself an option, it's also a link to its position in this list. - -## ClickHouse modes - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameDefault valueDescriptionComment
ENABLE_CLICKHOUSE_ALLONEnable all ClickHouse modes by defaultThe clickhouse binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), each of them may be built and linked as a separate library. If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only.
ENABLE_CLICKHOUSE_BENCHMARKENABLE_CLICKHOUSE_ALLQueries benchmarking modehttps://clickhouse.com/docs/en/operations/utilities/clickhouse-benchmark/
ENABLE_CLICKHOUSE_CLIENTENABLE_CLICKHOUSE_ALLClient mode (interactive tui/shell that connects to the server)
ENABLE_CLICKHOUSE_COMPRESSORENABLE_CLICKHOUSE_ALLData compressor and decompressorhttps://clickhouse.com/docs/en/operations/utilities/clickhouse-compressor/
ENABLE_CLICKHOUSE_COPIERENABLE_CLICKHOUSE_ALLInter-cluster data copying modehttps://clickhouse.com/docs/en/operations/utilities/clickhouse-copier/
ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIGENABLE_CLICKHOUSE_ALLConfigs processor (extract values etc.)
ENABLE_CLICKHOUSE_FORMATENABLE_CLICKHOUSE_ALLQueries pretty-printer and formatter with syntax highlighting
ENABLE_CLICKHOUSE_GIT_IMPORTENABLE_CLICKHOUSE_ALLA tool to analyze Git repositorieshttps://presentations.clickhouse.com/matemarketing_2020/
ENABLE_CLICKHOUSE_INSTALLOFFInstall ClickHouse without .deb/.rpm/.tgz packages (having the binary only)
ENABLE_CLICKHOUSE_KEEPERENABLE_CLICKHOUSE_ALLClickHouse alternative to ZooKeeper
ENABLE_CLICKHOUSE_KEEPER_CONVERTERENABLE_CLICKHOUSE_ALLUtil allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot
ENABLE_CLICKHOUSE_LIBRARY_BRIDGEENABLE_CLICKHOUSE_ALLHTTP-server working like a proxy to Library dictionary source
ENABLE_CLICKHOUSE_LOCALENABLE_CLICKHOUSE_ALLLocal files fast processing modehttps://clickhouse.com/docs/en/operations/utilities/clickhouse-local/
ENABLE_CLICKHOUSE_OBFUSCATORENABLE_CLICKHOUSE_ALLTable data obfuscator (convert real data to benchmark-ready one)https://clickhouse.com/docs/en/operations/utilities/clickhouse-obfuscator/
ENABLE_CLICKHOUSE_ODBC_BRIDGEENABLE_CLICKHOUSE_ALLHTTP-server working like a proxy to ODBC driver
ENABLE_CLICKHOUSE_SERVERENABLE_CLICKHOUSE_ALLServer mode (main mode)
ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADERENABLE_CLICKHOUSE_ALLA tool to export table data files to be later put to a static files web server
- - -## External libraries -Note that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameDefault valueDescriptionComment
ENABLE_AVX0Use AVX instructions on x86_64
ENABLE_AVX20Use AVX2 instructions on x86_64
ENABLE_AVX2_FOR_SPEC_OP0Use avx2 instructions for specific operations on x86_64
ENABLE_AVX5120Use AVX512 instructions on x86_64
ENABLE_AVX512_FOR_SPEC_OP0Use avx512 instructions for specific operations on x86_64
ENABLE_BMI0Use BMI instructions on x86_64
ENABLE_CCACHEENABLE_CCACHE_BY_DEFAULTSpeedup re-compilations using ccache (external tool)https://ccache.dev/
ENABLE_CLANG_TIDYOFFUse clang-tidy static analyzerhttps://clang.llvm.org/extra/clang-tidy/
ENABLE_PCLMULQDQ1Use pclmulqdq instructions on x86_64
ENABLE_POPCNT1Use popcnt instructions on x86_64
ENABLE_SSE411Use SSE4.1 instructions on x86_64
ENABLE_SSE421Use SSE4.2 instructions on x86_64
ENABLE_SSSE31Use SSSE3 instructions on x86_64
- - -## Other flags - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameDefault valueDescriptionComment
ADD_GDB_INDEX_FOR_GOLDOFFAdd .gdb-index to resulting binaries for gold linker.Ignored if lld is used
ARCH_NATIVE0Add -march=native compiler flag. This makes your binaries non-portable but more performant code may be generated. This option overrides ENABLE_* options for specific instruction set. Highly not recommended to use.
BUILD_STANDALONE_KEEPEROFFBuild keeper as small standalone binary
CLICKHOUSE_SPLIT_BINARYOFFMake several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled
COMPILER_PIPEON-pipe compiler optionLess /tmp usage, more RAM usage.
ENABLE_BUILD_PATH_MAPPINGONEnable remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE(). It's to generate reproducible builds. See https://reproducible-builds.org/docs/build-pathReproducible builds If turned ON, remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE().
ENABLE_CHECK_HEAVY_BUILDSOFFDon't allow C++ translation units to compile too long or to take too much memory while compiling.Take care to add prlimit in command line before ccache, or else ccache thinks that prlimit is compiler, and clang++ is its input file, and refuses to work with multiple inputs, e.g in ccache log: [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp Another way would be to use --ccache-skip option before clang++-11 to make ccache ignore it.
ENABLE_COLORED_BUILDONEnable colored diagnostics in build log.
ENABLE_EXAMPLESOFFBuild all example programs in 'examples' subdirectories
ENABLE_FUZZINGOFFFuzzy testing using libfuzzer
ENABLE_LIBRARIESONEnable all external libraries by defaultTurns on all external libs like s3, kafka, ODBC, ...
ENABLE_MULTITARGET_CODEONEnable platform-dependent codeClickHouse developers may use platform-dependent code under some macro (e.g. ifdef ENABLE_MULTITARGET). If turned ON, this option defines such macro. See src/Functions/TargetSpecific.h
ENABLE_TESTSONProvide unit_test_dbms target with Google.Test unit testsIf turned ON, assumes the user has either the system GTest library or the bundled one.
ENABLE_THINLTOONClang-specific link time optimizationhttps://clang.llvm.org/docs/ThinLTO.html Applies to clang only. Disabled when building with tests or sanitizers.
FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATIONONStop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfyIf turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.
GLIBC_COMPATIBILITYONEnable compatibility with older glibc libraries.Only for Linux, x86_64 or aarch64.
SPLIT_DEBUG_SYMBOLSOFFBuild stripped binaries with debug info in separate directory
LINKER_NAMEOFFLinker name or full pathExample values: lld-10, gold.
PARALLEL_COMPILE_JOBS""Maximum number of concurrent compilation jobs1 if not set
PARALLEL_LINK_JOBS""Maximum number of concurrent link jobs1 if not set
SANITIZE""Enable one of the code sanitizersPossible values: - address (ASan) - memory (MSan) - thread (TSan) - undefined (UBSan) - "" (no sanitizing)
SPLIT_SHARED_LIBRARIESOFFKeep all internal libraries as separate .so filesDEVELOPER ONLY. Faster linking if turned on.
STRIP_DEBUG_SYMBOLS_FUNCTIONSSTRIP_DSF_DEFAULTDo not generate debugger info for ClickHouse functionsProvides faster linking and lower binary size. Tradeoff is the inability to debug some source files with e.g. gdb (empty stack frames and no local variables)."
USE_DEBUG_HELPERSUSE_DEBUG_HELPERSEnable debug helpers
USE_STATIC_LIBRARIESONDisable to use shared libraries
USE_UNWINDENABLE_LIBRARIESEnable libunwind (better stacktraces)
WERROROFFEnable -Werror compiler optionUsing system libs can cause a lot of warnings in includes (on macro expansion).
WITH_COVERAGEOFFProfile the resulting binary/binariesCompiler-specific coverage flags e.g. -fcoverage-mapping for gcc
- -## Developer's guide for adding new CMake options - -#### Don't be obvious. Be informative. - -Bad: - -``` -option (ENABLE_TESTS "Enables testing" OFF) -``` - -This description is quite useless as it neither gives the viewer any additional information nor explains the option purpose. - -Better: - -``` -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -If the option's purpose can't be guessed by its name, or the purpose guess may be misleading, or option has some -pre-conditions, leave a comment above the option() line and explain what it does. -The best way would be linking the docs page (if it exists). -The comment is parsed into a separate column (see below). - -Even better: - -``` -# implies ${TESTS_ARE_ENABLED} -# see tests/CMakeLists.txt for implementation detail. -option(ENABLE_TESTS "Provide unit_test_dbms target with Google.test unit tests" OFF) -``` - -#### If the option's state could produce unwanted (or unusual) result, explicitly warn the user. - -Suppose you have an option that may strip debug symbols from the ClickHouse part. -This can speed up the linking process, but produces a binary that cannot be debugged. -In that case, prefer explicitly raising a warning telling the developer that he may be doing something wrong. -Also, such options should be disabled if applies. - -Bad: - -``` -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions. - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -Better: - -``` -# Provides faster linking and lower binary size. -# Tradeoff is the inability to debug some source files with e.g. gdb -# (empty stack frames and no local variables)." -option(STRIP_DEBUG_SYMBOLS_FUNCTIONS - "Do not generate debugger info for ClickHouse functions." - ${STRIP_DSF_DEFAULT}) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") - target_compile_options(clickhouse_functions PRIVATE "-g0") -endif() -``` - -#### In the option's description, explain WHAT the option does rather than WHY it does something. -The WHY explanation should be placed in the comment. You may find that the option's name is self-descriptive. - -Bad: - -``` -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better: - -``` -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -#### Don't assume other developers know as much as you do. -In ClickHouse, there are many tools used that an ordinary developer may not know. If you are in doubt, give a link to -the tool's docs. It won't take much of your time. - -Bad: - -``` -option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) -``` - -Better (combined with the above hint): - -``` -# https://clang.llvm.org/docs/ThinLTO.html -# Only applicable for clang. -# Turned off when building with tests or sanitizers. -option(ENABLE_THINLTO "Clang-specific link time optimisation" ON). -``` - -Other example, bad: - -``` -option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) -``` - -Better: - -``` -# https://github.com/include-what-you-use/include-what-you-use -option (USE_INCLUDE_WHAT_YOU_USE "Reduce unneeded #include s (external tool)" OFF) -``` - -#### Prefer consistent default values. -CMake allows you to pass a plethora of values representing boolean true/false, e.g. 1, ON, YES, .... - -Prefer the ON/OFF values, if possible. - From 8195aa768b1f70bcd09b504017289a3bd023bd61 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 1 Jul 2022 14:35:51 +0200 Subject: [PATCH 359/408] Move checking if parent profile is allowed to UsersConfigAccessStorage. --- src/Access/SettingsProfileElement.h | 3 -- src/Access/SettingsProfilesCache.cpp | 10 +------ src/Access/UsersConfigAccessStorage.cpp | 40 ++++++++++++++++++------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index e4be0821407..4343f4272d9 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -19,9 +19,6 @@ class AccessControl; struct SettingsProfileElement { std::optional parent_profile; - /// parent_profile_xml_name is set only when users are configured via XML - /// and is used for user-friendly error messages. - std::optional parent_profile_xml_name; String setting_name; Field value; diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index 959cbb01ea2..2a3dedbbd7a 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -171,15 +171,7 @@ void SettingsProfilesCache::substituteProfiles( auto profile_it = all_profiles.find(profile_id); if (profile_it == all_profiles.end()) - { - /// If the textual profile name is set, then users are configured via XML. - /// For these users we want to throw an exception when their profile can't - /// be found. Otherwise, these users are super admins. - if (element.parent_profile_xml_name) - throw Exception(ErrorCodes::THERE_IS_NO_PROFILE, "There is no profile '{}' in configuration file", *element.parent_profile_xml_name); - else - continue; - } + continue; const auto & profile = profile_it->second; const auto & profile_elements = profile->elements; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 6a2995f2579..d06befe0801 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -26,6 +26,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int UNKNOWN_ADDRESS_PATTERN_TYPE; + extern const int THERE_IS_NO_PROFILE; extern const int NOT_IMPLEMENTED; } @@ -47,7 +48,7 @@ namespace UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getName()); } - UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name, bool allow_no_password, bool allow_plaintext_password) + UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name, const std::unordered_set & allowed_profile_ids, bool allow_no_password, bool allow_plaintext_password) { auto user = std::make_shared(); user->setName(user_name); @@ -140,9 +141,11 @@ namespace if (config.has(profile_name_config)) { auto profile_name = config.getString(profile_name_config); + auto profile_id = generateID(AccessEntityType::SETTINGS_PROFILE, profile_name); + if (!allowed_profile_ids.contains(profile_id)) + throw Exception(ErrorCodes::THERE_IS_NO_PROFILE, "Profile {} was not found", profile_name); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(AccessEntityType::SETTINGS_PROFILE, profile_name); - profile_element.parent_profile_xml_name = profile_name; + profile_element.parent_profile = profile_id; user->settings.push_back(std::move(profile_element)); } @@ -232,7 +235,7 @@ namespace } - std::vector parseUsers(const Poco::Util::AbstractConfiguration & config, bool allow_no_password, bool allow_plaintext_password) + std::vector parseUsers(const Poco::Util::AbstractConfiguration & config, const std::unordered_set & allowed_profile_ids, bool allow_no_password, bool allow_plaintext_password) { Poco::Util::AbstractConfiguration::Keys user_names; config.keys("users", user_names); @@ -243,7 +246,7 @@ namespace { try { - users.push_back(parseUser(config, user_name, allow_no_password, allow_plaintext_password)); + users.push_back(parseUser(config, user_name, allowed_profile_ids, allow_no_password, allow_plaintext_password)); } catch (Exception & e) { @@ -458,6 +461,7 @@ namespace std::shared_ptr parseSettingsProfile( const Poco::Util::AbstractConfiguration & config, const String & profile_name, + const std::unordered_set & allowed_parent_profile_ids, const AccessControl & access_control) { auto profile = std::make_shared(); @@ -472,9 +476,11 @@ namespace if (key == "profile" || key.starts_with("profile[")) { String parent_profile_name = config.getString(profile_config + "." + key); + auto parent_profile_id = generateID(AccessEntityType::SETTINGS_PROFILE, parent_profile_name); + if (!allowed_parent_profile_ids.contains(parent_profile_id)) + throw Exception(ErrorCodes::THERE_IS_NO_PROFILE, "Parent profile '{}' was not found", parent_profile_name); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(AccessEntityType::SETTINGS_PROFILE, parent_profile_name); - profile_element.parent_profile_xml_name = parent_profile_name; + profile_element.parent_profile = parent_profile_id; profile->elements.emplace_back(std::move(profile_element)); continue; } @@ -500,6 +506,7 @@ namespace std::vector parseSettingsProfiles( const Poco::Util::AbstractConfiguration & config, + const std::unordered_set & allowed_parent_profile_ids, const AccessControl & access_control) { Poco::Util::AbstractConfiguration::Keys profile_names; @@ -512,7 +519,7 @@ namespace { try { - profiles.push_back(parseSettingsProfile(config, profile_name, access_control)); + profiles.push_back(parseSettingsProfile(config, profile_name, allowed_parent_profile_ids, access_control)); } catch (Exception & e) { @@ -523,6 +530,17 @@ namespace return profiles; } + + + std::unordered_set getAllowedSettingsProfileIDs(const Poco::Util::AbstractConfiguration & config) + { + Poco::Util::AbstractConfiguration::Keys profile_names; + config.keys("profiles", profile_names); + std::unordered_set ids; + for (const auto & profile_name : profile_names) + ids.emplace(generateID(AccessEntityType::SETTINGS_PROFILE, profile_name)); + return ids; + } } UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_, AccessControl & access_control_) @@ -568,16 +586,18 @@ void UsersConfigAccessStorage::parseFromConfig(const Poco::Util::AbstractConfigu { try { + auto allowed_profile_ids = getAllowedSettingsProfileIDs(config); bool no_password_allowed = access_control.isNoPasswordAllowed(); bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); + std::vector> all_entities; - for (const auto & entity : parseUsers(config, no_password_allowed, plaintext_password_allowed)) + for (const auto & entity : parseUsers(config, allowed_profile_ids, no_password_allowed, plaintext_password_allowed)) all_entities.emplace_back(generateID(*entity), entity); for (const auto & entity : parseQuotas(config)) all_entities.emplace_back(generateID(*entity), entity); for (const auto & entity : parseRowPolicies(config, access_control.isEnabledUsersWithoutRowPoliciesCanReadRows())) all_entities.emplace_back(generateID(*entity), entity); - for (const auto & entity : parseSettingsProfiles(config, access_control)) + for (const auto & entity : parseSettingsProfiles(config, allowed_profile_ids, access_control)) all_entities.emplace_back(generateID(*entity), entity); memory_storage.setAll(all_entities); } From 57284e6a9d64b8e3d387f7d1bdfd8e4498879eae Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 15:16:32 +0200 Subject: [PATCH 360/408] Fix possible deadlocks with MergeTreeData::Transaction --- src/Storages/MergeTree/MergeTreeSink.cpp | 7 +++---- .../MergeTree/ReplicatedMergeTreeSink.cpp | 2 ++ src/Storages/StorageMergeTree.cpp | 17 ++++++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 2f860b34fd5..7021d1ab51f 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -139,6 +139,9 @@ void MergeTreeSink::finishDelayedChunk() bool added = false; + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in desctructor and deadlock is possible. + MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); { auto lock = storage.lockParts(); storage.fillNewPartName(part, lock); @@ -155,19 +158,15 @@ void MergeTreeSink::finishDelayedChunk() } else { - MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); added = storage.renameTempPartAndAdd(part, transaction, lock); transaction.commit(&lock); - } } else { - MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); added = storage.renameTempPartAndAdd(part, transaction, lock); transaction.commit(&lock); } - } /// Part can be deduplicated, so increment counters and add to part log only if it's really added diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 07e21def184..473dad075b9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -471,6 +471,8 @@ void ReplicatedMergeTreeSink::commitPart( /// Information about the part. storage.getCommitPartOps(ops, part, block_id_path); + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in desctructor and deadlock is possible. MergeTreeData::Transaction transaction(storage, NO_TRANSACTION_RAW); /// If you can not add a part to ZK, we'll remove it back from the working set. bool renamed = false; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5fe7214194a..f6e62656c83 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1540,9 +1540,11 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition( loaded_parts[i]->storeVersionMetadata(); String old_name = renamed_parts.old_and_new_names[i].old_name; + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in desctructor and deadlock is possible. + MergeTreeData::Transaction transaction(*this, local_context->getCurrentTransaction().get()); { auto lock = lockParts(); - MergeTreeData::Transaction transaction(*this, local_context->getCurrentTransaction().get()); fillNewPartName(loaded_parts[i], lock); renameTempPartAndAdd(loaded_parts[i], transaction, lock); transaction.commit(&lock); @@ -1797,13 +1799,18 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) { + for (auto part : parts) { - auto lock = lockParts(); + /// It's important to create it outside of lock scope because + /// otherwise it can lock parts in desctructor and deadlock is possible. MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); - fillNewPartName(part, lock); - renameTempPartAndAdd(part, transaction, lock); - transaction.commit(&lock); + { + auto lock = lockParts(); + fillNewPartName(part, lock); + renameTempPartAndAdd(part, transaction, lock); + transaction.commit(&lock); + } } } From 0d7298f3a9c5410ca3f21ff40e2da8a54a714c37 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 15:26:27 +0200 Subject: [PATCH 361/408] Fix typos --- src/Storages/MergeTree/MergeTreeSink.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- src/Storages/StorageMergeTree.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 7021d1ab51f..c63c3e3bf94 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -140,7 +140,7 @@ void MergeTreeSink::finishDelayedChunk() bool added = false; /// It's important to create it outside of lock scope because - /// otherwise it can lock parts in desctructor and deadlock is possible. + /// otherwise it can lock parts in destructor and deadlock is possible. MergeTreeData::Transaction transaction(storage, context->getCurrentTransaction().get()); { auto lock = storage.lockParts(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 473dad075b9..27e54c457d8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -472,7 +472,7 @@ void ReplicatedMergeTreeSink::commitPart( storage.getCommitPartOps(ops, part, block_id_path); /// It's important to create it outside of lock scope because - /// otherwise it can lock parts in desctructor and deadlock is possible. + /// otherwise it can lock parts in destructor and deadlock is possible. MergeTreeData::Transaction transaction(storage, NO_TRANSACTION_RAW); /// If you can not add a part to ZK, we'll remove it back from the working set. bool renamed = false; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index f6e62656c83..a01dcafd078 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1541,7 +1541,7 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition( String old_name = renamed_parts.old_and_new_names[i].old_name; /// It's important to create it outside of lock scope because - /// otherwise it can lock parts in desctructor and deadlock is possible. + /// otherwise it can lock parts in destructor and deadlock is possible. MergeTreeData::Transaction transaction(*this, local_context->getCurrentTransaction().get()); { auto lock = lockParts(); @@ -1803,7 +1803,7 @@ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) for (auto part : parts) { /// It's important to create it outside of lock scope because - /// otherwise it can lock parts in desctructor and deadlock is possible. + /// otherwise it can lock parts in destructor and deadlock is possible. MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); { auto lock = lockParts(); From b8fa26f94f2a94067f21c1cfb98a98102173d24c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 1 Jul 2022 15:06:56 +0200 Subject: [PATCH 362/408] Fix wrong self.changed_files type --- tests/ci/pr_info.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index e206b8ed7b4..20487b667aa 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -2,6 +2,7 @@ import json import logging import os +from typing import Set from unidiff import PatchSet # type: ignore @@ -81,7 +82,7 @@ class PRInfo: else: github_event = PRInfo.default_event.copy() self.event = github_event - self.changed_files = set() + self.changed_files = set() # type: Set[str] self.body = "" ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): @@ -217,11 +218,11 @@ class PRInfo: diff = response.json() if "files" in diff: - self.changed_files = [f["filename"] for f in diff["files"]] + self.changed_files = {f["filename"] for f in diff["files"]} else: diff_object = PatchSet(response.text) self.changed_files = {f.path for f in diff_object} - print("Fetched info about %d changed files" % len(self.changed_files)) + print(f"Fetched info about {len(self.changed_files)} changed files") def get_dict(self): return { From 6a67be3195a68e175a1bdb854025f18e70c26329 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 1 Jul 2022 15:30:21 +0200 Subject: [PATCH 363/408] Get a proper set of changed files for backport PRs --- tests/ci/pr_info.py | 46 +++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 20487b667aa..4c17efa88cd 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -84,6 +84,7 @@ class PRInfo: self.event = github_event self.changed_files = set() # type: Set[str] self.body = "" + self.diff_urls = [] ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] @@ -149,7 +150,7 @@ class PRInfo: response_json = user_orgs_response.json() self.user_orgs = set(org["id"] for org in response_json) - self.diff_url = github_event["pull_request"]["diff_url"] + self.diff_urls.append(github_event["pull_request"]["diff_url"]) elif "commits" in github_event: self.sha = github_event["after"] pull_request = get_pr_for_commit(self.sha, github_event["ref"]) @@ -166,7 +167,7 @@ class PRInfo: self.base_name = self.repo_full_name self.head_ref = ref self.head_name = self.repo_full_name - self.diff_url = ( + self.diff_urls.append( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/" f"compare/{github_event['before']}...{self.sha}" ) @@ -180,12 +181,20 @@ class PRInfo: self.head_name = pull_request["head"]["repo"]["full_name"] self.pr_html_url = pull_request["html_url"] if "pr-backport" in self.labels: - self.diff_url = ( + # head1...head2 gives changes in head2 since merge base + # Thag's why we need {self.head_ref}...master to get + # files changed in upstream AND master...{self.head_ref} + # to get files, changed in current HEAD + self.diff_urls.append( f"https://github.com/{GITHUB_REPOSITORY}/" f"compare/master...{self.head_ref}.diff" ) + self.diff_urls.append( + f"https://github.com/{GITHUB_REPOSITORY}/" + f"compare/{self.head_ref}...master.diff" + ) else: - self.diff_url = pull_request["diff_url"] + self.diff_urls.append(pull_request["diff_url"]) else: print("event.json does not match pull_request or push:") print(json.dumps(github_event, sort_keys=True, indent=4)) @@ -206,22 +215,23 @@ class PRInfo: self.fetch_changed_files() def fetch_changed_files(self): - if not getattr(self, "diff_url", False): - raise TypeError("The event does not have diff URL") + if not getattr(self, "diff_urls", False): + raise TypeError("The event does not have diff URLs") - response = get_with_retries( - self.diff_url, - sleep=RETRY_SLEEP, - ) - response.raise_for_status() - if "commits" in self.event and self.number == 0: - diff = response.json() + for diff_url in self.diff_urls: + response = get_with_retries( + diff_url, + sleep=RETRY_SLEEP, + ) + response.raise_for_status() + if "commits" in self.event and self.number == 0: + diff = response.json() - if "files" in diff: - self.changed_files = {f["filename"] for f in diff["files"]} - else: - diff_object = PatchSet(response.text) - self.changed_files = {f.path for f in diff_object} + if "files" in diff: + self.changed_files = {f["filename"] for f in diff["files"]} + else: + diff_object = PatchSet(response.text) + self.changed_files.update({f.path for f in diff_object}) print(f"Fetched info about {len(self.changed_files)} changed files") def get_dict(self): From 919baf331e9a4e7f5ffefcf55c1805552edbc2b6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 15:57:24 +0200 Subject: [PATCH 364/408] Review fixes --- src/Coordination/KeeperDispatcher.cpp | 15 +------------- src/Coordination/KeeperServer.cpp | 20 +++++++++++++++++++ src/Coordination/KeeperServer.h | 3 +++ .../test_keeper_mntr_pressure/test.py | 7 ++++++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 623bc43f85e..213c924af6b 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -651,13 +651,7 @@ uint64_t KeeperDispatcher::getSnapDirSize() const Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const { - Keeper4LWInfo result; - result.is_follower = isFollower(); - result.is_standalone = !result.is_follower && server->getFollowerCount() == 0; - result.is_leader = isLeader(); - result.is_observer = isObserver(); - - result.has_leader = hasLeader(); + Keeper4LWInfo result = server->getPartiallyFilled4LWInfo(); { std::lock_guard lock(push_request_mutex); result.outstanding_requests_count = requests_queue->size(); @@ -666,13 +660,6 @@ Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const std::lock_guard lock(session_to_response_callback_mutex); result.alive_connections_count = session_to_response_callback.size(); } - if (result.is_leader) - { - result.follower_count = server->getFollowerCount(); - result.synced_follower_count = server->getSyncedFollowerCount(); - } - result.total_nodes_count = server->getKeeperStateMachine()->getNodesCount(); - result.last_zxid = server->getKeeperStateMachine()->getLastProcessedZxid(); return result; } diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index eba2e78d46a..9d556e6b4ad 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -804,4 +804,24 @@ bool KeeperServer::waitConfigurationUpdate(const ConfigUpdateAction & task) return true; } +Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const +{ + Keeper4LWInfo result; + result.is_leader = raft_instance->is_leader(); + + auto srv_config = state_manager->get_srv_config(); + result.is_observer = srv_config->is_learner(); + + result.is_follower = !result.is_leader && !result.is_observer; + result.has_leader = result.is_leader || isLeaderAlive(); + if (result.is_leader) + { + result.follower_count = getFollowerCount(); + result.synced_follower_count = getSyncedFollowerCount(); + } + result.total_nodes_count = getKeeperStateMachine()->getNodesCount(); + result.last_zxid = getKeeperStateMachine()->getLastProcessedZxid(); + return result; +} + } diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index 8c21cf47d94..f6524ce97a1 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { @@ -95,6 +96,8 @@ public: bool isLeaderAlive() const; + Keeper4LWInfo getPartiallyFilled4LWInfo() const; + /// @return follower count if node is not leader return 0 uint64_t getFollowerCount() const; diff --git a/tests/integration/test_keeper_mntr_pressure/test.py b/tests/integration/test_keeper_mntr_pressure/test.py index 59ea64d0561..6aa88b92aaa 100644 --- a/tests/integration/test_keeper_mntr_pressure/test.py +++ b/tests/integration/test_keeper_mntr_pressure/test.py @@ -23,6 +23,7 @@ node3 = cluster.add_instance( "node3", main_configs=["config/enable_keeper3.xml"], stay_alive=True ) +NOT_SERVING_REQUESTS_ERROR_MSG = "This instance is not currently serving requests" @pytest.fixture(scope="module") def started_cluster(): @@ -63,7 +64,7 @@ def send_4lw_cmd(node_name, cmd="ruok"): def test_aggressive_mntr(started_cluster): def go_mntr(node_name): - for i in range(100000): + for _ in range(100000): print(node_name, send_4lw_cmd(node_name, "mntr")) node1_thread = threading.Thread(target=lambda: go_mntr(node1.name)) @@ -75,6 +76,10 @@ def test_aggressive_mntr(started_cluster): node2.stop_clickhouse() node3.stop_clickhouse() + + while send_4lw_cmd(node1.name, "mntr") != NOT_SERVING_REQUESTS_ERROR_MSG: + time.sleep(0.2) + node1.stop_clickhouse() starters = [] for node in [node1, node2, node3]: From f47f761cc494e8453bb6ff76ef3ac248c2d9c98a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 16:04:20 +0200 Subject: [PATCH 365/408] Update src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp Co-authored-by: Dmitry Novik --- src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp index 2c3a7454d4b..2f919577c15 100644 --- a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp +++ b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp @@ -43,7 +43,7 @@ void registerDiskHDFS(DiskFactory & factory) auto metadata_storage = std::make_shared(metadata_disk, uri); uint64_t copy_thread_pool_size = config.getUInt(config_prefix + ".thread_pool_size", 16); - std::shared_ptr disk_result = std::make_shared( + auto disk_result = std::make_shared( name, uri, "DiskHDFS", From 8ba3d405de76ca715f93bb0dd6552c26cd07b803 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 1 Jul 2022 16:05:19 +0200 Subject: [PATCH 366/408] impl --- src/Processors/Transforms/FillingTransform.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index a41b5660e0d..b63e1dda084 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -215,6 +215,13 @@ IProcessor::Status FillingTransform::prepare() if (first || filling_row < next_row) { + /// Output if has data. + if (has_output) + { + output.pushData(std::move(output_data)); + has_output = false; + } + generate_suffix = true; return Status::Ready; } From 726aec1f51c48f5d5ad3c2f2f9251b8ded446b3a Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 1 Jul 2022 16:08:26 +0200 Subject: [PATCH 367/408] rm crutch in test --- tests/queries/0_stateless/02112_with_fill_interval.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/queries/0_stateless/02112_with_fill_interval.sql b/tests/queries/0_stateless/02112_with_fill_interval.sql index 16773780515..d2416f9a84b 100644 --- a/tests/queries/0_stateless/02112_with_fill_interval.sql +++ b/tests/queries/0_stateless/02112_with_fill_interval.sql @@ -1,5 +1,3 @@ -SET max_threads = 1; - DROP TABLE IF EXISTS with_fill_date; CREATE TABLE with_fill_date (d Date, d32 Date32) ENGINE = Memory; From 7c53c8bd2cc966d6b2ed6226ba261e64145b5d4c Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 16:10:06 +0200 Subject: [PATCH 368/408] Black --- tests/integration/test_keeper_mntr_pressure/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_keeper_mntr_pressure/test.py b/tests/integration/test_keeper_mntr_pressure/test.py index 6aa88b92aaa..471767210d6 100644 --- a/tests/integration/test_keeper_mntr_pressure/test.py +++ b/tests/integration/test_keeper_mntr_pressure/test.py @@ -25,6 +25,7 @@ node3 = cluster.add_instance( NOT_SERVING_REQUESTS_ERROR_MSG = "This instance is not currently serving requests" + @pytest.fixture(scope="module") def started_cluster(): try: From 4550acdc359d4981299146900b9c98bbf9b58baf Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 1 Jul 2022 16:23:03 +0200 Subject: [PATCH 369/408] Update src/Common/Epoll.cpp Co-authored-by: Azat Khuzhin --- src/Common/Epoll.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index e5c0b668150..a0cee01b425 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -72,7 +72,7 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR); if (errno == EINTR) - LOG_DEBUG(&Poco::Logger::get("Epoll"), "EINTR"); + LOG_TEST(&Poco::Logger::get("Epoll"), "EINTR"); } while (ready_size <= 0 && (ready_size != 0 || blocking)); From fe0dad5429669f969360d4f36e395b2a853da715 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 1 Jul 2022 16:23:20 +0200 Subject: [PATCH 370/408] Update src/Common/TimerDescriptor.cpp --- src/Common/TimerDescriptor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp index 53e7cefc358..1301ebce0ba 100644 --- a/src/Common/TimerDescriptor.cpp +++ b/src/Common/TimerDescriptor.cpp @@ -73,7 +73,7 @@ void TimerDescriptor::drain() const if (errno != EINTR) throwFromErrno("Cannot drain timer_fd", ErrorCodes::CANNOT_READ_FROM_SOCKET); else - LOG_DEBUG(&Poco::Logger::get("TimerDescriptor"), "EINTR"); + LOG_TEST(&Poco::Logger::get("TimerDescriptor"), "EINTR"); } } } From 27bd355525661e5f3289d84faeda6d32d02239fa Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 1 Jul 2022 16:45:00 +0200 Subject: [PATCH 371/408] Use cached image from the release branch for backport PRs --- tests/ci/docker_images_check.py | 35 ++++++++++++++++++++++++------- tests/ci/docker_test.py | 37 ++++++++++++++++++++++++++------- tests/ci/pr_info.py | 5 +++++ 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index a0d0e124f6d..2e181f678dd 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -210,6 +210,7 @@ def build_and_push_dummy_image( def build_and_push_one_image( image: DockerImage, version_string: str, + additional_cache: str, push: bool, child: bool, ) -> Tuple[bool, str]: @@ -232,6 +233,16 @@ def build_and_push_one_image( if child: from_tag_arg = f"--build-arg FROM_TAG={version_string} " + cache_from = ( + f"--cache-from type=registry,ref={image.repo}:{version_string} " + f"--cache-from type=registry,ref={image.repo}:latest" + ) + if additional_cache: + cache_from = ( + f"{cache_from} " + f"--cache-from type=registry,ref={image.repo}:{additional_cache}" + ) + with open(build_log, "wb") as bl: cmd = ( "docker buildx build --builder default " @@ -240,8 +251,7 @@ def build_and_push_one_image( # A hack to invalidate cache, grep for it in docker/ dir f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " - f"--cache-from type=registry,ref={image.repo}:{version_string} " - f"--cache-from type=registry,ref={image.repo}:latest " + f"{cache_from} " f"--cache-to type=inline,mode=max " f"{push_arg}" f"--progress plain {image.full_path}" @@ -260,6 +270,7 @@ def build_and_push_one_image( def process_single_image( image: DockerImage, versions: List[str], + additional_cache, push: bool, child: bool, ) -> List[Tuple[str, str, str]]: @@ -267,7 +278,9 @@ def process_single_image( result = [] for ver in versions: for i in range(5): - success, build_log = build_and_push_one_image(image, ver, push, child) + success, build_log = build_and_push_one_image( + image, ver, additional_cache, push, child + ) if success: result.append((image.repo + ":" + ver, build_log, "OK")) break @@ -284,17 +297,23 @@ def process_single_image( def process_image_with_parents( - image: DockerImage, versions: List[str], push: bool, child: bool = False + image: DockerImage, + versions: List[str], + additional_cache: str, + push: bool, + child: bool = False, ) -> List[Tuple[str, str, str]]: result = [] # type: List[Tuple[str,str,str]] if image.built: return result if image.parent is not None: - result += process_image_with_parents(image.parent, versions, push, False) + result += process_image_with_parents( + image.parent, versions, additional_cache, push, False + ) child = True - result += process_single_image(image, versions, push, child) + result += process_single_image(image, versions, additional_cache, push, child) return result @@ -423,8 +442,10 @@ def main(): result_images = {} images_processing_result = [] for image in changed_images: + # If we are in backport PR, then pr_info.release_pr is defined + # We use it as tag to reduce rebuilding time images_processing_result += process_image_with_parents( - image, image_versions, args.push + image, image_versions, pr_info.release_pr, args.push ) result_images[image.repo] = result_version diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 550d495939c..32df6d5f1d0 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -117,7 +117,7 @@ class TestDockerImageCheck(unittest.TestCase): mock_popen.return_value.__enter__.return_value.wait.return_value = 0 image = di.DockerImage("path", "name", False, gh_repo_path="") - result, _ = di.build_and_push_one_image(image, "version", True, True) + result, _ = di.build_and_push_one_image(image, "version", "", True, True) mock_open.assert_called_once() mock_popen.assert_called_once() mock_machine.assert_not_called() @@ -136,7 +136,7 @@ class TestDockerImageCheck(unittest.TestCase): mock_machine.reset_mock() mock_popen.return_value.__enter__.return_value.wait.return_value = 0 - result, _ = di.build_and_push_one_image(image, "version2", False, True) + result, _ = di.build_and_push_one_image(image, "version2", "", False, True) mock_open.assert_called_once() mock_popen.assert_called_once() mock_machine.assert_not_called() @@ -155,7 +155,7 @@ class TestDockerImageCheck(unittest.TestCase): mock_popen.reset_mock() mock_machine.reset_mock() mock_popen.return_value.__enter__.return_value.wait.return_value = 1 - result, _ = di.build_and_push_one_image(image, "version2", False, False) + result, _ = di.build_and_push_one_image(image, "version2", "", False, False) mock_open.assert_called_once() mock_popen.assert_called_once() mock_machine.assert_not_called() @@ -169,13 +169,36 @@ class TestDockerImageCheck(unittest.TestCase): ) self.assertFalse(result) + mock_open.reset_mock() + mock_popen.reset_mock() + mock_machine.reset_mock() + mock_popen.return_value.__enter__.return_value.wait.return_value = 1 + result, _ = di.build_and_push_one_image( + image, "version2", "cached-version", False, False + ) + mock_open.assert_called_once() + mock_popen.assert_called_once() + mock_machine.assert_not_called() + self.assertIn( + f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " + "--tag name:version2 --cache-from type=registry,ref=name:version2 " + "--cache-from type=registry,ref=name:latest " + "--cache-from type=registry,ref=name:cached-version " + "--cache-to type=inline,mode=max --progress plain path", + mock_popen.call_args.args, + ) + self.assertFalse(result) + mock_open.reset_mock() mock_popen.reset_mock() mock_machine.reset_mock() only_amd64_image = di.DockerImage("path", "name", True) mock_popen.return_value.__enter__.return_value.wait.return_value = 0 - result, _ = di.build_and_push_one_image(only_amd64_image, "version", True, True) + result, _ = di.build_and_push_one_image( + only_amd64_image, "version", "", True, True + ) mock_open.assert_called_once() mock_popen.assert_called_once() mock_machine.assert_called_once() @@ -186,7 +209,7 @@ class TestDockerImageCheck(unittest.TestCase): ) self.assertTrue(result) result, _ = di.build_and_push_one_image( - only_amd64_image, "version", False, True + only_amd64_image, "version", "", False, True ) self.assertIn( "docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; ", @@ -195,7 +218,7 @@ class TestDockerImageCheck(unittest.TestCase): @patch("docker_images_check.build_and_push_one_image") def test_process_image_with_parents(self, mock_build): - mock_build.side_effect = lambda w, x, y, z: (True, f"{w.repo}_{x}.log") + mock_build.side_effect = lambda v, w, x, y, z: (True, f"{v.repo}_{w}.log") im1 = di.DockerImage("path1", "repo1", False) im2 = di.DockerImage("path2", "repo2", False, im1) im3 = di.DockerImage("path3", "repo3", False, im2) @@ -203,7 +226,7 @@ class TestDockerImageCheck(unittest.TestCase): # We use list to have determined order of image builgings images = [im4, im1, im3, im2, im1] results = [ - di.process_image_with_parents(im, ["v1", "v2", "latest"], True) + di.process_image_with_parents(im, ["v1", "v2", "latest"], "", True) for im in images ] diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 4c17efa88cd..a06c5a75b0f 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -85,6 +85,7 @@ class PRInfo: self.changed_files = set() # type: Set[str] self.body = "" self.diff_urls = [] + self.release_pr = "" ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] @@ -193,6 +194,10 @@ class PRInfo: f"https://github.com/{GITHUB_REPOSITORY}/" f"compare/{self.head_ref}...master.diff" ) + # Get release PR number. + self.release_pr = get_pr_for_commit(self.base_ref, self.base_ref)[ + "number" + ] else: self.diff_urls.append(pull_request["diff_url"]) else: From 65956975e2ca571adfcebc090b56156beb63f628 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 1 Jul 2022 17:18:38 +0200 Subject: [PATCH 372/408] Update registerDiskHDFS.cpp --- src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp index 2f919577c15..21641aca392 100644 --- a/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp +++ b/src/Disks/ObjectStorages/HDFS/registerDiskHDFS.cpp @@ -43,7 +43,7 @@ void registerDiskHDFS(DiskFactory & factory) auto metadata_storage = std::make_shared(metadata_disk, uri); uint64_t copy_thread_pool_size = config.getUInt(config_prefix + ".thread_pool_size", 16); - auto disk_result = std::make_shared( + DiskPtr disk_result = std::make_shared( name, uri, "DiskHDFS", From b238ed409385d8534d73f2ef9062c4a4f7609ffe Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 1 Jul 2022 13:47:49 +0200 Subject: [PATCH 373/408] fix stress tests --- docker/test/stress/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 0b517fb4af8..fff720965b6 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -119,9 +119,10 @@ function start() counter=0 until clickhouse-client --query "SELECT 1" do - if [ "$counter" -gt ${1:-240} ] + if [ "$counter" -gt ${1:-120} ] then echo "Cannot start clickhouse-server" + echo -e "Cannot start clickhouse-server\tFAIL" >> /test_output/test_results.tsv cat /var/log/clickhouse-server/stdout.log tail -n1000 /var/log/clickhouse-server/stderr.log tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | grep -F -v -e ' RaftInstance:' -e ' RaftInstance' | tail -n1000 @@ -295,6 +296,10 @@ then # Start server from previous release configure + + # Avoid "Setting allow_deprecated_database_ordinary is neither a builtin setting..." + rm -f /etc/clickhouse-server/users.d/database_ordinary.xml ||: + start clickhouse-client --query="SELECT 'Server version: ', version()" From 53f47127e968518418e91ccbdd48c86c7dff6f42 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 1 Jul 2022 16:46:58 +0800 Subject: [PATCH 374/408] Fix only_merge header --- src/Interpreters/Aggregator.cpp | 10 +++++++++- .../01710_minmax_count_projection.reference | 2 ++ .../0_stateless/01710_minmax_count_projection.sql | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 511e5c9e031..a99ecee43bf 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -374,7 +374,15 @@ Block Aggregator::Params::getHeader( if (only_merge) { - res = header.cloneEmpty(); + NameSet needed_columns(keys.begin(), keys.end()); + for (const auto & aggregate : aggregates) + needed_columns.emplace(aggregate.column_name); + + for (const auto & column : header) + { + if (needed_columns.contains(column.name)) + res.insert(column.cloneEmpty()); + } if (final) { diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index 3e24cac4f6d..bbcec98fb74 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -2,6 +2,7 @@ 0 9998 5000 1 9999 5000 0 9998 5000 +0 9998 5000 1 0 9998 5000 0 @@ -15,6 +16,7 @@ 1 1 1 +2021-10-25 10:00:00 3 2021-10-27 10:00:00 3 \N 2021-10-27 10:00:00 3 0 2021-10-24 10:00:00 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index 4aea3cfd29e..7ceff6a2662 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -10,6 +10,7 @@ set max_rows_to_read = 2, allow_experimental_projection_optimization = 1; select min(i), max(i), count() from d; select min(i), max(i), count() from d group by _partition_id order by _partition_id; select min(i), max(i), count() from d where _partition_value.1 = 0 group by _partition_id order by _partition_id; +select min(i), max(i), count() from d where moduloLegacy(i, 2) = 0 group by _partition_id order by _partition_id; select min(i), max(i), count() from d where _partition_value.1 = 10 group by _partition_id order by _partition_id; -- fuzz crash @@ -57,6 +58,9 @@ select min(dt), max(dt), count() from d where toDate(dt) >= '2021-10-25'; select min(dt), max(dt), count(toDate(dt) >= '2021-10-25') from d where toDate(dt) >= '2021-10-25'; select count() from d group by toDate(dt); +-- fuzz crash +SELECT min(dt), count(ignore(ignore(ignore(tupleElement(_partition_value, NULL) = NULL), NULL, NULL, NULL), 0, '10485.76', NULL)), max(dt), count(toDate(dt) >= '2021-10-25') FROM d WHERE toDate(dt) >= '2021-10-25'; + -- fuzz crash SELECT pointInEllipses(min(j), NULL), max(dt), count('0.0000000007') FROM d WHERE toDate(dt) >= '2021-10-25'; SELECT min(j) FROM d PREWHERE ceil(j) <= 0; From eaf8397c139326c7ad2ba5ef14fd96989a4dae10 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 1 Jul 2022 19:51:07 +0200 Subject: [PATCH 375/408] Fix style --- src/Common/Epoll.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index a0cee01b425..9b2589f0589 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -70,7 +70,7 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki if (ready_size == -1 && errno != EINTR) throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR); - + if (errno == EINTR) LOG_TEST(&Poco::Logger::get("Epoll"), "EINTR"); } From 800baf5299aa029816821c42a9274b8389f8c879 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 1 Jul 2022 19:54:43 +0200 Subject: [PATCH 376/408] Fix test --- tests/queries/0_stateless/02322_sql_insert_format.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02322_sql_insert_format.sql b/tests/queries/0_stateless/02322_sql_insert_format.sql index adc28c1d01e..34cde1e56b6 100644 --- a/tests/queries/0_stateless/02322_sql_insert_format.sql +++ b/tests/queries/0_stateless/02322_sql_insert_format.sql @@ -6,8 +6,8 @@ select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInse select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_include_column_names=0; select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_use_replace=1; select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test'; -select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test', output_format_sql_quote_names=0; -insert into function file(02322_data.sql, 'SQLInsert') select number as x, number % 3 as y, 'Hello' as z from numbers(5) settings output_format_sql_insert_max_batch_size=2, output_format_sql_quote_names=0, engine_file_truncate_on_insert=1; +select number as x, number % 3 as y, 'Hello' as z from numbers(5) format SQLInsert settings output_format_sql_insert_table_name='test', output_format_sql_insert_quote_names=0; +insert into function file(02322_data.sql, 'SQLInsert') select number as x, number % 3 as y, 'Hello' as z from numbers(5) settings output_format_sql_insert_max_batch_size=2, output_format_sql_insert_quote_names=0, engine_file_truncate_on_insert=1; select * from file(02322_data.sql, 'MySQLDump'); insert into function file(02322_data.sql, 'SQLInsert') select number, number % 3, 'Hello' from numbers(5) settings output_format_sql_insert_max_batch_size=2, engine_file_truncate_on_insert=1; select * from file(02322_data.sql, 'MySQLDump'); From 8177608860275340eebf5dd4f1579c04189964c8 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 1 Jul 2022 22:35:20 +0200 Subject: [PATCH 377/408] SQL create drop index fix formatting --- src/Interpreters/InterpreterCreateIndexQuery.cpp | 3 +-- src/Interpreters/InterpreterDropIndexQuery.cpp | 5 ++--- src/Parsers/ParserCreateIndexQuery.cpp | 10 +++++----- src/Parsers/ParserDropIndexQuery.cpp | 8 ++++---- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index ef19eaa1c42..5117b92efdf 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -38,8 +38,7 @@ BlockIO InterpreterCreateIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !current_context->getClientInfo().is_replicated_database_internal) + if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 2339e0dc68e..70f35a92688 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -1,8 +1,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -36,8 +36,7 @@ BlockIO InterpreterDropIndexQuery::execute() query_ptr->as().setDatabase(table_id.database_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); - if (typeid_cast(database.get()) - && !current_context->getClientInfo().is_replicated_database_internal) + if (typeid_cast(database.get()) && !current_context->getClientInfo().is_replicated_database_internal) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp index af0d9064626..ab31d3f9b7a 100644 --- a/src/Parsers/ParserCreateIndexQuery.cpp +++ b/src/Parsers/ParserCreateIndexQuery.cpp @@ -1,10 +1,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -86,15 +86,15 @@ bool ParserCreateIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect if (!s_on.ignore(pos, expected)) return false; - if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) return false; /// [ON cluster_name] - if (s_on.ignore(pos, expected)) - { + if (s_on.ignore(pos, expected)) + { if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) return false; - } + } if (!parser_create_idx_decl.parse(pos, index_decl, expected)) return false; diff --git a/src/Parsers/ParserDropIndexQuery.cpp b/src/Parsers/ParserDropIndexQuery.cpp index 89ed4f01838..5843d8b94bf 100644 --- a/src/Parsers/ParserDropIndexQuery.cpp +++ b/src/Parsers/ParserDropIndexQuery.cpp @@ -38,17 +38,17 @@ bool ParserDropIndexQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected if (!s_on.ignore(pos, expected)) return false; - if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) + if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table)) return false; /// [ON cluster_name] - if (s_on.ignore(pos, expected)) - { + if (s_on.ignore(pos, expected)) + { if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) return false; query->cluster = std::move(cluster_str); - } + } if (query->index_name) query->children.push_back(query->index_name); From 1d665e9576f4cb6de0d5a79e4e4d622385f48a13 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 1 Jul 2022 21:28:01 +0000 Subject: [PATCH 378/408] Provide sort description for output stream in ReadFromMergeTree step --- .../QueryPlan/ReadFromMergeTree.cpp | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f377709a899..4a1772759bc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -71,6 +71,18 @@ static const PrewhereInfoPtr & getPrewhereInfo(const SelectQueryInfo & query_inf : query_info.prewhere_info; } +static int getSortDirection(const SelectQueryInfo & query_info) +{ + const InputOrderInfoPtr & order_info = query_info.input_order_info + ? query_info.input_order_info + : (query_info.projection ? query_info.projection->input_order_info : nullptr); + + if (!order_info) + return 1; + + return order_info->direction; +} + ReadFromMergeTree::ReadFromMergeTree( MergeTreeData::DataPartsVector parts_, Names real_column_names_, @@ -124,6 +136,22 @@ ReadFromMergeTree::ReadFromMergeTree( /// Add explicit description. setStepDescription(data.getStorageID().getFullNameNotQuoted()); + + { /// build sort description for output stream + SortDescription sort_description; + const Names & sorting_key_columns = storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(); + const Block & header = output_stream->header; + const int sort_direction = getSortDirection(query_info); + for (const auto & column_name : sorting_key_columns) + { + if (std::find_if(header.begin(), header.end(), [&](ColumnWithTypeAndName const & col) { return col.name == column_name; }) + == header.end()) + break; + sort_description.emplace_back(column_name, sort_direction); + } + output_stream->sort_description = std::move(sort_description); + output_stream->sort_mode = DataStream::SortMode::Chunk; + } } Pipe ReadFromMergeTree::readFromPool( From 9ef8ff5a3110c272b220bfdd26a92767a93f2a68 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 1 Jul 2022 22:50:00 +0000 Subject: [PATCH 379/408] Addressing review comments --- src/Processors/QueryPlan/DistinctStep.cpp | 8 +++++--- tests/performance/distinct_in_order.xml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 946af9ca4d6..553732fbcc5 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -82,7 +82,8 @@ DistinctStep::DistinctStep( void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - if (checkColumnsAlreadyDistinct(columns, input_streams.front().distinct_columns)) + const auto & input_stream = input_streams.back(); + if (checkColumnsAlreadyDistinct(columns, input_stream.distinct_columns)) return; if (!pre_distinct) @@ -90,7 +91,6 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil if (optimize_distinct_in_order) { - const auto & input_stream = input_streams.back(); SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns); if (!distinct_sort_desc.empty()) { @@ -109,8 +109,10 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil return; } /// final distinct for sorted stream (sorting inside and among chunks) - if (input_stream.has_single_port) + if (input_stream.sort_mode == DataStream::SortMode::Stream) { + assert(input_stream.has_single_port); + pipeline.addSimpleTransform( [&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr { diff --git a/tests/performance/distinct_in_order.xml b/tests/performance/distinct_in_order.xml index c4d09aa825b..834a6945622 100644 --- a/tests/performance/distinct_in_order.xml +++ b/tests/performance/distinct_in_order.xml @@ -1,7 +1,7 @@ CREATE TABLE distinct_cardinality_high (high UInt64, medium UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, medium) - INSERT INTO distinct_cardinality_high SELECT number % 10000, number % 1000, number % 100 FROM numbers_mt(1e8) + INSERT INTO distinct_cardinality_high SELECT number % 1e6, number % 1e4, number % 1e2 FROM numbers_mt(1e8) SELECT DISTINCT high FROM distinct_cardinality_high FORMAT Null SELECT DISTINCT high, low FROM distinct_cardinality_high FORMAT Null @@ -17,7 +17,7 @@ CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium) - INSERT INTO distinct_cardinality_low SELECT number % 100, number % 1000, number % 10000 FROM numbers_mt(1e8) + INSERT INTO distinct_cardinality_low SELECT number % 1e2, number % 1e4, number % 1e6 FROM numbers_mt(1e8) SELECT DISTINCT low FROM distinct_cardinality_low FORMAT Null SELECT DISTINCT low, medium FROM distinct_cardinality_low FORMAT Null From 16a4b6aa35f5cd0da41949d65b03e812cbdf2785 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 2 Jul 2022 03:38:18 +0200 Subject: [PATCH 380/408] Fix a bug in `rankCorr` function --- tests/queries/0_stateless/02347_rank_corr_nan.reference | 1 + tests/queries/0_stateless/02347_rank_corr_nan.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/02347_rank_corr_nan.reference create mode 100644 tests/queries/0_stateless/02347_rank_corr_nan.sql diff --git a/tests/queries/0_stateless/02347_rank_corr_nan.reference b/tests/queries/0_stateless/02347_rank_corr_nan.reference new file mode 100644 index 00000000000..6db331af725 --- /dev/null +++ b/tests/queries/0_stateless/02347_rank_corr_nan.reference @@ -0,0 +1 @@ +nan diff --git a/tests/queries/0_stateless/02347_rank_corr_nan.sql b/tests/queries/0_stateless/02347_rank_corr_nan.sql new file mode 100644 index 00000000000..0fd755259e6 --- /dev/null +++ b/tests/queries/0_stateless/02347_rank_corr_nan.sql @@ -0,0 +1 @@ +SELECT rankCorr(number, nan) FROM numbers(10); From 668f06b2dbefee3956a24723996422277e82b1e5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 2 Jul 2022 03:40:20 +0200 Subject: [PATCH 381/408] Fix error --- .../AggregateFunctionRankCorrelation.h | 4 ++-- src/AggregateFunctions/StatCommon.h | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionRankCorrelation.h b/src/AggregateFunctions/AggregateFunctionRankCorrelation.h index a9bf8254f35..670dd5948f7 100644 --- a/src/AggregateFunctions/AggregateFunctionRankCorrelation.h +++ b/src/AggregateFunctions/AggregateFunctionRankCorrelation.h @@ -31,8 +31,8 @@ struct RankCorrelationData : public StatisticalSample RanksArray ranks_y; std::tie(ranks_y, std::ignore) = computeRanksAndTieCorrection(this->y); - /// In our case sizes of both samples are equal. - const auto size = this->size_x; + /// Sizes can be non-equal due to skipped NaNs. + const auto size = std::min(this->size_x, this->size_y); /// Count d^2 sum Float64 answer = 0; diff --git a/src/AggregateFunctions/StatCommon.h b/src/AggregateFunctions/StatCommon.h index 29163b63f77..ff824ca11b8 100644 --- a/src/AggregateFunctions/StatCommon.h +++ b/src/AggregateFunctions/StatCommon.h @@ -31,8 +31,8 @@ std::pair computeRanksAndTieCorrection(const Values & value /// Save initial positions, than sort indices according to the values. std::vector indexes(size); std::iota(indexes.begin(), indexes.end(), 0); - ::sort(indexes.begin(), indexes.end(), - [&] (size_t lhs, size_t rhs) { return values[lhs] < values[rhs]; }); + std::sort(indexes.begin(), indexes.end(), + [&] (size_t lhs, size_t rhs) { return values[lhs] < values[rhs]; }); size_t left = 0; Float64 tie_numenator = 0; @@ -74,12 +74,18 @@ struct StatisticalSample void addX(X value, Arena * arena) { + if (isNaN(value)) + return; + ++size_x; x.push_back(value, arena); } void addY(Y value, Arena * arena) { + if (isNaN(value)) + return; + ++size_y; y.push_back(value, arena); } From 9056d80f47c1fd06e7bab50da8b7d23196d0cfe8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 2 Jul 2022 03:43:32 +0200 Subject: [PATCH 382/408] Add example query to hardware benchmark --- benchmark/hardware.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 0c3a1396440..da7b3c81f32 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -217,4 +217,31 @@ TO benchmark; GRANT INSERT ON benchmark_runs TO benchmark; GRANT INSERT ON benchmark_results TO benchmark; +Example query: + +SELECT + cpu_model, + threads, + instance, + k +FROM +( + SELECT + run_id, + exp(avg(log(adjusted_time / best_time))) AS k + FROM + ( + WITH greatest(time, 0.01) AS adjusted_time + SELECT + run_id, + adjusted_time, + min(adjusted_time) OVER (PARTITION BY query_num, try_num) AS best_time + FROM benchmark_results + WHERE try_num > 1 + ) + GROUP BY run_id + ORDER BY k ASC +) AS t +INNER JOIN benchmark_runs USING (run_id) + //// From dde76824ca53f14932d001826d697a46cdcfb6ca Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 2 Jul 2022 16:02:42 +0200 Subject: [PATCH 383/408] Fix ub --- src/Coordination/KeeperServer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 9d556e6b4ad..2a3d17af403 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -814,6 +814,7 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const result.is_follower = !result.is_leader && !result.is_observer; result.has_leader = result.is_leader || isLeaderAlive(); + result.is_standalone = !result.is_follower && getFollowerCount() == 0; if (result.is_leader) { result.follower_count = getFollowerCount(); From e4b81e6de9b5efb8ec8aa079741aaa05ece6719c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jul 2022 12:05:52 +0300 Subject: [PATCH 384/408] Add exp_internal for expect tests This is to catch possible issues like 01565_reconnect_after_client_error in [1]. [1]: https://s3.amazonaws.com/clickhouse-test-reports/38417/e33f236d502fe9f7906392f4ceffaee44e3c4ce4/stateless_tests__thread__actions__[3/3].html Signed-off-by: Azat Khuzhin --- tests/clickhouse-test | 52 ++++++++++++++++--- .../01176_mysql_client_interactive.expect | 5 +- .../01179_insert_values_semicolon.expect | 5 +- .../01180_client_syntax_errors.expect | 5 +- ...ient_interactive_vertical_multiline.expect | 5 +- ...ent_interactive_vertical_singleline.expect | 5 +- ...t_save_history_when_terminated_long.expect | 5 +- ..._autocomplete_word_break_characters.expect | 5 +- .../01520_client_print_query_id.expect | 5 +- .../01565_reconnect_after_client_error.expect | 5 +- ...light_multi_line_comment_regression.expect | 5 +- ...ient_replxx_container_overflow_long.expect | 5 +- ...01933_client_replxx_convert_history.expect | 5 +- .../01945_show_debug_warning.expect | 5 +- .../02003_memory_limit_in_client.expect | 6 ++- .../0_stateless/02047_client_exception.expect | 5 +- .../02049_clickhouse_local_merge_tree.expect | 5 +- .../02105_backslash_letter_commands.expect | 5 +- ...clickhouse_client_with_queries_file.expect | 6 ++- .../02112_delayed_clickhouse_local.expect | 5 +- ..._clickhouse_local_with_queries_file.expect | 6 ++- .../02116_interactive_hello.expect | 5 +- .../02132_client_history_navigation.expect | 5 +- ...160_client_autocomplete_parse_query.expect | 5 +- ...ckhouse_local_interactive_exception.expect | 5 +- utils/check-style/check-style | 6 +++ 26 files changed, 148 insertions(+), 33 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f21d1734029..083eb3830fa 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -499,6 +499,7 @@ class TestCase: database = testcase_args.database os.environ.setdefault("CLICKHOUSE_DATABASE", database) os.environ.setdefault("CLICKHOUSE_TMP", suite_tmp_dir) + testcase_args.test_tmp_dir = suite_tmp_dir else: # If --database is not specified, we will create temporary database with # unique name and we will recreate and drop it for each test @@ -529,6 +530,16 @@ class TestCase: testcase_args.testcase_database = database + # Printed only in case of failures + # + # NOTE: here we use "CLICKHOUSE_TMP" instead of "file_suffix", + # so it is installed in configure_testcase_args() unlike other files + # (stdout_file, stderr_file) in TestCase::__init__(). + # Since using CLICKHOUSE_TMP is easier to use in expect. + testcase_args.debug_log_file = ( + os.path.join(testcase_args.test_tmp_dir, testcase_basename) + ".debuglog" + ) + return testcase_args def cli_random_settings(self) -> str: @@ -699,7 +710,7 @@ class TestCase: return None - def process_result_impl(self, proc, stdout: str, stderr: str, total_time: float): + def process_result_impl(self, proc, stdout: str, stderr: str, debug_log: str, total_time: float): description = "" if proc: @@ -712,6 +723,9 @@ class TestCase: if stderr: description += stderr + if debug_log: + description += "\n" + description += debug_log return TestResult( self.name, TestStatus.FAIL, @@ -727,6 +741,9 @@ class TestCase: if stderr: description += "\n" description += stderr + if debug_log: + description += "\n" + description += debug_log # Stop on fatal errors like segmentation fault. They are sent to client via logs. if " " in stderr: @@ -757,6 +774,9 @@ class TestCase: if stderr: description += "\n{}\n".format("\n".join(stderr.splitlines()[:100])) description += f"\nstdout:\n{stdout}\n" + if debug_log: + description += "\n" + description += debug_log return TestResult( self.name, TestStatus.FAIL, @@ -767,6 +787,9 @@ class TestCase: if "Exception" in stdout: description += "\n{}\n".format("\n".join(stdout.splitlines()[:100])) + if debug_log: + description += "\n" + description += debug_log return TestResult( self.name, TestStatus.FAIL, @@ -813,6 +836,9 @@ class TestCase: universal_newlines=True, ).communicate()[0] description += f"\n{diff}\n" + if debug_log: + description += "\n" + description += debug_log return TestResult( self.name, TestStatus.FAIL, @@ -826,6 +852,9 @@ class TestCase: and total_time > 60 and "long" not in self.tags ): + if debug_log: + description += "\n" + description += debug_log # We're in Flaky Check mode, check the run time as well while we're at it. return TestResult( self.name, @@ -839,6 +868,8 @@ class TestCase: os.remove(self.stdout_file) if os.path.exists(self.stderr_file): os.remove(self.stderr_file) + if os.path.exists(self.testcase_args.debug_log_file): + os.remove(self.testcase_args.debug_log_file) return TestResult(self.name, TestStatus.OK, None, total_time, description) @@ -872,7 +903,7 @@ class TestCase: def run_single_test( self, server_logs_level, client_options - ) -> Tuple[Optional[Popen], str, str, float]: + ) -> Tuple[Optional[Popen], str, str, str, float]: args = self.testcase_args client = args.testcase_client start_time = args.testcase_start_time @@ -922,6 +953,13 @@ class TestCase: ) need_drop_database = not maybe_passed + debug_log = "" + if os.path.exists(self.testcase_args.debug_log_file): + with open(self.testcase_args.debug_log_file, "rb") as stream: + debug_log += self.testcase_args.debug_log_file + ":\n" + debug_log += str(stream.read(), errors="replace", encoding="utf-8") + debug_log += "\n" + if need_drop_database: seconds_left = max( args.timeout - (datetime.now() - start_time).total_seconds(), 20 @@ -941,6 +979,7 @@ class TestCase: None, "", f"Timeout dropping database {database} after test", + debug_log, total_time, ) shutil.rmtree(args.test_tmp_dir) @@ -964,12 +1003,13 @@ class TestCase: if os.path.exists(self.stdout_file): with open(self.stdout_file, "rb") as stdfd: stdout = str(stdfd.read(), errors="replace", encoding="utf-8") + stderr = "" if os.path.exists(self.stderr_file): with open(self.stderr_file, "rb") as stdfd: - stderr = str(stdfd.read(), errors="replace", encoding="utf-8") + stderr += str(stdfd.read(), errors="replace", encoding="utf-8") - return proc, stdout, stderr, total_time + return proc, stdout, stderr, debug_log, total_time def run(self, args, suite, client_options, server_logs_level): try: @@ -994,11 +1034,11 @@ class TestCase: args, self.case_file, suite.suite_tmp_path ) client_options = self.add_random_settings(args, client_options) - proc, stdout, stderr, total_time = self.run_single_test( + proc, stdout, stderr, debug_log, total_time = self.run_single_test( server_logs_level, client_options ) - result = self.process_result_impl(proc, stdout, stderr, total_time) + result = self.process_result_impl(proc, stdout, stderr, debug_log, total_time) result.check_if_need_retry(args, stdout, stderr, self.runs_count) if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings( diff --git a/tests/queries/0_stateless/01176_mysql_client_interactive.expect b/tests/queries/0_stateless/01176_mysql_client_interactive.expect index 5bbc77ccf14..8d23b3bef60 100755 --- a/tests/queries/0_stateless/01176_mysql_client_interactive.expect +++ b/tests/queries/0_stateless/01176_mysql_client_interactive.expect @@ -2,6 +2,10 @@ # Tags: no-fasttest # Tag no-fasttest: requires mysql client +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -13,7 +17,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$MYSQL_CLIENT_BINARY \$MYSQL_CLIENT_OPT" expect -nocase -re "mysql.*> " diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect index bf937c3a6a4..9d35941ae40 100755 --- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect +++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: long +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -11,7 +15,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01180_client_syntax_errors.expect b/tests/queries/0_stateless/01180_client_syntax_errors.expect index 6e4e975988e..da3dfbec6df 100755 --- a/tests/queries/0_stateless/01180_client_syntax_errors.expect +++ b/tests/queries/0_stateless/01180_client_syntax_errors.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index 07bdbcdac76..bab1dd224cf 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 10 match_max 100000 @@ -14,7 +18,6 @@ expect_after { # useful debugging configuration # exp_internal 1 -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect index 085a1140753..83eced841ce 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_singleline.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect index ad5b7625929..06a60ed95a2 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: long +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { # A default timeout action is to do nothing, change it to fail timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect index 9c20b7c517e..fff0dd015e1 100755 --- a/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect +++ b/tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" expect ":) " diff --git a/tests/queries/0_stateless/01520_client_print_query_id.expect b/tests/queries/0_stateless/01520_client_print_query_id.expect index 8b6e0e17a85..0e8f660041d 100755 --- a/tests/queries/0_stateless/01520_client_print_query_id.expect +++ b/tests/queries/0_stateless/01520_client_print_query_id.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect index 819450ffd30..035698f524b 100755 --- a/tests/queries/0_stateless/01565_reconnect_after_client_error.expect +++ b/tests/queries/0_stateless/01565_reconnect_after_client_error.expect @@ -4,6 +4,10 @@ # This is a separate test, because we want to test the interactive mode. # https://github.com/ClickHouse/ClickHouse/issues/19353 +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -15,7 +19,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn" expect "\n:) " diff --git a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect index 022320e2d4b..3d9c633eb44 100755 --- a/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect +++ b/tests/queries/0_stateless/01755_client_highlight_multi_line_comment_regression.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect b/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect index d5ce4c3cbf2..1be56675b33 100755 --- a/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect +++ b/tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: long +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -10,7 +14,6 @@ expect_after { # A default timeout action is to do nothing, change it to fail timeout { exit 1 } } -set basedir [file dirname $argv0] # history file is not required, in-memory history is enough spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$basedir/01910_client_replxx_container_overflow_long.history.log" diff --git a/tests/queries/0_stateless/01933_client_replxx_convert_history.expect b/tests/queries/0_stateless/01933_client_replxx_convert_history.expect index c5645179ab3..111389e49b2 100755 --- a/tests/queries/0_stateless/01933_client_replxx_convert_history.expect +++ b/tests/queries/0_stateless/01933_client_replxx_convert_history.expect @@ -2,6 +2,10 @@ # Tags: no-parallel # Tag no-parallel: Uses non unique history file +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -11,7 +15,6 @@ expect_after { # A default timeout action is to do nothing, change it to fail timeout { exit 1 } } -set basedir [file dirname $argv0] exec bash -c "echo select 1 > $argv0.txt" exec bash -c "echo select 1 >> $argv0.txt" diff --git a/tests/queries/0_stateless/01945_show_debug_warning.expect b/tests/queries/0_stateless/01945_show_debug_warning.expect index 2f74b6e33ae..ca423ee106c 100755 --- a/tests/queries/0_stateless/01945_show_debug_warning.expect +++ b/tests/queries/0_stateless/01945_show_debug_warning.expect @@ -3,6 +3,10 @@ # This is a test for system.warnings. Testing in interactive mode is necessary, # as we want to see certain warnings from client +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -14,7 +18,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] set Debug_type 0 spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.expect b/tests/queries/0_stateless/02003_memory_limit_in_client.expect index a3d6d04110a..a8e8c1d5786 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.expect +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.expect @@ -4,6 +4,10 @@ # This is a test for system.warnings. Testing in interactive mode is necessary, # as we want to see certain warnings from client +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -15,8 +19,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] - # # Check that the query will fail in clickhouse-client # diff --git a/tests/queries/0_stateless/02047_client_exception.expect b/tests/queries/0_stateless/02047_client_exception.expect index f7d4bfb555d..50ed09d03c5 100755 --- a/tests/queries/0_stateless/02047_client_exception.expect +++ b/tests/queries/0_stateless/02047_client_exception.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -11,7 +15,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect b/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect index ffa25b964db..a9905128ad5 100755 --- a/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect +++ b/tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -11,7 +15,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/02105_backslash_letter_commands.expect b/tests/queries/0_stateless/02105_backslash_letter_commands.expect index e67d60912fa..707a544f6bb 100755 --- a/tests/queries/0_stateless/02105_backslash_letter_commands.expect +++ b/tests/queries/0_stateless/02105_backslash_letter_commands.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 02 match_max 100000 @@ -10,7 +14,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect index 0abe25e60f4..4fd430a4a69 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_client_with_queries_file.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: no-parallel +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -12,8 +16,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] - system "$basedir/helpers/02112_prepare.sh" spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT --disable_suggestion --interactive --queries-file $basedir/file_02112" expect ":) " diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect index c846464b011..a90e85d1069 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -11,7 +15,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --query 'create table t(i Int32) engine=Memory; insert into t select 1'" expect ":) " diff --git a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect index c64f149a93c..34eac360132 100755 --- a/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect +++ b/tests/queries/0_stateless/02112_delayed_clickhouse_local_with_queries_file.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: no-parallel +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -12,8 +16,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] - system "$basedir/helpers/02112_prepare.sh" spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion --interactive --queries-file $basedir/file_02112" expect ":) " diff --git a/tests/queries/0_stateless/02116_interactive_hello.expect b/tests/queries/0_stateless/02116_interactive_hello.expect index e659cf8703c..5fa31d33e87 100755 --- a/tests/queries/0_stateless/02116_interactive_hello.expect +++ b/tests/queries/0_stateless/02116_interactive_hello.expect @@ -1,6 +1,10 @@ #!/usr/bin/expect -f # Tags: long +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 match_max 100000 @@ -12,7 +16,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect -re "ClickHouse client version \[\\d\]{2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\[\\d\]{1,2}.\r" diff --git a/tests/queries/0_stateless/02132_client_history_navigation.expect b/tests/queries/0_stateless/02132_client_history_navigation.expect index b722a0af04c..10167fb2e97 100755 --- a/tests/queries/0_stateless/02132_client_history_navigation.expect +++ b/tests/queries/0_stateless/02132_client_history_navigation.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 3 match_max 100000 @@ -14,7 +18,6 @@ expect_after { # useful debugging configuration # exp_internal 1 -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0" expect ":) " diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect index a77de874010..b95f85403e3 100755 --- a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 60 set uuid "" @@ -11,7 +15,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" expect ":) " diff --git a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect index 4f006b926bd..f70b699c71f 100755 --- a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect +++ b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect @@ -1,5 +1,9 @@ #!/usr/bin/expect -f +set basedir [file dirname $argv0] +set basename [file tail $argv0] +exp_internal -f $env(CLICKHOUSE_TMP)/$basename.debuglog 0 + log_user 0 set timeout 20 match_max 100000 @@ -9,7 +13,6 @@ expect_after { timeout { exit 1 } } -set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion" expect ":) " diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 84ce7ae5742..adae3068dcd 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -314,6 +314,12 @@ for test_case in "${tests_with_event_time_date[@]}"; do } done +expect_tests=( $(find $ROOT_PATH/tests/queries -name '*.expect') ) +for test_case in "${expect_tests[@]}"; do + pattern="^exp_internal -f \$env(CLICKHOUSE_TMP)/\$basename.debuglog 0$" + grep -q "$pattern" "$test_case" || echo "Missing '$pattern' in '$test_case'" +done + # Conflict markers find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -P '^(<<<<<<<|=======|>>>>>>>)$' | grep -P '.' && echo "Conflict markers are found in files" From 4ba4329e0de3e853ca3da7892d4aeedc7ff848df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 2 Jul 2022 19:48:42 +0200 Subject: [PATCH 385/408] Fix bad test --- .../queries/0_stateless/01848_http_insert_segfault.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01848_http_insert_segfault.sh b/tests/queries/0_stateless/01848_http_insert_segfault.sh index c766e9794ae..1f2e9eebcdc 100755 --- a/tests/queries/0_stateless/01848_http_insert_segfault.sh +++ b/tests/queries/0_stateless/01848_http_insert_segfault.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash +# Tags: no-tsan +# Sometimes is takes longer than 60 seconds under TSan. - CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - # shellcheck source=../shell_config.sh - . "$CUR_DIR"/../shell_config.sh +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh - ${CLICKHOUSE_LOCAL} -q "select col1, initializeAggregation('argMaxState', col2, insertTime) as col2, now() as insertTime FROM generateRandom('col1 String, col2 Array(Float64)') LIMIT 1000000 FORMAT CSV" | ${CLICKHOUSE_CURL} -s 'http://localhost:8123/?query=INSERT%20INTO%20non_existing_table%20SELECT%20col1%2C%20initializeAggregation(%27argMaxState%27%2C%20col2%2C%20insertTime)%20as%20col2%2C%20now()%20as%20insertTime%20FROM%20input(%27col1%20String%2C%20col2%20Array(Float64)%27)%20FORMAT%20CSV' --data-binary @- | grep -q "Table default.non_existing_table doesn't exist" && echo 'Ok.' || echo 'FAIL' ||: +${CLICKHOUSE_LOCAL} --query "select col1, initializeAggregation('argMaxState', col2, insertTime) as col2, now() as insertTime FROM generateRandom('col1 String, col2 Array(Float64)') LIMIT 1000000 FORMAT CSV" | ${CLICKHOUSE_CURL} -s 'http://localhost:8123/?query=INSERT%20INTO%20non_existing_table%20SELECT%20col1%2C%20initializeAggregation(%27argMaxState%27%2C%20col2%2C%20insertTime)%20as%20col2%2C%20now()%20as%20insertTime%20FROM%20input(%27col1%20String%2C%20col2%20Array(Float64)%27)%20FORMAT%20CSV' --data-binary @- | grep -q "Table default.non_existing_table doesn't exist" && echo 'Ok.' || echo 'FAIL' ||: From 9b8fc6a5a652271d661af015e53a84d1ebe19743 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jul 2022 21:46:04 +0300 Subject: [PATCH 386/408] Fix CLICKHOUSE_TMP in tests Previous it was not updated, and always uses the first CLICKHOUSE_TMP for all tests (that was run from one thread). Signed-off-by: Azat Khuzhin --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 083eb3830fa..cab6daf3a50 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -526,7 +526,7 @@ class TestCase: # collisions. testcase_args.test_tmp_dir = os.path.join(suite_tmp_dir, database) os.mkdir(testcase_args.test_tmp_dir) - os.environ.setdefault("CLICKHOUSE_TMP", testcase_args.test_tmp_dir) + os.environ["CLICKHOUSE_TMP"] = testcase_args.test_tmp_dir testcase_args.testcase_database = database From 92cbc2a3b5ce0fa192c31e3efe904456e06321fe Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 2 Jul 2022 22:13:48 +0200 Subject: [PATCH 387/408] Tests with long arrays --- .../02282_array_distance.reference | 66 ++++++++++++++----- .../0_stateless/02282_array_distance.sql | 54 +++++++++++++-- .../0_stateless/02283_array_norm.reference | 12 ++++ .../queries/0_stateless/02283_array_norm.sql | 6 +- 4 files changed, 111 insertions(+), 27 deletions(-) diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index e5bc46df295..a63ea0a634d 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -32,21 +32,51 @@ nan 0.020204102886728692 0.11808289631180313 0 -1 1 218.74642854227358 47850 -1 2 1348.2117786164013 1817675 -2 1 219.28064210048274 48084 -2 2 1347.4008312302617 1815489 -3 1 214.35251339790725 45947 -3 2 1342.8856987845243 1803342 -1 1 218.74642854227358 47850 -1 2 1348.2117786164013 1817675 -2 1 219.28064210048274 48084 -2 2 1347.4008312302617 1815489 -3 1 214.35251339790725 45947 -3 2 1342.8856987845243 1803342 -1 1 218.74642854227358 47850 -1 2 1348.2117786164013 1817675 -2 1 219.28064210048274 48084 -2 2 1347.4008312302617 1815489 -3 1 214.35251339790725 45947 -3 2 1342.8856987845243 1803342 +1 1 0 0 0 0 0 0 +1 2 2031 788 981.3289733414064 1182.129011571918 1397429 0.1939823640079572 +2 1 2031 788 981.3289733414064 1182.129011571918 1397429 0.1939823640079572 +2 2 0 0 0 0 0 0 +3 3 0 0 0 0 0 0 +3 4 68 2 6.238144819822315 11.661903789690601 136 0.0010041996325123037 +4 3 68 2 6.238144819822315 11.661903789690601 136 0.0010041996325123037 +4 4 0 0 0 0 0 0 +5 5 0 0 0 0 0 0 +5 6 268 2 9.70940985211152 23.15167380558045 536 0.00007815428961455151 +6 5 268 2 9.70940985211152 23.15167380558045 536 0.00007815428961455151 +6 6 0 0 0 0 0 0 +1 1 0 0 0 0 0 0 +1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 1 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 2 0 0 0 0 0 0 +3 3 0 0 0 0 0 0 +3 4 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 3 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 4 0 0 0 0 0 0 +5 5 0 0 0 0 0 0 +5 6 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 5 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 6 0 0 0 0 0 0 +1 1 0 0 0 0 0 0 +1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 1 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 2 0 0 0 0 0 0 +3 3 0 0 0 0 0 0 +3 4 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 3 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 4 0 0 0 0 0 0 +5 5 0 0 0 0 0 0 +5 6 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 5 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 6 0 0 0 0 0 0 +1 1 0 0 0 0 0 0 +1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 1 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +2 2 0 0 0 0 0 0 +3 3 0 0 0 0 0 0 +3 4 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 3 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +4 4 0 0 0 0 0 0 +5 5 0 0 0 0 0 0 +5 6 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 5 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +6 6 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/02282_array_distance.sql b/tests/queries/0_stateless/02282_array_distance.sql index 431704abd40..9c16071dc1f 100644 --- a/tests/queries/0_stateless/02282_array_distance.sql +++ b/tests/queries/0_stateless/02282_array_distance.sql @@ -37,14 +37,56 @@ SELECT cosineDistance([3, 2, 1], v) FROM vec1; SELECT LinfDistance(v, materialize([0, -2, 0])) FROM vec1; SELECT cosineDistance(v, materialize([1., 1., 1.])) FROM vec1; -INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2 v2; +INSERT INTO vec2 VALUES (1, [100, 200, 0]), (2, [888, 777, 666]), (3, range(1, 35, 1)), (4, range(3, 37, 1)), (5, range(1, 135, 1)), (6, range(3, 137, 1)); +SELECT + v1.id, + v2.id, + L1Distance(v1.v, v2.v), + LinfDistance(v1.v, v2.v), + LpDistance(v1.v, v2.v, 3.1), + L2Distance(v1.v, v2.v), + L2SquaredDistance(v1.v, v2.v), + cosineDistance(v1.v, v2.v) +FROM vec2 v1, vec2 v2 +WHERE length(v1.v) == length(v2.v); -INSERT INTO vec2f VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2f v2; +INSERT INTO vec2f VALUES (1, [100, 200, 0]), (2, [888, 777, 666]), (3, range(1, 35, 1)), (4, range(3, 37, 1)), (5, range(1, 135, 1)), (6, range(3, 137, 1)); +SELECT + v1.id, + v2.id, + L1Distance(v1.v, v2.v), + LinfDistance(v1.v, v2.v), + LpDistance(v1.v, v2.v, 3), + L2Distance(v1.v, v2.v), + L2SquaredDistance(v1.v, v2.v), + cosineDistance(v1.v, v2.v) +FROM vec2f v1, vec2f v2 +WHERE length(v1.v) == length(v2.v); -INSERT INTO vec2d VALUES (1, [100, 200, 0]), (2, [888, 777, 666]); -SELECT v1.id, v2.id, L2Distance(v1.v, v2.v), L2SquaredDistance(v1.v, v2.v) as dist FROM vec1 v1, vec2d v2; +INSERT INTO vec2d VALUES (1, [100, 200, 0]), (2, [888, 777, 666]), (3, range(1, 35, 1)), (4, range(3, 37, 1)), (5, range(1, 135, 1)), (6, range(3, 137, 1)); +SELECT + v1.id, + v2.id, + L1Distance(v1.v, v2.v), + LinfDistance(v1.v, v2.v), + LpDistance(v1.v, v2.v, 3), + L2Distance(v1.v, v2.v), + L2SquaredDistance(v1.v, v2.v), + cosineDistance(v1.v, v2.v) +FROM vec2d v1, vec2d v2 +WHERE length(v1.v) == length(v2.v); + +SELECT + v1.id, + v2.id, + L1Distance(v1.v, v2.v), + LinfDistance(v1.v, v2.v), + LpDistance(v1.v, v2.v, 3), + L2Distance(v1.v, v2.v), + L2SquaredDistance(v1.v, v2.v), + cosineDistance(v1.v, v2.v) +FROM vec2f v1, vec2d v2 +WHERE length(v1.v) == length(v2.v); SELECT L1Distance([0, 0], [1]); -- { serverError 190 } SELECT L2Distance([1, 2], (3,4)); -- { serverError 43 } diff --git a/tests/queries/0_stateless/02283_array_norm.reference b/tests/queries/0_stateless/02283_array_norm.reference index 86a77e27172..17abb889e24 100644 --- a/tests/queries/0_stateless/02283_array_norm.reference +++ b/tests/queries/0_stateless/02283_array_norm.reference @@ -8,23 +8,35 @@ 2 2 2 4 2 2 3 9 5.196152422706632 27 4.506432087111623 3 4 0 0 0 0 0 +5 330 78.16648898345122 6110 54.82161001608108 26 +6 5250 599.12436104702 358950 350.73959029428204 102 1 11 2 11 3 11 4 11 +5 11 +6 11 1 7 5 25 4.601724723020627 4 2 2 2 4 2 2 3 9 5.196152422706632 27 4.506432087111623 3 4 0 0 0 0 0 +5 330 78.16648898345122 6110 54.82161001608108 26 +6 5250 599.12436104702 358950 350.73959029428204 102 1 11 2 11 3 11 4 11 +5 11 +6 11 1 7 5 25 4.601724723020627 4 2 2 2 4 2 2 3 9 5.196152422706632 27 4.506432087111623 3 4 0 0 0 0 0 +5 330 78.16648898345122 6110 54.82161001608108 26 +6 5250 599.12436104702 358950 350.73959029428204 102 1 11 2 11 3 11 4 11 +5 11 +6 11 diff --git a/tests/queries/0_stateless/02283_array_norm.sql b/tests/queries/0_stateless/02283_array_norm.sql index 2f5753f9943..dcb5288a1ac 100644 --- a/tests/queries/0_stateless/02283_array_norm.sql +++ b/tests/queries/0_stateless/02283_array_norm.sql @@ -21,9 +21,9 @@ DROP TABLE IF EXISTS vec1d; CREATE TABLE vec1 (id UInt64, v Array(UInt8)) ENGINE = Memory; CREATE TABLE vec1f (id UInt64, v Array(Float32)) ENGINE = Memory; CREATE TABLE vec1d (id UInt64, v Array(Float64)) ENGINE = Memory; -INSERT INTO vec1 VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); -INSERT INTO vec1f VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); -INSERT INTO vec1d VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL); +INSERT INTO vec1 VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL), (5, range(7, 27)), (6, range(3, 103)); +INSERT INTO vec1f VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL), (5, range(7, 27)), (6, range(3, 103)); +INSERT INTO vec1d VALUES (1, [3, 4]), (2, [2]), (3, [3, 3, 3]), (4, NULL), (5, range(7, 27)), (6, range(3, 103)); SELECT id, L1Norm(v), L2Norm(v), L2SquaredNorm(v), LpNorm(v, 2.7), LinfNorm(v) FROM vec1; SELECT id, L1Norm(materialize([5., 6.])) FROM vec1; From c6691cc5f29222763d4f7fa37addf99f4ef8048b Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sat, 2 Jul 2022 22:34:06 +0200 Subject: [PATCH 388/408] Improved vectorized execution of main loop for array norm/distance --- src/Functions/array/arrayDistance.cpp | 69 +++++++++++++++++-- src/Functions/array/arrayNorm.cpp | 41 ++++++++++- .../02282_array_distance.reference | 8 +-- 3 files changed, 107 insertions(+), 11 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index 21e05916a5c..a8fea9b02fb 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -38,6 +38,12 @@ struct L1Distance state.sum += fabs(x - y); } + template + static void combine(State & state, const State & other_state, const ConstParams &) + { + state.sum += other_state.sum; + } + template static ResultType finalize(const State & state, const ConstParams &) { @@ -63,6 +69,12 @@ struct L2Distance state.sum += (x - y) * (x - y); } + template + static void combine(State & state, const State & other_state, const ConstParams &) + { + state.sum += other_state.sum; + } + template static ResultType finalize(const State & state, const ConstParams &) { @@ -103,6 +115,12 @@ struct LpDistance state.sum += std::pow(fabs(x - y), params.power); } + template + static void combine(State & state, const State & other_state, const ConstParams &) + { + state.sum += other_state.sum; + } + template static ResultType finalize(const State & state, const ConstParams & params) { @@ -128,6 +146,12 @@ struct LinfDistance state.dist = fmax(state.dist, fabs(x - y)); } + template + static void combine(State & state, const State & other_state, const ConstParams &) + { + state.dist = fmax(state.dist, other_state.dist); + } + template static ResultType finalize(const State & state, const ConstParams &) { @@ -157,6 +181,14 @@ struct CosineDistance state.y_squared += y * y; } + template + static void combine(State & state, const State & other_state, const ConstParams &) + { + state.dot_prod += other_state.dot_prod; + state.x_squared += other_state.x_squared; + state.y_squared += other_state.y_squared; + } + template static ResultType finalize(const State & state, const ConstParams &) { @@ -339,10 +371,23 @@ private: size_t row = 0; for (auto off : offsets_x) { - typename Kernel::template State state; + /// Process chunks in vectorized manner + static constexpr size_t VEC_SIZE = 4; + typename Kernel::template State states[VEC_SIZE]; + for (; prev + VEC_SIZE < off; prev += VEC_SIZE) + { + for (size_t s = 0; s < VEC_SIZE; ++s) + Kernel::template accumulate(states[s], data_x[prev+s], data_y[prev+s], kernel_params); + } + + typename Kernel::template State state; + for (const auto & other_state : states) + Kernel::template combine(state, other_state, kernel_params); + + /// Process the tail for (; prev < off; ++prev) { - Kernel::template accumulate(state, data_x[prev], data_y[prev], kernel_params); + Kernel::template accumulate(state, data_x[prev], data_y[prev], kernel_params); } result_data[row] = Kernel::finalize(state, kernel_params); row++; @@ -392,10 +437,24 @@ private: size_t row = 0; for (auto off : offsets_y) { - typename Kernel::template State state; - for (size_t i = 0; prev < off; ++i, ++prev) + /// Process chunks in vectorized manner + static constexpr size_t VEC_SIZE = 4; + typename Kernel::template State states[VEC_SIZE]; + size_t i = 0; + for (; prev + VEC_SIZE < off; i += VEC_SIZE, prev += VEC_SIZE) { - Kernel::template accumulate(state, data_x[i], data_y[prev], kernel_params); + for (size_t s = 0; s < VEC_SIZE; ++s) + Kernel::template accumulate(states[s], data_x[i+s], data_y[prev+s], kernel_params); + } + + typename Kernel::template State state; + for (const auto & other_state : states) + Kernel::template combine(state, other_state, kernel_params); + + /// Process the tail + for (; prev < off; ++i, ++prev) + { + Kernel::template accumulate(state, data_x[i], data_y[prev], kernel_params); } result_data[row] = Kernel::finalize(state, kernel_params); row++; diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 3ea16b23abd..5db330f9a2f 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -31,6 +31,12 @@ struct L1Norm return result + fabs(value); } + template + inline static ResultType combine(ResultType result, ResultType other_result, const ConstParams &) + { + return result + other_result; + } + template inline static ResultType finalize(ResultType result, const ConstParams &) { @@ -50,6 +56,12 @@ struct L2Norm return result + value * value; } + template + inline static ResultType combine(ResultType result, ResultType other_result, const ConstParams &) + { + return result + other_result; + } + template inline static ResultType finalize(ResultType result, const ConstParams &) { @@ -85,6 +97,12 @@ struct LpNorm return result + std::pow(fabs(value), params.power); } + template + inline static ResultType combine(ResultType result, ResultType other_result, const ConstParams &) + { + return result + other_result; + } + template inline static ResultType finalize(ResultType result, const ConstParams & params) { @@ -104,6 +122,12 @@ struct LinfNorm return fmax(result, fabs(value)); } + template + inline static ResultType combine(ResultType result, ResultType other_result, const ConstParams &) + { + return fmax(result, other_result); + } + template inline static ResultType finalize(ResultType result, const ConstParams &) { @@ -221,10 +245,23 @@ private: size_t row = 0; for (auto off : offsets) { - Float64 result = 0; + /// Process chunks in vectorized manner + static constexpr size_t VEC_SIZE = 4; + ResultType results[VEC_SIZE] = {0}; + for (; prev + VEC_SIZE < off; prev += VEC_SIZE) + { + for (size_t s = 0; s < VEC_SIZE; ++s) + results[s] = Kernel::template accumulate(results[s], data[prev+s], kernel_params); + } + + ResultType result = 0; + for (const auto & other_state : results) + result = Kernel::template combine(result, other_state, kernel_params); + + /// Process the tail for (; prev < off; ++prev) { - result = Kernel::template accumulate(result, data[prev], kernel_params); + result = Kernel::template accumulate(result, data[prev], kernel_params); } result_data[row] = Kernel::finalize(result, kernel_params); row++; diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index a63ea0a634d..dc40aaf128f 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -37,12 +37,12 @@ nan 2 1 2031 788 981.3289733414064 1182.129011571918 1397429 0.1939823640079572 2 2 0 0 0 0 0 0 3 3 0 0 0 0 0 0 -3 4 68 2 6.238144819822315 11.661903789690601 136 0.0010041996325123037 -4 3 68 2 6.238144819822315 11.661903789690601 136 0.0010041996325123037 +3 4 68 2 6.238144819822316 11.661903789690601 136 0.0010041996325123037 +4 3 68 2 6.238144819822316 11.661903789690601 136 0.0010041996325123037 4 4 0 0 0 0 0 0 5 5 0 0 0 0 0 0 -5 6 268 2 9.70940985211152 23.15167380558045 536 0.00007815428961455151 -6 5 268 2 9.70940985211152 23.15167380558045 536 0.00007815428961455151 +5 6 268 2 9.70940985211151 23.15167380558045 536 0.00007815428961455151 +6 5 268 2 9.70940985211151 23.15167380558045 536 0.00007815428961455151 6 6 0 0 0 0 0 0 1 1 0 0 0 0 0 0 1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 From 29ffee846409e6c93f48e9289c72a070142a2aa8 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Sat, 2 Jul 2022 19:24:26 -0400 Subject: [PATCH 389/408] typo --- docs/en/getting-started/example-datasets/metrica.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index da0286d8c05..300bbe58d3f 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -87,7 +87,7 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" The hits and visits dataset is used in the ClickHouse test routines, this is one of the queries from the test suite. The rest -of the tests are refernced in the *Next Steps* section at the +of the tests are referenced in the *Next Steps* section at the end of this page. ```sql From ca2829188d321fdf6d575f4ad5130ead2a3fcb5b Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 3 Jul 2022 08:01:49 +0200 Subject: [PATCH 390/408] Perf test for norm/distance with long arrays of floats --- tests/performance/norm_distance_float.xml | 95 +++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/performance/norm_distance_float.xml diff --git a/tests/performance/norm_distance_float.xml b/tests/performance/norm_distance_float.xml new file mode 100644 index 00000000000..e71d8eb6281 --- /dev/null +++ b/tests/performance/norm_distance_float.xml @@ -0,0 +1,95 @@ + + + + + element_type + + Float32 + Float64 + + + + + + CREATE TABLE vecs_{element_type} ( + v Array({element_type}) + ) ENGINE=Memory; + + + + + + INSERT INTO vecs_{element_type} + SELECT v FROM ( + SELECT + number AS n, + [ + rand(n*10), + rand(n*10+1), + rand(n*10+2), + rand(n*10+3), + rand(n*10+4), + rand(n*10+5), + rand(n*10+6), + rand(n*10+7), + rand(n*10+8), + rand(n*10+9), + rand(n*10), + rand(n*10+1), + rand(n*10+2), + rand(n*10+3), + rand(n*10+4), + rand(n*10+5), + rand(n*10+6), + rand(n*10+7), + rand(n*10+8), + rand(n*10+9), + rand(n*10), + rand(n*10+1), + rand(n*10+2), + rand(n*10+3), + rand(n*10+4), + rand(n*10+5), + rand(n*10+6), + rand(n*10+7), + rand(n*10+8), + rand(n*10+9), + rand(n*10), + rand(n*10+1), + rand(n*10+2), + rand(n*10+3), + rand(n*10+4), + rand(n*10+5), + rand(n*10+6), + rand(n*10+7) + ] AS v + FROM system.numbers + LIMIT 10000000 + ); + + + + 1 + + + + + + norm + + L1 + L2 + L2Squared + Linf + + + + + + SELECT sum(dist) FROM (SELECT {norm}Norm(v) AS dist FROM vecs_{element_type}) + WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT {norm}Distance(a, v) AS dist FROM vecs_{element_type}) + WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT cosineDistance(a, v) AS dist FROM vecs_{element_type}) + + DROP TABLE vecs_{element_type} + + From 8ce8158f7fa7b0cbc911d9e934d55ddb72fb9fcf Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Sun, 3 Jul 2022 10:33:11 +0200 Subject: [PATCH 391/408] Do computations in Float32 (not Float64) for arrays of Float32 --- src/Functions/array/arrayDistance.cpp | 6 +++++- src/Functions/array/arrayNorm.cpp | 6 +++++- .../0_stateless/02282_array_distance.reference | 12 ++++++------ tests/queries/0_stateless/02283_array_norm.reference | 8 ++++---- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Functions/array/arrayDistance.cpp b/src/Functions/array/arrayDistance.cpp index a8fea9b02fb..c3652e10644 100644 --- a/src/Functions/array/arrayDistance.cpp +++ b/src/Functions/array/arrayDistance.cpp @@ -228,11 +228,12 @@ public: case TypeIndex::Int8: case TypeIndex::Int16: case TypeIndex::Int32: - case TypeIndex::Float32: case TypeIndex::UInt64: case TypeIndex::Int64: case TypeIndex::Float64: return std::make_shared(); + case TypeIndex::Float32: + return std::make_shared(); default: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, @@ -247,6 +248,9 @@ public: { switch (result_type->getTypeId()) { + case TypeIndex::Float32: + return executeWithResultType(arguments, input_rows_count); + break; case TypeIndex::Float64: return executeWithResultType(arguments, input_rows_count); break; diff --git a/src/Functions/array/arrayNorm.cpp b/src/Functions/array/arrayNorm.cpp index 5db330f9a2f..e1e7935fcb1 100644 --- a/src/Functions/array/arrayNorm.cpp +++ b/src/Functions/array/arrayNorm.cpp @@ -162,11 +162,12 @@ public: case TypeIndex::Int8: case TypeIndex::Int16: case TypeIndex::Int32: - case TypeIndex::Float32: case TypeIndex::UInt64: case TypeIndex::Int64: case TypeIndex::Float64: return std::make_shared(); + case TypeIndex::Float32: + return std::make_shared(); default: throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, @@ -185,6 +186,9 @@ public: switch (result_type->getTypeId()) { + case TypeIndex::Float32: + return executeWithResultType(*arr, type, input_rows_count, arguments); + break; case TypeIndex::Float64: return executeWithResultType(*arr, type, input_rows_count, arguments); break; diff --git a/tests/queries/0_stateless/02282_array_distance.reference b/tests/queries/0_stateless/02282_array_distance.reference index dc40aaf128f..9758da9a833 100644 --- a/tests/queries/0_stateless/02282_array_distance.reference +++ b/tests/queries/0_stateless/02282_array_distance.reference @@ -45,16 +45,16 @@ nan 6 5 268 2 9.70940985211151 23.15167380558045 536 0.00007815428961455151 6 6 0 0 0 0 0 0 1 1 0 0 0 0 0 0 -1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 -2 1 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 +1 2 2031 788 992.2102 1182.129 1397429 0.19398236 +2 1 2031 788 992.2102 1182.129 1397429 0.19398236 2 2 0 0 0 0 0 0 3 3 0 0 0 0 0 0 -3 4 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 -4 3 68 2 6.479223602554966 11.661903789690601 136 0.0010041996325123037 +3 4 68 2 6.4792237 11.661903 136 0.0010041595 +4 3 68 2 6.4792237 11.661903 136 0.0010041595 4 4 0 0 0 0 0 0 5 5 0 0 0 0 0 0 -5 6 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 -6 5 268 2 10.234459893824097 23.15167380558045 536 0.00007815428961455151 +5 6 268 2 10.23446 23.151674 536 0.00007814169 +6 5 268 2 10.23446 23.151674 536 0.00007814169 6 6 0 0 0 0 0 0 1 1 0 0 0 0 0 0 1 2 2031 788 992.2102104083964 1182.129011571918 1397429 0.1939823640079572 diff --git a/tests/queries/0_stateless/02283_array_norm.reference b/tests/queries/0_stateless/02283_array_norm.reference index 17abb889e24..ed819e11255 100644 --- a/tests/queries/0_stateless/02283_array_norm.reference +++ b/tests/queries/0_stateless/02283_array_norm.reference @@ -16,12 +16,12 @@ 4 11 5 11 6 11 -1 7 5 25 4.601724723020627 4 +1 7 5 25 4.6017246 4 2 2 2 4 2 2 -3 9 5.196152422706632 27 4.506432087111623 3 +3 9 5.196152 27 4.506432 3 4 0 0 0 0 0 -5 330 78.16648898345122 6110 54.82161001608108 26 -6 5250 599.12436104702 358950 350.73959029428204 102 +5 330 78.16649 6110 54.82161 26 +6 5250 599.1244 358950 350.7396 102 1 11 2 11 3 11 From 4375a336fd773f5f0161ef3acacf6a3bead55ebf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 3 Jul 2022 11:15:10 +0300 Subject: [PATCH 392/408] Fix use-after-free for Map combinator that leads to incorrect result This use-after-free can be reproduced with distributed queries. Also note, that this is not sumMappedArray() and friends (that previously called sumMap()) but Map combinator. You will find ASan report in details.
READ of size 8 at 0x62d00012d218 thread T186 (QueryPipelineEx) 2022.07.03 05:09:40.000234 [ 31956 ] {} AsynchronousMetrics: MemoryTracking: was 1.23 GiB, peak 1.23 GiB, will set to 1.25 GiB (RSS), difference: 19.51 MiB 2022.07.03 05:09:41.000137 [ 31956 ] {} AsynchronousMetrics: MemoryTracking: was 1.25 GiB, peak 1.25 GiB, will set to 1.26 GiB (RSS), difference: 3.76 MiB #0 0x1233a0d8 in DB::AggregateFunctionSumData<>::get() const build_docker/../src/AggregateFunctions/AggregateFunctionSum.h:245:16 #1 0x1233a0d8 in DB::AggregateFunctionSum<>::insertResultInto(char*, DB::IColumn&, DB::Arena*) const build_docker/../src/AggregateFunctions/AggregateFunctionSum.h:536:70 #2 0x1470f910 in DB::AggregateFunctionMap::insertResultInto() const build_docker/../src/AggregateFunctions/AggregateFunctionMap.h:236:26 #3 0x147110ce in DB::IAggregateFunctionHelper<>::insertResultIntoBatch() const build_docker/../src/AggregateFunctions/IAggregateFunction.h:618:53 #4 0x2c4269d7 in void DB::Aggregator::convertToBlockImplFinal<>() const build_docker/../src/Interpreters/Aggregator.cpp:1878:49 #5 0x2c403b9f in void DB::Aggregator::convertToBlockImpl<>() const build_docker/../src/Interpreters/Aggregator.cpp:1714:13 #6 0x2be09b53 in DB::Aggregator::prepareBlockAndFillSingleLevel() const::$_2::operator()() const build_docker/../src/Interpreters/Aggregator.cpp:2144:9 #7 0x2be09b53 in DB::Block DB::Aggregator::prepareBlockAndFill<>() const build_docker/../src/Interpreters/Aggregator.cpp:2000:5 #8 0x2be09b53 in DB::Aggregator::prepareBlockAndFillSingleLevel() const build_docker/../src/Interpreters/Aggregator.cpp:2150:12 #9 0x2be37de3 in DB::Aggregator::mergeBlocks() build_docker/../src/Interpreters/Aggregator.cpp:3032:17 #10 0x308c27f8 in DB::MergingAggregatedBucketTransform::transform() build_docker/../src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp:360:37 0x62d00012d218 is located 3608 bytes inside of 32768-byte region [0x62d00012c400,0x62d000134400) freed by thread T186 (QueryPipelineEx) here: #0 0xd701312 in free (/work1/azat/tmp/upstream/clickhouse-asan+0xd701312) (BuildId: b7977aef37e9f720) ... #8 0x2e3c22eb in DB::ColumnAggregateFunction::~ColumnAggregateFunction() build_docker/../src/Columns/ColumnAggregateFunction.cpp:89:1 ... #18 0xd9fcdd4 in std::__1::vector >::~vector() build_docker/../contrib/libcxx/include/vector:401:9 #19 0x2be373f4 in DB::Aggregator::mergeBlocks() build_docker/../contrib/libcxx/include/__memory/unique_ptr.h #20 0x308c27f8 in DB::MergingAggregatedBucketTransform::transform() build_docker/../src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp:360:37 previously allocated by thread T186 (QueryPipelineEx) here: #0 0xd7015be in malloc (/work1/azat/tmp/upstream/clickhouse-asan+0xd7015be) (BuildId: b7977aef37e9f720) #1 0xd85190a in Allocator::allocNoTrack(unsigned long, unsigned long) build_docker/../src/Common/Allocator.h:227:27 #2 0xd988d45 in Allocator::alloc(unsigned long, unsigned long) build_docker/../src/Common/Allocator.h:96:16 #3 0xd988d45 in DB::Arena::MemoryChunk::MemoryChunk(unsigned long, DB::Arena::MemoryChunk*) build_docker/../src/Common/Arena.h:54:64 #4 0xd98904b in DB::Arena::addMemoryChunk(unsigned long) build_docker/../src/Common/Arena.h:122:20 #5 0xec9542c in DB::Arena::alignedAlloc(unsigned long, unsigned long) build_docker/../src/Common/Arena.h:171:13 #6 0x1470f123 in DB::AggregateFunctionMap::deserialize() const build_docker/../src/AggregateFunctions/AggregateFunctionMap.h:205:35
P.S. Thanks to @den-crane for the reproducer. Fixes: #35359 (cc @den-crane @dongxiao-yang) Signed-off-by: Azat Khuzhin --- src/AggregateFunctions/AggregateFunctionMap.h | 15 +++- .../02351_Map_combinator_dist.reference | 4 + .../0_stateless/02351_Map_combinator_dist.sql | 81 +++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02351_Map_combinator_dist.reference create mode 100644 tests/queries/0_stateless/02351_Map_combinator_dist.sql diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 8d77e22300b..5ccc9041c36 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -169,12 +169,21 @@ public: { const auto & it = merged_maps.find(elem.first); - if (it != merged_maps.end()) + AggregateDataPtr nested_place; + if (it == merged_maps.end()) { - nested_func->merge(it->second, elem.second, arena); + // elem.second cannot be copied since this it will be destroyed after merging, + // and lead to use-after-free. + nested_place = arena->alignedAlloc(nested_func->sizeOfData(), nested_func->alignOfData()); + nested_func->create(nested_place); + merged_maps.emplace(elem.first, nested_place); } else - merged_maps[elem.first] = elem.second; + { + nested_place = it->second; + } + + nested_func->merge(nested_place, elem.second, arena); } } diff --git a/tests/queries/0_stateless/02351_Map_combinator_dist.reference b/tests/queries/0_stateless/02351_Map_combinator_dist.reference new file mode 100644 index 00000000000..98fb6a68656 --- /dev/null +++ b/tests/queries/0_stateless/02351_Map_combinator_dist.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02351_Map_combinator_dist.sql b/tests/queries/0_stateless/02351_Map_combinator_dist.sql new file mode 100644 index 00000000000..937afa5480e --- /dev/null +++ b/tests/queries/0_stateless/02351_Map_combinator_dist.sql @@ -0,0 +1,81 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/35359 + +-- sumMap +SELECT x[67] +FROM +( + SELECT + A, + sumMap(CAST(arrayMap(x -> (x, 1), r), 'Map(UInt8,Int64)')) AS x + FROM remote('127.{1,1}', view( + SELECT + number AS A, + range(150) AS r + FROM numbers(60) + WHERE (A % 2) = shardNum() + )) + GROUP BY A + LIMIT 100000000 +) +WHERE A = 53 +SETTINGS prefer_localhost_replica = 0, distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 0, group_by_two_level_threshold_bytes = 0; + +-- minMap +SELECT x[0] +FROM +( + SELECT + A, + minMap(CAST(arrayMap(x -> (x, 1), r), 'Map(UInt8,Int64)')) AS x + FROM remote('127.{1,1}', view( + SELECT + number AS A, + range(150) AS r + FROM numbers(60) + WHERE (A % 2) = shardNum() + )) + GROUP BY A + LIMIT 100000000 +) +WHERE A = 41 +SETTINGS prefer_localhost_replica = 0, distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 0, group_by_two_level_threshold_bytes = 0; + +-- maxMap +SELECT x[0] +FROM +( + SELECT + A, + maxMap(CAST(arrayMap(x -> (x, 1), r), 'Map(UInt8,Int64)')) AS x + FROM remote('127.{1,1}', view( + SELECT + number AS A, + range(150) AS r + FROM numbers(60) + WHERE (A % 2) = shardNum() + )) + GROUP BY A + LIMIT 100000000 +) +WHERE A = 41 +SETTINGS prefer_localhost_replica = 0, distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 0, group_by_two_level_threshold_bytes = 0; + +-- avgMap +SELECT x[0] +FROM +( + SELECT + A, + avgMap(CAST(arrayMap(x -> (x, 1), r), 'Map(UInt8,Int64)')) AS x + FROM remote('127.{1,1}', view( + SELECT + number AS A, + range(150) AS r + FROM numbers(60) + WHERE (A % 2) = shardNum() + )) + GROUP BY A + LIMIT 100000000 +) +WHERE A = 41 +SETTINGS prefer_localhost_replica = 0, distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 0, group_by_two_level_threshold_bytes = 0; From 769017e1f508dd468628ac4520f46b89f6cfc97f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 28 Jun 2022 12:14:23 +0300 Subject: [PATCH 393/408] tests: move random_str() helper into shell_config.sh Signed-off-by: Azat Khuzhin --- .../0_stateless/01548_query_log_query_execution_ms.sh | 5 ----- .../01710_projections_optimize_aggregation_in_order.sh | 6 ------ ...710_projections_partial_optimize_aggregation_in_order.sh | 6 ------ tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh | 6 ------ tests/queries/shell_config.sh | 6 ++++++ 5 files changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh b/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh index c973612c80d..0d13a1d4eff 100755 --- a/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh +++ b/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh @@ -4,11 +4,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -function random_str() -{ - local n=$1 && shift - tr -cd '[:lower:]' < /dev/urandom | head -c"$n" -} function test_query_duration_ms() { local query_id diff --git a/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh b/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh index 0cafa904a71..2c6a6ef35eb 100755 --- a/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh +++ b/tests/queries/0_stateless/01710_projections_optimize_aggregation_in_order.sh @@ -29,12 +29,6 @@ $CLICKHOUSE_CLIENT -nm -q " INSERT INTO in_order_agg_01710 SELECT 1, number%2, number%4, number FROM numbers(100000); " -function random_str() -{ - local n=$1 && shift - tr -cd '[:lower:]' < /dev/urandom | head -c"$n" -} - function run_query() { local query=$1 && shift diff --git a/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh b/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh index f66dc9ff872..5a9c480c78c 100755 --- a/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh +++ b/tests/queries/0_stateless/01710_projections_partial_optimize_aggregation_in_order.sh @@ -35,12 +35,6 @@ $CLICKHOUSE_CLIENT -nm -q " INSERT INTO in_order_agg_partial_01710 SELECT 1, number%2, number%4, number FROM numbers(100000) LIMIT 50000, 100000; " -function random_str() -{ - local n=$1 && shift - tr -cd '[:lower:]' < /dev/urandom | head -c"$n" -} - function run_query() { local query=$1 && shift diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index 29b3b7b3d9d..4f3baa1f660 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -4,12 +4,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -function random_str() -{ - local n=$1 && shift - tr -cd '[:lower:]' < /dev/urandom | head -c"$n" -} - function check_refcnt_for_table() { local table=$1 && shift diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 866fba506e4..ab5d5ddc1b6 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -143,3 +143,9 @@ function wait_for_queries_to_finish() fi done } + +function random_str() +{ + local n=$1 && shift + tr -cd '[:lower:]' < /dev/urandom | head -c"$n" +} From f1eec0b6e809828443861f0c9b1cdbcaf11e7ff8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 3 Jul 2022 14:24:35 +0300 Subject: [PATCH 394/408] tests: remove unused stress script Signed-off-by: Azat Khuzhin --- tests/stress | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100755 tests/stress diff --git a/tests/stress b/tests/stress deleted file mode 100755 index 1aad49250c2..00000000000 --- a/tests/stress +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# https://stackoverflow.com/questions/360201/how-do-i-kill-background-processes-jobs-when-my-shell-script-exits -trap 'kill -9 $(jobs -p)' EXIT - -function thread() -{ - while true; do - ./clickhouse-test --client-option="query-fuzzer-runs=10" --order random 2>&1 | awk '/^\w+:/ { printf("\033[0;%s%sm \033[0m", ('$1' % 2 ? "4" : "10"), (int('$1' / 2) % 8)) }' - done -} - -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread; - -NUM_THREADS=${1:-"16"} -TIMEOUT=${2:-"300"} - -for i in $(seq 1 $NUM_THREADS); do - timeout $TIMEOUT bash -c "thread $i" 2> /dev/null & -done - -wait From 8e9c1eaa73b78390d78a25b2867bcfc0d55412a7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 3 Jul 2022 14:45:56 +0300 Subject: [PATCH 395/408] tests/stress: redirect gdb output on stop hang to gdb.log Signed-off-by: Azat Khuzhin --- docker/test/stress/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index fff720965b6..b3ee4e62f20 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -110,7 +110,8 @@ function stop() # We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces. kill -TERM "$(pidof gdb)" ||: sleep 5 - gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" ||: + echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log + gdb -batch -ex 'thread apply all backtrace' -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log clickhouse stop --force } From 4ae7db83697bba58b0669ff3229c3c62cedae133 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jul 2022 17:59:02 +0300 Subject: [PATCH 396/408] Fix max_insert_threads while pushing to views Signed-off-by: Azat Khuzhin --- src/Interpreters/InterpreterInsertQuery.cpp | 5 ----- tests/performance/views_max_insert_threads.xml | 11 +++++++++++ .../02350_views_max_insert_threads.reference | 1 + .../02350_views_max_insert_threads.sql | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 tests/performance/views_max_insert_threads.xml create mode 100644 tests/queries/0_stateless/02350_views_max_insert_threads.reference create mode 100644 tests/queries/0_stateless/02350_views_max_insert_threads.sql diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 4ed293e8530..f4394dc613d 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -457,17 +457,12 @@ BlockIO InterpreterInsertQuery::execute() }); size_t num_select_threads = pipeline.getNumThreads(); - size_t num_insert_threads = std::max_element(out_chains.begin(), out_chains.end(), [&](const auto &a, const auto &b) - { - return a.getNumThreads() < b.getNumThreads(); - })->getNumThreads(); for (auto & chain : out_chains) resources = chain.detachResources(); pipeline.addChains(std::move(out_chains)); - pipeline.setMaxThreads(num_insert_threads); /// Don't use more threads for insert then for select to reduce memory consumption. if (!settings.parallel_view_processing && pipeline.getNumThreads() > num_select_threads) pipeline.setMaxThreads(num_select_threads); diff --git a/tests/performance/views_max_insert_threads.xml b/tests/performance/views_max_insert_threads.xml new file mode 100644 index 00000000000..2988984f5d8 --- /dev/null +++ b/tests/performance/views_max_insert_threads.xml @@ -0,0 +1,11 @@ + + + create table views_max_insert_threads_null (a UInt64) Engine = Null + create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts + + insert into views_max_insert_threads_null select * from numbers_mt(3000000000) settings max_threads = 16, max_insert_threads=16 + + drop table if exists views_max_insert_threads_null + drop table if exists views_max_insert_threads_mv + + diff --git a/tests/queries/0_stateless/02350_views_max_insert_threads.reference b/tests/queries/0_stateless/02350_views_max_insert_threads.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02350_views_max_insert_threads.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02350_views_max_insert_threads.sql b/tests/queries/0_stateless/02350_views_max_insert_threads.sql new file mode 100644 index 00000000000..e19ad465b49 --- /dev/null +++ b/tests/queries/0_stateless/02350_views_max_insert_threads.sql @@ -0,0 +1,15 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/37900 + +drop table if exists t; +drop table if exists t_mv; +create table t (a UInt64) Engine = Null; +create materialized view t_mv Engine = Null AS select now() as ts, max(a) from t group by ts; + +insert into t select * from numbers_mt(10e6) settings max_threads = 16, max_insert_threads=16; +system flush logs; + +select arrayUniq(thread_ids)>=16 from system.query_log where + event_date >= yesterday() and + current_database = currentDatabase() and + type = 'QueryFinish' and + startsWith(query, 'insert'); From dd3515da9805ce12bee79953936e6d5461424ad7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jul 2022 20:02:21 +0300 Subject: [PATCH 397/408] Fix parallel_view_processing with optimize_trivial_insert_select=1 --- src/Interpreters/InterpreterInsertQuery.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index f4394dc613d..7b6066575ae 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -463,9 +463,20 @@ BlockIO InterpreterInsertQuery::execute() pipeline.addChains(std::move(out_chains)); - /// Don't use more threads for insert then for select to reduce memory consumption. - if (!settings.parallel_view_processing && pipeline.getNumThreads() > num_select_threads) - pipeline.setMaxThreads(num_select_threads); + if (!settings.parallel_view_processing) + { + /// Don't use more threads for INSERT than for SELECT to reduce memory consumption. + if (pipeline.getNumThreads() > num_select_threads) + pipeline.setMaxThreads(num_select_threads); + } + else if (pipeline.getNumThreads() < settings.max_threads) + { + /// It is possible for query to have max_threads=1, due to optimize_trivial_insert_select, + /// however in case of parallel_view_processing and multiple views, views can still be processed in parallel. + /// + /// Note, number of threads will be limited by buildPushingToViewsChain() to max_threads. + pipeline.setMaxThreads(settings.max_threads); + } pipeline.setSinks([&](const Block & cur_header, QueryPipelineBuilder::StreamType) -> ProcessorPtr { From 9225256dea808233feadc7e75265fbfeabfb0ffa Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 2 Jul 2022 19:49:41 +0300 Subject: [PATCH 398/408] tests: improve 01275_parallel_mv - cover optimize_trivial_insert_select=0/1 - cover max_insert_threads - convert parallel_view_processing into jinja2 Signed-off-by: Azat Khuzhin --- .../0_stateless/01275_parallel_mv.reference | 185 +++++++++++++++++- .../queries/0_stateless/01275_parallel_mv.sql | 39 ---- .../0_stateless/01275_parallel_mv.sql.j2 | 47 +++++ 3 files changed, 224 insertions(+), 47 deletions(-) delete mode 100644 tests/queries/0_stateless/01275_parallel_mv.sql create mode 100644 tests/queries/0_stateless/01275_parallel_mv.sql.j2 diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference index 9021ae2bb1a..a9801e3b910 100644 --- a/tests/queries/0_stateless/01275_parallel_mv.reference +++ b/tests/queries/0_stateless/01275_parallel_mv.reference @@ -1,9 +1,23 @@ -- { echoOn } -set parallel_view_processing=1; -insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } + + + +select 'optimize_trivial_insert_select=0', 'max_insert_threads=0'; +optimize_trivial_insert_select=0 max_insert_threads=0 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=0, + optimize_trivial_insert_select=0, + max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) >= 8 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; -1 +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '0' and + Settings['optimize_trivial_insert_select'] = '0' and + Settings['max_insert_threads'] = '0'; +2 select count() from testX; 10 select count() from testXA; @@ -12,11 +26,22 @@ select count() from testXB; 0 select count() from testXC; 10 -set parallel_view_processing=0; -insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +select 'optimize_trivial_insert_select=0', 'max_insert_threads=16'; +optimize_trivial_insert_select=0 max_insert_threads=16 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=0, + optimize_trivial_insert_select=0, + max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) >= 5 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; -1 +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '0' and + Settings['optimize_trivial_insert_select'] = '0' and + Settings['max_insert_threads'] = '16'; +2 select count() from testX; 20 select count() from testXA; @@ -25,3 +50,147 @@ select count() from testXB; 0 select count() from testXC; 20 +select 'optimize_trivial_insert_select=1', 'max_insert_threads=0'; +optimize_trivial_insert_select=1 max_insert_threads=0 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=0, + optimize_trivial_insert_select=1, + max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '0' and + Settings['optimize_trivial_insert_select'] = '1' and + Settings['max_insert_threads'] = '0'; +2 +select count() from testX; +30 +select count() from testXA; +30 +select count() from testXB; +0 +select count() from testXC; +30 +select 'optimize_trivial_insert_select=1', 'max_insert_threads=16'; +optimize_trivial_insert_select=1 max_insert_threads=16 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=0, + optimize_trivial_insert_select=1, + max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '0' and + Settings['optimize_trivial_insert_select'] = '1' and + Settings['max_insert_threads'] = '16'; +2 +select count() from testX; +40 +select count() from testXA; +40 +select count() from testXB; +0 +select count() from testXC; +40 +select 'optimize_trivial_insert_select=0', 'max_insert_threads=0'; +optimize_trivial_insert_select=0 max_insert_threads=0 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=1, + optimize_trivial_insert_select=0, + max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '1' and + Settings['optimize_trivial_insert_select'] = '0' and + Settings['max_insert_threads'] = '0'; +5 +select count() from testX; +50 +select count() from testXA; +50 +select count() from testXB; +0 +select count() from testXC; +50 +select 'optimize_trivial_insert_select=0', 'max_insert_threads=16'; +optimize_trivial_insert_select=0 max_insert_threads=16 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=1, + optimize_trivial_insert_select=0, + max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '1' and + Settings['optimize_trivial_insert_select'] = '0' and + Settings['max_insert_threads'] = '16'; +5 +select count() from testX; +60 +select count() from testXA; +60 +select count() from testXB; +0 +select count() from testXC; +60 +select 'optimize_trivial_insert_select=1', 'max_insert_threads=0'; +optimize_trivial_insert_select=1 max_insert_threads=0 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=1, + optimize_trivial_insert_select=1, + max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '1' and + Settings['optimize_trivial_insert_select'] = '1' and + Settings['max_insert_threads'] = '0'; +5 +select count() from testX; +70 +select count() from testXA; +70 +select count() from testXB; +0 +select count() from testXC; +70 +select 'optimize_trivial_insert_select=1', 'max_insert_threads=16'; +optimize_trivial_insert_select=1 max_insert_threads=16 +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing=1, + optimize_trivial_insert_select=1, + max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '1' and + Settings['optimize_trivial_insert_select'] = '1' and + Settings['max_insert_threads'] = '16'; +5 +select count() from testX; +80 +select count() from testXA; +80 +select count() from testXB; +0 +select count() from testXC; +80 diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql b/tests/queries/0_stateless/01275_parallel_mv.sql deleted file mode 100644 index 27b8ef96e0b..00000000000 --- a/tests/queries/0_stateless/01275_parallel_mv.sql +++ /dev/null @@ -1,39 +0,0 @@ -set max_threads = 0; - -drop table if exists testX; -drop table if exists testXA; -drop table if exists testXB; -drop table if exists testXC; - -create table testX (A Int64) engine=MergeTree order by tuple(); - -create materialized view testXA engine=MergeTree order by tuple() as select sleep(1) from testX; -create materialized view testXB engine=MergeTree order by tuple() as select sleep(2), throwIf(A=1) from testX; -create materialized view testXC engine=MergeTree order by tuple() as select sleep(1) from testX; - --- { echoOn } -set parallel_view_processing=1; -insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } -system flush logs; -select length(thread_ids) >= 8 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; - -select count() from testX; -select count() from testXA; -select count() from testXB; -select count() from testXC; - -set parallel_view_processing=0; -insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } -system flush logs; -select length(thread_ids) >= 5 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; - -select count() from testX; -select count() from testXA; -select count() from testXB; -select count() from testXC; --- { echoOff } - -drop table testX; -drop view testXA; -drop view testXB; -drop view testXC; diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 b/tests/queries/0_stateless/01275_parallel_mv.sql.j2 new file mode 100644 index 00000000000..3b7c414a44e --- /dev/null +++ b/tests/queries/0_stateless/01275_parallel_mv.sql.j2 @@ -0,0 +1,47 @@ +-- avoid settings randomization by clickhouse-test +set max_threads = 0; + +drop table if exists testX; +drop table if exists testXA; +drop table if exists testXB; +drop table if exists testXC; + +create table testX (A Int64) engine=MergeTree order by tuple(); + +create materialized view testXA engine=MergeTree order by tuple() as select sleep(0.1) from testX; +create materialized view testXB engine=MergeTree order by tuple() as select sleep(0.2), throwIf(A=1) from testX; +create materialized view testXC engine=MergeTree order by tuple() as select sleep(0.1) from testX; + +-- { echoOn } +{% for parallel_view_processing in [0, 1] %} +{% for optimize_trivial_insert_select in [0, 1] %} +{% for max_insert_threads in [0, 16] %} +select 'optimize_trivial_insert_select={{ optimize_trivial_insert_select }}', 'max_insert_threads={{ max_insert_threads }}'; + +insert into testX select number from numbers(10) settings + log_queries=1, + parallel_view_processing={{ parallel_view_processing }}, + optimize_trivial_insert_select={{ optimize_trivial_insert_select }}, + max_insert_threads={{ max_insert_threads }}; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +system flush logs; +select arrayUniq(thread_ids) from system.query_log where + current_database = currentDatabase() and + type != 'QueryStart' and + query like '%insert into testX %' and + Settings['parallel_view_processing'] = '{{ parallel_view_processing }}' and + Settings['optimize_trivial_insert_select'] = '{{ optimize_trivial_insert_select }}' and + Settings['max_insert_threads'] = '{{ max_insert_threads }}'; + +select count() from testX; +select count() from testXA; +select count() from testXB; +select count() from testXC; +{% endfor %} +{% endfor %} +{% endfor %} +-- { echoOff } + +drop table testX; +drop view testXA; +drop view testXB; +drop view testXC; From 7427adb600633008af5688e394eb287f370e1d41 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 3 Jul 2022 09:24:09 +0300 Subject: [PATCH 399/408] tests: disable 01275_parallel_mv under S3 (since it has thread pool for writes) Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/01275_parallel_mv.sql.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 b/tests/queries/0_stateless/01275_parallel_mv.sql.j2 index 3b7c414a44e..6b17a141d3e 100644 --- a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 +++ b/tests/queries/0_stateless/01275_parallel_mv.sql.j2 @@ -1,3 +1,6 @@ +-- Tags: no-s3-storage +-- no-s3-storage: s3 has 20 more threads + -- avoid settings randomization by clickhouse-test set max_threads = 0; From d863f6ce1ece810c66ddf8bd89575825d3f2595f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 3 Jul 2022 15:20:05 +0300 Subject: [PATCH 400/408] tests: add no-backward-compatibility-check for 02351_Map_combinator_dist Signed-off-by: Azat Khuzhin --- tests/queries/0_stateless/02351_Map_combinator_dist.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02351_Map_combinator_dist.sql b/tests/queries/0_stateless/02351_Map_combinator_dist.sql index 937afa5480e..30816061338 100644 --- a/tests/queries/0_stateless/02351_Map_combinator_dist.sql +++ b/tests/queries/0_stateless/02351_Map_combinator_dist.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check:22.6 + -- https://github.com/ClickHouse/ClickHouse/issues/35359 -- sumMap From 838fd1e41e2ae0233f2cc40b0c7e51d47e2d14d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Sun, 3 Jul 2022 15:11:25 +0200 Subject: [PATCH 401/408] Fix replication after improper merge process --- src/Storages/StorageReplicatedMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 458b5aedf7d..ae9f7640f66 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -599,7 +599,6 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() auto zookeeper = getZooKeeper(); std::vector futures; - futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/quorum/parallel", String(), zkutil::CreateMode::Persistent)); /// These 4 nodes used to be created in createNewZookeeperNodes() and they were moved to createTable() /// This means that if the first replica creating the table metadata has an older version of CH (22.3 or previous) From a772a091491a08894e3606d44ec365b1d84f981c Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 4 Jul 2022 00:14:15 +0200 Subject: [PATCH 402/408] Fix --- src/IO/ReadBufferFromS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 53831e02cb2..cf19b6f1980 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -227,7 +227,7 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; - auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id, false); + auto object_size = S3::getObjectSize(client_ptr, bucket, key, version_id); file_size = object_size; return *file_size; From 19516c768f4bcdda14ec88508dbf2c2934e54b91 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 4 Jul 2022 10:15:27 +0200 Subject: [PATCH 403/408] Revert "Upload to S3 compressed self-extracting clickhouse" --- tests/ci/build_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 5dcc0e3e49f..3976e2ba916 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -231,7 +231,7 @@ def upload_master_static_binaries( return s3_path = "/".join((pr_info.base_ref, static_binary_name, "clickhouse")) - binary = os.path.join(build_output_path, "self-extracting", "clickhouse") + binary = os.path.join(build_output_path, "clickhouse") url = s3_helper.upload_build_file_to_s3(binary, s3_path) print(f"::notice ::Binary static URL: {url}") From 7388b6fb55da74c2b67c78197dd4e2bfee719306 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 18:52:59 +0200 Subject: [PATCH 404/408] Test and build clickhouse-diagnostics in release builds --- docker/packager/binary/Dockerfile | 11 +++++++++++ docker/packager/binary/build.sh | 15 +++++++++++++++ packages/clickhouse-common-static.yaml | 10 ++++++---- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index ab603cd9ea4..1dff4b1a2d4 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -104,6 +104,17 @@ RUN arch=${TARGETARCH:-amd64} \ && dpkg -i /tmp/nfpm.deb \ && rm /tmp/nfpm.deb +ARG GO_VERSION=1.18.3 +# We need go for clickhouse-diagnostics +RUN arch=${TARGETARCH:-amd64} \ + && curl -Lo /tmp/go.tgz "https://go.dev/dl/go${GO_VERSION}.linux-${arch}.tar.gz" \ + && tar -xzf /tmp/go.tgz -C /usr/local/ \ + && rm /tmp/go.tgz + +ENV PATH="$PATH:/usr/local/go/bin" +ENV GOPATH=/workdir/go +ENV GOCACHE=/workdir/ + RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index b8d11e9c293..270c93c105c 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -29,8 +29,21 @@ env if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root + if [ -z "$SANITIZER" ]; then + # We need to check if clickhouse-diagnostics is fine and build it + ( + cd /build/programs/diagnostics + make test-no-docker + GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build + mv clickhouse-diagnostics .. + ) + else + echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics + chmod +x /build/programs/clickhouse-diagnostics + fi fi + cache_status # clear cache stats ccache --zero-stats ||: @@ -81,6 +94,8 @@ if [ -n "$MAKE_DEB" ]; then # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 DESTDIR=/build/packages/root ninja $NINJA_FLAGS install + cp /build/programs/clickhouse-diagnostics /build/packages/root/usr/bin + cp /build/programs/clickhouse-diagnostics /output bash -x /build/packages/build fi diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml index 269d4318e5e..527b6a24703 100644 --- a/packages/clickhouse-common-static.yaml +++ b/packages/clickhouse-common-static.yaml @@ -29,12 +29,14 @@ description: | contents: - src: root/usr/bin/clickhouse dst: /usr/bin/clickhouse -- src: root/usr/bin/clickhouse-odbc-bridge - dst: /usr/bin/clickhouse-odbc-bridge -- src: root/usr/bin/clickhouse-library-bridge - dst: /usr/bin/clickhouse-library-bridge +- src: root/usr/bin/clickhouse-diagnostics + dst: /usr/bin/clickhouse-diagnostics - src: root/usr/bin/clickhouse-extract-from-config dst: /usr/bin/clickhouse-extract-from-config +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge - src: root/usr/share/bash-completion/completions dst: /usr/share/bash-completion/completions # docs From 664d43e34898106d4ddb40376e4b934154fb475f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 30 Jun 2022 23:20:24 +0200 Subject: [PATCH 405/408] Add disabling tag no_docker to some go tests --- programs/diagnostics/Makefile | 3 +++ programs/diagnostics/internal/platform/database/native_test.go | 2 ++ programs/diagnostics/internal/platform/manager_test.go | 2 ++ programs/diagnostics/internal/platform/utils/process_test.go | 2 ++ programs/diagnostics/internal/runner_test.go | 2 ++ 5 files changed, 11 insertions(+) diff --git a/programs/diagnostics/Makefile b/programs/diagnostics/Makefile index 0a12630ca6d..2e85002b871 100644 --- a/programs/diagnostics/Makefile +++ b/programs/diagnostics/Makefile @@ -40,6 +40,9 @@ vendor: ## Copy of all packages needed to support builds and tests in the vendor test: ## Run the tests of the project CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race `go list ./... | grep -v ./internal/platform/test` +test-no-docker: ## Don't run tests depending on dockerd + CLICKHOUSE_VERSION=$(CLICKHOUSE_VERSION) $(GOTEST) -v -race -tags no_docker `go list ./... | grep -v ./internal/platform/test` + lint-go: ## Use golintci-lint docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:latest-alpine golangci-lint run diff --git a/programs/diagnostics/internal/platform/database/native_test.go b/programs/diagnostics/internal/platform/database/native_test.go index 8c317ab7f12..4814310f182 100644 --- a/programs/diagnostics/internal/platform/database/native_test.go +++ b/programs/diagnostics/internal/platform/database/native_test.go @@ -1,3 +1,5 @@ +//go:build !no_docker + package database_test import ( diff --git a/programs/diagnostics/internal/platform/manager_test.go b/programs/diagnostics/internal/platform/manager_test.go index e63ec6af6a7..49efee49ce3 100644 --- a/programs/diagnostics/internal/platform/manager_test.go +++ b/programs/diagnostics/internal/platform/manager_test.go @@ -1,3 +1,5 @@ +//go:build !no_docker + package platform_test import ( diff --git a/programs/diagnostics/internal/platform/utils/process_test.go b/programs/diagnostics/internal/platform/utils/process_test.go index 0c7541f4abb..ed54d16cc72 100644 --- a/programs/diagnostics/internal/platform/utils/process_test.go +++ b/programs/diagnostics/internal/platform/utils/process_test.go @@ -1,3 +1,5 @@ +//go:build !no_docker + package utils_test import ( diff --git a/programs/diagnostics/internal/runner_test.go b/programs/diagnostics/internal/runner_test.go index 81fe9b70a2b..8cf29a140ec 100644 --- a/programs/diagnostics/internal/runner_test.go +++ b/programs/diagnostics/internal/runner_test.go @@ -1,3 +1,5 @@ +//go:build !no_docker + package internal_test import ( From 446ead1af464a666bdd3f1ce79fad42331ff5c93 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 1 Jul 2022 00:26:56 +0200 Subject: [PATCH 406/408] Fix typos issues it clickhouse-diagnostics --- programs/diagnostics/README.md | 4 ++-- programs/diagnostics/internal/outputs/file/simple.go | 2 +- programs/diagnostics/internal/platform/data/file.go | 4 ++-- programs/diagnostics/internal/platform/utils/file.go | 5 +++-- programs/diagnostics/internal/runner.go | 2 +- utils/check-style/codespell-ignore-lines.list | 1 + 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/programs/diagnostics/README.md b/programs/diagnostics/README.md index b69a6e3a996..f800bb0648e 100644 --- a/programs/diagnostics/README.md +++ b/programs/diagnostics/README.md @@ -9,7 +9,7 @@ This tool provides a means of obtaining a diagnostic bundle from a ClickHouse in - **No local dependencies** to run. We compile to a platform-independent binary, hence Go. - **Minimize resource overhead**. Improvements always welcome. - **Extendable framework**. At its core, the tool provides collectors and outputs. Collectors are independent and are responsible for collecting a specific dataset e.g. system configuration. Outputs produce the diagnostic bundle in a specific format. It should be trivial to add both for contributors. See [Collectors](#collectors) and [Outputs](#outputs) for more details. -- **Convertable output formats**. Outputs produce diagnostic bundles in different formats e.g. archive, simple report etc. Where possible, it should be possible to convert between these formats. For example, an administrator may provide a bundle as an archive to their support provider who in turn wishes to visualise this as a report or even in ClickHouse itself... +- **Convertible output formats**. Outputs produce diagnostic bundles in different formats e.g. archive, simple report etc. Where possible, it should be possible to convert between these formats. For example, an administrator may provide a bundle as an archive to their support provider who in turn wishes to visualise this as a report or even in ClickHouse itself... - **Something is better than nothing**. Collectors execute independently. We never fail a collection because one fails - preferring to warn the user only. There are good reasons for a collector failure e.g. insufficient permissions or missing data. - **Execute anywhere** - Ideally, this tool is executed on a ClickHouse host. Some collectors e.g. configuration file collection or system information, rely on this. However, collectors will obtain as much information remotely from the database as possible if executed remotely from the cluster - warning where collection fails. **We do currently require ClickHouse to be running, connecting over the native port**. @@ -25,7 +25,7 @@ The `collect` command allows the collection of a diagnostic bundle. In its simpl clickhouse-diagnostics collect ``` -This will use the default collectors and the simple output. This output produces a timestamped archive bundle in `gz` format in a sub folder named after the host. This folder name can be controlled via the parameter `--id` or configured directly for the simple output parameter `output.simple.folder` (this allows a specific diretory to be specified). +This will use the default collectors and the simple output. This output produces a timestamped archive bundle in `gz` format in a sub folder named after the host. This folder name can be controlled via the parameter `--id` or configured directly for the simple output parameter `output.simple.folder` (this allows a specific directory to be specified). Collectors, Outputs and ClickHouse connection credentials can be specified as shown below: diff --git a/programs/diagnostics/internal/outputs/file/simple.go b/programs/diagnostics/internal/outputs/file/simple.go index f91ec9f74ee..63847b3addd 100644 --- a/programs/diagnostics/internal/outputs/file/simple.go +++ b/programs/diagnostics/internal/outputs/file/simple.go @@ -149,7 +149,7 @@ func writeDatabaseFrame(frameId string, frame data.Frame, baseDir string) ([]str errs = append(errs, errors.Wrapf(err, "unable to write columns for frame %s", frameId)) return []string{}, errs } - // we don't collect an error for every line here like configs and logs - could mean alot of unnecessary noise + // we don't collect an error for every line here like configs and logs - could mean a lot of unnecessary noise for { values, ok, err := frame.Next() if err != nil { diff --git a/programs/diagnostics/internal/platform/data/file.go b/programs/diagnostics/internal/platform/data/file.go index 19bf5884876..9760b4b6906 100644 --- a/programs/diagnostics/internal/platform/data/file.go +++ b/programs/diagnostics/internal/platform/data/file.go @@ -303,7 +303,7 @@ func (x XmlConfigFile) FindLogPaths() ([]string, error) { func (x XmlConfigFile) FindIncludedConfig() (ConfigFile, error) { if x.Included { - //cant recurse + //can't recurse return XmlConfigFile{}, nil } config, err := x.UnmarshallConfig() @@ -385,7 +385,7 @@ func (y YamlConfigFile) FindLogPaths() ([]string, error) { func (y YamlConfigFile) FindIncludedConfig() (ConfigFile, error) { if y.Included { - //cant recurse + //can't recurse return YamlConfigFile{}, nil } inputFile, err := ioutil.ReadFile(y.Path) diff --git a/programs/diagnostics/internal/platform/utils/file.go b/programs/diagnostics/internal/platform/utils/file.go index 608e45be74a..71af4b32658 100644 --- a/programs/diagnostics/internal/platform/utils/file.go +++ b/programs/diagnostics/internal/platform/utils/file.go @@ -2,11 +2,12 @@ package utils import ( "fmt" - "github.com/pkg/errors" "io" "io/fs" "os" "path/filepath" + + "github.com/pkg/errors" ) func FileExists(name string) (bool, error) { @@ -64,7 +65,7 @@ func CopyFile(sourceFilename string, destFilename string) error { return err } -// patterns passed are an OR - any can be satisified and the file will be listed +// patterns passed are an OR - any can be satisfied and the file will be listed func ListFilesInDirectory(directory string, patterns []string) ([]string, []error) { var files []string diff --git a/programs/diagnostics/internal/runner.go b/programs/diagnostics/internal/runner.go index 6960cf1cd23..9386a1d178b 100644 --- a/programs/diagnostics/internal/runner.go +++ b/programs/diagnostics/internal/runner.go @@ -96,7 +96,7 @@ func output(config *runConfiguration, bundles map[string]*data.DiagnosticBundle) return err } frameErrors, err := output.Write(config.id, bundles, config.outputConfig) - // we report over failing hard on frame errors - upto the output to determine what is fatal via error + // we report over failing hard on frame errors - up to the output to determine what is fatal via error for _, fError := range frameErrors.Errors { log.Warn().Msgf("failure to write frame in output %s - %s", config.output, fError) } diff --git a/utils/check-style/codespell-ignore-lines.list b/utils/check-style/codespell-ignore-lines.list index 7c2959e9468..78b0639989f 100644 --- a/utils/check-style/codespell-ignore-lines.list +++ b/utils/check-style/codespell-ignore-lines.list @@ -4,3 +4,4 @@ The TRE regular expression implementation (src/regex/reg* and src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed pullRequests(first: {min_page_size} baseRefName: "{base}" headRefName: "{head}") {{ uint64_t time_to_wait = nanoseconds * timebase_info.denom / timebase_info.numer; + REPLACE_ME From 0e4c65a378f08622e24a5d570754a06e18ab24e8 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 4 Jul 2022 12:57:55 +0200 Subject: [PATCH 407/408] Update tests/integration/test_postgresql_database_engine/test.py Co-authored-by: alesapin --- tests/integration/test_postgresql_database_engine/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index f6cfa1b230f..5619c551c71 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -345,7 +345,7 @@ def test_postgres_database_old_syntax(started_cluster): ) create_postgres_table(cursor, "test_table") assert "test_table" in node1.query("SHOW TABLES FROM postgres_database") - cursor.execute(f"DROP TABLE test_table ") + cursor.execute(f"DROP TABLE test_table") node1.query("DROP DATABASE IF EXISTS postgres_database;") From a7604a19e92507b6fcade27c9c8a704ab8febd93 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 4 Jul 2022 14:23:30 +0300 Subject: [PATCH 408/408] Update 01710_projection_fetch_long.sql --- tests/queries/0_stateless/01710_projection_fetch_long.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_projection_fetch_long.sql b/tests/queries/0_stateless/01710_projection_fetch_long.sql index fd12b84c817..29effc53745 100644 --- a/tests/queries/0_stateless/01710_projection_fetch_long.sql +++ b/tests/queries/0_stateless/01710_projection_fetch_long.sql @@ -1,4 +1,4 @@ --- Tags: long, no-s3-storage +-- Tags: long, no-s3-storage, no-backward-compatibility-check drop table if exists tp_1; drop table if exists tp_2;